From ee09313b155a3ae29868e0bcb4513242eba1e169 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 22 Jul 2024 12:58:49 +0200 Subject: [PATCH 01/33] Init parallelization --- .../memory_related/decommit_code.rs | 106 ++++---- .../memory_related/ecrecover.rs | 74 +++--- .../keccak256_round_function.rs | 76 ++++-- .../individual_circuits/memory_related/mod.rs | 250 ++++++++++++++++++ .../memory_related/ram_permutation.rs | 83 +++--- .../memory_related/secp256r1_verify.rs | 73 ++--- .../memory_related/sha256_round_function.rs | 74 ++++-- src/witness/oracle.rs | 133 ++++++++-- 8 files changed, 615 insertions(+), 254 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/decommit_code.rs b/src/witness/individual_circuits/memory_related/decommit_code.rs index e77bb879..2a156fe8 100644 --- a/src/witness/individual_circuits/memory_related/decommit_code.rs +++ b/src/witness/individual_circuits/memory_related/decommit_code.rs @@ -26,6 +26,34 @@ pub(crate) fn decommitter_memory_queries_amount( .fold(0, |inner, (_, writes)| inner + writes.len()) } +pub(crate) fn decommitter_memory_queries(deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>) -> Vec { + let mut result = vec![]; + for (query, writes) in deduplicated_decommit_requests_with_data.iter() { + assert!(query.is_fresh); + + // now feed the queries into it + let as_queries_it = writes + .iter() + .enumerate() + .map(|(idx, el)| MemoryQuery { + timestamp: query.timestamp, + location: zk_evm::aux_structures::MemoryLocation { + memory_type: zk_evm::abstractions::MemoryType::Code, + page: query.memory_page, + index: MemoryIndex(idx as u32), + }, + rw_flag: true, + value: *el, + value_is_pointer: false, + }); + + // and plain test memory queues + result.extend(as_queries_it); + } + + result +} + pub(crate) struct DecommiterCircuitProcessingInputs { pub deduplicated_decommittment_queue_simulator: DecommittmentQueueSimulator, pub deduplicated_decommittment_queue_states: Vec>, @@ -37,25 +65,26 @@ pub(crate) fn compute_decommitter_circuit_snapshots< R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, + implicit_memory_queries: &ImplicitMemoryQueries, + implicit_memory_states: &ImplicitMemoryStates, memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, decommiter_circuit_inputs: DecommiterCircuitProcessingInputs, round_function: &R, decommiter_circuit_capacity: usize, ) -> Vec> { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + implicit_memory_queries.decommitter_memory_queries.len(), + implicit_memory_states.decommitter_memory_states.len() ); + + let memory_simulator_before = &implicit_memory_states.decommitter_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries, + memory_simulator_before.num_items as usize ); + let start_idx_for_memory_accumulator = 0; - let start_idx_for_memory_accumulator = implicit_memory_artifacts.memory_queue_states.len(); - - let initial_memory_queue_state = &memory_queue_simulator.take_sponge_like_queue_state(); + let initial_memory_queue_state = &memory_simulator_before.take_sponge_like_queue_state(); let DecommiterCircuitProcessingInputs { deduplicated_decommit_requests_with_data, @@ -70,46 +99,6 @@ pub(crate) fn compute_decommitter_circuit_snapshots< "we must have some decommitment requests" ); - for (query, writes) in deduplicated_decommit_requests_with_data.iter() { - assert!(query.is_fresh); - - // now feed the queries into it - let as_queries: Vec<_> = writes - .iter() - .cloned() - .enumerate() - .map(|(idx, el)| MemoryQuery { - timestamp: query.timestamp, - location: zk_evm::aux_structures::MemoryLocation { - memory_type: zk_evm::abstractions::MemoryType::Code, - page: query.memory_page, - index: MemoryIndex(idx as u32), - }, - rw_flag: true, - value: el, - value_is_pointer: false, - }) - .collect(); - - // fill up the memory queue - for query in as_queries.iter() { - let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, round_function); - - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - } - - // and plain test memory queues - implicit_memory_artifacts.memory_queries.extend(as_queries); - } - - assert_eq!( - implicit_memory_artifacts.memory_queries.len(), - implicit_memory_artifacts.memory_queue_states.len() - ); - // our simulator is simple: it will try to take an element from the queue, run some number of rounds, and compare the results let mut results: Vec> = vec![]; @@ -191,8 +180,8 @@ pub(crate) fn compute_decommitter_circuit_snapshots< let wintess_state = if start_idx_for_memory_accumulator + memory_queue_state_offset == 0 { memory_queue_states_accumulator.last().unwrap() } else { - implicit_memory_artifacts - .memory_queue_states + implicit_memory_states + .decommitter_memory_states .get(start_idx_for_memory_accumulator + memory_queue_state_offset - 1) .unwrap() }; @@ -407,8 +396,8 @@ pub(crate) fn compute_decommitter_circuit_snapshots< let wintess_state = if start_idx_for_memory_accumulator + memory_queue_state_offset == 0 { memory_queue_states_accumulator.last().unwrap() } else { - implicit_memory_artifacts - .memory_queue_states + implicit_memory_states + .decommitter_memory_states .get(start_idx_for_memory_accumulator + memory_queue_state_offset - 1) .unwrap() }; @@ -449,13 +438,12 @@ pub(crate) fn compute_decommitter_circuit_snapshots< } } + let memory_simulator_after = &implicit_memory_states.decommitter_simulator_snapshots[1]; + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.decommitter_memory_queries.len(), + memory_simulator_after.num_items as usize ); results diff --git a/src/witness/individual_circuits/memory_related/ecrecover.rs b/src/witness/individual_circuits/memory_related/ecrecover.rs index e3f55daf..168fd116 100644 --- a/src/witness/individual_circuits/memory_related/ecrecover.rs +++ b/src/witness/individual_circuits/memory_related/ecrecover.rs @@ -3,6 +3,7 @@ use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQ use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::ecrecover::ECRecoverRoundWitness; use crate::zkevm_circuits::base_structures::log_query::*; use crate::zkevm_circuits::ecrecover::*; @@ -20,6 +21,21 @@ pub(crate) fn ecrecover_memory_queries_amount( }) } +pub(crate) fn ecrecover_memory_queries(ecrecover_witnesses: &Vec<(u32, LogQuery_, ECRecoverRoundWitness)>) -> Vec { + let mut ecrecover_memory_queries = Vec::with_capacity(ecrecover_memory_queries_amount(&ecrecover_witnesses)); + + for (_cycle, _query, witness) in ecrecover_witnesses.iter() { + let initial_memory_len = ecrecover_memory_queries.len(); + + // we read, then write + ecrecover_memory_queries.extend_from_slice(&witness.reads); + ecrecover_memory_queries.extend_from_slice(&witness.writes); + + assert_eq!(ecrecover_memory_queries.len() - initial_memory_len, 6); + } + ecrecover_memory_queries +} + // we want to simulate splitting of data into many separate instances of the same circuit. // So we basically need to reconstruct the FSM state on input/output, and passthrough data. // In practice the only difficulty is buffer state, everything else is provided by out-of-circuit VM @@ -29,9 +45,8 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + implicit_memory_queries: &ImplicitMemoryQueries, + implicit_memory_states: &ImplicitMemoryStates, ecrecover_witnesses: Vec<(u32, LogQuery_, ECRecoverRoundWitness)>, ecrecover_queries: Vec, mut demuxed_ecrecover_queue: LogQueueStates, @@ -39,12 +54,17 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< round_function: &R, ) -> Vec> { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + implicit_memory_queries.ecrecover_memory_queries.len(), + implicit_memory_states.ecrecover_memory_states.len() ); + + let memory_simulator_before = &implicit_memory_states.ecrecover_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.decommitter_memory_queries.len() + + implicit_memory_queries.keccak256_memory_queries.len() + + implicit_memory_queries.sha256_memory_queries.len(), + memory_simulator_before.num_items as usize ); // split into aux witness, don't mix with the memory @@ -94,9 +114,11 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< let mut memory_read_witnesses = vec![]; let mut starting_request_idx = 0; - let mut memory_queue_input_state = memory_queue_simulator.take_sponge_like_queue_state(); + let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); + let mut memory_queue_states_it = implicit_memory_states.ecrecover_memory_states.iter(); + for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() .zip(round_function_witness.into_iter()) @@ -105,7 +127,6 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< let _ = demuxed_ecrecover_queue .simulator .pop_and_output_intermediate_data(round_function); - let initial_memory_len = memory_queue_simulator.num_items; let mut memory_reads_per_request = vec![]; @@ -116,6 +137,7 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< let mut precompile_request = precompile_abi_in_log(request); let is_last_request = request_idx == num_requests - 1; + let mut amount_of_queries = 0; // we have 4 reads for (_query_index, read) in round_witness.reads.into_iter().enumerate() { let read_query = memory_queries_it.next().unwrap(); @@ -123,15 +145,10 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< assert!(read_query.rw_flag == false); memory_reads_per_request.push(read_query.value); - implicit_memory_artifacts.memory_queries.push(read); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(read, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; + amount_of_queries += 1; } // and 2 writes @@ -140,18 +157,13 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< assert!(write == write_query); assert!(write_query.rw_flag == true); - implicit_memory_artifacts.memory_queries.push(write); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(write, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.output_memory_offset += 1; + amount_of_queries += 1; } - assert_eq!(memory_queue_simulator.num_items - initial_memory_len, 6); + assert_eq!(amount_of_queries, 6); round_counter += 1; if round_counter == num_rounds_per_circuit || is_last_request { @@ -230,13 +242,15 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< } } + let memory_simulator_after = &implicit_memory_states.ecrecover_simulator_snapshots[1]; + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.decommitter_memory_queries.len() + + implicit_memory_queries.keccak256_memory_queries.len() + + implicit_memory_queries.sha256_memory_queries.len() + + implicit_memory_queries.ecrecover_memory_queries.len(), + memory_simulator_after.num_items as usize ); result diff --git a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs index d84c8acd..7be25fb3 100644 --- a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs @@ -3,6 +3,7 @@ use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQ use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::keccak256::Keccak256RoundWitness; use crate::zkevm_circuits::base_structures::log_query::*; use crate::zkevm_circuits::keccak256_round_function::{ @@ -42,6 +43,33 @@ pub(crate) fn keccak256_memory_queries_amount( result } +pub(crate) fn keccak256_memory_queries(keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>) -> Vec { + let mut keccak_256_memory_queries = Vec::with_capacity(keccak256_memory_queries_amount(&keccak_round_function_witnesses)); + + for (_cycle, _query, witness) in keccak_round_function_witnesses.iter() { + for el in witness.iter() { + let Keccak256RoundWitness { + new_request: _, + reads, + writes, + } = el; + + // we read, then write + reads.iter().for_each(|read| { + if let Some(read) = read { + keccak_256_memory_queries.push(*read); + } + }); + + if let Some(writes) = writes.as_ref() { + keccak_256_memory_queries.extend_from_slice(writes); + } + } + } + + keccak_256_memory_queries +} + #[derive(Derivative)] #[derivative(Clone, Copy, Debug, PartialEq, Eq)] pub enum Keccak256PrecompileState { @@ -60,9 +88,8 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + implicit_memory_queries: &ImplicitMemoryQueries, + implicit_memory_states: &ImplicitMemoryStates, keccak_round_function_witnesses: Vec<(u32, LogQuery_, Vec)>, keccak_precompile_queries: Vec, mut demuxed_keccak_precompile_queue: LogQueueStates, @@ -70,12 +97,15 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< round_function: &R, ) -> Vec> { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + implicit_memory_queries.keccak256_memory_queries.len(), + implicit_memory_states.keccak256_memory_states.len() ); + + let memory_simulator_before = &implicit_memory_states.keccak256_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.decommitter_memory_queries.len(), + memory_simulator_before.num_items as usize ); // split into aux witness, don't mix with the memory @@ -153,7 +183,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let mut memory_queries_it = memory_queries.into_iter(); let mut precompile_state = Keccak256PrecompileState::GetRequestFromQueue; - let mut memory_queue_input_state = memory_queue_simulator.take_sponge_like_queue_state(); + let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); let mut memory_reads_per_circuit = VecDeque::new(); @@ -210,6 +240,8 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< precompile_state = Keccak256PrecompileState::RunPaddingRound; } + let mut memory_queue_states_it = implicit_memory_states.keccak256_memory_states.iter(); + for (round_idx, round) in round_witness.into_iter().enumerate() { // we proceed the request as long as we can if round_idx == 0 { @@ -263,13 +295,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< assert_eq!(read, read_query); memory_reads_per_circuit.push_back(read_query.value); - implicit_memory_artifacts.memory_queries.push(read); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(read, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); input_buffer.fill_with_bytes( &bytes32_buffer, @@ -319,13 +345,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let write_query = memory_queries_it.next().unwrap(); assert_eq!(write, write_query); - implicit_memory_artifacts.memory_queries.push(write); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(write, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); if is_last_request { precompile_state = Keccak256PrecompileState::Finished; @@ -482,13 +502,13 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< } } + let memory_simulator_after = &implicit_memory_states.keccak256_simulator_snapshots[1]; + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.decommitter_memory_queries.len() + + implicit_memory_queries.keccak256_memory_queries.len(), + memory_simulator_after.num_items as usize ); result diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index a2b8c357..12eea6b7 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -1,3 +1,9 @@ +use decommit_code::decommitter_memory_queries; +use ecrecover::ecrecover_memory_queries; +use keccak256_round_function::keccak256_memory_queries; +use secp256r1_verify::secp256r1_memory_queries; +use sha256_round_function::sha256_memory_queries; + use super::*; use crate::ethereum_types::U256; @@ -8,11 +14,14 @@ use crate::witness::individual_circuits::memory_related::secp256r1_verify::secp2 use crate::witness::individual_circuits::memory_related::sha256_round_function::sha256_memory_queries_amount; use crate::zk_evm::aux_structures::DecommittmentQuery; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::ecrecover::ECRecoverRoundWitness; use crate::zk_evm::zk_evm_abstractions::precompiles::keccak256::Keccak256RoundWitness; use crate::zk_evm::zk_evm_abstractions::precompiles::secp256r1_verify::Secp256r1VerifyRoundWitness; use crate::zk_evm::zk_evm_abstractions::precompiles::sha256::Sha256RoundWitness; +use circuit_definitions::encodings::memory_query::MemoryQueueState; + pub(crate) mod decommit_code; pub(crate) mod ecrecover; pub(crate) mod keccak256_round_function; @@ -34,3 +43,244 @@ pub(crate) fn amount_of_implicit_memory_queries( + secp256r1_memory_queries_amount(secp256r1_verify_witnesses) + sha256_memory_queries_amount(sha256_round_function_witnesses) } + +#[derive(Clone)] +pub(crate) struct ImplicitMemoryQueries { + decommitter_memory_queries: Vec, + ecrecover_memory_queries: Vec, + keccak256_memory_queries: Vec, + secp256r1_memory_queries: Vec, + sha256_memory_queries: Vec, +} + +impl ImplicitMemoryQueries { + pub fn amount_of_queries(&self) -> usize { + self.decommitter_memory_queries.len() + + self.ecrecover_memory_queries.len() + + self.keccak256_memory_queries.len() + + self.secp256r1_memory_queries.len() + + self.sha256_memory_queries.len() + } + + fn get_vector(&self, index: usize) -> Option<&Vec> { + match index { + 0 => Some(&self.decommitter_memory_queries), + 1 => Some(&self.keccak256_memory_queries), + 2 => Some(&self.sha256_memory_queries), + 3 => Some(&self.ecrecover_memory_queries), + 4 => Some(&self.secp256r1_memory_queries), + _ => None + } + } + + pub fn iter(&self) -> ImplicitMemoryQueriesIter { + ImplicitMemoryQueriesIter { + inner: &self, + circuit_index: 0, + iter: self.decommitter_memory_queries.iter() + } + } +} + +pub struct ImplicitMemoryQueriesIter<'a> { + inner: &'a ImplicitMemoryQueries, + circuit_index: usize, + iter: Iter<'a, MemoryQuery>, +} + +impl<'a> Iterator for ImplicitMemoryQueriesIter<'a> { + type Item = &'a MemoryQuery; + + fn next(&mut self) -> Option { + let mut next = self.iter.next(); + + while next.is_none() { + self.circuit_index += 1; + let inner_vec = &self.inner.get_vector(self.circuit_index); + if inner_vec.is_none() { + return None; + } + + self.iter = inner_vec.unwrap().iter(); + next = self.iter.next(); + } + + next + } +} + +use crate::witness::oracle::PrecompilesInputData; +pub fn get_implicit_memory_queries( + deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>, + precompiles_inputs: &PrecompilesInputData +) -> ImplicitMemoryQueries{ + ImplicitMemoryQueries { + decommitter_memory_queries: decommitter_memory_queries(deduplicated_decommit_requests_with_data), + ecrecover_memory_queries: ecrecover_memory_queries(&precompiles_inputs.ecrecover_witnesses), + keccak256_memory_queries: keccak256_memory_queries(&precompiles_inputs.keccak_round_function_witnesses), + secp256r1_memory_queries: secp256r1_memory_queries(&precompiles_inputs.secp256r1_verify_witnesses), + sha256_memory_queries: sha256_memory_queries(&precompiles_inputs.sha256_round_function_witnesses), + } +} + + +pub(crate) struct SimulatorSnapshot { + pub head: [F; SW], + pub tail: [F; SW], + pub num_items: u32, +} +use crate::boojum::gadgets::queue::QueueStateWitness; +use crate::boojum::gadgets::queue::QueueTailStateWitness; + +impl SimulatorSnapshot { + pub fn take_sponge_like_queue_state(&self) -> QueueStateWitness { + let result = QueueStateWitness { + head: self.head, + tail: QueueTailStateWitness { + tail: self.tail, + length: self.num_items, + }, + }; + + result + } +} + +#[derive(Default)] +pub(crate) struct ImplicitMemoryStates { + decommitter_simulator_snapshots: Vec>, + decommitter_memory_states: Vec>, + ecrecover_simulator_snapshots: Vec>, + ecrecover_memory_states: Vec>, + keccak256_simulator_snapshots: Vec>, + keccak256_memory_states: Vec>, + secp256r1_simulator_snapshots: Vec>, + secp256r1_memory_states: Vec>, + sha256_simulator_snapshots: Vec>, + sha256_memory_states: Vec>, +} + +impl ImplicitMemoryStates { + pub fn amount_of_states(&self) -> usize { + self.decommitter_memory_states.len() + + self.ecrecover_memory_states.len() + + self.keccak256_memory_states.len() + + self.secp256r1_memory_states.len() + + self.sha256_memory_states.len() + } + + fn get_vectors(self) -> [Vec>; 5] { + [ + self.decommitter_memory_states, + self.keccak256_memory_states, + self.sha256_memory_states, + self.ecrecover_memory_states, + self.secp256r1_memory_states + ] + } + + fn into_iter(self) -> ImplicitMemoryStatesIntoIter { + let mut outer_iter = self.get_vectors().into_iter(); + let last_vector_iter = outer_iter.next().unwrap().into_iter(); + ImplicitMemoryStatesIntoIter { + last_vector: last_vector_iter, + outer_iter + } + } +} + +use core::slice::Iter; +use core::array::IntoIter as ArrayIntoIter; +use std::vec::IntoIter as VecIntoIter; +pub struct ImplicitMemoryStatesIntoIter { + last_vector: VecIntoIter>, + outer_iter: ArrayIntoIter>, 5>, +} + +impl Iterator for ImplicitMemoryStatesIntoIter { + type Item = MemoryQueueState; + + fn next(&mut self) -> Option { + + let mut next = self.last_vector.next(); + + while next.is_none() { + let new_last_vector = self.outer_iter.next(); + if new_last_vector.is_none() { + return None; + } + self.last_vector = new_last_vector.unwrap().into_iter(); + next = self.last_vector.next(); + } + + next + } +} + +use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; + +fn get_simulator_snapshot (memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator) -> SimulatorSnapshot { + SimulatorSnapshot{ + head: memory_queue_simulator.head, + tail: memory_queue_simulator.tail, + num_items: memory_queue_simulator.num_items + } +} + +pub(crate) fn simulate_implicit_memory_queues< +F: SmallField, +R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, +>( + memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + implicit_memory_queries: &ImplicitMemoryQueries, + round_function: R +) -> ImplicitMemoryStates { + let mut implicit_memory_states = ImplicitMemoryStates::default(); + + implicit_memory_states.decommitter_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + for query in implicit_memory_queries.decommitter_memory_queries.iter() { + let (_old_tail, intermediate_info) = + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + + implicit_memory_states.decommitter_memory_states.push(intermediate_info); + } + implicit_memory_states.decommitter_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + + implicit_memory_states.keccak256_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + for query in implicit_memory_queries.keccak256_memory_queries.iter() { + let (_old_tail, intermediate_info) = + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + + implicit_memory_states.keccak256_memory_states.push(intermediate_info); + } + implicit_memory_states.keccak256_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + + implicit_memory_states.sha256_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + for query in implicit_memory_queries.sha256_memory_queries.iter() { + let (_old_tail, intermediate_info) = + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + + implicit_memory_states.sha256_memory_states.push(intermediate_info); + } + implicit_memory_states.sha256_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + + implicit_memory_states.ecrecover_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + for query in implicit_memory_queries.ecrecover_memory_queries.iter() { + let (_old_tail, intermediate_info) = + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + + implicit_memory_states.ecrecover_memory_states.push(intermediate_info); + } + implicit_memory_states.ecrecover_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + + implicit_memory_states.secp256r1_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + for query in implicit_memory_queries.secp256r1_memory_queries.iter() { + let (_old_tail, intermediate_info) = + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + + implicit_memory_states.secp256r1_memory_states.push(intermediate_info); + } + implicit_memory_states.secp256r1_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + + implicit_memory_states +} diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index fd775395..38253027 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -38,9 +38,12 @@ pub(crate) fn compute_ram_circuit_snapshots< QSCB: FnMut(u64, RecursionQueueSimulator, Vec>), >( memory_queries: &Vec<(u32, MemoryQuery)>, - implicit_memory_artifacts: ImplicitMemoryArtifacts, mut memory_queue_states_accumulator: LastPerCircuitAccumulator>, + sorted_memory_queue_states_accumulator: LastPerCircuitAccumulator>, + implicit_memory_queries: ImplicitMemoryQueries, + implicit_memory_states: ImplicitMemoryStates, memory_queue_simulator: MemoryQueuePerCircuitSimulator, + sorted_memory_queries_simulator: MemoryQueuePerCircuitSimulator, round_function: &RoundFunction, num_non_deterministic_heap_queries: usize, per_circuit_capacity: usize, @@ -52,16 +55,37 @@ pub(crate) fn compute_ram_circuit_snapshots< FirstAndLastCircuitWitness>, Vec>, ) { - assert_eq!(memory_queries.len(), memory_queue_states_accumulator.len()); - assert_eq!( - implicit_memory_artifacts.memory_queries.len(), - implicit_memory_artifacts.memory_queue_states.len() + implicit_memory_queries.amount_of_queries(), + implicit_memory_states.amount_of_states() ); // including additional queries from precompiles let total_amount_of_queries = - memory_queries.len() + implicit_memory_artifacts.memory_queries.len(); + memory_queries.len() + implicit_memory_queries.amount_of_queries(); + + assert_eq!( + memory_queries.len(), + memory_queue_states_accumulator.len() + ); + + + // push implicit queries + + memory_queue_states_accumulator.reserve_exact_flat(implicit_memory_states.amount_of_states()); + for state in implicit_memory_states.into_iter() { + memory_queue_states_accumulator.push(state); + } + + assert_eq!( + total_amount_of_queries, + memory_queue_states_accumulator.len() + ); + + assert_eq!( + total_amount_of_queries, + sorted_memory_queue_states_accumulator.len() + ); assert!( total_amount_of_queries > 0, @@ -71,11 +95,6 @@ pub(crate) fn compute_ram_circuit_snapshots< let amount_of_circuits = (total_amount_of_queries + per_circuit_capacity - 1) / per_circuit_capacity; - memory_queue_states_accumulator - .reserve_exact_flat(implicit_memory_artifacts.memory_queue_states.len()); - for state in implicit_memory_artifacts.memory_queue_states.into_iter() { - memory_queue_states_accumulator.push(state); - } let unsorted_memory_queue_chunk_final_states = memory_queue_states_accumulator.into_circuits(); assert_eq!( @@ -83,47 +102,7 @@ pub(crate) fn compute_ram_circuit_snapshots< amount_of_circuits ); - let mut sorted_memory_queue_chunk_final_states = Vec::with_capacity(amount_of_circuits); - - let mut sorted_memory_queries_simulator = - MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( - per_circuit_capacity, - memory_queries.len() + implicit_memory_artifacts.memory_queries.len(), - )); - { - let mut sorted_memory_queries_accumulated: Vec<&MemoryQuery> = memory_queries - .iter() - .map(|(_, query)| query) - .chain(implicit_memory_artifacts.memory_queries.iter()) - .collect(); - - // sort by memory location, and then by timestamp - sorted_memory_queries_accumulated.par_sort_by(|a, b| match a.location.cmp(&b.location) { - Ordering::Equal => a.timestamp.cmp(&b.timestamp), - a @ _ => a, - }); - - // those two thins are parallelizable, and can be internally parallelized too - - // now we can finish reconstruction of each sorted and unsorted memory queries - - // reconstruct sorted one in full - - for chunk in sorted_memory_queries_accumulated.chunks(per_circuit_capacity) { - let intermediate_info = chunk - .iter() - .map(|query| { - let (_, _intermediate_info) = sorted_memory_queries_simulator - .push_and_output_intermediate_data(**query, round_function); - _intermediate_info - }) - .last() - .unwrap(); - sorted_memory_queue_chunk_final_states.push(intermediate_info); - } - } - - drop(implicit_memory_artifacts.memory_queries); + let sorted_memory_queue_chunk_final_states = sorted_memory_queue_states_accumulator.into_circuits(); assert_eq!( unsorted_memory_queue_chunk_final_states.len(), diff --git a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs index b5314980..787a805f 100644 --- a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs +++ b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs @@ -3,6 +3,7 @@ use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQ use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::secp256r1_verify::Secp256r1VerifyRoundWitness; use crate::zkevm_circuits::base_structures::log_query::*; use crate::zkevm_circuits::secp256r1_verify::*; @@ -20,6 +21,21 @@ pub(crate) fn secp256r1_memory_queries_amount( }) } +pub(crate) fn secp256r1_memory_queries(secp256r1_verify_witnesses: &Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>) -> Vec { + let mut secp256r1_memory_queries = Vec::with_capacity(secp256r1_memory_queries_amount(&secp256r1_verify_witnesses)); + + for (_cycle, _query, witness) in secp256r1_verify_witnesses.iter() { + let initial_memory_len = secp256r1_memory_queries.len(); + // we read, then write + secp256r1_memory_queries.extend_from_slice(&witness.reads); + secp256r1_memory_queries.extend_from_slice(&witness.writes); + + assert_eq!(secp256r1_memory_queries.len() - initial_memory_len, 7); + } + + secp256r1_memory_queries +} + // we want to simulate splitting of data into many separate instances of the same circuit. // So we basically need to reconstruct the FSM state on input/output, and passthrough data. // In practice the only difficulty is buffer state, everything else is provided by out-of-circuit VM @@ -29,9 +45,8 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + implicit_memory_queries: &ImplicitMemoryQueries, + implicit_memory_states: &ImplicitMemoryStates, secp256r1_verify_witnesses: Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>, secp256r1_verify_queries: Vec, mut demuxed_secp256r1_verify_queue: LogQueueStates, @@ -39,12 +54,18 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< round_function: &R, ) -> Vec> { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + implicit_memory_queries.secp256r1_memory_queries.len(), + implicit_memory_states.secp256r1_memory_states.len() ); + + let memory_simulator_before = &implicit_memory_states.secp256r1_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.decommitter_memory_queries.len() + + implicit_memory_queries.keccak256_memory_queries.len() + + implicit_memory_queries.sha256_memory_queries.len() + + implicit_memory_queries.ecrecover_memory_queries.len(), + memory_simulator_before.num_items as usize ); // split into aux witness, don't mix with the memory @@ -96,9 +117,11 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< let mut memory_read_witnesses = vec![]; let mut starting_request_idx = 0; - let mut memory_queue_input_state = memory_queue_simulator.take_sponge_like_queue_state(); + let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); + let mut memory_queue_states_it = implicit_memory_states.ecrecover_memory_states.iter(); + for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() .zip(round_function_witness.into_iter()) @@ -107,7 +130,6 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< let _ = demuxed_secp256r1_verify_queue .simulator .pop_and_output_intermediate_data(round_function); - let initial_memory_len = memory_queue_simulator.num_items; let mut memory_reads_per_request = vec![]; @@ -118,6 +140,8 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< let mut precompile_request = precompile_abi_in_log(request); let is_last_request = request_idx == num_requests - 1; + let mut amount_of_queries = 0; + // we have reads for (_query_index, read) in round_witness.reads.into_iter().enumerate() { let read_query = memory_queries_it.next().unwrap(); @@ -125,15 +149,10 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< assert!(read_query.rw_flag == false); memory_reads_per_request.push(read_query.value); - implicit_memory_artifacts.memory_queries.push(read); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(read, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; + amount_of_queries += 1; } // and writes @@ -142,18 +161,13 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< assert!(write == write_query); assert!(write_query.rw_flag == true); - implicit_memory_artifacts.memory_queries.push(write); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(write, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.output_memory_offset += 1; + amount_of_queries += 1; } - assert_eq!(memory_queue_simulator.num_items - initial_memory_len, 7); + assert_eq!(amount_of_queries, 7); round_counter += 1; if round_counter == num_rounds_per_circuit || is_last_request { @@ -232,13 +246,12 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< } } + let memory_simulator_after = &implicit_memory_states.secp256r1_simulator_snapshots[1]; + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.amount_of_queries(), + memory_simulator_after.num_items as usize ); result diff --git a/src/witness/individual_circuits/memory_related/sha256_round_function.rs b/src/witness/individual_circuits/memory_related/sha256_round_function.rs index 67e20e47..f8cbf92a 100644 --- a/src/witness/individual_circuits/memory_related/sha256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/sha256_round_function.rs @@ -4,6 +4,7 @@ use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQ use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; +use crate::zk_evm::aux_structures::MemoryQuery; use crate::zk_evm::zk_evm_abstractions::precompiles::sha256::Sha256RoundWitness; use crate::zk_evm::zkevm_opcode_defs::ethereum_types::U256; use crate::zkevm_circuits::base_structures::log_query::*; @@ -39,6 +40,29 @@ pub(crate) fn sha256_memory_queries_amount( result } +pub(crate) fn sha256_memory_queries(sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>) -> Vec { + let mut sha256_memory_queries = Vec::with_capacity(sha256_memory_queries_amount(&sha256_round_function_witnesses)); + + for (_cycle, _query, witness) in sha256_round_function_witnesses.iter() { + for el in witness.iter() { + let Sha256RoundWitness { + new_request: _, + reads, + writes, + } = el; + + // we read, then write + sha256_memory_queries.extend_from_slice(reads); + + if let Some(writes) = writes.as_ref() { + sha256_memory_queries.extend_from_slice(writes); + } + } + } + + sha256_memory_queries +} + #[derive(Derivative)] #[derivative(Clone, Copy, Debug, PartialEq, Eq)] pub enum Sha256PrecompileState { @@ -56,9 +80,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( amount_of_memory_queries: usize, - implicit_memory_artifacts: &mut ImplicitMemoryArtifacts, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, - memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + implicit_memory_queries: &ImplicitMemoryQueries, + implicit_memory_states: &ImplicitMemoryStates, sha256_round_function_witnesses: Vec<(u32, LogQuery_, Vec)>, sha256_precompile_queries: Vec, mut demuxed_sha256_precompile_queue: LogQueueStates, @@ -66,12 +89,16 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< round_function: &R, ) -> Vec> { assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() + implicit_memory_queries.sha256_memory_queries.len(), + implicit_memory_states.sha256_memory_states.len() ); + + let memory_simulator_before = &implicit_memory_states.sha256_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.decommitter_memory_queries.len() + + implicit_memory_queries.keccak256_memory_queries.len(), + memory_simulator_before.num_items as usize ); // split into aux witness, don't mix with the memory @@ -136,9 +163,11 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< let mut request_ranges = vec![]; let mut starting_request_idx = 0; - let mut memory_queue_input_state = memory_queue_simulator.take_sponge_like_queue_state(); + let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); + let mut memory_queue_states_it = implicit_memory_states.sha256_memory_states.iter(); + for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() .zip(round_function_witness.into_iter()) @@ -184,13 +213,7 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< assert_eq!(read, read_query); memory_reads_per_request.push(read_query.value); - implicit_memory_artifacts.memory_queries.push(read); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(read, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; } @@ -208,13 +231,7 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< let write_query = memory_queries_it.next().unwrap(); assert_eq!(write, write_query); - implicit_memory_artifacts.memory_queries.push(write); - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(write, round_function); - implicit_memory_artifacts - .memory_queue_states - .push(intermediate_info); - current_memory_queue_state = memory_queue_simulator.take_sponge_like_queue_state(); + current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); if is_last_request { precompile_state = Sha256PrecompileState::Finished; @@ -359,13 +376,14 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< } } + let memory_simulator_after = &implicit_memory_states.sha256_simulator_snapshots[1]; + assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_states_accumulator.len() + implicit_memory_artifacts.memory_queue_states.len() - ); - assert_eq!( - amount_of_memory_queries + implicit_memory_artifacts.memory_queries.len(), - memory_queue_simulator.num_items as usize + amount_of_memory_queries + + implicit_memory_queries.decommitter_memory_queries.len() + + implicit_memory_queries.keccak256_memory_queries.len() + + implicit_memory_queries.sha256_memory_queries.len(), + memory_simulator_after.num_items as usize ); result diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 43fe73c5..57155792 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -4,6 +4,7 @@ use super::artifacts::LogCircuitsArtifacts; use super::individual_circuits::main_vm::CallstackSimulationResult; +use super::individual_circuits::memory_related::{ImplicitMemoryQueries, ImplicitMemoryStates}; use super::postprocessing::{ BlockFirstAndLastBasicCircuitsObservableWitnesses, CsForWitnessGeneration, FirstAndLastCircuitWitness, @@ -832,12 +833,13 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; fn simulate_memory_queue( geometry: GeometryConfig, memory_queries: Vec<(Cycle, MemoryQuery)>, - amount_of_implicit_memory_queries: usize, + implicit_memory_queries: &ImplicitMemoryQueries, round_function: Poseidon2Goldilocks, ) -> ( MemoryArtifacts, LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, + ImplicitMemoryStates ) { let mut memory_artifacts_for_main_vm = MemoryArtifacts { memory_queries, @@ -859,7 +861,7 @@ fn simulate_memory_queue( let mut memory_queue_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, - memory_artifacts_for_main_vm.memory_queries.len() + amount_of_implicit_memory_queries, + memory_artifacts_for_main_vm.memory_queries.len() + implicit_memory_queries.amount_of_queries(), )); // very slow @@ -884,23 +886,86 @@ fn simulate_memory_queue( ); } + use crate::witness::individual_circuits::memory_related::simulate_implicit_memory_queues; + let implicit_memory_states = simulate_implicit_memory_queues( + &mut memory_queue_simulator, + &implicit_memory_queries, + round_function + ); + ( memory_artifacts_for_main_vm, memory_queue_states_accumulator, memory_queue_simulator, + implicit_memory_states ) } +fn simulate_sorted_memory_queue( + geometry: GeometryConfig, + memory_queries: &Vec<(Cycle, MemoryQuery)>, + implicit_memory_queries: &ImplicitMemoryQueries, + round_function: Poseidon2Goldilocks, +) -> ( + LastPerCircuitAccumulator>, + MemoryQueuePerCircuitSimulator +) { + let mut all_memory_queries_sorted: Vec<&MemoryQuery> = memory_queries + .iter() + .map(|(_, query)| query) + .chain(implicit_memory_queries.iter()) + .collect(); + + use std::cmp::Ordering; + use rayon::prelude::*; + use crate::witness::aux_data_structs::per_circuit_accumulator::PerCircuitAccumulator; + + // sort by memory location, and then by timestamp + all_memory_queries_sorted.par_sort_by(|a, b| match a.location.cmp(&b.location) { + Ordering::Equal => a.timestamp.cmp(&b.timestamp), + a @ _ => a, + }); + + // those two things are parallelizable, and can be internally parallelized too + + // now we can finish reconstruction of each sorted and unsorted memory queries + + // reconstruct sorted one in full + let amount_of_queries = memory_queries.len() + implicit_memory_queries.amount_of_queries(); + //let sorted_handle = thread::spawn(move || { + let mut sorted_memory_queries_simulator = + MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + amount_of_queries, + )); + + // for RAM permutation circuits + let mut sorted_memory_queue_states_accumulator = + LastPerCircuitAccumulator::>::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + amount_of_queries, + ); + + for query in all_memory_queries_sorted.into_iter() { + let (_, intermediate_info) = sorted_memory_queries_simulator + .push_and_output_intermediate_data(*query, &round_function); + sorted_memory_queue_states_accumulator.push(intermediate_info); + } + + (sorted_memory_queue_states_accumulator, sorted_memory_queries_simulator) + //}); +} + use crate::witness::artifacts::DemuxedPrecompilesLogQueries; use crate::witness::individual_circuits::log_demux::PrecompilesQueuesStates; -struct PrecompilesInputData { - keccak_round_function_witnesses: Vec<(Cycle, LogQuery, Vec)>, - sha256_round_function_witnesses: Vec<(Cycle, LogQuery, Vec)>, - ecrecover_witnesses: Vec<(Cycle, LogQuery, ECRecoverRoundWitness)>, - secp256r1_verify_witnesses: Vec<(Cycle, LogQuery, Secp256r1VerifyRoundWitness)>, - logs_queues_states: PrecompilesQueuesStates, - logs_queries: DemuxedPrecompilesLogQueries, +pub(crate) struct PrecompilesInputData { + pub keccak_round_function_witnesses: Vec<(Cycle, LogQuery, Vec)>, + pub sha256_round_function_witnesses: Vec<(Cycle, LogQuery, Vec)>, + pub ecrecover_witnesses: Vec<(Cycle, LogQuery, ECRecoverRoundWitness)>, + pub secp256r1_verify_witnesses: Vec<(Cycle, LogQuery, Secp256r1VerifyRoundWitness)>, + pub logs_queues_states: PrecompilesQueuesStates, + pub logs_queries: DemuxedPrecompilesLogQueries, } fn process_memory_related_circuits< @@ -976,6 +1041,13 @@ fn process_memory_related_circuits< tracing::debug!("Running unsorted memory queue simulation"); + use crate::witness::individual_circuits::memory_related::get_implicit_memory_queries; + + let implicit_memory_queries = get_implicit_memory_queries( + &decommiter_circuit_inputs.deduplicated_decommit_requests_with_data, + &precompiles_data + ); + use crate::witness::individual_circuits::memory_related::amount_of_implicit_memory_queries; let amount_of_memory_queries = memory_queries.len(); @@ -987,11 +1059,18 @@ fn process_memory_related_circuits< &precompiles_data.sha256_round_function_witnesses, ); - let (memory_artifacts_for_main_vm, memory_queue_states_accumulator, mut memory_queue_simulator) = + let (sorted_memory_queue_states_accumulator, sorted_memory_queue_simulator) = simulate_sorted_memory_queue( + *geometry, + &memory_queries, + &implicit_memory_queries, + *round_function + ); + + let (memory_artifacts_for_main_vm, memory_queue_states_accumulator, memory_queue_simulator, implicit_memory_states) = simulate_memory_queue( *geometry, memory_queries, - amount_of_implicit_memory_queries, + &implicit_memory_queries, *round_function, ); @@ -1010,9 +1089,9 @@ fn process_memory_related_circuits< let code_decommitter_circuits_data = compute_decommitter_circuit_snapshots( amount_of_memory_queries, - &mut implicit_memory_artifacts, + &implicit_memory_queries, + &implicit_memory_states, &memory_queue_states_accumulator, - &mut memory_queue_simulator, decommiter_circuit_inputs, round_function, geometry.cycles_per_code_decommitter as usize, @@ -1030,9 +1109,8 @@ fn process_memory_related_circuits< let keccak256_circuits_data = keccak256_decompose_into_per_circuit_witness( amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, + &implicit_memory_queries, + &implicit_memory_states, precompiles_data.keccak_round_function_witnesses, precompiles_data.logs_queries.keccak, precompiles_data.logs_queues_states.keccak, @@ -1049,9 +1127,8 @@ fn process_memory_related_circuits< let sha256_circuits_data = sha256_decompose_into_per_circuit_witness( amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, + &implicit_memory_queries, + &implicit_memory_states, precompiles_data.sha256_round_function_witnesses, precompiles_data.logs_queries.sha256, precompiles_data.logs_queues_states.sha256, @@ -1068,9 +1145,8 @@ fn process_memory_related_circuits< let ecrecover_circuits_data = ecrecover_decompose_into_per_circuit_witness( amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, + &implicit_memory_queries, + &implicit_memory_states, precompiles_data.ecrecover_witnesses, precompiles_data.logs_queries.ecrecover, precompiles_data.logs_queues_states.ecrecover, @@ -1085,9 +1161,8 @@ fn process_memory_related_circuits< let secp256r1_verify_circuits_data = secp256r1_verify_decompose_into_per_circuit_witness( amount_of_memory_queries, - &mut implicit_memory_artifacts, - &memory_queue_states_accumulator, - &mut memory_queue_simulator, + &implicit_memory_queries, + &implicit_memory_states, precompiles_data.secp256r1_verify_witnesses, precompiles_data.logs_queries.secp256r1_verify, precompiles_data.logs_queues_states.secp256r1_verify, @@ -1096,7 +1171,8 @@ fn process_memory_related_circuits< ); circuits_data.secp256r1_verify_circuits_data = secp256r1_verify_circuits_data; - assert!(implicit_memory_artifacts.memory_queries.len() == amount_of_implicit_memory_queries); + assert_eq!(implicit_memory_queries.amount_of_queries(), amount_of_implicit_memory_queries); + assert_eq!(implicit_memory_states.amount_of_states(), amount_of_implicit_memory_queries); use crate::witness::individual_circuits::memory_related::ram_permutation::compute_ram_circuit_snapshots; @@ -1105,9 +1181,12 @@ fn process_memory_related_circuits< let (ram_permutation_circuits, ram_permutation_circuits_compact_forms_witnesses) = compute_ram_circuit_snapshots( &memory_artifacts_for_main_vm.memory_queries, - implicit_memory_artifacts, memory_queue_states_accumulator, + sorted_memory_queue_states_accumulator, + implicit_memory_queries, + implicit_memory_states, memory_queue_simulator, + sorted_memory_queue_simulator, round_function, num_non_deterministic_heap_queries, geometry.cycles_per_ram_permutation as usize, From dd5d583e96fb282ca69eeb2614d951b62fbae638 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 22 Jul 2024 13:19:38 +0200 Subject: [PATCH 02/33] Run sorted queue simulation and callstack simulation in separate threads --- src/witness/oracle.rs | 71 ++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 57155792..632375e9 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -903,8 +903,8 @@ fn simulate_memory_queue( fn simulate_sorted_memory_queue( geometry: GeometryConfig, - memory_queries: &Vec<(Cycle, MemoryQuery)>, - implicit_memory_queries: &ImplicitMemoryQueries, + memory_queries: Vec<(Cycle, MemoryQuery)>, + implicit_memory_queries: ImplicitMemoryQueries, round_function: Poseidon2Goldilocks, ) -> ( LastPerCircuitAccumulator>, @@ -953,7 +953,6 @@ fn simulate_sorted_memory_queue( } (sorted_memory_queue_states_accumulator, sorted_memory_queries_simulator) - //}); } use crate::witness::artifacts::DemuxedPrecompilesLogQueries; @@ -1043,6 +1042,7 @@ fn process_memory_related_circuits< use crate::witness::individual_circuits::memory_related::get_implicit_memory_queries; + // precompiles and decommiter will produce additional implicit memory queries let implicit_memory_queries = get_implicit_memory_queries( &decommiter_circuit_inputs.deduplicated_decommit_requests_with_data, &precompiles_data @@ -1059,13 +1059,22 @@ fn process_memory_related_circuits< &precompiles_data.sha256_round_function_witnesses, ); - let (sorted_memory_queue_states_accumulator, sorted_memory_queue_simulator) = simulate_sorted_memory_queue( - *geometry, - &memory_queries, - &implicit_memory_queries, - *round_function - ); - + use std::thread; + let sorted_handle = { + let memory_queries = memory_queries.clone(); + let implicit_memory_queries = implicit_memory_queries.clone(); + let geometry = *geometry; + let round_function = *round_function; + thread::spawn(move || { + simulate_sorted_memory_queue( + geometry, + memory_queries, + implicit_memory_queries, + round_function + ) + }) + }; + let (memory_artifacts_for_main_vm, memory_queue_states_accumulator, memory_queue_simulator, implicit_memory_states) = simulate_memory_queue( *geometry, @@ -1074,17 +1083,13 @@ fn process_memory_related_circuits< *round_function, ); + let (sorted_memory_queue_states_accumulator, sorted_memory_queue_simulator) = sorted_handle.join().unwrap(); + // direct VM related part is done, other subcircuit's functionality is moved to other functions // that should properly do sorts and memory writes use crate::witness::individual_circuits::memory_related::decommit_code::compute_decommitter_circuit_snapshots; - // precompiles and decommiter will produce additional implicit memory queries - let mut implicit_memory_artifacts: ImplicitMemoryArtifacts = - ImplicitMemoryArtifacts::default(); - implicit_memory_artifacts.memory_queries = - Vec::with_capacity(amount_of_implicit_memory_queries); - tracing::debug!("Running code code decommitter simulation"); let code_decommitter_circuits_data = compute_decommitter_circuit_snapshots( @@ -1295,6 +1300,26 @@ pub(crate) fn create_artifacts_from_tracer< *round_function, ); + use std::thread; + let callstack_handle = { + let log_rollback_tails_for_frames = log_rollback_tails_for_frames.clone(); + let geometry = *geometry; + let round_function = *round_function; + thread::spawn(move || { + // We need to simulate all callstack states and prepare for each MainVM circuit: + // - entry value of callstack sponge + // - callstack witnesses (for every callstack state change) + // - detailed log queue state for entry call frame (frame index, log queue state) + callstack_simulation( + &geometry, + full_callstack_history, + log_states_data, + &log_rollback_tails_for_frames, + &round_function, + ) + }) + }; + // Scratch-space constraint system for circuits processing // Used when creating circuit instances and compact form witnesses let mut cs_for_witness_generation = CsForWitnessGeneration::new(); @@ -1375,19 +1400,9 @@ pub(crate) fn create_artifacts_from_tracer< &mut recursion_queue_callback, ); - tracing::debug!("Running callstack sumulation"); + tracing::debug!("Waiting for callstack sumulation"); - // We need to simulate all callstack states and prepare for each MainVM circuit: - // - entry value of callstack sponge - // - callstack witnesses (for every callstack state change) - // - detailed log queue state for entry call frame (frame index, log queue state) - let callstack_simulation_result = callstack_simulation( - geometry, - full_callstack_history, - log_states_data, - &log_rollback_tails_for_frames, - round_function, - ); + let callstack_simulation_result = callstack_handle.join().unwrap(); tracing::debug!( "Processing VM snapshots queue (total {:?})", From 35a4827aae1cff4ce60c98bcb97abf5704d35fc4 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 22 Jul 2024 18:20:11 +0200 Subject: [PATCH 03/33] Remove unused implicit memory queries amount calculation --- .../memory_related/decommit_code.rs | 9 ---- .../memory_related/ecrecover.rs | 20 ++++---- .../keccak256_round_function.rs | 47 +++++++------------ .../individual_circuits/memory_related/mod.rs | 19 -------- .../memory_related/secp256r1_verify.rs | 20 ++++---- .../memory_related/sha256_round_function.rs | 42 ++++++----------- src/witness/oracle.rs | 12 +---- 7 files changed, 49 insertions(+), 120 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/decommit_code.rs b/src/witness/individual_circuits/memory_related/decommit_code.rs index 2a156fe8..16a4ed97 100644 --- a/src/witness/individual_circuits/memory_related/decommit_code.rs +++ b/src/witness/individual_circuits/memory_related/decommit_code.rs @@ -17,15 +17,6 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::zk_evm::aux_structures::DecommittmentQuery; use std::collections::VecDeque; -// TODO docs -pub(crate) fn decommitter_memory_queries_amount( - deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>, -) -> usize { - deduplicated_decommit_requests_with_data - .iter() - .fold(0, |inner, (_, writes)| inner + writes.len()) -} - pub(crate) fn decommitter_memory_queries(deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>) -> Vec { let mut result = vec![]; for (query, writes) in deduplicated_decommit_requests_with_data.iter() { diff --git a/src/witness/individual_circuits/memory_related/ecrecover.rs b/src/witness/individual_circuits/memory_related/ecrecover.rs index 168fd116..d0abb261 100644 --- a/src/witness/individual_circuits/memory_related/ecrecover.rs +++ b/src/witness/individual_circuits/memory_related/ecrecover.rs @@ -11,18 +11,14 @@ use circuit_definitions::encodings::memory_query::MemoryQueueSimulator; use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; -pub(crate) fn ecrecover_memory_queries_amount( - ecrecover_witnesses: &Vec<(u32, LogQuery_, ECRecoverRoundWitness)>, -) -> usize { - ecrecover_witnesses - .iter() - .fold(0, |inner, (_, _, witness)| { - inner + witness.reads.len() + witness.writes.len() - }) -} - pub(crate) fn ecrecover_memory_queries(ecrecover_witnesses: &Vec<(u32, LogQuery_, ECRecoverRoundWitness)>) -> Vec { - let mut ecrecover_memory_queries = Vec::with_capacity(ecrecover_memory_queries_amount(&ecrecover_witnesses)); + let amount_of_queries = ecrecover_witnesses + .iter() + .fold(0, |inner, (_, _, witness)| { + inner + witness.reads.len() + witness.writes.len() + }); + + let mut ecrecover_memory_queries = Vec::with_capacity(amount_of_queries); for (_cycle, _query, witness) in ecrecover_witnesses.iter() { let initial_memory_len = ecrecover_memory_queries.len(); @@ -71,7 +67,7 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< use crate::zk_evm::zk_evm_abstractions::precompiles::ecrecover::ECRecoverRoundWitness; let mut ecrecover_memory_queries = - Vec::with_capacity(ecrecover_memory_queries_amount(&ecrecover_witnesses)); + Vec::with_capacity(implicit_memory_queries.ecrecover_memory_queries.len()); for (_cycle, _query, witness) in ecrecover_witnesses.iter() { let ECRecoverRoundWitness { diff --git a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs index 7be25fb3..3f9f395e 100644 --- a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs @@ -14,37 +14,26 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; use derivative::*; -pub(crate) fn keccak256_memory_queries_amount( - keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, -) -> usize { - let result = keccak_round_function_witnesses - .iter() - .fold(0, |mut inner, (_, _, witness)| { - for el in witness.iter() { - let Keccak256RoundWitness { - new_request: _, - reads, - writes, - } = el; - - reads.iter().for_each(|read| { - if read.is_some() { - inner += 1; - } - }); - - if let Some(writes) = writes.as_ref() { - inner += writes.len() +pub(crate) fn keccak256_memory_queries(keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>) -> Vec { + let amount_of_queries = keccak_round_function_witnesses + .iter() + .fold(0, |mut inner, (_, _, witness)| { + witness.iter().for_each(|el | { + el.reads.iter().for_each(|read| { + if read.is_some() { + inner += 1; } + }); + + if el.writes.is_some() { + inner += 1; } - inner }); - result -} + inner + }); -pub(crate) fn keccak256_memory_queries(keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>) -> Vec { - let mut keccak_256_memory_queries = Vec::with_capacity(keccak256_memory_queries_amount(&keccak_round_function_witnesses)); + let mut keccak_256_memory_queries = Vec::with_capacity(amount_of_queries); for (_cycle, _query, witness) in keccak_round_function_witnesses.iter() { for el in witness.iter() { @@ -67,6 +56,8 @@ pub(crate) fn keccak256_memory_queries(keccak_round_function_witnesses: &Vec<(u3 } } + keccak_256_memory_queries.shrink_to_fit(); + keccak_256_memory_queries } @@ -110,9 +101,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< // split into aux witness, don't mix with the memory - let mut keccak_256_memory_queries = Vec::with_capacity(keccak256_memory_queries_amount( - &keccak_round_function_witnesses, - )); + let mut keccak_256_memory_queries = Vec::with_capacity(implicit_memory_queries.keccak256_memory_queries.len()); for (_cycle, _query, witness) in keccak_round_function_witnesses.iter() { for el in witness.iter() { diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index 12eea6b7..fec422a2 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -7,11 +7,6 @@ use sha256_round_function::sha256_memory_queries; use super::*; use crate::ethereum_types::U256; -use crate::witness::individual_circuits::memory_related::decommit_code::decommitter_memory_queries_amount; -use crate::witness::individual_circuits::memory_related::ecrecover::ecrecover_memory_queries_amount; -use crate::witness::individual_circuits::memory_related::keccak256_round_function::keccak256_memory_queries_amount; -use crate::witness::individual_circuits::memory_related::secp256r1_verify::secp256r1_memory_queries_amount; -use crate::witness::individual_circuits::memory_related::sha256_round_function::sha256_memory_queries_amount; use crate::zk_evm::aux_structures::DecommittmentQuery; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; use crate::zk_evm::aux_structures::MemoryQuery; @@ -30,20 +25,6 @@ pub(crate) mod secp256r1_verify; pub(crate) mod sha256_round_function; pub(crate) mod sort_decommit_requests; -pub(crate) fn amount_of_implicit_memory_queries( - deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>, - ecrecover_witnesses: &Vec<(u32, LogQuery_, ECRecoverRoundWitness)>, - keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, - secp256r1_verify_witnesses: &Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>, - sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, -) -> usize { - decommitter_memory_queries_amount(deduplicated_decommit_requests_with_data) - + ecrecover_memory_queries_amount(ecrecover_witnesses) - + keccak256_memory_queries_amount(keccak_round_function_witnesses) - + secp256r1_memory_queries_amount(secp256r1_verify_witnesses) - + sha256_memory_queries_amount(sha256_round_function_witnesses) -} - #[derive(Clone)] pub(crate) struct ImplicitMemoryQueries { decommitter_memory_queries: Vec, diff --git a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs index 787a805f..86e5c93c 100644 --- a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs +++ b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs @@ -11,18 +11,14 @@ use circuit_definitions::encodings::memory_query::MemoryQueueSimulator; use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; -pub(crate) fn secp256r1_memory_queries_amount( - secp256r1_verify_witnesses: &Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>, -) -> usize { - secp256r1_verify_witnesses - .iter() - .fold(0, |inner, (_, _, witness)| { - inner + witness.reads.len() + witness.writes.len() - }) -} - pub(crate) fn secp256r1_memory_queries(secp256r1_verify_witnesses: &Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>) -> Vec { - let mut secp256r1_memory_queries = Vec::with_capacity(secp256r1_memory_queries_amount(&secp256r1_verify_witnesses)); + let amount_of_queries = secp256r1_verify_witnesses + .iter() + .fold(0, |inner, (_, _, witness)| { + inner + witness.reads.len() + witness.writes.len() + }); + + let mut secp256r1_memory_queries = Vec::with_capacity(amount_of_queries); for (_cycle, _query, witness) in secp256r1_verify_witnesses.iter() { let initial_memory_len = secp256r1_memory_queries.len(); @@ -72,7 +68,7 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< use crate::zk_evm::zk_evm_abstractions::precompiles::secp256r1_verify::Secp256r1VerifyRoundWitness; let mut memory_queries = - Vec::with_capacity(secp256r1_memory_queries_amount(&secp256r1_verify_witnesses)); + Vec::with_capacity(implicit_memory_queries.secp256r1_memory_queries.len()); for (_cycle, _query, witness) in secp256r1_verify_witnesses.iter() { let Secp256r1VerifyRoundWitness { diff --git a/src/witness/individual_circuits/memory_related/sha256_round_function.rs b/src/witness/individual_circuits/memory_related/sha256_round_function.rs index f8cbf92a..18133df2 100644 --- a/src/witness/individual_circuits/memory_related/sha256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/sha256_round_function.rs @@ -15,33 +15,21 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; use derivative::*; -pub(crate) fn sha256_memory_queries_amount( - sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, -) -> usize { - let result = sha256_round_function_witnesses - .iter() - .fold(0, |mut inner, (_, _, witness)| { - for el in witness.iter() { - let Sha256RoundWitness { - new_request: _, - reads, - writes, - } = el; - - inner += reads.len(); - - if let Some(writes) = writes.as_ref() { - inner += writes.len() - } - } - inner - }); +pub(crate) fn sha256_memory_queries(sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>) -> Vec { + let amount_of_queries = sha256_round_function_witnesses + .iter() + .fold(0, |mut inner, (_, _, witness)| { + for el in witness.iter() { + inner += el.reads.len(); - result -} + if let Some(writes) = el.writes.as_ref() { + inner += writes.len() + } + } + inner + }); -pub(crate) fn sha256_memory_queries(sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>) -> Vec { - let mut sha256_memory_queries = Vec::with_capacity(sha256_memory_queries_amount(&sha256_round_function_witnesses)); + let mut sha256_memory_queries = Vec::with_capacity(amount_of_queries); for (_cycle, _query, witness) in sha256_round_function_witnesses.iter() { for el in witness.iter() { @@ -103,9 +91,7 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< // split into aux witness, don't mix with the memory use crate::zk_evm::zk_evm_abstractions::precompiles::sha256::Sha256RoundWitness; - let mut sha256_memory_queries = Vec::with_capacity(sha256_memory_queries_amount( - &sha256_round_function_witnesses, - )); + let mut sha256_memory_queries = Vec::with_capacity(implicit_memory_queries.sha256_memory_queries.len()); for (_cycle, _query, witness) in sha256_round_function_witnesses.iter() { for el in witness.iter() { diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 632375e9..6b5ced2a 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -1048,16 +1048,7 @@ fn process_memory_related_circuits< &precompiles_data ); - use crate::witness::individual_circuits::memory_related::amount_of_implicit_memory_queries; - let amount_of_memory_queries = memory_queries.len(); - let amount_of_implicit_memory_queries = amount_of_implicit_memory_queries( - &decommiter_circuit_inputs.deduplicated_decommit_requests_with_data, - &precompiles_data.ecrecover_witnesses, - &precompiles_data.keccak_round_function_witnesses, - &precompiles_data.secp256r1_verify_witnesses, - &precompiles_data.sha256_round_function_witnesses, - ); use std::thread; let sorted_handle = { @@ -1176,8 +1167,7 @@ fn process_memory_related_circuits< ); circuits_data.secp256r1_verify_circuits_data = secp256r1_verify_circuits_data; - assert_eq!(implicit_memory_queries.amount_of_queries(), amount_of_implicit_memory_queries); - assert_eq!(implicit_memory_states.amount_of_states(), amount_of_implicit_memory_queries); + assert_eq!(implicit_memory_queries.amount_of_queries(), implicit_memory_states.amount_of_states()); use crate::witness::individual_circuits::memory_related::ram_permutation::compute_ram_circuit_snapshots; From 6f4b1dc2e2056f990bad18d4e556f1d22d6797b9 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 22 Jul 2024 18:20:30 +0200 Subject: [PATCH 04/33] Cargo fmt --- .../memory_related/decommit_code.rs | 34 ++--- .../memory_related/ecrecover.rs | 18 ++- .../keccak256_round_function.rs | 51 ++++--- .../individual_circuits/memory_related/mod.rs | 142 +++++++++++------- .../memory_related/ram_permutation.rs | 11 +- .../memory_related/secp256r1_verify.rs | 23 +-- .../memory_related/sha256_round_function.rs | 40 ++--- src/witness/oracle.rs | 78 ++++++---- 8 files changed, 230 insertions(+), 167 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/decommit_code.rs b/src/witness/individual_circuits/memory_related/decommit_code.rs index 16a4ed97..b0d5cd56 100644 --- a/src/witness/individual_circuits/memory_related/decommit_code.rs +++ b/src/witness/individual_circuits/memory_related/decommit_code.rs @@ -17,26 +17,25 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::zk_evm::aux_structures::DecommittmentQuery; use std::collections::VecDeque; -pub(crate) fn decommitter_memory_queries(deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>) -> Vec { +pub(crate) fn decommitter_memory_queries( + deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>, +) -> Vec { let mut result = vec![]; for (query, writes) in deduplicated_decommit_requests_with_data.iter() { assert!(query.is_fresh); // now feed the queries into it - let as_queries_it = writes - .iter() - .enumerate() - .map(|(idx, el)| MemoryQuery { - timestamp: query.timestamp, - location: zk_evm::aux_structures::MemoryLocation { - memory_type: zk_evm::abstractions::MemoryType::Code, - page: query.memory_page, - index: MemoryIndex(idx as u32), - }, - rw_flag: true, - value: *el, - value_is_pointer: false, - }); + let as_queries_it = writes.iter().enumerate().map(|(idx, el)| MemoryQuery { + timestamp: query.timestamp, + location: zk_evm::aux_structures::MemoryLocation { + memory_type: zk_evm::abstractions::MemoryType::Code, + page: query.memory_page, + index: MemoryIndex(idx as u32), + }, + rw_flag: true, + value: *el, + value_is_pointer: false, + }); // and plain test memory queues result.extend(as_queries_it); @@ -432,9 +431,8 @@ pub(crate) fn compute_decommitter_circuit_snapshots< let memory_simulator_after = &implicit_memory_states.decommitter_simulator_snapshots[1]; assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.decommitter_memory_queries.len(), - memory_simulator_after.num_items as usize + amount_of_memory_queries + implicit_memory_queries.decommitter_memory_queries.len(), + memory_simulator_after.num_items as usize ); results diff --git a/src/witness/individual_circuits/memory_related/ecrecover.rs b/src/witness/individual_circuits/memory_related/ecrecover.rs index d0abb261..6ea55031 100644 --- a/src/witness/individual_circuits/memory_related/ecrecover.rs +++ b/src/witness/individual_circuits/memory_related/ecrecover.rs @@ -11,12 +11,14 @@ use circuit_definitions::encodings::memory_query::MemoryQueueSimulator; use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; -pub(crate) fn ecrecover_memory_queries(ecrecover_witnesses: &Vec<(u32, LogQuery_, ECRecoverRoundWitness)>) -> Vec { +pub(crate) fn ecrecover_memory_queries( + ecrecover_witnesses: &Vec<(u32, LogQuery_, ECRecoverRoundWitness)>, +) -> Vec { let amount_of_queries = ecrecover_witnesses - .iter() - .fold(0, |inner, (_, _, witness)| { - inner + witness.reads.len() + witness.writes.len() - }); + .iter() + .fold(0, |inner, (_, _, witness)| { + inner + witness.reads.len() + witness.writes.len() + }); let mut ecrecover_memory_queries = Vec::with_capacity(amount_of_queries); @@ -141,7 +143,8 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< assert!(read_query.rw_flag == false); memory_reads_per_request.push(read_query.value); - current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; amount_of_queries += 1; @@ -153,7 +156,8 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< assert!(write == write_query); assert!(write_query.rw_flag == true); - current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.output_memory_offset += 1; amount_of_queries += 1; diff --git a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs index 3f9f395e..5816a450 100644 --- a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs @@ -14,24 +14,27 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; use derivative::*; -pub(crate) fn keccak256_memory_queries(keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>) -> Vec { - let amount_of_queries = keccak_round_function_witnesses - .iter() - .fold(0, |mut inner, (_, _, witness)| { - witness.iter().for_each(|el | { - el.reads.iter().for_each(|read| { - if read.is_some() { - inner += 1; - } - }); - - if el.writes.is_some() { - inner += 1; - } - }); +pub(crate) fn keccak256_memory_queries( + keccak_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, +) -> Vec { + let amount_of_queries = + keccak_round_function_witnesses + .iter() + .fold(0, |mut inner, (_, _, witness)| { + witness.iter().for_each(|el| { + el.reads.iter().for_each(|read| { + if read.is_some() { + inner += 1; + } + }); + + if el.writes.is_some() { + inner += 1; + } + }); - inner - }); + inner + }); let mut keccak_256_memory_queries = Vec::with_capacity(amount_of_queries); @@ -94,14 +97,14 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let memory_simulator_before = &implicit_memory_states.keccak256_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.decommitter_memory_queries.len(), + amount_of_memory_queries + implicit_memory_queries.decommitter_memory_queries.len(), memory_simulator_before.num_items as usize ); // split into aux witness, don't mix with the memory - let mut keccak_256_memory_queries = Vec::with_capacity(implicit_memory_queries.keccak256_memory_queries.len()); + let mut keccak_256_memory_queries = + Vec::with_capacity(implicit_memory_queries.keccak256_memory_queries.len()); for (_cycle, _query, witness) in keccak_round_function_witnesses.iter() { for el in witness.iter() { @@ -284,7 +287,8 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< assert_eq!(read, read_query); memory_reads_per_circuit.push_back(read_query.value); - current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); input_buffer.fill_with_bytes( &bytes32_buffer, @@ -334,7 +338,8 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let write_query = memory_queries_it.next().unwrap(); assert_eq!(write, write_query); - current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); if is_last_request { precompile_state = Keccak256PrecompileState::Finished; @@ -497,7 +502,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< amount_of_memory_queries + implicit_memory_queries.decommitter_memory_queries.len() + implicit_memory_queries.keccak256_memory_queries.len(), - memory_simulator_after.num_items as usize + memory_simulator_after.num_items as usize ); result diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index fec422a2..8c147025 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -36,11 +36,11 @@ pub(crate) struct ImplicitMemoryQueries { impl ImplicitMemoryQueries { pub fn amount_of_queries(&self) -> usize { - self.decommitter_memory_queries.len() + - self.ecrecover_memory_queries.len() + - self.keccak256_memory_queries.len() + - self.secp256r1_memory_queries.len() + - self.sha256_memory_queries.len() + self.decommitter_memory_queries.len() + + self.ecrecover_memory_queries.len() + + self.keccak256_memory_queries.len() + + self.secp256r1_memory_queries.len() + + self.sha256_memory_queries.len() } fn get_vector(&self, index: usize) -> Option<&Vec> { @@ -50,7 +50,7 @@ impl ImplicitMemoryQueries { 2 => Some(&self.sha256_memory_queries), 3 => Some(&self.ecrecover_memory_queries), 4 => Some(&self.secp256r1_memory_queries), - _ => None + _ => None, } } @@ -58,7 +58,7 @@ impl ImplicitMemoryQueries { ImplicitMemoryQueriesIter { inner: &self, circuit_index: 0, - iter: self.decommitter_memory_queries.iter() + iter: self.decommitter_memory_queries.iter(), } } } @@ -93,18 +93,25 @@ impl<'a> Iterator for ImplicitMemoryQueriesIter<'a> { use crate::witness::oracle::PrecompilesInputData; pub fn get_implicit_memory_queries( deduplicated_decommit_requests_with_data: &Vec<(DecommittmentQuery, Vec)>, - precompiles_inputs: &PrecompilesInputData -) -> ImplicitMemoryQueries{ + precompiles_inputs: &PrecompilesInputData, +) -> ImplicitMemoryQueries { ImplicitMemoryQueries { - decommitter_memory_queries: decommitter_memory_queries(deduplicated_decommit_requests_with_data), + decommitter_memory_queries: decommitter_memory_queries( + deduplicated_decommit_requests_with_data, + ), ecrecover_memory_queries: ecrecover_memory_queries(&precompiles_inputs.ecrecover_witnesses), - keccak256_memory_queries: keccak256_memory_queries(&precompiles_inputs.keccak_round_function_witnesses), - secp256r1_memory_queries: secp256r1_memory_queries(&precompiles_inputs.secp256r1_verify_witnesses), - sha256_memory_queries: sha256_memory_queries(&precompiles_inputs.sha256_round_function_witnesses), + keccak256_memory_queries: keccak256_memory_queries( + &precompiles_inputs.keccak_round_function_witnesses, + ), + secp256r1_memory_queries: secp256r1_memory_queries( + &precompiles_inputs.secp256r1_verify_witnesses, + ), + sha256_memory_queries: sha256_memory_queries( + &precompiles_inputs.sha256_round_function_witnesses, + ), } } - pub(crate) struct SimulatorSnapshot { pub head: [F; SW], pub tail: [F; SW], @@ -113,7 +120,7 @@ pub(crate) struct SimulatorSnapshot { use crate::boojum::gadgets::queue::QueueStateWitness; use crate::boojum::gadgets::queue::QueueTailStateWitness; -impl SimulatorSnapshot { +impl SimulatorSnapshot { pub fn take_sponge_like_queue_state(&self) -> QueueStateWitness { let result = QueueStateWitness { head: self.head, @@ -122,7 +129,7 @@ impl SimulatorSnapshot { length: self.num_items, }, }; - + result } } @@ -143,11 +150,11 @@ pub(crate) struct ImplicitMemoryStates { impl ImplicitMemoryStates { pub fn amount_of_states(&self) -> usize { - self.decommitter_memory_states.len() + - self.ecrecover_memory_states.len() + - self.keccak256_memory_states.len() + - self.secp256r1_memory_states.len() + - self.sha256_memory_states.len() + self.decommitter_memory_states.len() + + self.ecrecover_memory_states.len() + + self.keccak256_memory_states.len() + + self.secp256r1_memory_states.len() + + self.sha256_memory_states.len() } fn get_vectors(self) -> [Vec>; 5] { @@ -156,7 +163,7 @@ impl ImplicitMemoryStates { self.keccak256_memory_states, self.sha256_memory_states, self.ecrecover_memory_states, - self.secp256r1_memory_states + self.secp256r1_memory_states, ] } @@ -165,24 +172,23 @@ impl ImplicitMemoryStates { let last_vector_iter = outer_iter.next().unwrap().into_iter(); ImplicitMemoryStatesIntoIter { last_vector: last_vector_iter, - outer_iter + outer_iter, } } } -use core::slice::Iter; use core::array::IntoIter as ArrayIntoIter; +use core::slice::Iter; use std::vec::IntoIter as VecIntoIter; pub struct ImplicitMemoryStatesIntoIter { last_vector: VecIntoIter>, outer_iter: ArrayIntoIter>, 5>, } -impl Iterator for ImplicitMemoryStatesIntoIter { +impl Iterator for ImplicitMemoryStatesIntoIter { type Item = MemoryQueueState; fn next(&mut self) -> Option { - let mut next = self.last_vector.next(); while next.is_none() { @@ -200,68 +206,100 @@ impl Iterator for ImplicitMemoryStatesIntoIter { use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; -fn get_simulator_snapshot (memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator) -> SimulatorSnapshot { - SimulatorSnapshot{ +fn get_simulator_snapshot( + memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, +) -> SimulatorSnapshot { + SimulatorSnapshot { head: memory_queue_simulator.head, tail: memory_queue_simulator.tail, - num_items: memory_queue_simulator.num_items + num_items: memory_queue_simulator.num_items, } } pub(crate) fn simulate_implicit_memory_queues< -F: SmallField, -R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, + F: SmallField, + R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, implicit_memory_queries: &ImplicitMemoryQueries, - round_function: R + round_function: R, ) -> ImplicitMemoryStates { let mut implicit_memory_states = ImplicitMemoryStates::default(); - implicit_memory_states.decommitter_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .decommitter_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); for query in implicit_memory_queries.decommitter_memory_queries.iter() { let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - implicit_memory_states.decommitter_memory_states.push(intermediate_info); + implicit_memory_states + .decommitter_memory_states + .push(intermediate_info); } - implicit_memory_states.decommitter_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .decommitter_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); - implicit_memory_states.keccak256_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .keccak256_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); for query in implicit_memory_queries.keccak256_memory_queries.iter() { let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - implicit_memory_states.keccak256_memory_states.push(intermediate_info); + implicit_memory_states + .keccak256_memory_states + .push(intermediate_info); } - implicit_memory_states.keccak256_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .keccak256_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); - implicit_memory_states.sha256_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .sha256_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); for query in implicit_memory_queries.sha256_memory_queries.iter() { let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - implicit_memory_states.sha256_memory_states.push(intermediate_info); + implicit_memory_states + .sha256_memory_states + .push(intermediate_info); } - implicit_memory_states.sha256_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .sha256_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); - implicit_memory_states.ecrecover_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .ecrecover_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); for query in implicit_memory_queries.ecrecover_memory_queries.iter() { let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - implicit_memory_states.ecrecover_memory_states.push(intermediate_info); + implicit_memory_states + .ecrecover_memory_states + .push(intermediate_info); } - implicit_memory_states.ecrecover_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .ecrecover_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); - implicit_memory_states.secp256r1_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .secp256r1_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); for query in implicit_memory_queries.secp256r1_memory_queries.iter() { let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - implicit_memory_states.secp256r1_memory_states.push(intermediate_info); + implicit_memory_states + .secp256r1_memory_states + .push(intermediate_info); } - implicit_memory_states.secp256r1_simulator_snapshots.push(get_simulator_snapshot(memory_queue_simulator)); + implicit_memory_states + .secp256r1_simulator_snapshots + .push(get_simulator_snapshot(memory_queue_simulator)); implicit_memory_states } diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index 38253027..546fce6c 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -64,13 +64,9 @@ pub(crate) fn compute_ram_circuit_snapshots< let total_amount_of_queries = memory_queries.len() + implicit_memory_queries.amount_of_queries(); - assert_eq!( - memory_queries.len(), - memory_queue_states_accumulator.len() - ); - + assert_eq!(memory_queries.len(), memory_queue_states_accumulator.len()); - // push implicit queries + // push implicit queries memory_queue_states_accumulator.reserve_exact_flat(implicit_memory_states.amount_of_states()); for state in implicit_memory_states.into_iter() { @@ -102,7 +98,8 @@ pub(crate) fn compute_ram_circuit_snapshots< amount_of_circuits ); - let sorted_memory_queue_chunk_final_states = sorted_memory_queue_states_accumulator.into_circuits(); + let sorted_memory_queue_chunk_final_states = + sorted_memory_queue_states_accumulator.into_circuits(); assert_eq!( unsorted_memory_queue_chunk_final_states.len(), diff --git a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs index 86e5c93c..70a6e253 100644 --- a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs +++ b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs @@ -11,12 +11,14 @@ use circuit_definitions::encodings::memory_query::MemoryQueueSimulator; use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; -pub(crate) fn secp256r1_memory_queries(secp256r1_verify_witnesses: &Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>) -> Vec { +pub(crate) fn secp256r1_memory_queries( + secp256r1_verify_witnesses: &Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>, +) -> Vec { let amount_of_queries = secp256r1_verify_witnesses - .iter() - .fold(0, |inner, (_, _, witness)| { - inner + witness.reads.len() + witness.writes.len() - }); + .iter() + .fold(0, |inner, (_, _, witness)| { + inner + witness.reads.len() + witness.writes.len() + }); let mut secp256r1_memory_queries = Vec::with_capacity(amount_of_queries); @@ -145,7 +147,8 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< assert!(read_query.rw_flag == false); memory_reads_per_request.push(read_query.value); - current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; amount_of_queries += 1; @@ -157,7 +160,8 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< assert!(write == write_query); assert!(write_query.rw_flag == true); - current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.output_memory_offset += 1; amount_of_queries += 1; @@ -245,9 +249,8 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< let memory_simulator_after = &implicit_memory_states.secp256r1_simulator_snapshots[1]; assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.amount_of_queries(), - memory_simulator_after.num_items as usize + amount_of_memory_queries + implicit_memory_queries.amount_of_queries(), + memory_simulator_after.num_items as usize ); result diff --git a/src/witness/individual_circuits/memory_related/sha256_round_function.rs b/src/witness/individual_circuits/memory_related/sha256_round_function.rs index 18133df2..75bf3e60 100644 --- a/src/witness/individual_circuits/memory_related/sha256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/sha256_round_function.rs @@ -15,19 +15,22 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; use circuit_definitions::encodings::*; use derivative::*; -pub(crate) fn sha256_memory_queries(sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>) -> Vec { - let amount_of_queries = sha256_round_function_witnesses - .iter() - .fold(0, |mut inner, (_, _, witness)| { - for el in witness.iter() { - inner += el.reads.len(); - - if let Some(writes) = el.writes.as_ref() { - inner += writes.len() - } - } - inner - }); +pub(crate) fn sha256_memory_queries( + sha256_round_function_witnesses: &Vec<(u32, LogQuery_, Vec)>, +) -> Vec { + let amount_of_queries = + sha256_round_function_witnesses + .iter() + .fold(0, |mut inner, (_, _, witness)| { + for el in witness.iter() { + inner += el.reads.len(); + + if let Some(writes) = el.writes.as_ref() { + inner += writes.len() + } + } + inner + }); let mut sha256_memory_queries = Vec::with_capacity(amount_of_queries); @@ -91,7 +94,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< // split into aux witness, don't mix with the memory use crate::zk_evm::zk_evm_abstractions::precompiles::sha256::Sha256RoundWitness; - let mut sha256_memory_queries = Vec::with_capacity(implicit_memory_queries.sha256_memory_queries.len()); + let mut sha256_memory_queries = + Vec::with_capacity(implicit_memory_queries.sha256_memory_queries.len()); for (_cycle, _query, witness) in sha256_round_function_witnesses.iter() { for el in witness.iter() { @@ -199,7 +203,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< assert_eq!(read, read_query); memory_reads_per_request.push(read_query.value); - current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); precompile_request.input_memory_offset += 1; } @@ -217,7 +222,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< let write_query = memory_queries_it.next().unwrap(); assert_eq!(write, write_query); - current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); + current_memory_queue_state = + transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); if is_last_request { precompile_state = Sha256PrecompileState::Finished; @@ -369,7 +375,7 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< + implicit_memory_queries.decommitter_memory_queries.len() + implicit_memory_queries.keccak256_memory_queries.len() + implicit_memory_queries.sha256_memory_queries.len(), - memory_simulator_after.num_items as usize + memory_simulator_after.num_items as usize ); result diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 6b5ced2a..df3f6c0a 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -839,7 +839,7 @@ fn simulate_memory_queue( MemoryArtifacts, LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, - ImplicitMemoryStates + ImplicitMemoryStates, ) { let mut memory_artifacts_for_main_vm = MemoryArtifacts { memory_queries, @@ -861,7 +861,8 @@ fn simulate_memory_queue( let mut memory_queue_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, - memory_artifacts_for_main_vm.memory_queries.len() + implicit_memory_queries.amount_of_queries(), + memory_artifacts_for_main_vm.memory_queries.len() + + implicit_memory_queries.amount_of_queries(), )); // very slow @@ -888,16 +889,16 @@ fn simulate_memory_queue( use crate::witness::individual_circuits::memory_related::simulate_implicit_memory_queues; let implicit_memory_states = simulate_implicit_memory_queues( - &mut memory_queue_simulator, - &implicit_memory_queries, - round_function + &mut memory_queue_simulator, + &implicit_memory_queries, + round_function, ); ( memory_artifacts_for_main_vm, memory_queue_states_accumulator, memory_queue_simulator, - implicit_memory_states + implicit_memory_states, ) } @@ -908,17 +909,17 @@ fn simulate_sorted_memory_queue( round_function: Poseidon2Goldilocks, ) -> ( LastPerCircuitAccumulator>, - MemoryQueuePerCircuitSimulator + MemoryQueuePerCircuitSimulator, ) { let mut all_memory_queries_sorted: Vec<&MemoryQuery> = memory_queries - .iter() - .map(|(_, query)| query) - .chain(implicit_memory_queries.iter()) - .collect(); + .iter() + .map(|(_, query)| query) + .chain(implicit_memory_queries.iter()) + .collect(); - use std::cmp::Ordering; - use rayon::prelude::*; use crate::witness::aux_data_structs::per_circuit_accumulator::PerCircuitAccumulator; + use rayon::prelude::*; + use std::cmp::Ordering; // sort by memory location, and then by timestamp all_memory_queries_sorted.par_sort_by(|a, b| match a.location.cmp(&b.location) { @@ -933,7 +934,7 @@ fn simulate_sorted_memory_queue( // reconstruct sorted one in full let amount_of_queries = memory_queries.len() + implicit_memory_queries.amount_of_queries(); //let sorted_handle = thread::spawn(move || { - let mut sorted_memory_queries_simulator = + let mut sorted_memory_queries_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, amount_of_queries, @@ -945,14 +946,17 @@ fn simulate_sorted_memory_queue( geometry.cycles_per_ram_permutation as usize, amount_of_queries, ); - - for query in all_memory_queries_sorted.into_iter() { - let (_, intermediate_info) = sorted_memory_queries_simulator + + for query in all_memory_queries_sorted.into_iter() { + let (_, intermediate_info) = sorted_memory_queries_simulator .push_and_output_intermediate_data(*query, &round_function); - sorted_memory_queue_states_accumulator.push(intermediate_info); - } + sorted_memory_queue_states_accumulator.push(intermediate_info); + } - (sorted_memory_queue_states_accumulator, sorted_memory_queries_simulator) + ( + sorted_memory_queue_states_accumulator, + sorted_memory_queries_simulator, + ) } use crate::witness::artifacts::DemuxedPrecompilesLogQueries; @@ -1044,8 +1048,8 @@ fn process_memory_related_circuits< // precompiles and decommiter will produce additional implicit memory queries let implicit_memory_queries = get_implicit_memory_queries( - &decommiter_circuit_inputs.deduplicated_decommit_requests_with_data, - &precompiles_data + &decommiter_circuit_inputs.deduplicated_decommit_requests_with_data, + &precompiles_data, ); let amount_of_memory_queries = memory_queries.len(); @@ -1061,20 +1065,25 @@ fn process_memory_related_circuits< geometry, memory_queries, implicit_memory_queries, - round_function + round_function, ) }) }; - - let (memory_artifacts_for_main_vm, memory_queue_states_accumulator, memory_queue_simulator, implicit_memory_states) = - simulate_memory_queue( - *geometry, - memory_queries, - &implicit_memory_queries, - *round_function, - ); - let (sorted_memory_queue_states_accumulator, sorted_memory_queue_simulator) = sorted_handle.join().unwrap(); + let ( + memory_artifacts_for_main_vm, + memory_queue_states_accumulator, + memory_queue_simulator, + implicit_memory_states, + ) = simulate_memory_queue( + *geometry, + memory_queries, + &implicit_memory_queries, + *round_function, + ); + + let (sorted_memory_queue_states_accumulator, sorted_memory_queue_simulator) = + sorted_handle.join().unwrap(); // direct VM related part is done, other subcircuit's functionality is moved to other functions // that should properly do sorts and memory writes @@ -1167,7 +1176,10 @@ fn process_memory_related_circuits< ); circuits_data.secp256r1_verify_circuits_data = secp256r1_verify_circuits_data; - assert_eq!(implicit_memory_queries.amount_of_queries(), implicit_memory_states.amount_of_states()); + assert_eq!( + implicit_memory_queries.amount_of_queries(), + implicit_memory_states.amount_of_states() + ); use crate::witness::individual_circuits::memory_related::ram_permutation::compute_ram_circuit_snapshots; From bec123d719e015cc20eae6c30205d6ddfdfa800d Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 22 Jul 2024 18:45:55 +0200 Subject: [PATCH 05/33] Simplify memory queue states accumulation --- .../memory_related/decommit_code.rs | 6 ++-- .../individual_circuits/memory_related/mod.rs | 29 +++++++++---------- .../memory_related/ram_permutation.rs | 16 +--------- src/witness/oracle.rs | 24 ++++++++++----- 4 files changed, 34 insertions(+), 41 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/decommit_code.rs b/src/witness/individual_circuits/memory_related/decommit_code.rs index b0d5cd56..f93a8915 100644 --- a/src/witness/individual_circuits/memory_related/decommit_code.rs +++ b/src/witness/individual_circuits/memory_related/decommit_code.rs @@ -57,7 +57,7 @@ pub(crate) fn compute_decommitter_circuit_snapshots< amount_of_memory_queries: usize, implicit_memory_queries: &ImplicitMemoryQueries, implicit_memory_states: &ImplicitMemoryStates, - memory_queue_states_accumulator: &LastPerCircuitAccumulator>, + final_explicit_memory_queue_state: MemoryQueueState, decommiter_circuit_inputs: DecommiterCircuitProcessingInputs, round_function: &R, decommiter_circuit_capacity: usize, @@ -168,7 +168,7 @@ pub(crate) fn compute_decommitter_circuit_snapshots< }; let wintess_state = if start_idx_for_memory_accumulator + memory_queue_state_offset == 0 { - memory_queue_states_accumulator.last().unwrap() + &final_explicit_memory_queue_state } else { implicit_memory_states .decommitter_memory_states @@ -384,7 +384,7 @@ pub(crate) fn compute_decommitter_circuit_snapshots< ); let wintess_state = if start_idx_for_memory_accumulator + memory_queue_state_offset == 0 { - memory_queue_states_accumulator.last().unwrap() + &final_explicit_memory_queue_state } else { implicit_memory_states .decommitter_memory_states diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index 8c147025..0f62ec04 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -157,20 +157,20 @@ impl ImplicitMemoryStates { + self.sha256_memory_states.len() } - fn get_vectors(self) -> [Vec>; 5] { + fn get_vectors(&self) -> [&Vec>; 5] { [ - self.decommitter_memory_states, - self.keccak256_memory_states, - self.sha256_memory_states, - self.ecrecover_memory_states, - self.secp256r1_memory_states, + &self.decommitter_memory_states, + &self.keccak256_memory_states, + &self.sha256_memory_states, + &self.ecrecover_memory_states, + &self.secp256r1_memory_states, ] } - fn into_iter(self) -> ImplicitMemoryStatesIntoIter { + pub fn iter(&self) -> ImplicitMemoryStatesIter { let mut outer_iter = self.get_vectors().into_iter(); - let last_vector_iter = outer_iter.next().unwrap().into_iter(); - ImplicitMemoryStatesIntoIter { + let last_vector_iter = outer_iter.next().unwrap().iter(); + ImplicitMemoryStatesIter { last_vector: last_vector_iter, outer_iter, } @@ -179,14 +179,13 @@ impl ImplicitMemoryStates { use core::array::IntoIter as ArrayIntoIter; use core::slice::Iter; -use std::vec::IntoIter as VecIntoIter; -pub struct ImplicitMemoryStatesIntoIter { - last_vector: VecIntoIter>, - outer_iter: ArrayIntoIter>, 5>, +pub struct ImplicitMemoryStatesIter<'a, F: SmallField> { + last_vector: Iter<'a, MemoryQueueState>, + outer_iter: ArrayIntoIter<&'a Vec>, 5>, } -impl Iterator for ImplicitMemoryStatesIntoIter { - type Item = MemoryQueueState; +impl<'a, F: SmallField> Iterator for ImplicitMemoryStatesIter<'a, F> { + type Item = &'a MemoryQueueState; fn next(&mut self) -> Option { let mut next = self.last_vector.next(); diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index 546fce6c..9a672bec 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -38,10 +38,9 @@ pub(crate) fn compute_ram_circuit_snapshots< QSCB: FnMut(u64, RecursionQueueSimulator, Vec>), >( memory_queries: &Vec<(u32, MemoryQuery)>, - mut memory_queue_states_accumulator: LastPerCircuitAccumulator>, + memory_queue_states_accumulator: LastPerCircuitAccumulator>, sorted_memory_queue_states_accumulator: LastPerCircuitAccumulator>, implicit_memory_queries: ImplicitMemoryQueries, - implicit_memory_states: ImplicitMemoryStates, memory_queue_simulator: MemoryQueuePerCircuitSimulator, sorted_memory_queries_simulator: MemoryQueuePerCircuitSimulator, round_function: &RoundFunction, @@ -55,24 +54,11 @@ pub(crate) fn compute_ram_circuit_snapshots< FirstAndLastCircuitWitness>, Vec>, ) { - assert_eq!( - implicit_memory_queries.amount_of_queries(), - implicit_memory_states.amount_of_states() - ); // including additional queries from precompiles let total_amount_of_queries = memory_queries.len() + implicit_memory_queries.amount_of_queries(); - assert_eq!(memory_queries.len(), memory_queue_states_accumulator.len()); - - // push implicit queries - - memory_queue_states_accumulator.reserve_exact_flat(implicit_memory_states.amount_of_states()); - for state in implicit_memory_states.into_iter() { - memory_queue_states_accumulator.push(state); - } - assert_eq!( total_amount_of_queries, memory_queue_states_accumulator.len() diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index df3f6c0a..a955da22 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -837,6 +837,7 @@ fn simulate_memory_queue( round_function: Poseidon2Goldilocks, ) -> ( MemoryArtifacts, + MemoryQueueState, LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, ImplicitMemoryStates, @@ -887,6 +888,8 @@ fn simulate_memory_queue( ); } + let final_explicit_memory_queue_state = memory_queue_states_accumulator.last().unwrap().clone(); + use crate::witness::individual_circuits::memory_related::simulate_implicit_memory_queues; let implicit_memory_states = simulate_implicit_memory_queues( &mut memory_queue_simulator, @@ -894,8 +897,15 @@ fn simulate_memory_queue( round_function, ); + // push implicit queries + memory_queue_states_accumulator.reserve_exact_flat(implicit_memory_states.amount_of_states()); + for state in implicit_memory_states.iter() { + memory_queue_states_accumulator.push(*state); + } + ( memory_artifacts_for_main_vm, + final_explicit_memory_queue_state, memory_queue_states_accumulator, memory_queue_simulator, implicit_memory_states, @@ -927,13 +937,9 @@ fn simulate_sorted_memory_queue( a @ _ => a, }); - // those two things are parallelizable, and can be internally parallelized too - - // now we can finish reconstruction of each sorted and unsorted memory queries - - // reconstruct sorted one in full + // can be internally parallelized + let amount_of_queries = memory_queries.len() + implicit_memory_queries.amount_of_queries(); - //let sorted_handle = thread::spawn(move || { let mut sorted_memory_queries_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, @@ -1072,6 +1078,7 @@ fn process_memory_related_circuits< let ( memory_artifacts_for_main_vm, + final_explicit_memory_queue_state, memory_queue_states_accumulator, memory_queue_simulator, implicit_memory_states, @@ -1096,7 +1103,7 @@ fn process_memory_related_circuits< amount_of_memory_queries, &implicit_memory_queries, &implicit_memory_states, - &memory_queue_states_accumulator, + final_explicit_memory_queue_state, decommiter_circuit_inputs, round_function, geometry.cycles_per_code_decommitter as usize, @@ -1181,6 +1188,8 @@ fn process_memory_related_circuits< implicit_memory_states.amount_of_states() ); + drop(implicit_memory_states); + use crate::witness::individual_circuits::memory_related::ram_permutation::compute_ram_circuit_snapshots; tracing::debug!("Running RAM permutation simulation"); @@ -1191,7 +1200,6 @@ fn process_memory_related_circuits< memory_queue_states_accumulator, sorted_memory_queue_states_accumulator, implicit_memory_queries, - implicit_memory_states, memory_queue_simulator, sorted_memory_queue_simulator, round_function, From d9ec995d7f5a6b71ce1bf5732d8ebfd1e23a4dc9 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 23 Jul 2024 10:49:49 +0200 Subject: [PATCH 06/33] Cargo fmt --- .../individual_circuits/memory_related/ram_permutation.rs | 1 - src/witness/oracle.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index 9a672bec..1b5eadd6 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -54,7 +54,6 @@ pub(crate) fn compute_ram_circuit_snapshots< FirstAndLastCircuitWitness>, Vec>, ) { - // including additional queries from precompiles let total_amount_of_queries = memory_queries.len() + implicit_memory_queries.amount_of_queries(); diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index a955da22..b75d3afd 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -938,7 +938,7 @@ fn simulate_sorted_memory_queue( }); // can be internally parallelized - + let amount_of_queries = memory_queries.len() + implicit_memory_queries.amount_of_queries(); let mut sorted_memory_queries_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( From 6105dd669d4f852ca23dee0655e59de5c2d9aa55 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 23 Jul 2024 11:57:26 +0200 Subject: [PATCH 07/33] Fix memory-related circuits --- .../keccak256_round_function.rs | 38 +++---------------- .../memory_related/secp256r1_verify.rs | 2 +- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs index 5816a450..bde0c07d 100644 --- a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs @@ -101,39 +101,11 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< memory_simulator_before.num_items as usize ); - // split into aux witness, don't mix with the memory - - let mut keccak_256_memory_queries = - Vec::with_capacity(implicit_memory_queries.keccak256_memory_queries.len()); - - for (_cycle, _query, witness) in keccak_round_function_witnesses.iter() { - for el in witness.iter() { - let Keccak256RoundWitness { - new_request: _, - reads, - writes, - } = el; - - // we read, then write - reads.iter().for_each(|read| { - if let Some(read) = read { - keccak_256_memory_queries.push(*read); - } - }); - - if let Some(writes) = writes.as_ref() { - keccak_256_memory_queries.extend_from_slice(writes); - } - } - } - let mut result = vec![]; let keccak_precompile_calls = keccak_precompile_queries; let round_function_witness = keccak_round_function_witnesses; - let memory_queries = keccak_256_memory_queries; - // check basic consistency assert_eq!( keccak_precompile_calls.len(), @@ -172,7 +144,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let mut hidden_fsm_input_state = Keccak256RoundFunctionFSM::::placeholder_witness(); hidden_fsm_input_state.read_precompile_call = true; - let mut memory_queries_it = memory_queries.into_iter(); + let mut memory_queries_it = implicit_memory_queries.keccak256_memory_queries.iter(); let mut precompile_state = Keccak256PrecompileState::GetRequestFromQueue; let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); @@ -180,6 +152,8 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let mut memory_reads_per_circuit = VecDeque::new(); + let mut memory_queue_states_it = implicit_memory_states.keccak256_memory_states.iter(); + for (request_idx, (request, per_request_work)) in keccak_precompile_calls .into_iter() .zip(round_function_witness.into_iter()) @@ -232,8 +206,6 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< precompile_state = Keccak256PrecompileState::RunPaddingRound; } - let mut memory_queue_states_it = implicit_memory_states.keccak256_memory_states.iter(); - for (round_idx, round) in round_witness.into_iter().enumerate() { // we proceed the request as long as we can if round_idx == 0 { @@ -284,7 +256,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< data.to_big_endian(&mut bytes32_buffer[..]); let read_query = memory_queries_it.next().unwrap(); - assert_eq!(read, read_query); + assert_eq!(read, *read_query); memory_reads_per_circuit.push_back(read_query.value); current_memory_queue_state = @@ -336,7 +308,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< assert!(round.writes.is_some()); let [write] = round.writes.unwrap(); let write_query = memory_queries_it.next().unwrap(); - assert_eq!(write, write_query); + assert_eq!(write, *write_query); current_memory_queue_state = transform_sponge_like_queue_state(*memory_queue_states_it.next().unwrap()); diff --git a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs index 70a6e253..09084ded 100644 --- a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs +++ b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs @@ -118,7 +118,7 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); - let mut memory_queue_states_it = implicit_memory_states.ecrecover_memory_states.iter(); + let mut memory_queue_states_it = implicit_memory_states.secp256r1_memory_states.iter(); for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() From a1f00723b0660ca904f1fd6607286a3b05ed0c55 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 23 Jul 2024 12:14:56 +0200 Subject: [PATCH 08/33] Cleanup in simulate_implicit_memory_queues --- .../individual_circuits/memory_related/mod.rs | 112 ++++++------------ 1 file changed, 38 insertions(+), 74 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index 0f62ec04..ad2ba695 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -225,80 +225,44 @@ pub(crate) fn simulate_implicit_memory_queues< ) -> ImplicitMemoryStates { let mut implicit_memory_states = ImplicitMemoryStates::default(); - implicit_memory_states - .decommitter_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - for query in implicit_memory_queries.decommitter_memory_queries.iter() { - let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - - implicit_memory_states - .decommitter_memory_states - .push(intermediate_info); - } - implicit_memory_states - .decommitter_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - - implicit_memory_states - .keccak256_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - for query in implicit_memory_queries.keccak256_memory_queries.iter() { - let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - - implicit_memory_states - .keccak256_memory_states - .push(intermediate_info); - } - implicit_memory_states - .keccak256_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - - implicit_memory_states - .sha256_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - for query in implicit_memory_queries.sha256_memory_queries.iter() { - let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - - implicit_memory_states - .sha256_memory_states - .push(intermediate_info); - } - implicit_memory_states - .sha256_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - - implicit_memory_states - .ecrecover_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - for query in implicit_memory_queries.ecrecover_memory_queries.iter() { - let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - - implicit_memory_states - .ecrecover_memory_states - .push(intermediate_info); - } - implicit_memory_states - .ecrecover_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - - implicit_memory_states - .secp256r1_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); - for query in implicit_memory_queries.secp256r1_memory_queries.iter() { - let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - - implicit_memory_states - .secp256r1_memory_states - .push(intermediate_info); - } - implicit_memory_states - .secp256r1_simulator_snapshots - .push(get_simulator_snapshot(memory_queue_simulator)); + let mut simulate_subqueue = |memory_queries: &Vec, memory_states: &mut Vec>| { + let mut snapshots = vec![]; + snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // before + for query in memory_queries.iter() { + let (_old_tail, intermediate_info) = + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + + memory_states.push(intermediate_info); + } + snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // after + + snapshots + }; + + implicit_memory_states.decommitter_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.decommitter_memory_queries, + &mut implicit_memory_states.decommitter_memory_states + ); + + implicit_memory_states.keccak256_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.keccak256_memory_queries, + &mut implicit_memory_states.keccak256_memory_states + ); + + implicit_memory_states.sha256_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.sha256_memory_queries, + &mut implicit_memory_states.sha256_memory_states + ); + + implicit_memory_states.ecrecover_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.ecrecover_memory_queries, + &mut implicit_memory_states.ecrecover_memory_states + ); + + implicit_memory_states.secp256r1_simulator_snapshots = simulate_subqueue( + &implicit_memory_queries.secp256r1_memory_queries, + &mut implicit_memory_states.secp256r1_memory_states + ); implicit_memory_states } From bf879fe637f2a19eaef2e6b9448841340d6b08ef Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 23 Jul 2024 13:18:18 +0200 Subject: [PATCH 09/33] Cargo fmt --- .../individual_circuits/memory_related/mod.rs | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index ad2ba695..3267cd0b 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -225,43 +225,44 @@ pub(crate) fn simulate_implicit_memory_queues< ) -> ImplicitMemoryStates { let mut implicit_memory_states = ImplicitMemoryStates::default(); - let mut simulate_subqueue = |memory_queries: &Vec, memory_states: &mut Vec>| { - let mut snapshots = vec![]; - snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // before - for query in memory_queries.iter() { - let (_old_tail, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - - memory_states.push(intermediate_info); - } - snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // after + let mut simulate_subqueue = + |memory_queries: &Vec, memory_states: &mut Vec>| { + let mut snapshots = vec![]; + snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // before + for query in memory_queries.iter() { + let (_old_tail, intermediate_info) = memory_queue_simulator + .push_and_output_intermediate_data(*query, &round_function); + + memory_states.push(intermediate_info); + } + snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // after - snapshots - }; + snapshots + }; implicit_memory_states.decommitter_simulator_snapshots = simulate_subqueue( &implicit_memory_queries.decommitter_memory_queries, - &mut implicit_memory_states.decommitter_memory_states + &mut implicit_memory_states.decommitter_memory_states, ); implicit_memory_states.keccak256_simulator_snapshots = simulate_subqueue( &implicit_memory_queries.keccak256_memory_queries, - &mut implicit_memory_states.keccak256_memory_states + &mut implicit_memory_states.keccak256_memory_states, ); implicit_memory_states.sha256_simulator_snapshots = simulate_subqueue( &implicit_memory_queries.sha256_memory_queries, - &mut implicit_memory_states.sha256_memory_states + &mut implicit_memory_states.sha256_memory_states, ); implicit_memory_states.ecrecover_simulator_snapshots = simulate_subqueue( &implicit_memory_queries.ecrecover_memory_queries, - &mut implicit_memory_states.ecrecover_memory_states + &mut implicit_memory_states.ecrecover_memory_states, ); implicit_memory_states.secp256r1_simulator_snapshots = simulate_subqueue( &implicit_memory_queries.secp256r1_memory_queries, - &mut implicit_memory_states.secp256r1_memory_states + &mut implicit_memory_states.secp256r1_memory_states, ); implicit_memory_states From a6353b4ac7837a3efbf4a67cb9dfce414e2c44d7 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 23 Jul 2024 15:19:36 +0200 Subject: [PATCH 10/33] Unpack structures for memory-related circuits --- .../memory_related/decommit_code.rs | 26 ++++---- .../memory_related/ecrecover.rs | 59 ++++++------------ .../keccak256_round_function.rs | 32 +++++----- .../individual_circuits/memory_related/mod.rs | 30 +++++----- .../memory_related/ram_permutation.rs | 10 +--- .../memory_related/secp256r1_verify.rs | 54 ++++++----------- .../memory_related/sha256_round_function.rs | 60 ++++++------------- src/witness/oracle.rs | 57 +++++++++--------- 8 files changed, 128 insertions(+), 200 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/decommit_code.rs b/src/witness/individual_circuits/memory_related/decommit_code.rs index f93a8915..7f84aae8 100644 --- a/src/witness/individual_circuits/memory_related/decommit_code.rs +++ b/src/witness/individual_circuits/memory_related/decommit_code.rs @@ -55,19 +55,20 @@ pub(crate) fn compute_decommitter_circuit_snapshots< R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( amount_of_memory_queries: usize, - implicit_memory_queries: &ImplicitMemoryQueries, - implicit_memory_states: &ImplicitMemoryStates, + decommitter_memory_queries: Vec, + decommitter_simulator_snapshots: Vec>, + decommitter_memory_states: Vec>, final_explicit_memory_queue_state: MemoryQueueState, decommiter_circuit_inputs: DecommiterCircuitProcessingInputs, round_function: &R, decommiter_circuit_capacity: usize, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - implicit_memory_queries.decommitter_memory_queries.len(), - implicit_memory_states.decommitter_memory_states.len() + decommitter_memory_queries.len(), + decommitter_memory_states.len() ); - let memory_simulator_before = &implicit_memory_states.decommitter_simulator_snapshots[0]; + let memory_simulator_before = &decommitter_simulator_snapshots[0]; assert_eq!( amount_of_memory_queries, memory_simulator_before.num_items as usize @@ -170,8 +171,7 @@ pub(crate) fn compute_decommitter_circuit_snapshots< let wintess_state = if start_idx_for_memory_accumulator + memory_queue_state_offset == 0 { &final_explicit_memory_queue_state } else { - implicit_memory_states - .decommitter_memory_states + decommitter_memory_states .get(start_idx_for_memory_accumulator + memory_queue_state_offset - 1) .unwrap() }; @@ -386,8 +386,7 @@ pub(crate) fn compute_decommitter_circuit_snapshots< let wintess_state = if start_idx_for_memory_accumulator + memory_queue_state_offset == 0 { &final_explicit_memory_queue_state } else { - implicit_memory_states - .decommitter_memory_states + decommitter_memory_states .get(start_idx_for_memory_accumulator + memory_queue_state_offset - 1) .unwrap() }; @@ -428,12 +427,13 @@ pub(crate) fn compute_decommitter_circuit_snapshots< } } - let memory_simulator_after = &implicit_memory_states.decommitter_simulator_snapshots[1]; + let memory_simulator_after = &decommitter_simulator_snapshots[1]; + let amount_of_memory_queries_after = amount_of_memory_queries + decommitter_memory_queries.len(); assert_eq!( - amount_of_memory_queries + implicit_memory_queries.decommitter_memory_queries.len(), + amount_of_memory_queries_after, memory_simulator_after.num_items as usize ); - results + (results, amount_of_memory_queries_after) } diff --git a/src/witness/individual_circuits/memory_related/ecrecover.rs b/src/witness/individual_circuits/memory_related/ecrecover.rs index 6ea55031..85eb7edf 100644 --- a/src/witness/individual_circuits/memory_related/ecrecover.rs +++ b/src/witness/individual_circuits/memory_related/ecrecover.rs @@ -42,63 +42,40 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( - amount_of_memory_queries: usize, - implicit_memory_queries: &ImplicitMemoryQueries, - implicit_memory_states: &ImplicitMemoryStates, + amount_of_memory_queries_before: usize, + ecrecover_memory_queries: Vec, + ecrecover_simulator_snapshots: Vec>, + ecrecover_memory_states: Vec>, ecrecover_witnesses: Vec<(u32, LogQuery_, ECRecoverRoundWitness)>, ecrecover_queries: Vec, mut demuxed_ecrecover_queue: LogQueueStates, num_rounds_per_circuit: usize, round_function: &R, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - implicit_memory_queries.ecrecover_memory_queries.len(), - implicit_memory_states.ecrecover_memory_states.len() + ecrecover_memory_queries.len(), + ecrecover_memory_states.len() ); - let memory_simulator_before = &implicit_memory_states.ecrecover_simulator_snapshots[0]; + let memory_simulator_before = &ecrecover_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.decommitter_memory_queries.len() - + implicit_memory_queries.keccak256_memory_queries.len() - + implicit_memory_queries.sha256_memory_queries.len(), + amount_of_memory_queries_before, memory_simulator_before.num_items as usize ); - // split into aux witness, don't mix with the memory - - use crate::zk_evm::zk_evm_abstractions::precompiles::ecrecover::ECRecoverRoundWitness; - let mut ecrecover_memory_queries = - Vec::with_capacity(implicit_memory_queries.ecrecover_memory_queries.len()); - - for (_cycle, _query, witness) in ecrecover_witnesses.iter() { - let ECRecoverRoundWitness { - new_request: _, - reads, - writes, - } = witness; - - // we read, then write - ecrecover_memory_queries.extend_from_slice(reads); - - ecrecover_memory_queries.extend_from_slice(writes); - } - let mut result = vec![]; let precompile_calls = ecrecover_queries; let simulator_witness: Vec<_> = demuxed_ecrecover_queue.simulator.witness.clone().into(); let round_function_witness = ecrecover_witnesses; - let memory_queries = ecrecover_memory_queries; - // check basic consistency assert!(precompile_calls.len() == demuxed_ecrecover_queue.states_accumulator.len()); drop(demuxed_ecrecover_queue.states_accumulator); assert!(precompile_calls.len() == round_function_witness.len()); if precompile_calls.len() == 0 { - return vec![]; + return (vec![], amount_of_memory_queries_before); } let mut round_counter = 0; @@ -107,7 +84,8 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< // convension let mut log_queue_input_state = take_queue_state_from_simulator(&demuxed_ecrecover_queue.simulator); - let mut memory_queries_it = memory_queries.into_iter(); + let amount_ecrecover_memory_queries = ecrecover_memory_queries.len(); + let mut memory_queries_it = ecrecover_memory_queries.into_iter(); let mut memory_read_witnesses = vec![]; let mut starting_request_idx = 0; @@ -115,7 +93,7 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); - let mut memory_queue_states_it = implicit_memory_states.ecrecover_memory_states.iter(); + let mut memory_queue_states_it = ecrecover_memory_states.iter(); for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() @@ -242,16 +220,13 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< } } - let memory_simulator_after = &implicit_memory_states.ecrecover_simulator_snapshots[1]; + let memory_simulator_after = &ecrecover_simulator_snapshots[1]; + let amount_of_memory_queries_after = amount_of_memory_queries_before + amount_ecrecover_memory_queries; assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.decommitter_memory_queries.len() - + implicit_memory_queries.keccak256_memory_queries.len() - + implicit_memory_queries.sha256_memory_queries.len() - + implicit_memory_queries.ecrecover_memory_queries.len(), + amount_of_memory_queries_after, memory_simulator_after.num_items as usize ); - result + (result, amount_of_memory_queries_after) } diff --git a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs index bde0c07d..1512fb81 100644 --- a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs @@ -81,23 +81,24 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( - amount_of_memory_queries: usize, - implicit_memory_queries: &ImplicitMemoryQueries, - implicit_memory_states: &ImplicitMemoryStates, + amount_of_memory_queries_before: usize, + keccak256_memory_queries: Vec, + keccak256_simulator_snapshots: Vec>, + keccak256_memory_states: Vec>, keccak_round_function_witnesses: Vec<(u32, LogQuery_, Vec)>, keccak_precompile_queries: Vec, mut demuxed_keccak_precompile_queue: LogQueueStates, num_rounds_per_circuit: usize, round_function: &R, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - implicit_memory_queries.keccak256_memory_queries.len(), - implicit_memory_states.keccak256_memory_states.len() + keccak256_memory_queries.len(), + keccak256_memory_states.len() ); - let memory_simulator_before = &implicit_memory_states.keccak256_simulator_snapshots[0]; + let memory_simulator_before = &keccak256_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries + implicit_memory_queries.decommitter_memory_queries.len(), + amount_of_memory_queries_before, memory_simulator_before.num_items as usize ); @@ -120,7 +121,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< ); if keccak_precompile_calls.len() == 0 { - return vec![]; + return (vec![], amount_of_memory_queries_before); } let mut round_counter = 0; @@ -144,7 +145,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let mut hidden_fsm_input_state = Keccak256RoundFunctionFSM::::placeholder_witness(); hidden_fsm_input_state.read_precompile_call = true; - let mut memory_queries_it = implicit_memory_queries.keccak256_memory_queries.iter(); + let mut memory_queries_it = keccak256_memory_queries.iter(); let mut precompile_state = Keccak256PrecompileState::GetRequestFromQueue; let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); @@ -152,7 +153,7 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< let mut memory_reads_per_circuit = VecDeque::new(); - let mut memory_queue_states_it = implicit_memory_states.keccak256_memory_states.iter(); + let mut memory_queue_states_it = keccak256_memory_states.iter(); for (request_idx, (request, per_request_work)) in keccak_precompile_calls .into_iter() @@ -468,16 +469,15 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< } } - let memory_simulator_after = &implicit_memory_states.keccak256_simulator_snapshots[1]; + let memory_simulator_after = &keccak256_simulator_snapshots[1]; + let amount_of_memory_queries_after = amount_of_memory_queries_before + keccak256_memory_queries.len(); assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.decommitter_memory_queries.len() - + implicit_memory_queries.keccak256_memory_queries.len(), + amount_of_memory_queries_after, memory_simulator_after.num_items as usize ); - result + (result, amount_of_memory_queries_after) } pub(crate) fn encode_keccak256_inner_state(state: [u64; 25]) -> [[[u8; 8]; 5]; 5] { diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index 3267cd0b..ec09f7c9 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -27,11 +27,11 @@ pub(crate) mod sort_decommit_requests; #[derive(Clone)] pub(crate) struct ImplicitMemoryQueries { - decommitter_memory_queries: Vec, - ecrecover_memory_queries: Vec, - keccak256_memory_queries: Vec, - secp256r1_memory_queries: Vec, - sha256_memory_queries: Vec, + pub decommitter_memory_queries: Vec, + pub ecrecover_memory_queries: Vec, + pub keccak256_memory_queries: Vec, + pub secp256r1_memory_queries: Vec, + pub sha256_memory_queries: Vec, } impl ImplicitMemoryQueries { @@ -136,16 +136,16 @@ impl SimulatorSnapshot { #[derive(Default)] pub(crate) struct ImplicitMemoryStates { - decommitter_simulator_snapshots: Vec>, - decommitter_memory_states: Vec>, - ecrecover_simulator_snapshots: Vec>, - ecrecover_memory_states: Vec>, - keccak256_simulator_snapshots: Vec>, - keccak256_memory_states: Vec>, - secp256r1_simulator_snapshots: Vec>, - secp256r1_memory_states: Vec>, - sha256_simulator_snapshots: Vec>, - sha256_memory_states: Vec>, + pub decommitter_simulator_snapshots: Vec>, + pub decommitter_memory_states: Vec>, + pub ecrecover_simulator_snapshots: Vec>, + pub ecrecover_memory_states: Vec>, + pub keccak256_simulator_snapshots: Vec>, + pub keccak256_memory_states: Vec>, + pub secp256r1_simulator_snapshots: Vec>, + pub secp256r1_memory_states: Vec>, + pub sha256_simulator_snapshots: Vec>, + pub sha256_memory_states: Vec>, } impl ImplicitMemoryStates { diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index 1b5eadd6..789fd459 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -37,15 +37,13 @@ pub(crate) fn compute_ram_circuit_snapshots< CB: FnMut(ZkSyncBaseLayerCircuit), QSCB: FnMut(u64, RecursionQueueSimulator, Vec>), >( - memory_queries: &Vec<(u32, MemoryQuery)>, + total_amount_of_queries: usize, // including additional queries from precompiles memory_queue_states_accumulator: LastPerCircuitAccumulator>, sorted_memory_queue_states_accumulator: LastPerCircuitAccumulator>, - implicit_memory_queries: ImplicitMemoryQueries, memory_queue_simulator: MemoryQueuePerCircuitSimulator, sorted_memory_queries_simulator: MemoryQueuePerCircuitSimulator, round_function: &RoundFunction, num_non_deterministic_heap_queries: usize, - per_circuit_capacity: usize, geometry: &GeometryConfig, cs_for_witness_generation: &mut CsForWitnessGeneration, mut circuit_callback: CB, @@ -54,10 +52,6 @@ pub(crate) fn compute_ram_circuit_snapshots< FirstAndLastCircuitWitness>, Vec>, ) { - // including additional queries from precompiles - let total_amount_of_queries = - memory_queries.len() + implicit_memory_queries.amount_of_queries(); - assert_eq!( total_amount_of_queries, memory_queue_states_accumulator.len() @@ -73,6 +67,8 @@ pub(crate) fn compute_ram_circuit_snapshots< "VM should have made some memory requests" ); + let per_circuit_capacity = geometry.cycles_per_ram_permutation as usize; + let amount_of_circuits = (total_amount_of_queries + per_circuit_capacity - 1) / per_circuit_capacity; diff --git a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs index 09084ded..d0b39c8d 100644 --- a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs +++ b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs @@ -42,49 +42,27 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( - amount_of_memory_queries: usize, - implicit_memory_queries: &ImplicitMemoryQueries, - implicit_memory_states: &ImplicitMemoryStates, + amount_of_memory_queries_before: usize, + secp256r1_memory_queries: Vec, + secp256r1_simulator_snapshots: Vec>, + secp256r1_memory_states: Vec>, secp256r1_verify_witnesses: Vec<(u32, LogQuery_, Secp256r1VerifyRoundWitness)>, secp256r1_verify_queries: Vec, mut demuxed_secp256r1_verify_queue: LogQueueStates, num_rounds_per_circuit: usize, round_function: &R, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - implicit_memory_queries.secp256r1_memory_queries.len(), - implicit_memory_states.secp256r1_memory_states.len() + secp256r1_memory_queries.len(), + secp256r1_memory_states.len() ); - let memory_simulator_before = &implicit_memory_states.secp256r1_simulator_snapshots[0]; + let memory_simulator_before = &secp256r1_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.decommitter_memory_queries.len() - + implicit_memory_queries.keccak256_memory_queries.len() - + implicit_memory_queries.sha256_memory_queries.len() - + implicit_memory_queries.ecrecover_memory_queries.len(), + amount_of_memory_queries_before, memory_simulator_before.num_items as usize ); - // split into aux witness, don't mix with the memory - - use crate::zk_evm::zk_evm_abstractions::precompiles::secp256r1_verify::Secp256r1VerifyRoundWitness; - let mut memory_queries = - Vec::with_capacity(implicit_memory_queries.secp256r1_memory_queries.len()); - - for (_cycle, _query, witness) in secp256r1_verify_witnesses.iter() { - let Secp256r1VerifyRoundWitness { - new_request: _, - reads, - writes, - } = witness; - - // we read, then write - memory_queries.extend_from_slice(reads); - - memory_queries.extend_from_slice(writes); - } - let mut result = vec![]; let precompile_calls = secp256r1_verify_queries; @@ -101,7 +79,7 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< assert!(precompile_calls.len() == round_function_witness.len()); if precompile_calls.len() == 0 { - return vec![]; + return (vec![], amount_of_memory_queries_before); } let mut round_counter = 0; @@ -110,7 +88,8 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< // convension let mut log_queue_input_state = take_queue_state_from_simulator(&demuxed_secp256r1_verify_queue.simulator); - let mut memory_queries_it = memory_queries.into_iter(); + let amount_secp256r1_memory_queries = secp256r1_memory_queries.len(); + let mut memory_queries_it = secp256r1_memory_queries.into_iter(); let mut memory_read_witnesses = vec![]; let mut starting_request_idx = 0; @@ -118,7 +97,7 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); - let mut memory_queue_states_it = implicit_memory_states.secp256r1_memory_states.iter(); + let mut memory_queue_states_it = secp256r1_memory_states.iter(); for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() @@ -246,12 +225,13 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< } } - let memory_simulator_after = &implicit_memory_states.secp256r1_simulator_snapshots[1]; + let memory_simulator_after = &secp256r1_simulator_snapshots[1]; + let amount_of_memory_queries_after = amount_of_memory_queries_before + amount_secp256r1_memory_queries; assert_eq!( - amount_of_memory_queries + implicit_memory_queries.amount_of_queries(), + amount_of_memory_queries_after, memory_simulator_after.num_items as usize ); - result + (result, amount_of_memory_queries_after) } diff --git a/src/witness/individual_circuits/memory_related/sha256_round_function.rs b/src/witness/individual_circuits/memory_related/sha256_round_function.rs index 75bf3e60..cb212a48 100644 --- a/src/witness/individual_circuits/memory_related/sha256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/sha256_round_function.rs @@ -70,50 +70,27 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( - amount_of_memory_queries: usize, - implicit_memory_queries: &ImplicitMemoryQueries, - implicit_memory_states: &ImplicitMemoryStates, + amount_of_memory_queries_before: usize, + sha256_memory_queries: Vec, + sha256_simulator_snapshots: Vec>, + sha256_memory_states: Vec>, sha256_round_function_witnesses: Vec<(u32, LogQuery_, Vec)>, sha256_precompile_queries: Vec, mut demuxed_sha256_precompile_queue: LogQueueStates, num_rounds_per_circuit: usize, round_function: &R, -) -> Vec> { +) -> (Vec>, usize) { assert_eq!( - implicit_memory_queries.sha256_memory_queries.len(), - implicit_memory_states.sha256_memory_states.len() + sha256_memory_queries.len(), + sha256_memory_states.len() ); - let memory_simulator_before = &implicit_memory_states.sha256_simulator_snapshots[0]; + let memory_simulator_before = &sha256_simulator_snapshots[0]; assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.decommitter_memory_queries.len() - + implicit_memory_queries.keccak256_memory_queries.len(), + amount_of_memory_queries_before, memory_simulator_before.num_items as usize ); - // split into aux witness, don't mix with the memory - use crate::zk_evm::zk_evm_abstractions::precompiles::sha256::Sha256RoundWitness; - let mut sha256_memory_queries = - Vec::with_capacity(implicit_memory_queries.sha256_memory_queries.len()); - - for (_cycle, _query, witness) in sha256_round_function_witnesses.iter() { - for el in witness.iter() { - let Sha256RoundWitness { - new_request: _, - reads, - writes, - } = el; - - // we read, then write - sha256_memory_queries.extend_from_slice(reads); - - if let Some(writes) = writes.as_ref() { - sha256_memory_queries.extend_from_slice(writes); - } - } - } - let mut result = vec![]; let precompile_calls = sha256_precompile_queries; @@ -124,15 +101,13 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< .into(); let round_function_witness = sha256_round_function_witnesses; - let memory_queries = sha256_memory_queries; - // check basic consistency assert!(precompile_calls.len() == demuxed_sha256_precompile_queue.states_accumulator.len()); drop(demuxed_sha256_precompile_queue.states_accumulator); assert!(precompile_calls.len() == round_function_witness.len()); if precompile_calls.len() == 0 { - return vec![]; + return (vec![], amount_of_memory_queries_before); } let mut round_counter = 0; @@ -144,7 +119,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< let mut hidden_fsm_input_state = Sha256RoundFunctionFSM::::placeholder_witness(); hidden_fsm_input_state.read_precompile_call = true; - let mut memory_queries_it = memory_queries.into_iter(); + let amount_sha256_memory_queries = sha256_memory_queries.len(); + let mut memory_queries_it = sha256_memory_queries.into_iter(); let mut memory_read_witnesses = vec![]; @@ -156,7 +132,7 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< let mut memory_queue_input_state = memory_simulator_before.take_sponge_like_queue_state(); let mut current_memory_queue_state = memory_queue_input_state.clone(); - let mut memory_queue_states_it = implicit_memory_states.sha256_memory_states.iter(); + let mut memory_queue_states_it = sha256_memory_states.iter(); for (request_idx, (request, per_request_work)) in precompile_calls .into_iter() @@ -368,15 +344,13 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< } } - let memory_simulator_after = &implicit_memory_states.sha256_simulator_snapshots[1]; + let memory_simulator_after = &sha256_simulator_snapshots[1]; + let amount_of_memory_queries_after = amount_of_memory_queries_before + amount_sha256_memory_queries; assert_eq!( - amount_of_memory_queries - + implicit_memory_queries.decommitter_memory_queries.len() - + implicit_memory_queries.keccak256_memory_queries.len() - + implicit_memory_queries.sha256_memory_queries.len(), + amount_of_memory_queries_after, memory_simulator_after.num_items as usize ); - result + (result, amount_of_memory_queries_after) } diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index b75d3afd..29af12a8 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -1058,7 +1058,7 @@ fn process_memory_related_circuits< &precompiles_data, ); - let amount_of_memory_queries = memory_queries.len(); + let amount_of_explicit_memory_queries = memory_queries.len(); use std::thread; let sorted_handle = { @@ -1095,14 +1095,22 @@ fn process_memory_related_circuits< // direct VM related part is done, other subcircuit's functionality is moved to other functions // that should properly do sorts and memory writes + let amount_of_implicit_queries = implicit_memory_queries.amount_of_queries(); + + assert_eq!( + amount_of_implicit_queries, + implicit_memory_states.amount_of_states() + ); + use crate::witness::individual_circuits::memory_related::decommit_code::compute_decommitter_circuit_snapshots; tracing::debug!("Running code code decommitter simulation"); - let code_decommitter_circuits_data = compute_decommitter_circuit_snapshots( - amount_of_memory_queries, - &implicit_memory_queries, - &implicit_memory_states, + let (code_decommitter_circuits_data, amount_of_memory_queries) = compute_decommitter_circuit_snapshots( + amount_of_explicit_memory_queries, + implicit_memory_queries.decommitter_memory_queries, + implicit_memory_states.decommitter_simulator_snapshots, + implicit_memory_states.decommitter_memory_states, final_explicit_memory_queue_state, decommiter_circuit_inputs, round_function, @@ -1119,10 +1127,11 @@ fn process_memory_related_circuits< tracing::debug!("Running keccak simulation"); - let keccak256_circuits_data = keccak256_decompose_into_per_circuit_witness( + let (keccak256_circuits_data, amount_of_memory_queries) = keccak256_decompose_into_per_circuit_witness( amount_of_memory_queries, - &implicit_memory_queries, - &implicit_memory_states, + implicit_memory_queries.keccak256_memory_queries, + implicit_memory_states.keccak256_simulator_snapshots, + implicit_memory_states.keccak256_memory_states, precompiles_data.keccak_round_function_witnesses, precompiles_data.logs_queries.keccak, precompiles_data.logs_queues_states.keccak, @@ -1137,10 +1146,11 @@ fn process_memory_related_circuits< tracing::debug!("Running sha256 simulation"); - let sha256_circuits_data = sha256_decompose_into_per_circuit_witness( + let (sha256_circuits_data, amount_of_memory_queries) = sha256_decompose_into_per_circuit_witness( amount_of_memory_queries, - &implicit_memory_queries, - &implicit_memory_states, + implicit_memory_queries.sha256_memory_queries, + implicit_memory_states.sha256_simulator_snapshots, + implicit_memory_states.sha256_memory_states, precompiles_data.sha256_round_function_witnesses, precompiles_data.logs_queries.sha256, precompiles_data.logs_queues_states.sha256, @@ -1155,10 +1165,11 @@ fn process_memory_related_circuits< tracing::debug!("Running ecrecover simulation"); - let ecrecover_circuits_data = ecrecover_decompose_into_per_circuit_witness( + let (ecrecover_circuits_data, amount_of_memory_queries) = ecrecover_decompose_into_per_circuit_witness( amount_of_memory_queries, - &implicit_memory_queries, - &implicit_memory_states, + implicit_memory_queries.ecrecover_memory_queries, + implicit_memory_states.ecrecover_simulator_snapshots, + implicit_memory_states.ecrecover_memory_states, precompiles_data.ecrecover_witnesses, precompiles_data.logs_queries.ecrecover, precompiles_data.logs_queues_states.ecrecover, @@ -1171,10 +1182,11 @@ fn process_memory_related_circuits< tracing::debug!("Running secp256r1_simulation simulation"); - let secp256r1_verify_circuits_data = secp256r1_verify_decompose_into_per_circuit_witness( + let (secp256r1_verify_circuits_data, amount_of_memory_queries) = secp256r1_verify_decompose_into_per_circuit_witness( amount_of_memory_queries, - &implicit_memory_queries, - &implicit_memory_states, + implicit_memory_queries.secp256r1_memory_queries, + implicit_memory_states.secp256r1_simulator_snapshots, + implicit_memory_states.secp256r1_memory_states, precompiles_data.secp256r1_verify_witnesses, precompiles_data.logs_queries.secp256r1_verify, precompiles_data.logs_queues_states.secp256r1_verify, @@ -1183,28 +1195,19 @@ fn process_memory_related_circuits< ); circuits_data.secp256r1_verify_circuits_data = secp256r1_verify_circuits_data; - assert_eq!( - implicit_memory_queries.amount_of_queries(), - implicit_memory_states.amount_of_states() - ); - - drop(implicit_memory_states); - use crate::witness::individual_circuits::memory_related::ram_permutation::compute_ram_circuit_snapshots; tracing::debug!("Running RAM permutation simulation"); let (ram_permutation_circuits, ram_permutation_circuits_compact_forms_witnesses) = compute_ram_circuit_snapshots( - &memory_artifacts_for_main_vm.memory_queries, + amount_of_memory_queries, memory_queue_states_accumulator, sorted_memory_queue_states_accumulator, - implicit_memory_queries, memory_queue_simulator, sorted_memory_queue_simulator, round_function, num_non_deterministic_heap_queries, - geometry.cycles_per_ram_permutation as usize, geometry, cs_for_witness_generation, &mut circuit_callback, From a5b26054d30d76cd5d59efdde930ca5c101172f1 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 23 Jul 2024 15:19:51 +0200 Subject: [PATCH 11/33] Cargo fmt --- .../memory_related/decommit_code.rs | 3 +- .../memory_related/ecrecover.rs | 3 +- .../keccak256_round_function.rs | 3 +- .../memory_related/secp256r1_verify.rs | 3 +- .../memory_related/sha256_round_function.rs | 8 +- src/witness/oracle.rs | 113 +++++++++--------- 6 files changed, 70 insertions(+), 63 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/decommit_code.rs b/src/witness/individual_circuits/memory_related/decommit_code.rs index 7f84aae8..baf22d3e 100644 --- a/src/witness/individual_circuits/memory_related/decommit_code.rs +++ b/src/witness/individual_circuits/memory_related/decommit_code.rs @@ -429,7 +429,8 @@ pub(crate) fn compute_decommitter_circuit_snapshots< let memory_simulator_after = &decommitter_simulator_snapshots[1]; - let amount_of_memory_queries_after = amount_of_memory_queries + decommitter_memory_queries.len(); + let amount_of_memory_queries_after = + amount_of_memory_queries + decommitter_memory_queries.len(); assert_eq!( amount_of_memory_queries_after, memory_simulator_after.num_items as usize diff --git a/src/witness/individual_circuits/memory_related/ecrecover.rs b/src/witness/individual_circuits/memory_related/ecrecover.rs index 85eb7edf..7db17f09 100644 --- a/src/witness/individual_circuits/memory_related/ecrecover.rs +++ b/src/witness/individual_circuits/memory_related/ecrecover.rs @@ -221,7 +221,8 @@ pub(crate) fn ecrecover_decompose_into_per_circuit_witness< } let memory_simulator_after = &ecrecover_simulator_snapshots[1]; - let amount_of_memory_queries_after = amount_of_memory_queries_before + amount_ecrecover_memory_queries; + let amount_of_memory_queries_after = + amount_of_memory_queries_before + amount_ecrecover_memory_queries; assert_eq!( amount_of_memory_queries_after, diff --git a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs index 1512fb81..61adbd90 100644 --- a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs @@ -470,7 +470,8 @@ pub(crate) fn keccak256_decompose_into_per_circuit_witness< } let memory_simulator_after = &keccak256_simulator_snapshots[1]; - let amount_of_memory_queries_after = amount_of_memory_queries_before + keccak256_memory_queries.len(); + let amount_of_memory_queries_after = + amount_of_memory_queries_before + keccak256_memory_queries.len(); assert_eq!( amount_of_memory_queries_after, diff --git a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs index d0b39c8d..0117ac3e 100644 --- a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs +++ b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs @@ -226,7 +226,8 @@ pub(crate) fn secp256r1_verify_decompose_into_per_circuit_witness< } let memory_simulator_after = &secp256r1_simulator_snapshots[1]; - let amount_of_memory_queries_after = amount_of_memory_queries_before + amount_secp256r1_memory_queries; + let amount_of_memory_queries_after = + amount_of_memory_queries_before + amount_secp256r1_memory_queries; assert_eq!( amount_of_memory_queries_after, diff --git a/src/witness/individual_circuits/memory_related/sha256_round_function.rs b/src/witness/individual_circuits/memory_related/sha256_round_function.rs index cb212a48..4142101d 100644 --- a/src/witness/individual_circuits/memory_related/sha256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/sha256_round_function.rs @@ -80,10 +80,7 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< num_rounds_per_circuit: usize, round_function: &R, ) -> (Vec>, usize) { - assert_eq!( - sha256_memory_queries.len(), - sha256_memory_states.len() - ); + assert_eq!(sha256_memory_queries.len(), sha256_memory_states.len()); let memory_simulator_before = &sha256_simulator_snapshots[0]; assert_eq!( @@ -345,7 +342,8 @@ pub(crate) fn sha256_decompose_into_per_circuit_witness< } let memory_simulator_after = &sha256_simulator_snapshots[1]; - let amount_of_memory_queries_after = amount_of_memory_queries_before + amount_sha256_memory_queries; + let amount_of_memory_queries_after = + amount_of_memory_queries_before + amount_sha256_memory_queries; assert_eq!( amount_of_memory_queries_after, diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 29af12a8..5330e71b 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -1106,16 +1106,17 @@ fn process_memory_related_circuits< tracing::debug!("Running code code decommitter simulation"); - let (code_decommitter_circuits_data, amount_of_memory_queries) = compute_decommitter_circuit_snapshots( - amount_of_explicit_memory_queries, - implicit_memory_queries.decommitter_memory_queries, - implicit_memory_states.decommitter_simulator_snapshots, - implicit_memory_states.decommitter_memory_states, - final_explicit_memory_queue_state, - decommiter_circuit_inputs, - round_function, - geometry.cycles_per_code_decommitter as usize, - ); + let (code_decommitter_circuits_data, amount_of_memory_queries) = + compute_decommitter_circuit_snapshots( + amount_of_explicit_memory_queries, + implicit_memory_queries.decommitter_memory_queries, + implicit_memory_states.decommitter_simulator_snapshots, + implicit_memory_states.decommitter_memory_states, + final_explicit_memory_queue_state, + decommiter_circuit_inputs, + round_function, + geometry.cycles_per_code_decommitter as usize, + ); circuits_data.code_decommitter_circuits_data = code_decommitter_circuits_data; @@ -1127,17 +1128,18 @@ fn process_memory_related_circuits< tracing::debug!("Running keccak simulation"); - let (keccak256_circuits_data, amount_of_memory_queries) = keccak256_decompose_into_per_circuit_witness( - amount_of_memory_queries, - implicit_memory_queries.keccak256_memory_queries, - implicit_memory_states.keccak256_simulator_snapshots, - implicit_memory_states.keccak256_memory_states, - precompiles_data.keccak_round_function_witnesses, - precompiles_data.logs_queries.keccak, - precompiles_data.logs_queues_states.keccak, - geometry.cycles_per_keccak256_circuit as usize, - round_function, - ); + let (keccak256_circuits_data, amount_of_memory_queries) = + keccak256_decompose_into_per_circuit_witness( + amount_of_memory_queries, + implicit_memory_queries.keccak256_memory_queries, + implicit_memory_states.keccak256_simulator_snapshots, + implicit_memory_states.keccak256_memory_states, + precompiles_data.keccak_round_function_witnesses, + precompiles_data.logs_queries.keccak, + precompiles_data.logs_queues_states.keccak, + geometry.cycles_per_keccak256_circuit as usize, + round_function, + ); circuits_data.keccak256_circuits_data = keccak256_circuits_data; // sha256 precompile @@ -1146,17 +1148,18 @@ fn process_memory_related_circuits< tracing::debug!("Running sha256 simulation"); - let (sha256_circuits_data, amount_of_memory_queries) = sha256_decompose_into_per_circuit_witness( - amount_of_memory_queries, - implicit_memory_queries.sha256_memory_queries, - implicit_memory_states.sha256_simulator_snapshots, - implicit_memory_states.sha256_memory_states, - precompiles_data.sha256_round_function_witnesses, - precompiles_data.logs_queries.sha256, - precompiles_data.logs_queues_states.sha256, - geometry.cycles_per_sha256_circuit as usize, - round_function, - ); + let (sha256_circuits_data, amount_of_memory_queries) = + sha256_decompose_into_per_circuit_witness( + amount_of_memory_queries, + implicit_memory_queries.sha256_memory_queries, + implicit_memory_states.sha256_simulator_snapshots, + implicit_memory_states.sha256_memory_states, + precompiles_data.sha256_round_function_witnesses, + precompiles_data.logs_queries.sha256, + precompiles_data.logs_queues_states.sha256, + geometry.cycles_per_sha256_circuit as usize, + round_function, + ); circuits_data.sha256_circuits_data = sha256_circuits_data; // ecrecover precompile @@ -1165,34 +1168,36 @@ fn process_memory_related_circuits< tracing::debug!("Running ecrecover simulation"); - let (ecrecover_circuits_data, amount_of_memory_queries) = ecrecover_decompose_into_per_circuit_witness( - amount_of_memory_queries, - implicit_memory_queries.ecrecover_memory_queries, - implicit_memory_states.ecrecover_simulator_snapshots, - implicit_memory_states.ecrecover_memory_states, - precompiles_data.ecrecover_witnesses, - precompiles_data.logs_queries.ecrecover, - precompiles_data.logs_queues_states.ecrecover, - geometry.cycles_per_ecrecover_circuit as usize, - round_function, - ); + let (ecrecover_circuits_data, amount_of_memory_queries) = + ecrecover_decompose_into_per_circuit_witness( + amount_of_memory_queries, + implicit_memory_queries.ecrecover_memory_queries, + implicit_memory_states.ecrecover_simulator_snapshots, + implicit_memory_states.ecrecover_memory_states, + precompiles_data.ecrecover_witnesses, + precompiles_data.logs_queries.ecrecover, + precompiles_data.logs_queues_states.ecrecover, + geometry.cycles_per_ecrecover_circuit as usize, + round_function, + ); circuits_data.ecrecover_circuits_data = ecrecover_circuits_data; use crate::witness::individual_circuits::memory_related::secp256r1_verify::secp256r1_verify_decompose_into_per_circuit_witness; tracing::debug!("Running secp256r1_simulation simulation"); - let (secp256r1_verify_circuits_data, amount_of_memory_queries) = secp256r1_verify_decompose_into_per_circuit_witness( - amount_of_memory_queries, - implicit_memory_queries.secp256r1_memory_queries, - implicit_memory_states.secp256r1_simulator_snapshots, - implicit_memory_states.secp256r1_memory_states, - precompiles_data.secp256r1_verify_witnesses, - precompiles_data.logs_queries.secp256r1_verify, - precompiles_data.logs_queues_states.secp256r1_verify, - geometry.cycles_per_secp256r1_verify_circuit as usize, - round_function, - ); + let (secp256r1_verify_circuits_data, amount_of_memory_queries) = + secp256r1_verify_decompose_into_per_circuit_witness( + amount_of_memory_queries, + implicit_memory_queries.secp256r1_memory_queries, + implicit_memory_states.secp256r1_simulator_snapshots, + implicit_memory_states.secp256r1_memory_states, + precompiles_data.secp256r1_verify_witnesses, + precompiles_data.logs_queries.secp256r1_verify, + precompiles_data.logs_queues_states.secp256r1_verify, + geometry.cycles_per_secp256r1_verify_circuit as usize, + round_function, + ); circuits_data.secp256r1_verify_circuits_data = secp256r1_verify_circuits_data; use crate::witness::individual_circuits::memory_related::ram_permutation::compute_ram_circuit_snapshots; From c69281692598bf691e1b132768bf97275f64bcd4 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 23 Jul 2024 15:59:08 +0200 Subject: [PATCH 12/33] Cleanup --- src/witness/artifacts.rs | 7 ------ .../memory_related/decommit_code.rs | 1 - .../memory_related/ecrecover.rs | 2 +- .../keccak256_round_function.rs | 2 +- .../individual_circuits/memory_related/mod.rs | 23 ++++--------------- .../memory_related/ram_permutation.rs | 2 +- .../memory_related/secp256r1_verify.rs | 2 +- .../memory_related/sha256_round_function.rs | 2 +- src/witness/oracle.rs | 13 +++-------- 9 files changed, 12 insertions(+), 42 deletions(-) diff --git a/src/witness/artifacts.rs b/src/witness/artifacts.rs index d8fad4c3..0b818265 100644 --- a/src/witness/artifacts.rs +++ b/src/witness/artifacts.rs @@ -121,13 +121,6 @@ pub struct MemoryArtifacts { CircuitsEntryAccumulatorSparse<(u32, QueueStateWitness)>, } -#[derive(Derivative)] -#[derivative(Default)] -pub struct ImplicitMemoryArtifacts { - pub memory_queries: Vec, - pub memory_queue_states: Vec>, -} - #[derive(Derivative)] #[derivative(Default)] pub struct LogCircuitsArtifacts { diff --git a/src/witness/individual_circuits/memory_related/decommit_code.rs b/src/witness/individual_circuits/memory_related/decommit_code.rs index baf22d3e..49a3d1a7 100644 --- a/src/witness/individual_circuits/memory_related/decommit_code.rs +++ b/src/witness/individual_circuits/memory_related/decommit_code.rs @@ -8,7 +8,6 @@ use crate::zk_evm::ethereum_types::U256; use crate::zkevm_circuits::base_structures::decommit_query::DecommitQueryWitness; use crate::zkevm_circuits::base_structures::decommit_query::DECOMMIT_QUERY_PACKED_WIDTH; use crate::zkevm_circuits::code_unpacker_sha256::input::*; -use artifacts::ImplicitMemoryArtifacts; use circuit_definitions::encodings::decommittment_request::normalized_preimage_as_u256; use circuit_definitions::encodings::decommittment_request::DecommittmentQueueSimulator; use circuit_definitions::encodings::decommittment_request::DecommittmentQueueState; diff --git a/src/witness/individual_circuits/memory_related/ecrecover.rs b/src/witness/individual_circuits/memory_related/ecrecover.rs index 7db17f09..bb31b44a 100644 --- a/src/witness/individual_circuits/memory_related/ecrecover.rs +++ b/src/witness/individual_circuits/memory_related/ecrecover.rs @@ -1,5 +1,5 @@ use super::*; -use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQueueStates}; +use crate::witness::artifacts::{DemuxedLogQueries, LogQueueStates}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; diff --git a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs index 61adbd90..2b83ad3e 100644 --- a/src/witness/individual_circuits/memory_related/keccak256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/keccak256_round_function.rs @@ -1,5 +1,5 @@ use super::*; -use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQueueStates}; +use crate::witness::artifacts::{DemuxedLogQueries, LogQueueStates}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index ec09f7c9..52ce4957 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -156,25 +156,6 @@ impl ImplicitMemoryStates { + self.secp256r1_memory_states.len() + self.sha256_memory_states.len() } - - fn get_vectors(&self) -> [&Vec>; 5] { - [ - &self.decommitter_memory_states, - &self.keccak256_memory_states, - &self.sha256_memory_states, - &self.ecrecover_memory_states, - &self.secp256r1_memory_states, - ] - } - - pub fn iter(&self) -> ImplicitMemoryStatesIter { - let mut outer_iter = self.get_vectors().into_iter(); - let last_vector_iter = outer_iter.next().unwrap().iter(); - ImplicitMemoryStatesIter { - last_vector: last_vector_iter, - outer_iter, - } - } } use core::array::IntoIter as ArrayIntoIter; @@ -215,11 +196,14 @@ fn get_simulator_snapshot( } } +use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; + pub(crate) fn simulate_implicit_memory_queues< F: SmallField, R: BuildableCircuitRoundFunction + AlgebraicRoundFunction, >( memory_queue_simulator: &mut MemoryQueuePerCircuitSimulator, + memory_queue_states_accumulator: &mut LastPerCircuitAccumulator>, implicit_memory_queries: &ImplicitMemoryQueries, round_function: R, ) -> ImplicitMemoryStates { @@ -234,6 +218,7 @@ pub(crate) fn simulate_implicit_memory_queues< .push_and_output_intermediate_data(*query, &round_function); memory_states.push(intermediate_info); + memory_queue_states_accumulator.push(intermediate_info); } snapshots.push(get_simulator_snapshot(memory_queue_simulator)); // after diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index 789fd459..69ce6c00 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -12,7 +12,7 @@ use crate::zk_evm::ethereum_types::U256; use crate::zkevm_circuits::{ base_structures::memory_query::MEMORY_QUERY_PACKED_WIDTH, ram_permutation::input::*, }; -use artifacts::{ImplicitMemoryArtifacts, MemoryArtifacts}; +use artifacts::MemoryArtifacts; use circuit_definitions::circuit_definitions::base_layer::{ RAMPermutationInstanceSynthesisFunction, ZkSyncBaseLayerCircuit, }; diff --git a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs index 0117ac3e..67af787b 100644 --- a/src/witness/individual_circuits/memory_related/secp256r1_verify.rs +++ b/src/witness/individual_circuits/memory_related/secp256r1_verify.rs @@ -1,5 +1,5 @@ use super::*; -use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQueueStates}; +use crate::witness::artifacts::{DemuxedLogQueries, LogQueueStates}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; diff --git a/src/witness/individual_circuits/memory_related/sha256_round_function.rs b/src/witness/individual_circuits/memory_related/sha256_round_function.rs index 4142101d..23d372b6 100644 --- a/src/witness/individual_circuits/memory_related/sha256_round_function.rs +++ b/src/witness/individual_circuits/memory_related/sha256_round_function.rs @@ -1,6 +1,6 @@ use super::*; use crate::boojum::gadgets::traits::allocatable::CSAllocatable; -use crate::witness::artifacts::{DemuxedLogQueries, ImplicitMemoryArtifacts, LogQueueStates}; +use crate::witness::artifacts::{DemuxedLogQueries, LogQueueStates}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::LastPerCircuitAccumulator; use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; use crate::zk_evm::aux_structures::LogQuery as LogQuery_; diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 5330e71b..3f3acf5f 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -17,7 +17,7 @@ use crate::boojum::gadgets::traits::allocatable::CSAllocatable; use crate::ethereum_types::U256; use crate::toolset::GeometryConfig; use crate::witness::artifacts::{ - DemuxedLogQueries, ImplicitMemoryArtifacts, MemoryArtifacts, MemoryCircuitsArtifacts, + DemuxedLogQueries, MemoryArtifacts, MemoryCircuitsArtifacts, }; use crate::witness::aux_data_structs::one_per_circuit_accumulator::{ CircuitsEntryAccumulatorSparse, LastPerCircuitAccumulator, @@ -893,16 +893,11 @@ fn simulate_memory_queue( use crate::witness::individual_circuits::memory_related::simulate_implicit_memory_queues; let implicit_memory_states = simulate_implicit_memory_queues( &mut memory_queue_simulator, + &mut memory_queue_states_accumulator, &implicit_memory_queries, round_function, ); - // push implicit queries - memory_queue_states_accumulator.reserve_exact_flat(implicit_memory_states.amount_of_states()); - for state in implicit_memory_states.iter() { - memory_queue_states_accumulator.push(*state); - } - ( memory_artifacts_for_main_vm, final_explicit_memory_queue_state, @@ -1095,10 +1090,8 @@ fn process_memory_related_circuits< // direct VM related part is done, other subcircuit's functionality is moved to other functions // that should properly do sorts and memory writes - let amount_of_implicit_queries = implicit_memory_queries.amount_of_queries(); - assert_eq!( - amount_of_implicit_queries, + implicit_memory_queries.amount_of_queries(), implicit_memory_states.amount_of_states() ); From 389812ed480010edf55441ef202e4a5441409095 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Thu, 8 Aug 2024 15:21:38 +0200 Subject: [PATCH 13/33] Unify callbacks --- Cargo.toml | 1 + src/external_calls.rs | 17 +-- src/run_vms.rs | 17 +-- src/tests/complex_tests/mod.rs | 24 ++-- src/tests/run_manually.rs | 9 +- src/utils.rs | 94 +++++++++++++ src/witness/individual_circuits/log_demux.rs | 21 ++- src/witness/individual_circuits/main_vm.rs | 19 +-- .../memory_related/ram_permutation.rs | 17 +-- .../storage_application.rs | 21 +-- src/witness/oracle.rs | 126 ++++++++---------- src/witness/postprocessing/mod.rs | 51 +++++-- src/witness/utils.rs | 2 +- 13 files changed, 251 insertions(+), 168 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fc1776d0..3c1416f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ smallvec = "1.13" structopt = "0.3.26" codegen = "0.2.0" regex = { version = "1.10.6", features = ["pattern"] } +peak_alloc = "*" [dev-dependencies] rand = "0.4" diff --git a/src/external_calls.rs b/src/external_calls.rs index a566ca62..a0e2aa50 100644 --- a/src/external_calls.rs +++ b/src/external_calls.rs @@ -5,6 +5,7 @@ pub use crate::run_vms::SCHEDULER_TIMESTAMP; use crate::snark_wrapper::boojum::field::goldilocks::GoldilocksExt2; use crate::snark_wrapper::boojum::gadgets::recursion::recursive_tree_hasher::CircuitGoldilocksPoseidon2Sponge; use crate::toolset::GeometryConfig; +use crate::witness::oracle::WitnessGenerationArtifact; use crate::witness::tree::BinarySparseStorageTree; use crate::witness::tree::ZkSyncStorageLeaf; use crate::zk_evm::abstractions::Storage; @@ -28,15 +29,7 @@ use circuit_definitions::Field as MainField; /// /// This function will setup the environment and will run out-of-circuit and then in-circuit. /// GenericNoopTracer will be used as out-of-circuit tracer -pub fn run< - S: Storage, - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub fn run( caller: Address, // for real block must be zero entry_point_address: Address, // for real block must be the bootloader entry_point_code: Vec<[u8; 32]>, // for read block must be a bootloader code @@ -52,8 +45,7 @@ pub fn run< tree: impl BinarySparseStorageTree<256, 32, 32, 8, 32, Blake2s256, ZkSyncStorageLeaf>, trusted_setup_path: &str, eip_4844_repack_inputs: [Option>; MAX_4844_BLOBS_PER_BLOCK], - circuit_callback: CB, - queue_simulator_callback: QSCB, + artifacts_callback: CB, ) -> ( SchedulerCircuitInstanceWitness, BlockAuxilaryOutputWitness, @@ -75,8 +67,7 @@ pub fn run< tree, trusted_setup_path, eip_4844_repack_inputs, - circuit_callback, - queue_simulator_callback, + artifacts_callback, &mut out_of_circuit_tracer, ) { Ok((scheduler_circuit_witness, aux_data)) => (scheduler_circuit_witness, aux_data), diff --git a/src/run_vms.rs b/src/run_vms.rs index a7821266..3dce3fad 100644 --- a/src/run_vms.rs +++ b/src/run_vms.rs @@ -7,6 +7,7 @@ use crate::snark_wrapper::boojum::gadgets::recursion::recursive_tree_hasher::Cir use crate::toolset::create_tools; use crate::toolset::GeometryConfig; use crate::witness::oracle::create_artifacts_from_tracer; +use crate::witness::oracle::WitnessGenerationArtifact; use crate::witness::tracer::tracer::WitnessTracer; use crate::witness::tree::BinarySparseStorageTree; use crate::witness::tree::ZkSyncStorageLeaf; @@ -61,15 +62,7 @@ pub type RunVMsResult = ( /// - witness with AUX data (with information that might be useful during verification to generate the public input) /// /// This function will setup the environment and will run out-of-circuit and then in-circuit -pub fn run_vms< - S: Storage, - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub fn run_vms( caller: Address, // for real block must be zero entry_point_address: Address, // for real block must be the bootloader entry_point_code: Vec<[u8; 32]>, // for read block must be a bootloader code @@ -85,8 +78,7 @@ pub fn run_vms< tree: impl BinarySparseStorageTree<256, 32, 32, 8, 32, Blake2s256, ZkSyncStorageLeaf>, trusted_setup_path: &str, eip_4844_repack_inputs: [Option>; MAX_4844_BLOBS_PER_BLOCK], - circuit_callback: CB, - queue_simulator_callback: QSCB, + artifacts_callback: CB, out_of_circuit_tracer: &mut impl Tracer, ) -> Result { let round_function = ZkSyncDefaultRoundFunction::default(); @@ -254,8 +246,7 @@ pub fn run_vms< evm_simulator_code_hash, eip_4844_repack_inputs.clone(), trusted_setup_path, - circuit_callback, - queue_simulator_callback, + artifacts_callback, ); let (scheduler_circuit_witness, aux_data) = { diff --git a/src/tests/complex_tests/mod.rs b/src/tests/complex_tests/mod.rs index c9459288..c16813c7 100644 --- a/src/tests/complex_tests/mod.rs +++ b/src/tests/complex_tests/mod.rs @@ -58,6 +58,7 @@ use circuit_definitions::{ use circuit_definitions::{Field, RoundFunction}; use utils::read_basic_test_artifact; +use witness::oracle::WitnessGenerationArtifact; use zkevm_assembly::Assembly; #[ignore = "Too slow"] @@ -255,6 +256,18 @@ pub(crate) fn generate_base_layer( let mut basic_block_circuits = vec![]; let mut recursion_queues = vec![]; + + let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { + WitnessGenerationArtifact::BaseLayerCircuit(circuit) => basic_block_circuits.push(circuit), + WitnessGenerationArtifact::RecursionQueue((a, b, c)) => recursion_queues.push(( + a, + b, + c.into_iter() + .map(|x| ZkSyncBaseLayerStorage::from_inner(a as u8, x)) + .collect(), + )), + }; + let (scheduler_partial_input, _aux_data) = run( Address::zero(), test_artifact.entry_point_address, @@ -271,16 +284,7 @@ pub(crate) fn generate_base_layer( tree, "kzg/src/trusted_setup.json", blobs, - |circuit| basic_block_circuits.push(circuit), - |a, b, c| { - recursion_queues.push(( - a, - b, - c.into_iter() - .map(|x| ZkSyncBaseLayerStorage::from_inner(a as u8, x)) - .collect(), - )) - }, + artifacts_callback, ); ( diff --git a/src/tests/run_manually.rs b/src/tests/run_manually.rs index 6d3311cf..3a2135b6 100644 --- a/src/tests/run_manually.rs +++ b/src/tests/run_manually.rs @@ -23,6 +23,7 @@ use crate::zkevm_circuits::main_vm::main_vm_entry_point; use circuit_definitions::aux_definitions::witness_oracle::VmWitnessOracle; use circuit_definitions::zk_evm::vm_state::cycle; use storage::{InMemoryCustomRefundStorage, StorageRefund}; +use witness::oracle::WitnessGenerationArtifact; use zkevm_assembly::Assembly; #[test] @@ -265,6 +266,11 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt let mut out_of_circuit_tracer = TestingTracer::new(Some(storage_impl.create_refund_controller())); + let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { + WitnessGenerationArtifact::BaseLayerCircuit(circuit) => basic_block_circuits.push(circuit), + _ => {} + }; + if let Err(err) = run_vms( Address::zero(), *BOOTLOADER_FORMAL_ADDRESS, @@ -281,8 +287,7 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt tree, "kzg/src/trusted_setup.json", std::array::from_fn(|_| None), - |circuit| basic_block_circuits.push(circuit), - |_, _, _| {}, + artifacts_callback, &mut out_of_circuit_tracer, ) { let error_text = match err { diff --git a/src/utils.rs b/src/utils.rs index ecc29038..2005de17 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,6 +4,100 @@ use crate::witness::tree::BinaryHasher; use circuit_definitions::encodings::BytesSerializable; +use peak_alloc::PeakAlloc; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; +use std::time::Instant; + +#[global_allocator] +static PEAK_ALLOC: PeakAlloc = PeakAlloc; + +fn mem_array() -> &'static Mutex> { + static MEM_ARRAY: OnceLock>> = OnceLock::new(); + MEM_ARRAY.get_or_init(|| Mutex::new(vec![])) +} + +fn time_instant() -> &'static Mutex { + static TIME_INSTANT: OnceLock> = OnceLock::new(); + TIME_INSTANT.get_or_init(|| Mutex::new(Instant::now())) +} + +fn time_array() -> &'static Mutex> { + static TIME_ARRAY: OnceLock>> = OnceLock::new(); + TIME_ARRAY.get_or_init(|| Mutex::new(vec![])) +} + +fn peak_mem_array() -> &'static Mutex> { + static PEAK_MEM_ARRAY: OnceLock>> = OnceLock::new(); + PEAK_MEM_ARRAY.get_or_init(|| Mutex::new(vec![])) +} + +fn peak_snapshot_mem(label: &str) { + let peak_mem = PEAK_ALLOC.peak_usage_as_mb(); + println!("PEAK MEM: {}: {}", label.to_owned(), peak_mem); + peak_mem_array() + .lock() + .unwrap() + .push((label.to_owned(), peak_mem)); +} + +pub fn print_mem_snapshots() { + let mem = mem_array().lock().unwrap(); + println!("MEMORY SNAPSHOTS"); + for snapshot in mem.clone() { + println!("{}: {}", snapshot.0, snapshot.1); + } + println!(); +} + +pub fn print_peak_mem_snapshots() { + let mem = peak_mem_array().lock().unwrap(); + println!("PEAK MEMORY SNAPSHOTS"); + for snapshot in mem.clone() { + println!("{}: {}", snapshot.0, snapshot.1); + } + println!(); +} + +pub fn print_time_snapshots() { + let time = time_array().lock().unwrap(); + println!("TIME SNAPSHOTS"); + for snapshot in time.clone() { + println!("{}: {:.2?}", snapshot.0, snapshot.1); + } + println!(); +} + +pub fn snapshot_mem(label: &str) { + let current_mem = PEAK_ALLOC.current_usage_as_mb(); + println!("MEM: {}: {}", label.to_owned(), current_mem); + mem_array() + .lock() + .unwrap() + .push((label.to_owned(), current_mem)); +} + +pub fn snapshot_prof(label: &str) { + snapshot_mem(label); + peak_snapshot_mem(label); + snapshot_time(label); + reset_peak_snapshot_mem(); +} + +pub fn snapshot_time(label: &str) { + let mut instant = time_instant().lock().unwrap(); + println!("TIME: {}: {:.2?}", label.to_owned(), instant.elapsed()); + time_array() + .lock() + .unwrap() + .push((label.to_owned(), instant.elapsed())); + *instant = Instant::now(); +} + +pub fn reset_peak_snapshot_mem() { + PEAK_ALLOC.reset_peak_usage(); +} + pub fn u64_as_u32_le(value: u64) -> [u32; 2] { [value as u32, (value >> 32) as u32] } diff --git a/src/witness/individual_circuits/log_demux.rs b/src/witness/individual_circuits/log_demux.rs index 0c11f550..5602c4a1 100644 --- a/src/witness/individual_circuits/log_demux.rs +++ b/src/witness/individual_circuits/log_demux.rs @@ -19,6 +19,7 @@ use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; use circuit_definitions::zkevm_circuits::demux_log_queue::DemuxOutput; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; use circuit_definitions::{encodings::*, Field, RoundFunction}; +use oracle::WitnessGenerationArtifact; use postprocessing::CsForWitnessGeneration; use zk_evm::zkevm_opcode_defs::SECP256R1_VERIFY_PRECOMPILE_ADDRESS; @@ -173,18 +174,14 @@ impl DemuxedQueuesStatesSimulator { } /// Take a storage log, output logs separately for events, l1 messages, storage, etc -pub(crate) fn process_logs_demux_and_make_circuits< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut(u64, RecursionQueueSimulator, Vec>), ->( +pub(crate) fn process_logs_demux_and_make_circuits( mut log_demux_artifacts: LogDemuxCircuitArtifacts, demuxed_queues: &DemuxedLogQueries, per_circuit_capacity: usize, round_function: &RoundFunction, geometry: &GeometryConfig, cs_for_witness_generation: &mut CsForWitnessGeneration, - mut circuit_callback: CB, - mut recursion_queue_callback: QSCB, + mut artifacts_callback: CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, @@ -214,11 +211,11 @@ pub(crate) fn process_logs_demux_and_make_circuits< { let (log_demux_circuits, queue_simulator, log_demux_circuits_compact_forms_witnesses) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, queue_simulator, log_demux_circuits_compact_forms_witnesses.clone(), - ); + ))); let (io_queues_states, precompiles_queues_states) = DemuxedQueuesStatesSimulator::build_empty(*round_function); @@ -397,18 +394,18 @@ pub(crate) fn process_logs_demux_and_make_circuits< } previous_hidden_fsm_output = Some(witness.closed_form_input.hidden_fsm_output.clone()); - circuit_callback(ZkSyncBaseLayerCircuit::LogDemuxer( - maker.process(witness, circuit_type), + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit( + ZkSyncBaseLayerCircuit::LogDemuxer(maker.process(witness, circuit_type)), )); } let (log_demux_circuits, queue_simulator, log_demux_circuits_compact_forms_witnesses) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, queue_simulator, log_demux_circuits_compact_forms_witnesses.clone(), - ); + ))); for (sub_queue, mut iter) in queries_iterators { assert!( diff --git a/src/witness/individual_circuits/main_vm.rs b/src/witness/individual_circuits/main_vm.rs index 4025a65f..886f9597 100644 --- a/src/witness/individual_circuits/main_vm.rs +++ b/src/witness/individual_circuits/main_vm.rs @@ -316,18 +316,12 @@ fn repack_input_for_main_vm( use crate::witness::postprocessing::observable_witness::VmObservableWitness; use crate::zkevm_circuits::fsm_input_output::circuit_inputs::main_vm::VmCircuitWitness; +use super::oracle::WitnessGenerationArtifact; use super::{ simulate_public_input_value_from_witness, vm_instance_witness_to_circuit_formal_input, }; -pub(crate) fn process_main_vm< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub(crate) fn process_main_vm( geometry: &GeometryConfig, in_circuit_global_context: GlobalContextWitness, memory_artifacts_for_main_vm: MemoryArtifacts, @@ -345,8 +339,7 @@ pub(crate) fn process_main_vm< mut vm_snapshots: Vec, round_function: Poseidon2Goldilocks, cs_for_witness_generation: &mut CsForWitnessGeneration, - circuit_callback: &mut CB, - recursion_queue_callback: &mut QSCB, + artifacts_callback: &mut CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, @@ -412,7 +405,7 @@ pub(crate) fn process_main_vm< }; queue_simulator.push(recursive_request, &round_function); - circuit_callback(instance); + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit(instance)); main_vm_circuits_compact_forms_witnesses.push(compact_form_witness); }; @@ -544,11 +537,11 @@ pub(crate) fn process_main_vm< } } - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( BaseLayerCircuitType::VM as u64, queue_simulator, main_vm_circuits_compact_forms_witnesses.clone(), - ); + ))); (main_vm_circuits, main_vm_circuits_compact_forms_witnesses) } diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index 69ce6c00..2bf671c8 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -22,6 +22,7 @@ use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; use circuit_definitions::{encodings::*, Field, RoundFunction}; use memory_query::{CustomMemoryQueueSimulator, QueueWitness}; +use oracle::WitnessGenerationArtifact; use postprocessing::{CsForWitnessGeneration, FirstAndLastCircuitWitness}; use rayon::prelude::*; @@ -33,10 +34,7 @@ use zkevm_circuits::base_structures::vm_state::QUEUE_STATE_WIDTH; use crate::zk_evm::zkevm_opcode_defs::BOOTLOADER_HEAP_PAGE; -pub(crate) fn compute_ram_circuit_snapshots< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut(u64, RecursionQueueSimulator, Vec>), ->( +pub(crate) fn compute_ram_circuit_snapshots( total_amount_of_queries: usize, // including additional queries from precompiles memory_queue_states_accumulator: LastPerCircuitAccumulator>, sorted_memory_queue_states_accumulator: LastPerCircuitAccumulator>, @@ -46,8 +44,7 @@ pub(crate) fn compute_ram_circuit_snapshots< num_non_deterministic_heap_queries: usize, geometry: &GeometryConfig, cs_for_witness_generation: &mut CsForWitnessGeneration, - mut circuit_callback: CB, - mut recursion_queue_callback: QSCB, + mut artifacts_callback: CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, @@ -428,8 +425,8 @@ pub(crate) fn compute_ram_circuit_snapshots< tmp.current_sorted_queue_state.clone(), ); - circuit_callback(ZkSyncBaseLayerCircuit::RAMPermutation( - maker.process(instance_witness, circuit_type), + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit( + ZkSyncBaseLayerCircuit::RAMPermutation(maker.process(instance_witness, circuit_type)), )); } @@ -441,11 +438,11 @@ pub(crate) fn compute_ram_circuit_snapshots< queue_simulator, ram_permutation_circuits_compact_forms_witnesses, ) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, queue_simulator, ram_permutation_circuits_compact_forms_witnesses.clone(), - ); + ))); ( ram_permutation_circuits, diff --git a/src/witness/individual_circuits/storage_application.rs b/src/witness/individual_circuits/storage_application.rs index 997a8215..11bc9ef6 100644 --- a/src/witness/individual_circuits/storage_application.rs +++ b/src/witness/individual_circuits/storage_application.rs @@ -21,20 +21,14 @@ use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; use circuit_definitions::encodings::state_diff_record::StateDiffRecord; use circuit_definitions::encodings::LogQueueSimulator; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; +use oracle::WitnessGenerationArtifact; use postprocessing::CsForWitnessGeneration; use tracing; use zk_evm::aux_structures::LogQuery; use crate::sha3::Digest; -pub(crate) fn decompose_into_storage_application_witnesses< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub(crate) fn decompose_into_storage_application_witnesses( deduplicated_rollup_storage_queue_simulator: LogQueueSimulator, deduplicated_rollup_storage_queries: Vec, mut tree: impl BinarySparseStorageTree<256, 32, 32, 8, 32, Blake2s256, ZkSyncStorageLeaf>, @@ -42,8 +36,7 @@ pub(crate) fn decompose_into_storage_application_witnesses< num_rounds_per_circuit: usize, geometry: &GeometryConfig, cs_for_witness_generation: &mut CsForWitnessGeneration, - mut circuit_callback: CB, - mut recursion_queue_callback: QSCB, + mut artifacts_callback: CB, ) -> ( FirstAndLastCircuitWitness>, Vec>, @@ -264,8 +257,8 @@ pub(crate) fn decompose_into_storage_application_witnesses< initial_fsm_state = final_fsm_state.clone(); - circuit_callback(ZkSyncBaseLayerCircuit::StorageApplication( - maker.process(input, circuit_type), + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit( + ZkSyncBaseLayerCircuit::StorageApplication(maker.process(input, circuit_type)), )); } @@ -274,11 +267,11 @@ pub(crate) fn decompose_into_storage_application_witnesses< queue_simulator, storage_application_circuits_compact_forms_witnesses, ) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, queue_simulator, storage_application_circuits_compact_forms_witnesses.clone(), - ); + ))); tracing::debug!( "Final enumeration index = {}", diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 3f3acf5f..917858fe 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -15,10 +15,9 @@ use crate::boojum::field::SmallField; use crate::boojum::gadgets::queue::QueueState; use crate::boojum::gadgets::traits::allocatable::CSAllocatable; use crate::ethereum_types::U256; +use crate::snapshot_prof; use crate::toolset::GeometryConfig; -use crate::witness::artifacts::{ - DemuxedLogQueries, MemoryArtifacts, MemoryCircuitsArtifacts, -}; +use crate::witness::artifacts::{DemuxedLogQueries, MemoryArtifacts, MemoryCircuitsArtifacts}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::{ CircuitsEntryAccumulatorSparse, LastPerCircuitAccumulator, }; @@ -689,22 +688,14 @@ use crate::zkevm_circuits::demux_log_queue::DemuxOutput; /// Process log circuits that do not use memory. /// Storage, transient storage, events, l2 to l1 queries /// Precompiles use memory and are processed in 'process_memory_related_circuits' -fn process_io_log_circuits< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +fn process_io_log_circuits( geometry: &GeometryConfig, tree: impl BinarySparseStorageTree<256, 32, 32, 8, 32, Blake2s256, ZkSyncStorageLeaf>, demuxed_log_queues_states: IOLogsQueuesStates, demuxed_log_queries: DemuxedIOLogQueries, round_function: &Poseidon2Goldilocks, cs_for_witness_generation: &mut CsForWitnessGeneration, - mut circuit_callback: &mut CB, - mut recursion_queue_callback: &mut QSCB, + mut artifacts_callback: &mut CB, ) -> ( LogCircuitsArtifacts, FirstAndLastCircuitWitness>, @@ -801,8 +792,7 @@ fn process_io_log_circuits< geometry.cycles_per_storage_application as usize, geometry, cs_for_witness_generation, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); ( @@ -972,14 +962,7 @@ pub(crate) struct PrecompilesInputData { pub logs_queries: DemuxedPrecompilesLogQueries, } -fn process_memory_related_circuits< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +fn process_memory_related_circuits( geometry: &GeometryConfig, vm_snapshots: &Vec, memory_queries: Vec<(Cycle, MemoryQuery)>, @@ -989,8 +972,7 @@ fn process_memory_related_circuits< precompiles_data: PrecompilesInputData, round_function: &Poseidon2Goldilocks, cs_for_witness_generation: &mut CsForWitnessGeneration, - mut circuit_callback: &mut CB, - mut recursion_queue_callback: &mut QSCB, + mut artifacts_callback: &mut CB, ) -> ( MemoryCircuitsArtifacts, MemoryArtifacts, @@ -1055,6 +1037,8 @@ fn process_memory_related_circuits< let amount_of_explicit_memory_queries = memory_queries.len(); + snapshot_prof("BEFORE QUEUES SIMULATION"); + use std::thread; let sorted_handle = { let memory_queries = memory_queries.clone(); @@ -1095,6 +1079,8 @@ fn process_memory_related_circuits< implicit_memory_states.amount_of_states() ); + snapshot_prof("AFTER QUEUES SIMULATION"); + use crate::witness::individual_circuits::memory_related::decommit_code::compute_decommitter_circuit_snapshots; tracing::debug!("Running code code decommitter simulation"); @@ -1197,6 +1183,8 @@ fn process_memory_related_circuits< tracing::debug!("Running RAM permutation simulation"); + snapshot_prof("BEFORE RAM"); + let (ram_permutation_circuits, ram_permutation_circuits_compact_forms_witnesses) = compute_ram_circuit_snapshots( amount_of_memory_queries, @@ -1208,10 +1196,11 @@ fn process_memory_related_circuits< num_non_deterministic_heap_queries, geometry, cs_for_witness_generation, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); + snapshot_prof("AFTER RAM"); + ( circuits_data, memory_artifacts_for_main_vm, @@ -1221,16 +1210,20 @@ fn process_memory_related_circuits< ) } +pub enum WitnessGenerationArtifact { + BaseLayerCircuit(ZkSyncBaseLayerCircuit), + RecursionQueue( + ( + u64, + RecursionQueueSimulator, + Vec>, + ), + ), +} + /// Make basic circuits instances and witnesses, /// create artifacts for recursion layer and scheduler -pub(crate) fn create_artifacts_from_tracer< - CB: FnMut(ZkSyncBaseLayerCircuit), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), ->( +pub(crate) fn create_artifacts_from_tracer( tracer: WitnessTracer, round_function: &Poseidon2Goldilocks, geometry: &GeometryConfig, @@ -1242,8 +1235,7 @@ pub(crate) fn create_artifacts_from_tracer< evm_simulator_code_hash: U256, eip_4844_repack_inputs: [Option>; MAX_4844_BLOBS_PER_BLOCK], trusted_setup_path: &str, - mut circuit_callback: CB, - mut recursion_queue_callback: QSCB, + mut artifacts_callback: CB, ) -> ( BlockFirstAndLastBasicCircuitsObservableWitnesses, Vec>, @@ -1354,8 +1346,7 @@ pub(crate) fn create_artifacts_from_tracer< round_function, geometry, &mut cs_for_witness_generation, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); tracing::debug!("Processing log circuits"); @@ -1371,8 +1362,7 @@ pub(crate) fn create_artifacts_from_tracer< demuxed_log_queries.io, round_function, &mut cs_for_witness_generation, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); tracing::debug!("Processing memory-related circuits"); @@ -1407,8 +1397,7 @@ pub(crate) fn create_artifacts_from_tracer< precompiles_data, round_function, &mut cs_for_witness_generation, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); tracing::debug!("Waiting for callstack sumulation"); @@ -1445,8 +1434,7 @@ pub(crate) fn create_artifacts_from_tracer< vm_snapshots, *round_function, &mut cs_for_witness_generation, - &mut circuit_callback, - &mut recursion_queue_callback, + &mut artifacts_callback, ); tracing::debug!("Making remaining circuits"); @@ -1480,8 +1468,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::DecommitmentsFilter, decommittments_deduplicator_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::CodeDecommittmentsSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::CodeDecommittmentsSorter(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1492,8 +1480,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::Decommiter, code_decommitter_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::CodeDecommitter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::CodeDecommitter(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1504,8 +1492,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::KeccakPrecompile, keccak256_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::KeccakRoundFunction(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::KeccakRoundFunction(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1516,8 +1504,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::Sha256Precompile, sha256_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::Sha256RoundFunction(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::Sha256RoundFunction(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1528,8 +1516,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::EcrecoverPrecompile, ecrecover_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::ECRecover(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::ECRecover(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1540,8 +1528,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::Secp256r1Verify, secp256r1_verify_circuits_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::Secp256r1Verify(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::Secp256r1Verify(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1551,8 +1539,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::StorageFilter, storage_deduplicator_circuit_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::StorageSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::StorageSorter(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1562,8 +1550,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::EventsRevertsFilter, events_deduplicator_circuit_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::EventsSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::EventsSorter(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1574,8 +1562,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::L1MessagesRevertsFilter, l1_messages_deduplicator_circuit_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::L1MessagesSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::L1MessagesSorter(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1586,8 +1574,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::L1MessagesHasher, l1_messages_linear_hash_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::L1MessagesHasher(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::L1MessagesHasher(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1600,8 +1588,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::TransientStorageChecker, transient_storage_sorter_circuit_data, *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::TransientStorageSorter(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::TransientStorageSorter(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); @@ -1615,8 +1603,8 @@ pub(crate) fn create_artifacts_from_tracer< BaseLayerCircuitType::EIP4844Repack, eip_4844_circuits.clone(), *round_function, - |x| circuit_callback(ZkSyncBaseLayerCircuit::EIP4844Repack(x)), - &mut recursion_queue_callback, + |x| ZkSyncBaseLayerCircuit::EIP4844Repack(x), + &mut artifacts_callback, &mut cs_for_witness_generation, ); diff --git a/src/witness/postprocessing/mod.rs b/src/witness/postprocessing/mod.rs index 4e0dc759..082d1d46 100644 --- a/src/witness/postprocessing/mod.rs +++ b/src/witness/postprocessing/mod.rs @@ -2,6 +2,7 @@ use super::*; use crate::witness::utils::*; use crate::zkevm_circuits::eip_4844::input::EIP4844OutputData; +use boojum::gadgets::queue::full_state_queue::FullStateCircuitQueueRawWitness; use circuit_definitions::aux_definitions::witness_oracle::VmWitnessOracle; use circuit_definitions::boojum::field::U64Representable; use circuit_definitions::boojum::gadgets::traits::allocatable::CSAllocatable; @@ -62,7 +63,12 @@ use circuit_definitions::zkevm_circuits::transient_storage_validity_by_grand_pro use circuit_definitions::zkevm_circuits::transient_storage_validity_by_grand_product::input::*; use circuit_definitions::Field; use crossbeam::atomic::AtomicCell; +use derivative::Derivative; use observable_witness::ObservableWitness; +use oracle::WitnessGenerationArtifact; +use zkevm_circuits::base_structures::memory_query::{MemoryQuery, MEMORY_QUERY_PACKED_WIDTH}; +use zkevm_circuits::base_structures::vm_state::FULL_SPONGE_QUEUE_STATE_WIDTH; +use zkevm_circuits::ram_permutation::input::RamPermutationCycleInputOutputWitness; use std::sync::Arc; @@ -497,19 +503,15 @@ where pub(crate) fn make_circuits< T: ClosedFormInputField, S: ZkSyncUniformSynthesisFunction, - CB: FnMut(ZkSyncUniformCircuitInstance), - QSCB: FnMut( - u64, - RecursionQueueSimulator, - Vec>, - ), + WCB: Fn(ZkSyncUniformCircuitInstance) -> ZkSyncBaseLayerCircuit, + CB: FnMut(WitnessGenerationArtifact), >( geometry: u32, circuit_type: BaseLayerCircuitType, circuits_data: Vec, round_function: Poseidon2Goldilocks, - mut circuit_callback: CB, - recursion_queue_callback: &mut QSCB, + wrap_circuit: WCB, + artifacts_callback: &mut CB, cs_for_witness_generation: &mut CsForWitnessGeneration, ) -> ( FirstAndLastCircuitWitness>, @@ -532,7 +534,9 @@ where let mut maker = CircuitMaker::new(geometry, round_function.clone(), cs_for_witness_generation); for circuit_input in circuits_data.into_iter() { - circuit_callback(maker.process(circuit_input, circuit_type)); + artifacts_callback(WitnessGenerationArtifact::BaseLayerCircuit(wrap_circuit( + maker.process(circuit_input, circuit_type), + ))); } let ( @@ -540,14 +544,39 @@ where recursion_queue_simulator, circuits_compact_forms_witnesses, ) = maker.into_results(); - recursion_queue_callback( + artifacts_callback(WitnessGenerationArtifact::RecursionQueue(( circuit_type as u64, recursion_queue_simulator, circuits_compact_forms_witnesses.clone(), - ); + ))); ( first_and_last_observable_witnesses, circuits_compact_forms_witnesses, ) } + +#[derive(Derivative, serde::Serialize, serde::Deserialize)] +#[derivative(Clone, Debug, Default)] +#[serde(bound = "")] +pub struct RamPermutationQueuesWitness { + pub unsorted_queue_witness: FullStateCircuitQueueRawWitness< + F, + MemoryQuery, + FULL_SPONGE_QUEUE_STATE_WIDTH, + MEMORY_QUERY_PACKED_WIDTH, + >, + pub sorted_queue_witness: FullStateCircuitQueueRawWitness< + F, + MemoryQuery, + FULL_SPONGE_QUEUE_STATE_WIDTH, + MEMORY_QUERY_PACKED_WIDTH, + >, +} + +#[derive(Derivative, serde::Serialize, serde::Deserialize)] +#[derivative(Clone, Debug, Default)] +#[serde(bound = "")] +pub struct RamPermutationCircuitInstancePartialWitness { + pub closed_form_input: RamPermutationCycleInputOutputWitness, +} diff --git a/src/witness/utils.rs b/src/witness/utils.rs index 709e69fb..37572811 100644 --- a/src/witness/utils.rs +++ b/src/witness/utils.rs @@ -132,7 +132,7 @@ pub fn transform_queue_witness< const N: usize, D: CircuitEncodable, >( - witness_iter: impl Iterator +'a, + witness_iter: impl Iterator + 'a, ) -> CircuitQueueWitness { let wit: VecDeque<_> = witness_iter .map(|(_enc, old_tail, el)| (el.reflect(), *old_tail)) From 2dc6e6780550d6ab276d3d48114e927f7a49d56d Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Thu, 8 Aug 2024 17:38:25 +0200 Subject: [PATCH 14/33] Simulate queues in parts --- circuit_encodings/src/lib.rs | 14 +- src/tests/complex_tests/mod.rs | 1 + .../memory_related/ram_permutation.rs | 76 +++---- src/witness/oracle.rs | 209 ++++++++++++++++-- 4 files changed, 233 insertions(+), 67 deletions(-) diff --git a/circuit_encodings/src/lib.rs b/circuit_encodings/src/lib.rs index a1021cf2..7aa9e5bc 100644 --- a/circuit_encodings/src/lib.rs +++ b/circuit_encodings/src/lib.rs @@ -316,8 +316,6 @@ impl< pub struct FullWidthQueueIntermediateStates { pub head: [F; SW], pub tail: [F; SW], - pub old_head: [F; SW], - pub old_tail: [F; SW], pub num_items: u32, pub round_function_execution_pairs: [([F; SW], [F; SW]); ROUNDS], } @@ -433,8 +431,6 @@ impl< let intermediate_info = FullWidthQueueIntermediateStates { head: self.head, tail: new_tail, - old_head: self.head, - old_tail, num_items: self.num_items, round_function_execution_pairs: states, }; @@ -473,8 +469,6 @@ impl< let intermediate_info = FullWidthQueueIntermediateStates { head: self.head, tail: self.tail, - old_head, - old_tail: self.tail, num_items: self.num_items, round_function_execution_pairs: states, }; @@ -562,6 +556,12 @@ impl< } } + pub fn replace_container(mut self, container: C) -> (Self, C) { + let prev_container = self.witness; + self.witness = container; + (self, prev_container) + } + pub fn take_sponge_like_queue_state(&self) -> QueueStateWitness { let result = QueueStateWitness { head: self.head, @@ -605,8 +605,6 @@ impl< let intermediate_info = FullWidthQueueIntermediateStates { head: self.head, tail: new_tail, - old_head: self.head, - old_tail, num_items: self.num_items, round_function_execution_pairs: states, }; diff --git a/src/tests/complex_tests/mod.rs b/src/tests/complex_tests/mod.rs index c16813c7..a1fb7ed1 100644 --- a/src/tests/complex_tests/mod.rs +++ b/src/tests/complex_tests/mod.rs @@ -266,6 +266,7 @@ pub(crate) fn generate_base_layer( .map(|x| ZkSyncBaseLayerStorage::from_inner(a as u8, x)) .collect(), )), + _ => unimplemented!(), }; let (scheduler_partial_input, _aux_data) = run( diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index 2bf671c8..4ef4a3d9 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -40,6 +40,9 @@ pub(crate) fn compute_ram_circuit_snapshots>, memory_queue_simulator: MemoryQueuePerCircuitSimulator, sorted_memory_queries_simulator: MemoryQueuePerCircuitSimulator, + sorted_queries_aux_data_for_chunks: Vec<(u32, MemoryQuery, usize)>, + sorted_encodings: Vec<[Field; 8]>, // TODO + unsorted_encodings: Vec<[Field; 8]>, round_function: &RoundFunction, num_non_deterministic_heap_queries: usize, geometry: &GeometryConfig, @@ -99,6 +102,8 @@ pub(crate) fn compute_ram_circuit_snapshots = memory_queue_simulator - .witness - .iter() - .map(|el| &el.0) - .collect(); - let rhs_contributions: Vec<_> = sorted_memory_queries_simulator - .witness - .iter() - .map(|el| &el.0) - .collect(); + let lhs_contributions = unsorted_encodings; + let rhs_contributions = sorted_encodings; for idx in 0..DEFAULT_NUM_PERMUTATION_ARGUMENT_REPETITIONS { let (lhs_grand_product_chain, rhs_grand_product_chain) = compute_grand_product_chains( - &lhs_contributions, - &rhs_contributions, + &lhs_contributions.iter().collect(), // TODO + &rhs_contributions.iter().collect(), &challenges[idx], ); @@ -140,11 +137,11 @@ pub(crate) fn compute_ram_circuit_snapshots, - implicit_memory_queries: &ImplicitMemoryQueries, + implicit_memory_queries: ImplicitMemoryQueries, round_function: Poseidon2Goldilocks, + channel_sender: Sender, ) -> ( MemoryArtifacts, MemoryQueueState, LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, ImplicitMemoryStates, + Vec<[GoldilocksField; 8]>, // TODO ) { let mut memory_artifacts_for_main_vm = MemoryArtifacts { memory_queries, @@ -852,10 +859,11 @@ fn simulate_memory_queue( let mut memory_queue_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, - memory_artifacts_for_main_vm.memory_queries.len() - + implicit_memory_queries.amount_of_queries(), + geometry.cycles_per_ram_permutation as usize, )); + let mut encodings_witnesses = vec![]; + // very slow for (cycle, query) in memory_artifacts_for_main_vm.memory_queries.iter() { let (_, intermediate_info) = @@ -865,6 +873,27 @@ fn simulate_memory_queue( memory_artifacts_for_main_vm .memory_queue_entry_states .push((*cycle, transform_sponge_like_queue_state(intermediate_info))); + + if memory_queue_simulator.witness.len() == geometry.cycles_per_ram_permutation as usize { + let witnesses; + (memory_queue_simulator, witnesses) = memory_queue_simulator.replace_container( + PerCircuitAccumulator::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + geometry.cycles_per_ram_permutation as usize, + ), + ); + + for witness in witnesses.iter() { + encodings_witnesses.push(witness.0); + } + + let w = witnesses.into_circuits(1); + channel_sender + .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( + w[0].clone(), + )) + .unwrap(); + } } { @@ -888,12 +917,34 @@ fn simulate_memory_queue( round_function, ); + let witnesses; + (memory_queue_simulator, witnesses) = + memory_queue_simulator.replace_container(PerCircuitAccumulator::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + geometry.cycles_per_ram_permutation as usize, + )); + + let amount_of_ram_circuits = (witnesses.len() as u32 + geometry.cycles_per_ram_permutation - 1) + / geometry.cycles_per_ram_permutation; + for unsorted_witnesses_for_chunk in witnesses.into_circuits(amount_of_ram_circuits as usize) { + for witness in unsorted_witnesses_for_chunk.iter() { + encodings_witnesses.push(witness.0) + } + + channel_sender + .send(WitnessGenerationArtifact::SortedMemoryQueueWitness( + unsorted_witnesses_for_chunk, + )) + .unwrap(); + } + ( memory_artifacts_for_main_vm, final_explicit_memory_queue_state, memory_queue_states_accumulator, memory_queue_simulator, implicit_memory_states, + encodings_witnesses, ) } @@ -902,9 +953,12 @@ fn simulate_sorted_memory_queue( memory_queries: Vec<(Cycle, MemoryQuery)>, implicit_memory_queries: ImplicitMemoryQueries, round_function: Poseidon2Goldilocks, + channel_sender: Sender, ) -> ( LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, + Vec<[GoldilocksField; 8]>, + Vec<(u32, MemoryQuery, usize)>, ) { let mut all_memory_queries_sorted: Vec<&MemoryQuery> = memory_queries .iter() @@ -938,15 +992,104 @@ fn simulate_sorted_memory_queue( amount_of_queries, ); + let mut sorted_encodings = Vec::with_capacity(all_memory_queries_sorted.len()); + let amount_of_ram_circuits = + (all_memory_queries_sorted.len() as u32 + geometry.cycles_per_ram_permutation - 1) + / geometry.cycles_per_ram_permutation; + let mut sorted_queries_aux_data_for_chunks = + Vec::with_capacity(amount_of_ram_circuits as usize); + for query in all_memory_queries_sorted.into_iter() { let (_, intermediate_info) = sorted_memory_queries_simulator .push_and_output_intermediate_data(*query, &round_function); sorted_memory_queue_states_accumulator.push(intermediate_info); + + if sorted_memory_queries_simulator.witness.len() + == geometry.cycles_per_ram_permutation as usize + { + let witnesses; + (sorted_memory_queries_simulator, witnesses) = sorted_memory_queries_simulator + .replace_container(PerCircuitAccumulator::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + geometry.cycles_per_ram_permutation as usize, + )); + + let sorted_witnesses_for_chunk = witnesses.into_circuits(1)[0].clone(); + + let sorted_states_len = sorted_witnesses_for_chunk.len(); + let num_nondet_writes_in_chunk = sorted_witnesses_for_chunk + .iter() + .filter(|el| { + let query = &el.2; + query.rw_flag == true + && query.timestamp.0 == 0 + && query.location.page.0 == BOOTLOADER_HEAP_PAGE + }) + .count(); + let last_sorted_query = sorted_witnesses_for_chunk.last().unwrap().2; + + sorted_queries_aux_data_for_chunks.push(( + num_nondet_writes_in_chunk as u32, + last_sorted_query, + sorted_states_len, + )); + + for witness in sorted_witnesses_for_chunk.iter() { + sorted_encodings.push(witness.0); + } + + channel_sender + .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( + sorted_witnesses_for_chunk, + )) + .unwrap(); + } + } + + if sorted_memory_queries_simulator.witness.len() != 0 as usize { + let witnesses; + (sorted_memory_queries_simulator, witnesses) = sorted_memory_queries_simulator + .replace_container(PerCircuitAccumulator::with_flat_capacity( + geometry.cycles_per_ram_permutation as usize, + geometry.cycles_per_ram_permutation as usize, + )); + + let sorted_witnesses_for_chunk = witnesses.into_circuits(1)[0].clone(); + + let sorted_states_len = sorted_witnesses_for_chunk.len(); + let num_nondet_writes_in_chunk = sorted_witnesses_for_chunk + .iter() + .filter(|el| { + let query = &el.2; + query.rw_flag == true + && query.timestamp.0 == 0 + && query.location.page.0 == BOOTLOADER_HEAP_PAGE + }) + .count(); + let last_sorted_query = sorted_witnesses_for_chunk.last().unwrap().2; + + sorted_queries_aux_data_for_chunks.push(( + num_nondet_writes_in_chunk as u32, + last_sorted_query, + sorted_states_len, + )); + + for witness in sorted_witnesses_for_chunk.iter() { + sorted_encodings.push(witness.0); + } + + channel_sender + .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( + sorted_witnesses_for_chunk, + )) + .unwrap(); } ( sorted_memory_queue_states_accumulator, sorted_memory_queries_simulator, + sorted_encodings, + sorted_queries_aux_data_for_chunks, ) } @@ -1039,37 +1182,72 @@ fn process_memory_related_circuits( snapshot_prof("BEFORE QUEUES SIMULATION"); + let (tx, rx): ( + Sender, + Receiver, + ) = mpsc::channel(); + use std::thread; let sorted_handle = { let memory_queries = memory_queries.clone(); let implicit_memory_queries = implicit_memory_queries.clone(); let geometry = *geometry; let round_function = *round_function; + let tx_thread = tx.clone(); thread::spawn(move || { simulate_sorted_memory_queue( geometry, memory_queries, implicit_memory_queries, round_function, + tx_thread, + ) + }) + }; + + let unsorted_handle = { + let memory_queries = memory_queries; + let implicit_memory_queries = implicit_memory_queries.clone(); + let geometry = *geometry; + let round_function = *round_function; + let tx_thread = tx.clone(); + thread::spawn(move || { + simulate_memory_queue( + geometry, + memory_queries, + implicit_memory_queries, + round_function, + tx_thread, ) }) }; + let amount_of_ram_circuits = ((amount_of_explicit_memory_queries + + implicit_memory_queries.amount_of_queries()) as u32 + + geometry.cycles_per_ram_permutation + - 1) + / geometry.cycles_per_ram_permutation; + + for _ in 0..amount_of_ram_circuits * 2 { + let artifact = rx.recv().unwrap(); + artifacts_callback(artifact); + } + let ( memory_artifacts_for_main_vm, final_explicit_memory_queue_state, memory_queue_states_accumulator, memory_queue_simulator, implicit_memory_states, - ) = simulate_memory_queue( - *geometry, - memory_queries, - &implicit_memory_queries, - *round_function, - ); + unsorted_encodings, + ) = unsorted_handle.join().unwrap(); - let (sorted_memory_queue_states_accumulator, sorted_memory_queue_simulator) = - sorted_handle.join().unwrap(); + let ( + sorted_memory_queue_states_accumulator, + sorted_memory_queue_simulator, + sorted_encodings, + sorted_queries_aux_data_for_chunks, + ) = sorted_handle.join().unwrap(); // direct VM related part is done, other subcircuit's functionality is moved to other functions // that should properly do sorts and memory writes @@ -1192,6 +1370,9 @@ fn process_memory_related_circuits( sorted_memory_queue_states_accumulator, memory_queue_simulator, sorted_memory_queue_simulator, + sorted_queries_aux_data_for_chunks, + sorted_encodings, + unsorted_encodings, round_function, num_non_deterministic_heap_queries, geometry, @@ -1219,6 +1400,8 @@ pub enum WitnessGenerationArtifact { Vec>, ), ), + SortedMemoryQueueWitness(Vec<([GoldilocksField; 8], [GoldilocksField; 12], MemoryQuery)>), + UnsortedMemoryQueueWitness(Vec<([GoldilocksField; 8], [GoldilocksField; 12], MemoryQuery)>), } /// Make basic circuits instances and witnesses, From 4bb9690fbe37bb1e326399c753a790a8729ccbab Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Fri, 9 Aug 2024 12:10:20 +0200 Subject: [PATCH 15/33] cleanup --- src/witness/oracle.rs | 98 +++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 64 deletions(-) diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index ba1946d6..4985b7cf 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -42,6 +42,7 @@ use circuit_definitions::circuit_definitions::base_layer::ZkSyncBaseLayerCircuit use circuit_definitions::encodings::callstack_entry::ExtendedCallstackEntry; use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; use circuit_definitions::encodings::{CircuitEquivalentReflection, LogQueueSimulator}; +use circuit_definitions::zkevm_circuits::base_structures::memory_query::MemoryQueryWitness; use circuit_definitions::zkevm_circuits::eip_4844::input::EIP4844CircuitInstanceWitness; use circuit_definitions::zkevm_circuits::fsm_input_output::ClosedFormInputCompactFormWitness; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; @@ -883,29 +884,27 @@ fn simulate_memory_queue( ), ); + let mut unsorted_witnesses_for_circuit = Vec::with_capacity(witnesses.len()); for witness in witnesses.iter() { encodings_witnesses.push(witness.0); + unsorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); } - - let w = witnesses.into_circuits(1); channel_sender .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( - w[0].clone(), + unsorted_witnesses_for_circuit, )) .unwrap(); } } - { - assert_eq!( - memory_artifacts_for_main_vm.memory_queries.len(), - memory_queue_states_accumulator.len() - ); - assert_eq!( - memory_artifacts_for_main_vm.memory_queries.len(), - memory_queue_simulator.num_items as usize - ); - } + assert_eq!( + memory_artifacts_for_main_vm.memory_queries.len(), + memory_queue_states_accumulator.len() + ); + assert_eq!( + memory_artifacts_for_main_vm.memory_queries.len(), + memory_queue_simulator.num_items as usize + ); let final_explicit_memory_queue_state = memory_queue_states_accumulator.last().unwrap().clone(); @@ -926,14 +925,19 @@ fn simulate_memory_queue( let amount_of_ram_circuits = (witnesses.len() as u32 + geometry.cycles_per_ram_permutation - 1) / geometry.cycles_per_ram_permutation; + for unsorted_witnesses_for_chunk in witnesses.into_circuits(amount_of_ram_circuits as usize) { - for witness in unsorted_witnesses_for_chunk.iter() { - encodings_witnesses.push(witness.0) + let mut unsorted_witnesses_for_circuit = + Vec::with_capacity(unsorted_witnesses_for_chunk.len()); + + for witness in unsorted_witnesses_for_chunk.into_iter() { + encodings_witnesses.push(witness.0); + unsorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); } channel_sender .send(WitnessGenerationArtifact::SortedMemoryQueueWitness( - unsorted_witnesses_for_chunk, + unsorted_witnesses_for_circuit, )) .unwrap(); } @@ -992,20 +996,21 @@ fn simulate_sorted_memory_queue( amount_of_queries, ); - let mut sorted_encodings = Vec::with_capacity(all_memory_queries_sorted.len()); - let amount_of_ram_circuits = - (all_memory_queries_sorted.len() as u32 + geometry.cycles_per_ram_permutation - 1) - / geometry.cycles_per_ram_permutation; + let mut sorted_encodings = Vec::with_capacity(amount_of_queries); + let amount_of_ram_circuits = (amount_of_queries as u32 + geometry.cycles_per_ram_permutation + - 1) + / geometry.cycles_per_ram_permutation; let mut sorted_queries_aux_data_for_chunks = Vec::with_capacity(amount_of_ram_circuits as usize); - for query in all_memory_queries_sorted.into_iter() { + for (idx, query) in all_memory_queries_sorted.into_iter().enumerate() { let (_, intermediate_info) = sorted_memory_queries_simulator .push_and_output_intermediate_data(*query, &round_function); sorted_memory_queue_states_accumulator.push(intermediate_info); if sorted_memory_queries_simulator.witness.len() == geometry.cycles_per_ram_permutation as usize + || idx == amount_of_queries - 1 { let witnesses; (sorted_memory_queries_simulator, witnesses) = sorted_memory_queries_simulator @@ -1034,57 +1039,22 @@ fn simulate_sorted_memory_queue( sorted_states_len, )); - for witness in sorted_witnesses_for_chunk.iter() { + let mut sorted_witnesses_for_circuit = + Vec::with_capacity(sorted_witnesses_for_chunk.len()); + + for witness in sorted_witnesses_for_chunk.into_iter() { sorted_encodings.push(witness.0); + sorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); } channel_sender .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( - sorted_witnesses_for_chunk, + sorted_witnesses_for_circuit, )) .unwrap(); } } - if sorted_memory_queries_simulator.witness.len() != 0 as usize { - let witnesses; - (sorted_memory_queries_simulator, witnesses) = sorted_memory_queries_simulator - .replace_container(PerCircuitAccumulator::with_flat_capacity( - geometry.cycles_per_ram_permutation as usize, - geometry.cycles_per_ram_permutation as usize, - )); - - let sorted_witnesses_for_chunk = witnesses.into_circuits(1)[0].clone(); - - let sorted_states_len = sorted_witnesses_for_chunk.len(); - let num_nondet_writes_in_chunk = sorted_witnesses_for_chunk - .iter() - .filter(|el| { - let query = &el.2; - query.rw_flag == true - && query.timestamp.0 == 0 - && query.location.page.0 == BOOTLOADER_HEAP_PAGE - }) - .count(); - let last_sorted_query = sorted_witnesses_for_chunk.last().unwrap().2; - - sorted_queries_aux_data_for_chunks.push(( - num_nondet_writes_in_chunk as u32, - last_sorted_query, - sorted_states_len, - )); - - for witness in sorted_witnesses_for_chunk.iter() { - sorted_encodings.push(witness.0); - } - - channel_sender - .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( - sorted_witnesses_for_chunk, - )) - .unwrap(); - } - ( sorted_memory_queue_states_accumulator, sorted_memory_queries_simulator, @@ -1400,8 +1370,8 @@ pub enum WitnessGenerationArtifact { Vec>, ), ), - SortedMemoryQueueWitness(Vec<([GoldilocksField; 8], [GoldilocksField; 12], MemoryQuery)>), - UnsortedMemoryQueueWitness(Vec<([GoldilocksField; 8], [GoldilocksField; 12], MemoryQuery)>), + SortedMemoryQueueWitness(Vec<(MemoryQueryWitness, [GoldilocksField; 12])>), + UnsortedMemoryQueueWitness(Vec<(MemoryQueryWitness, [GoldilocksField; 12])>), } /// Make basic circuits instances and witnesses, From f18e14405f8b8f2596ce7d839e2ff951536bb329 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Fri, 9 Aug 2024 12:32:40 +0200 Subject: [PATCH 16/33] Remove excessive cloning --- src/witness/oracle.rs | 54 ++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 4985b7cf..1558e015 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -828,8 +828,8 @@ use circuit_definitions::encodings::memory_query::MemoryQueueState; fn simulate_memory_queue( geometry: GeometryConfig, - memory_queries: Vec<(Cycle, MemoryQuery)>, - implicit_memory_queries: ImplicitMemoryQueries, + memory_queries: Arc>, + implicit_memory_queries: Arc, round_function: Poseidon2Goldilocks, channel_sender: Sender, ) -> ( @@ -841,7 +841,7 @@ fn simulate_memory_queue( Vec<[GoldilocksField; 8]>, // TODO ) { let mut memory_artifacts_for_main_vm = MemoryArtifacts { - memory_queries, + memory_queries: vec![], memory_queue_entry_states: CircuitsEntryAccumulatorSparse::new( geometry.cycles_per_vm_snapshot as usize, (0, QueueState::placeholder_witness()), @@ -852,7 +852,7 @@ fn simulate_memory_queue( let mut memory_queue_states_accumulator = LastPerCircuitAccumulator::>::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, - memory_artifacts_for_main_vm.memory_queries.len(), + memory_queries.len(), ); use crate::witness::aux_data_structs::per_circuit_accumulator::PerCircuitAccumulator; @@ -866,7 +866,7 @@ fn simulate_memory_queue( let mut encodings_witnesses = vec![]; // very slow - for (cycle, query) in memory_artifacts_for_main_vm.memory_queries.iter() { + for (cycle, query) in memory_queries.iter() { let (_, intermediate_info) = memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); @@ -898,11 +898,11 @@ fn simulate_memory_queue( } assert_eq!( - memory_artifacts_for_main_vm.memory_queries.len(), + memory_queries.len(), memory_queue_states_accumulator.len() ); assert_eq!( - memory_artifacts_for_main_vm.memory_queries.len(), + memory_queries.len(), memory_queue_simulator.num_items as usize ); @@ -954,8 +954,8 @@ fn simulate_memory_queue( fn simulate_sorted_memory_queue( geometry: GeometryConfig, - memory_queries: Vec<(Cycle, MemoryQuery)>, - implicit_memory_queries: ImplicitMemoryQueries, + memory_queries: Arc>, + implicit_memory_queries: Arc, round_function: Poseidon2Goldilocks, channel_sender: Sender, ) -> ( @@ -1149,6 +1149,11 @@ fn process_memory_related_circuits( ); let amount_of_explicit_memory_queries = memory_queries.len(); + let amount_of_ram_circuits = ((amount_of_explicit_memory_queries + + implicit_memory_queries.amount_of_queries()) as u32 + + geometry.cycles_per_ram_permutation + - 1) + / geometry.cycles_per_ram_permutation; snapshot_prof("BEFORE QUEUES SIMULATION"); @@ -1157,18 +1162,21 @@ fn process_memory_related_circuits( Receiver, ) = mpsc::channel(); + let implicit_memory_queries_arc = Arc::new(implicit_memory_queries); + let memory_queries_arc = Arc::new(memory_queries); + use std::thread; let sorted_handle = { - let memory_queries = memory_queries.clone(); - let implicit_memory_queries = implicit_memory_queries.clone(); + let memory_queries_arc = memory_queries_arc.clone(); + let implicit_memory_queries_arc = implicit_memory_queries_arc.clone(); let geometry = *geometry; let round_function = *round_function; let tx_thread = tx.clone(); thread::spawn(move || { simulate_sorted_memory_queue( geometry, - memory_queries, - implicit_memory_queries, + memory_queries_arc, + implicit_memory_queries_arc, round_function, tx_thread, ) @@ -1176,35 +1184,29 @@ fn process_memory_related_circuits( }; let unsorted_handle = { - let memory_queries = memory_queries; - let implicit_memory_queries = implicit_memory_queries.clone(); + let memory_queries_arc = memory_queries_arc.clone(); + let implicit_memory_queries_arc = implicit_memory_queries_arc.clone(); let geometry = *geometry; let round_function = *round_function; let tx_thread = tx.clone(); thread::spawn(move || { simulate_memory_queue( geometry, - memory_queries, - implicit_memory_queries, + memory_queries_arc, + implicit_memory_queries_arc, round_function, tx_thread, ) }) }; - let amount_of_ram_circuits = ((amount_of_explicit_memory_queries - + implicit_memory_queries.amount_of_queries()) as u32 - + geometry.cycles_per_ram_permutation - - 1) - / geometry.cycles_per_ram_permutation; - for _ in 0..amount_of_ram_circuits * 2 { let artifact = rx.recv().unwrap(); artifacts_callback(artifact); } let ( - memory_artifacts_for_main_vm, + mut memory_artifacts_for_main_vm, final_explicit_memory_queue_state, memory_queue_states_accumulator, memory_queue_simulator, @@ -1219,6 +1221,10 @@ fn process_memory_related_circuits( sorted_queries_aux_data_for_chunks, ) = sorted_handle.join().unwrap(); + let memory_queries = std::sync::Arc::>::into_inner(memory_queries_arc).unwrap(); + memory_artifacts_for_main_vm.memory_queries = memory_queries; + + let implicit_memory_queries = std::sync::Arc::::into_inner(implicit_memory_queries_arc).unwrap(); // direct VM related part is done, other subcircuit's functionality is moved to other functions // that should properly do sorts and memory writes From c63514dfa072f7c7c706478536c6acd44414ccb5 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Fri, 9 Aug 2024 12:49:24 +0200 Subject: [PATCH 17/33] Move MemoryArtifacts out of simulate_memory_queue --- src/witness/oracle.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 1558e015..1ed82e30 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -37,6 +37,7 @@ use crate::zkevm_circuits::base_structures::vm_state::{ use crate::zkevm_circuits::scheduler::block_header::MAX_4844_BLOBS_PER_BLOCK; use circuit_definitions::boojum::field::goldilocks::GoldilocksField; use circuit_definitions::boojum::field::{Field, U64Representable}; +use circuit_definitions::boojum::gadgets::queue::QueueStateWitness; use circuit_definitions::boojum::implementations::poseidon2::Poseidon2Goldilocks; use circuit_definitions::circuit_definitions::base_layer::ZkSyncBaseLayerCircuit; use circuit_definitions::encodings::callstack_entry::ExtendedCallstackEntry; @@ -833,20 +834,17 @@ fn simulate_memory_queue( round_function: Poseidon2Goldilocks, channel_sender: Sender, ) -> ( - MemoryArtifacts, + CircuitsEntryAccumulatorSparse<(u32, QueueStateWitness)>, MemoryQueueState, LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, ImplicitMemoryStates, Vec<[GoldilocksField; 8]>, // TODO ) { - let mut memory_artifacts_for_main_vm = MemoryArtifacts { - memory_queries: vec![], - memory_queue_entry_states: CircuitsEntryAccumulatorSparse::new( - geometry.cycles_per_vm_snapshot as usize, - (0, QueueState::placeholder_witness()), - ), - }; + let mut memory_queue_entry_states = CircuitsEntryAccumulatorSparse::new( + geometry.cycles_per_vm_snapshot as usize, + (0, QueueState::placeholder_witness()), + ); // for RAM permutation circuits let mut memory_queue_states_accumulator = @@ -871,8 +869,7 @@ fn simulate_memory_queue( memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); memory_queue_states_accumulator.push(intermediate_info); - memory_artifacts_for_main_vm - .memory_queue_entry_states + memory_queue_entry_states .push((*cycle, transform_sponge_like_queue_state(intermediate_info))); if memory_queue_simulator.witness.len() == geometry.cycles_per_ram_permutation as usize { @@ -943,7 +940,7 @@ fn simulate_memory_queue( } ( - memory_artifacts_for_main_vm, + memory_queue_entry_states, final_explicit_memory_queue_state, memory_queue_states_accumulator, memory_queue_simulator, @@ -1206,7 +1203,7 @@ fn process_memory_related_circuits( } let ( - mut memory_artifacts_for_main_vm, + memory_queue_entry_states_for_main_vm, final_explicit_memory_queue_state, memory_queue_states_accumulator, memory_queue_simulator, @@ -1221,10 +1218,12 @@ fn process_memory_related_circuits( sorted_queries_aux_data_for_chunks, ) = sorted_handle.join().unwrap(); - let memory_queries = std::sync::Arc::>::into_inner(memory_queries_arc).unwrap(); - memory_artifacts_for_main_vm.memory_queries = memory_queries; + let memory_artifacts_for_main_vm = MemoryArtifacts { + memory_queries: Arc::into_inner(memory_queries_arc).unwrap(), + memory_queue_entry_states: memory_queue_entry_states_for_main_vm + }; + let implicit_memory_queries = Arc::into_inner(implicit_memory_queries_arc).unwrap(); - let implicit_memory_queries = std::sync::Arc::::into_inner(implicit_memory_queries_arc).unwrap(); // direct VM related part is done, other subcircuit's functionality is moved to other functions // that should properly do sorts and memory writes From 186108038afc017d2cb5455ec359b2b3a72a99ec Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Fri, 9 Aug 2024 13:27:34 +0200 Subject: [PATCH 18/33] fix tests --- src/tests/complex_tests/mod.rs | 25 ++++++++++++++++++++++++- src/tests/run_manually.rs | 21 +++++++++++++++++++++ src/witness/oracle.rs | 4 ++-- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/tests/complex_tests/mod.rs b/src/tests/complex_tests/mod.rs index a1fb7ed1..de4901c6 100644 --- a/src/tests/complex_tests/mod.rs +++ b/src/tests/complex_tests/mod.rs @@ -39,6 +39,7 @@ use crate::zk_evm::GenericNoopTracer; use crate::zkevm_circuits::eip_4844::input::*; use crate::zkevm_circuits::scheduler::block_header::MAX_4844_BLOBS_PER_BLOCK; use crate::zkevm_circuits::scheduler::input::SchedulerCircuitInstanceWitness; +use boojum::gadgets::queue::full_state_queue::FullStateCircuitQueueRawWitness; use circuit_definitions::aux_definitions::witness_oracle::VmWitnessOracle; use circuit_definitions::circuit_definitions::aux_layer::compression::{ self, CompressionMode1Circuit, @@ -256,6 +257,8 @@ pub(crate) fn generate_base_layer( let mut basic_block_circuits = vec![]; let mut recursion_queues = vec![]; + let mut unsorted_memory_queue_witnesses = vec![]; + let mut sorted_memory_queue_witnesses = vec![]; let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { WitnessGenerationArtifact::BaseLayerCircuit(circuit) => basic_block_circuits.push(circuit), @@ -266,7 +269,8 @@ pub(crate) fn generate_base_layer( .map(|x| ZkSyncBaseLayerStorage::from_inner(a as u8, x)) .collect(), )), - _ => unimplemented!(), + WitnessGenerationArtifact::UnsortedMemoryQueueWitness(witnesses) => unsorted_memory_queue_witnesses.push(witnesses), + WitnessGenerationArtifact::SortedMemoryQueueWitness(witnesses) => sorted_memory_queue_witnesses.push(witnesses), }; let (scheduler_partial_input, _aux_data) = run( @@ -288,6 +292,25 @@ pub(crate) fn generate_base_layer( artifacts_callback, ); + let mut unsorted_memory_queue_witnesses_it = unsorted_memory_queue_witnesses.into_iter(); + let mut sorted_memory_queue_witnesses = sorted_memory_queue_witnesses.into_iter(); + for el in basic_block_circuits.iter_mut() { + match &el { + ZkSyncBaseLayerCircuit::RAMPermutation(inner) => { + let mut witness = inner.witness.take().unwrap(); + witness.sorted_queue_witness = FullStateCircuitQueueRawWitness { + elements: sorted_memory_queue_witnesses.next().unwrap().into() + }; + witness.unsorted_queue_witness = FullStateCircuitQueueRawWitness { + elements: unsorted_memory_queue_witnesses_it.next().unwrap().into() + }; + + inner.witness.store(Some(witness)); + }, + _ => {} + } + } + ( basic_block_circuits, recursion_queues, diff --git a/src/tests/run_manually.rs b/src/tests/run_manually.rs index 3a2135b6..862839d0 100644 --- a/src/tests/run_manually.rs +++ b/src/tests/run_manually.rs @@ -20,6 +20,7 @@ use crate::zk_evm::witness_trace::VmWitnessTracer; use crate::zk_evm::GenericNoopTracer; use crate::zkevm_circuits::base_structures::vm_state::GlobalContextWitness; use crate::zkevm_circuits::main_vm::main_vm_entry_point; +use boojum::gadgets::queue::full_state_queue::FullStateCircuitQueueRawWitness; use circuit_definitions::aux_definitions::witness_oracle::VmWitnessOracle; use circuit_definitions::zk_evm::vm_state::cycle; use storage::{InMemoryCustomRefundStorage, StorageRefund}; @@ -261,6 +262,8 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt save_predeployed_contracts(&mut storage_impl.storage, &mut tree, &known_contracts); let mut basic_block_circuits = vec![]; + let mut unsorted_memory_queue_witnesses = vec![]; + let mut sorted_memory_queue_witnesses = vec![]; // we are using TestingTracer to track prints and exceptions inside out_of_circuit_vm cycles let mut out_of_circuit_tracer = @@ -268,6 +271,8 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { WitnessGenerationArtifact::BaseLayerCircuit(circuit) => basic_block_circuits.push(circuit), + WitnessGenerationArtifact::UnsortedMemoryQueueWitness(witnesses) => unsorted_memory_queue_witnesses.push(witnesses), + WitnessGenerationArtifact::SortedMemoryQueueWitness(witnesses) => sorted_memory_queue_witnesses.push(witnesses), _ => {} }; @@ -308,8 +313,24 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt println!("Simulation and witness creation are completed"); + let mut unsorted_memory_queue_witnesses_it = unsorted_memory_queue_witnesses.into_iter(); + let mut sorted_memory_queue_witnesses = sorted_memory_queue_witnesses.into_iter(); for el in basic_block_circuits { println!("Doing {} circuit", el.short_description()); + match &el { + ZkSyncBaseLayerCircuit::RAMPermutation(inner) => { + let mut witness = inner.witness.take().unwrap(); + witness.sorted_queue_witness = FullStateCircuitQueueRawWitness { + elements: sorted_memory_queue_witnesses.next().unwrap().into() + }; + witness.unsorted_queue_witness = FullStateCircuitQueueRawWitness { + elements: unsorted_memory_queue_witnesses_it.next().unwrap().into() + }; + + inner.witness.store(Some(witness)); + }, + _ => {} + } base_test_circuit(el); } diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 1ed82e30..b496f7e0 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -933,7 +933,7 @@ fn simulate_memory_queue( } channel_sender - .send(WitnessGenerationArtifact::SortedMemoryQueueWitness( + .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( unsorted_witnesses_for_circuit, )) .unwrap(); @@ -1045,7 +1045,7 @@ fn simulate_sorted_memory_queue( } channel_sender - .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( + .send(WitnessGenerationArtifact::SortedMemoryQueueWitness( sorted_witnesses_for_circuit, )) .unwrap(); From 7ccde37fd7e3646d1d4646675c69e0a5c650b3c7 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Fri, 9 Aug 2024 15:33:09 +0200 Subject: [PATCH 19/33] Do not store unneeded encodings in stack simulator --- circuit_encodings/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/circuit_encodings/src/lib.rs b/circuit_encodings/src/lib.rs index 7aa9e5bc..43556abf 100644 --- a/circuit_encodings/src/lib.rs +++ b/circuit_encodings/src/lib.rs @@ -642,7 +642,7 @@ pub struct FullWidthStackSimulator< > { pub state: [F; SW], pub num_items: u32, - pub witness: Vec<([F; N], [F; SW], I)>, + pub witness: Vec<([F; SW], I)>, } impl< @@ -695,7 +695,7 @@ impl< let states = make_round_function_pairs(old_state, states); - self.witness.push((encoding, self.state, element)); + self.witness.push((self.state, element)); self.num_items += 1; self.state = new_state; @@ -725,7 +725,7 @@ impl< let popped = self.witness.pop().unwrap(); self.num_items -= 1; - let (_element_encoding, previous_state, element) = popped; + let (previous_state, element) = popped; let encoding = element.encoding_witness(); let mut state = previous_state; From 0bdab6e6177131f5e8625ead0c5952d9b763920f Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Fri, 9 Aug 2024 17:50:26 +0200 Subject: [PATCH 20/33] Add more profiling --- src/witness/oracle.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index b496f7e0..d555d869 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -1444,6 +1444,8 @@ pub(crate) fn create_artifacts_from_tracer std::mem::take(&mut callstack_with_aux_data.flat_new_frames_history); drop(callstack_with_aux_data); + snapshot_prof("BEFORE MUX LOG QUEUE CIRCUITS"); + tracing::debug!("Running multiplexed log queue simulation"); // We have all log queries in one multiplexed queue. We need to simulate this queue, @@ -1461,6 +1463,8 @@ pub(crate) fn create_artifacts_from_tracer *round_function, ); + snapshot_prof("AFTER MUX LOG QUEUE CIRCUITS"); + use std::thread; let callstack_handle = { let log_rollback_tails_for_frames = log_rollback_tails_for_frames.clone(); @@ -1509,6 +1513,8 @@ pub(crate) fn create_artifacts_from_tracer tracing::debug!("Processing log circuits"); + snapshot_prof("AFTER LOG DEMUX CIRCUITS"); + // Process part of log circuits that do not use memory (I/O-like). // Precompiles will be processed in process_memory_related_circuits. // Also makes storage application circuits and compact form witnesses. @@ -1523,6 +1529,8 @@ pub(crate) fn create_artifacts_from_tracer &mut artifacts_callback, ); + snapshot_prof("AFTER LOG CIRCUITS"); + tracing::debug!("Processing memory-related circuits"); let precompiles_data = PrecompilesInputData { From bdacaa5afd246da7de29a9887c7bcc214dcd4b29 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 12 Aug 2024 10:30:31 +0200 Subject: [PATCH 21/33] Fix encodings test --- circuit_encodings/src/lib.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/circuit_encodings/src/lib.rs b/circuit_encodings/src/lib.rs index 43556abf..e9f4c313 100644 --- a/circuit_encodings/src/lib.rs +++ b/circuit_encodings/src/lib.rs @@ -865,6 +865,9 @@ mod tests { } assert_eq!(queue.num_items, 10); + let old_head = queue.head; + let old_tail = queue.tail; + // pop one element let (element, data) = queue.pop_and_output_intermediate_data(&round_function); // it should return the first one that we entered (with circuit 0). @@ -874,8 +877,8 @@ mod tests { assert_eq!(data.num_items, 9); assert_eq!(data.head, tail_after_first); - assert_eq!(data.old_head, empty_head); - assert_eq!(data.old_tail, data.tail); + assert_eq!(old_head, empty_head); + assert_eq!(data.tail, old_tail); let mut parts = queue.split_by(3, &round_function); From 6f6c62908f7fd955f2c3df6b4b14be1a1857466b Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 12 Aug 2024 10:31:07 +0200 Subject: [PATCH 22/33] Cargo fmt --- src/tests/complex_tests/mod.rs | 14 +++++++++----- src/tests/run_manually.rs | 14 +++++++++----- src/witness/oracle.rs | 12 ++++++------ 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/tests/complex_tests/mod.rs b/src/tests/complex_tests/mod.rs index de4901c6..758a616c 100644 --- a/src/tests/complex_tests/mod.rs +++ b/src/tests/complex_tests/mod.rs @@ -269,8 +269,12 @@ pub(crate) fn generate_base_layer( .map(|x| ZkSyncBaseLayerStorage::from_inner(a as u8, x)) .collect(), )), - WitnessGenerationArtifact::UnsortedMemoryQueueWitness(witnesses) => unsorted_memory_queue_witnesses.push(witnesses), - WitnessGenerationArtifact::SortedMemoryQueueWitness(witnesses) => sorted_memory_queue_witnesses.push(witnesses), + WitnessGenerationArtifact::UnsortedMemoryQueueWitness(witnesses) => { + unsorted_memory_queue_witnesses.push(witnesses) + } + WitnessGenerationArtifact::SortedMemoryQueueWitness(witnesses) => { + sorted_memory_queue_witnesses.push(witnesses) + } }; let (scheduler_partial_input, _aux_data) = run( @@ -299,14 +303,14 @@ pub(crate) fn generate_base_layer( ZkSyncBaseLayerCircuit::RAMPermutation(inner) => { let mut witness = inner.witness.take().unwrap(); witness.sorted_queue_witness = FullStateCircuitQueueRawWitness { - elements: sorted_memory_queue_witnesses.next().unwrap().into() + elements: sorted_memory_queue_witnesses.next().unwrap().into(), }; witness.unsorted_queue_witness = FullStateCircuitQueueRawWitness { - elements: unsorted_memory_queue_witnesses_it.next().unwrap().into() + elements: unsorted_memory_queue_witnesses_it.next().unwrap().into(), }; inner.witness.store(Some(witness)); - }, + } _ => {} } } diff --git a/src/tests/run_manually.rs b/src/tests/run_manually.rs index 862839d0..b814988f 100644 --- a/src/tests/run_manually.rs +++ b/src/tests/run_manually.rs @@ -271,8 +271,12 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { WitnessGenerationArtifact::BaseLayerCircuit(circuit) => basic_block_circuits.push(circuit), - WitnessGenerationArtifact::UnsortedMemoryQueueWitness(witnesses) => unsorted_memory_queue_witnesses.push(witnesses), - WitnessGenerationArtifact::SortedMemoryQueueWitness(witnesses) => sorted_memory_queue_witnesses.push(witnesses), + WitnessGenerationArtifact::UnsortedMemoryQueueWitness(witnesses) => { + unsorted_memory_queue_witnesses.push(witnesses) + } + WitnessGenerationArtifact::SortedMemoryQueueWitness(witnesses) => { + sorted_memory_queue_witnesses.push(witnesses) + } _ => {} }; @@ -321,14 +325,14 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt ZkSyncBaseLayerCircuit::RAMPermutation(inner) => { let mut witness = inner.witness.take().unwrap(); witness.sorted_queue_witness = FullStateCircuitQueueRawWitness { - elements: sorted_memory_queue_witnesses.next().unwrap().into() + elements: sorted_memory_queue_witnesses.next().unwrap().into(), }; witness.unsorted_queue_witness = FullStateCircuitQueueRawWitness { - elements: unsorted_memory_queue_witnesses_it.next().unwrap().into() + elements: unsorted_memory_queue_witnesses_it.next().unwrap().into(), }; inner.witness.store(Some(witness)); - }, + } _ => {} } base_test_circuit(el); diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index d555d869..75ea94f2 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -834,7 +834,10 @@ fn simulate_memory_queue( round_function: Poseidon2Goldilocks, channel_sender: Sender, ) -> ( - CircuitsEntryAccumulatorSparse<(u32, QueueStateWitness)>, + CircuitsEntryAccumulatorSparse<( + u32, + QueueStateWitness, + )>, MemoryQueueState, LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, @@ -894,10 +897,7 @@ fn simulate_memory_queue( } } - assert_eq!( - memory_queries.len(), - memory_queue_states_accumulator.len() - ); + assert_eq!(memory_queries.len(), memory_queue_states_accumulator.len()); assert_eq!( memory_queries.len(), memory_queue_simulator.num_items as usize @@ -1220,7 +1220,7 @@ fn process_memory_related_circuits( let memory_artifacts_for_main_vm = MemoryArtifacts { memory_queries: Arc::into_inner(memory_queries_arc).unwrap(), - memory_queue_entry_states: memory_queue_entry_states_for_main_vm + memory_queue_entry_states: memory_queue_entry_states_for_main_vm, }; let implicit_memory_queries = Arc::into_inner(implicit_memory_queries_arc).unwrap(); From 8bda472b523b5cbc6fac798f914fc6fdd8d77c73 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 13 Aug 2024 12:07:26 +0200 Subject: [PATCH 23/33] Add comments; Cleanup --- .../per_circuit_accumulator.rs | 8 + src/witness/oracle.rs | 200 ++++++++++-------- 2 files changed, 122 insertions(+), 86 deletions(-) diff --git a/src/witness/aux_data_structs/per_circuit_accumulator.rs b/src/witness/aux_data_structs/per_circuit_accumulator.rs index 3dc484c5..27ac3fd4 100644 --- a/src/witness/aux_data_structs/per_circuit_accumulator.rs +++ b/src/witness/aux_data_structs/per_circuit_accumulator.rs @@ -32,6 +32,10 @@ impl PerCircuitAccumulatorContainer { self.accumulated } + pub fn amount_of_circuits_accumulated(&self) -> usize { + self.circuits_data.len() + } + pub fn last(&self) -> Option<&T> { let last_batch = self.circuits_data.last(); last_batch?; @@ -181,6 +185,10 @@ impl PerCircuitAccumulator { self.container.len() } + pub fn amount_of_circuits_accumulated(&self) -> usize { + self.container.amount_of_circuits_accumulated() + } + pub fn push(&mut self, val: T) { assert!(self.container.cycles_per_circuit != 0); let idx = self.container.len(); diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 747f1876..62d2fb56 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -42,7 +42,9 @@ use circuit_definitions::circuit_definitions::base_layer::ZkSyncBaseLayerCircuit use circuit_definitions::encodings::callstack_entry::ExtendedCallstackEntry; use circuit_definitions::encodings::recursion_request::RecursionQueueSimulator; use circuit_definitions::encodings::{CircuitEquivalentReflection, LogQueueSimulator}; -use circuit_definitions::zkevm_circuits::base_structures::memory_query::MemoryQueryWitness; +use circuit_definitions::zkevm_circuits::base_structures::memory_query::{ + MemoryQueryWitness, MEMORY_QUERY_PACKED_WIDTH, +}; use circuit_definitions::zkevm_circuits::eip_4844::input::EIP4844CircuitInstanceWitness; use circuit_definitions::zkevm_circuits::fsm_input_output::ClosedFormInputCompactFormWitness; use circuit_definitions::zkevm_circuits::scheduler::aux::BaseLayerCircuitType; @@ -839,14 +841,15 @@ fn simulate_memory_queue( LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, ImplicitMemoryStates, - Vec<[GoldilocksField; 8]>, // TODO + Vec<[GoldilocksField; MEMORY_QUERY_PACKED_WIDTH]>, ) { + // for MainVM circuits let mut memory_queue_entry_states = CircuitsEntryAccumulatorSparse::new( geometry.cycles_per_vm_snapshot as usize, (0, QueueState::placeholder_witness()), ); - // for RAM permutation circuits + // for RAM permutation circuits, only last per circuit let mut memory_queue_states_accumulator = LastPerCircuitAccumulator::>::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, @@ -854,25 +857,19 @@ fn simulate_memory_queue( ); use crate::witness::aux_data_structs::per_circuit_accumulator::PerCircuitAccumulator; - // very big data struct inside + // simulator states are very RAM-heavy let mut memory_queue_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, geometry.cycles_per_ram_permutation as usize, )); - let mut encodings_witnesses = vec![]; - - // very slow - for (cycle, query) in memory_queries.iter() { - let (_, intermediate_info) = - memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); - - memory_queue_states_accumulator.push(intermediate_info); - memory_queue_entry_states - .push((*cycle, transform_sponge_like_queue_state(intermediate_info))); + // for fs challenges in RAM permutation circuits + let mut encodings_witnesses_for_fs = vec![]; - if memory_queue_simulator.witness.len() == geometry.cycles_per_ram_permutation as usize { + // move accumulated full state witnesses from simulator, and process them + let mut process_simulation_result = + |mut memory_queue_simulator: MemoryQueuePerCircuitSimulator| { let witnesses; (memory_queue_simulator, witnesses) = memory_queue_simulator.replace_container( PerCircuitAccumulator::with_flat_capacity( @@ -880,17 +877,44 @@ fn simulate_memory_queue( geometry.cycles_per_ram_permutation as usize, ), ); + let amount_of_circuits_accumulated = witnesses.amount_of_circuits_accumulated(); + for full_witnesses_for_circuit in + witnesses.into_circuits(amount_of_circuits_accumulated) + { + let mut unsorted_witnesses_for_circuit = + Vec::with_capacity(full_witnesses_for_circuit.len()); + + // split full witnesses + for witness in full_witnesses_for_circuit.into_iter() { + encodings_witnesses_for_fs.push(witness.0); + unsorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); + } - let mut unsorted_witnesses_for_circuit = Vec::with_capacity(witnesses.len()); - for witness in witnesses.iter() { - encodings_witnesses.push(witness.0); - unsorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); + // send to storage + channel_sender + .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( + unsorted_witnesses_for_circuit, + )) + .unwrap(); } - channel_sender - .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( - unsorted_witnesses_for_circuit, - )) - .unwrap(); + + memory_queue_simulator + }; + + // the simulation is mostly a sequential computation of hashes + // for this reason it is one of the slowest parts + // we are simulating explicit part of queue (direct memory queries) + for (cycle, query) in memory_queries.iter() { + let (_, intermediate_info) = + memory_queue_simulator.push_and_output_intermediate_data(*query, &round_function); + + memory_queue_states_accumulator.push(intermediate_info); + memory_queue_entry_states + .push((*cycle, transform_sponge_like_queue_state(intermediate_info))); + + // if we have collected witnesses for the circuit, we process and send part of them to the storage to free up RAM + if memory_queue_simulator.witness.len() == geometry.cycles_per_ram_permutation as usize { + memory_queue_simulator = process_simulation_result(memory_queue_simulator); } } @@ -902,6 +926,8 @@ fn simulate_memory_queue( let final_explicit_memory_queue_state = memory_queue_states_accumulator.last().unwrap().clone(); + // now we need to handle implicit memory queries produced by decomitter, precompiles etc. + use crate::witness::individual_circuits::memory_related::simulate_implicit_memory_queues; let implicit_memory_states = simulate_implicit_memory_queues( &mut memory_queue_simulator, @@ -910,31 +936,12 @@ fn simulate_memory_queue( round_function, ); - let witnesses; - (memory_queue_simulator, witnesses) = - memory_queue_simulator.replace_container(PerCircuitAccumulator::with_flat_capacity( - geometry.cycles_per_ram_permutation as usize, - geometry.cycles_per_ram_permutation as usize, - )); - - let amount_of_ram_circuits = (witnesses.len() as u32 + geometry.cycles_per_ram_permutation - 1) - / geometry.cycles_per_ram_permutation; - - for unsorted_witnesses_for_chunk in witnesses.into_circuits(amount_of_ram_circuits as usize) { - let mut unsorted_witnesses_for_circuit = - Vec::with_capacity(unsorted_witnesses_for_chunk.len()); - - for witness in unsorted_witnesses_for_chunk.into_iter() { - encodings_witnesses.push(witness.0); - unsorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); - } + memory_queue_simulator = process_simulation_result(memory_queue_simulator); - channel_sender - .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( - unsorted_witnesses_for_circuit, - )) - .unwrap(); - } + assert_eq!( + memory_queries.len() + implicit_memory_queries.amount_of_queries(), + encodings_witnesses_for_fs.len() + ); ( memory_queue_entry_states, @@ -942,7 +949,7 @@ fn simulate_memory_queue( memory_queue_states_accumulator, memory_queue_simulator, implicit_memory_states, - encodings_witnesses, + encodings_witnesses_for_fs, ) } @@ -955,7 +962,7 @@ fn simulate_sorted_memory_queue( ) -> ( LastPerCircuitAccumulator>, MemoryQueuePerCircuitSimulator, - Vec<[GoldilocksField; 8]>, + Vec<[GoldilocksField; MEMORY_QUERY_PACKED_WIDTH]>, Vec<(u32, MemoryQuery, usize)>, ) { let mut all_memory_queries_sorted: Vec<&MemoryQuery> = memory_queries @@ -974,9 +981,13 @@ fn simulate_sorted_memory_queue( a @ _ => a, }); - // can be internally parallelized + let amount_of_queries = all_memory_queries_sorted.len(); + assert_eq!( + memory_queries.len() + implicit_memory_queries.amount_of_queries(), + amount_of_queries + ); - let amount_of_queries = memory_queries.len() + implicit_memory_queries.amount_of_queries(); + // simulator states are very RAM-heavy let mut sorted_memory_queries_simulator = MemoryQueuePerCircuitSimulator::using_container(PerCircuitAccumulator::with_flat_capacity( geometry.cycles_per_ram_permutation as usize, @@ -990,18 +1001,24 @@ fn simulate_sorted_memory_queue( amount_of_queries, ); - let mut sorted_encodings = Vec::with_capacity(amount_of_queries); + // for fs challenges in RAM permutation circuits + let mut encodings_witnesses_for_fs = Vec::with_capacity(amount_of_queries); + let amount_of_ram_circuits = (amount_of_queries as u32 + geometry.cycles_per_ram_permutation - 1) / geometry.cycles_per_ram_permutation; + // for RAM permutation circuits let mut sorted_queries_aux_data_for_chunks = Vec::with_capacity(amount_of_ram_circuits as usize); + // the simulation is mostly a sequential computation of hashes + // for this reason it is one of the slowest parts for (idx, query) in all_memory_queries_sorted.into_iter().enumerate() { let (_, intermediate_info) = sorted_memory_queries_simulator .push_and_output_intermediate_data(*query, &round_function); sorted_memory_queue_states_accumulator.push(intermediate_info); + // if we have collected witnesses for the circuit, we process and send part of them to the storage to free up RAM if sorted_memory_queries_simulator.witness.len() == geometry.cycles_per_ram_permutation as usize || idx == amount_of_queries - 1 @@ -1013,46 +1030,51 @@ fn simulate_sorted_memory_queue( geometry.cycles_per_ram_permutation as usize, )); - let sorted_witnesses_for_chunk = witnesses.into_circuits(1)[0].clone(); - - let sorted_states_len = sorted_witnesses_for_chunk.len(); - let num_nondet_writes_in_chunk = sorted_witnesses_for_chunk - .iter() - .filter(|el| { - let query = &el.2; - query.rw_flag == true - && query.timestamp.0 == 0 - && query.location.page.0 == BOOTLOADER_HEAP_PAGE - }) - .count(); - let last_sorted_query = sorted_witnesses_for_chunk.last().unwrap().2; - - sorted_queries_aux_data_for_chunks.push(( - num_nondet_writes_in_chunk as u32, - last_sorted_query, - sorted_states_len, - )); - - let mut sorted_witnesses_for_circuit = - Vec::with_capacity(sorted_witnesses_for_chunk.len()); - - for witness in sorted_witnesses_for_chunk.into_iter() { - sorted_encodings.push(witness.0); - sorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); - } + assert_eq!(witnesses.amount_of_circuits_accumulated(), 1); + + // should be only one iteration + for full_witnesses_for_circuit in witnesses.into_circuits(1) { + let sorted_states_len = full_witnesses_for_circuit.len(); + let num_nondet_writes_in_chunk = full_witnesses_for_circuit + .iter() + .filter(|el| { + let query = &el.2; + query.rw_flag == true + && query.timestamp.0 == 0 + && query.location.page.0 == BOOTLOADER_HEAP_PAGE + }) + .count(); + let last_sorted_query = full_witnesses_for_circuit.last().unwrap().2; + + sorted_queries_aux_data_for_chunks.push(( + num_nondet_writes_in_chunk as u32, + last_sorted_query, + sorted_states_len, + )); - channel_sender - .send(WitnessGenerationArtifact::SortedMemoryQueueWitness( - sorted_witnesses_for_circuit, - )) - .unwrap(); + let mut sorted_witnesses_for_circuit = + Vec::with_capacity(full_witnesses_for_circuit.len()); + + // split full witnesses + for witness in full_witnesses_for_circuit.into_iter() { + encodings_witnesses_for_fs.push(witness.0); + sorted_witnesses_for_circuit.push((witness.2.reflect(), witness.1)); + } + + // send to storage + channel_sender + .send(WitnessGenerationArtifact::SortedMemoryQueueWitness( + sorted_witnesses_for_circuit, + )) + .unwrap(); + } } } ( sorted_memory_queue_states_accumulator, sorted_memory_queries_simulator, - sorted_encodings, + encodings_witnesses_for_fs, sorted_queries_aux_data_for_chunks, ) } @@ -1150,6 +1172,10 @@ fn process_memory_related_circuits( snapshot_prof("BEFORE QUEUES SIMULATION"); + // Memory queues simulation is a slowest part in basic witness generation. + // Each queue simulation is sequential single-threaded computation of hashes. + // We will simulate unsorted and sorted queues in separate threads. + let (tx, rx): ( Sender, Receiver, @@ -1193,6 +1219,8 @@ fn process_memory_related_circuits( }) }; + // send "finalized" part of RAM permutations circuits witnesses to storage (parts of simulator states, RAM-heavy) + // the rest will be processed further for _ in 0..amount_of_ram_circuits * 2 { let artifact = rx.recv().unwrap(); artifacts_callback(artifact); From d78428fd68444cb9abaef456f66e02f96f2997ce Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 13 Aug 2024 12:14:32 +0200 Subject: [PATCH 24/33] Remove unused code --- .../individual_circuits/memory_related/mod.rs | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/mod.rs b/src/witness/individual_circuits/memory_related/mod.rs index 52ce4957..bc7e4359 100644 --- a/src/witness/individual_circuits/memory_related/mod.rs +++ b/src/witness/individual_circuits/memory_related/mod.rs @@ -69,6 +69,8 @@ pub struct ImplicitMemoryQueriesIter<'a> { iter: Iter<'a, MemoryQuery>, } +use core::slice::Iter; + impl<'a> Iterator for ImplicitMemoryQueriesIter<'a> { type Item = &'a MemoryQuery; @@ -157,33 +159,6 @@ impl ImplicitMemoryStates { + self.sha256_memory_states.len() } } - -use core::array::IntoIter as ArrayIntoIter; -use core::slice::Iter; -pub struct ImplicitMemoryStatesIter<'a, F: SmallField> { - last_vector: Iter<'a, MemoryQueueState>, - outer_iter: ArrayIntoIter<&'a Vec>, 5>, -} - -impl<'a, F: SmallField> Iterator for ImplicitMemoryStatesIter<'a, F> { - type Item = &'a MemoryQueueState; - - fn next(&mut self) -> Option { - let mut next = self.last_vector.next(); - - while next.is_none() { - let new_last_vector = self.outer_iter.next(); - if new_last_vector.is_none() { - return None; - } - self.last_vector = new_last_vector.unwrap().into_iter(); - next = self.last_vector.next(); - } - - next - } -} - use crate::witness::aux_data_structs::MemoryQueuePerCircuitSimulator; fn get_simulator_snapshot( From 8ee83490be533a4a83cfd1ed50da1b1397b45e83 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 13 Aug 2024 12:30:51 +0200 Subject: [PATCH 25/33] Cleanup in RAM permutation circuit --- .../memory_related/ram_permutation.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index e7949474..6fb7fff6 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -41,8 +41,8 @@ pub(crate) fn compute_ram_circuit_snapshots, sorted_memory_queries_simulator: MemoryQueuePerCircuitSimulator, sorted_queries_aux_data_for_chunks: Vec<(u32, MemoryQuery, usize)>, - sorted_encodings: Vec<[Field; 8]>, // TODO - unsorted_encodings: Vec<[Field; 8]>, + sorted_encodings: Vec<[Field; MEMORY_QUERY_PACKED_WIDTH]>, + unsorted_encodings: Vec<[Field; MEMORY_QUERY_PACKED_WIDTH]>, round_function: &RoundFunction, num_non_deterministic_heap_queries: usize, geometry: &GeometryConfig, @@ -91,9 +91,7 @@ pub(crate) fn compute_ram_circuit_snapshots Date: Tue, 13 Aug 2024 13:39:38 +0200 Subject: [PATCH 26/33] Update comments in ram_permutation --- .../memory_related/ram_permutation.rs | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index 6fb7fff6..c16998b9 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -225,7 +225,7 @@ pub(crate) fn compute_ram_circuit_snapshots Date: Tue, 13 Aug 2024 14:17:51 +0200 Subject: [PATCH 27/33] Update WitnessGenerationArtifact definition --- src/witness/oracle.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 62d2fb56..914cdab8 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -1389,6 +1389,8 @@ fn process_memory_related_circuits( ) } +pub type MemoryQueueWitness = + Vec<(MemoryQueryWitness, [F; FULL_SPONGE_QUEUE_STATE_WIDTH])>; pub enum WitnessGenerationArtifact { BaseLayerCircuit(ZkSyncBaseLayerCircuit), RecursionQueue( @@ -1398,8 +1400,8 @@ pub enum WitnessGenerationArtifact { Vec>, ), ), - SortedMemoryQueueWitness(Vec<(MemoryQueryWitness, [GoldilocksField; 12])>), - UnsortedMemoryQueueWitness(Vec<(MemoryQueryWitness, [GoldilocksField; 12])>), + SortedMemoryQueueWitness(MemoryQueueWitness), + UnsortedMemoryQueueWitness(MemoryQueueWitness), } /// Make basic circuits instances and witnesses, From d7fd7546a045e547a12e7b9ff827f64479e50948 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 13 Aug 2024 16:30:45 +0200 Subject: [PATCH 28/33] Move MemoryQueueStateWitnesses definition to circuit_encodings --- circuit_encodings/src/memory_query.rs | 4 ++++ src/witness/oracle.rs | 8 +++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/circuit_encodings/src/memory_query.rs b/circuit_encodings/src/memory_query.rs index d65d6b30..a9faef5e 100644 --- a/circuit_encodings/src/memory_query.rs +++ b/circuit_encodings/src/memory_query.rs @@ -157,3 +157,7 @@ impl CircuitEquivalentReflection for MemoryQuery { } } } + +use zkevm_circuits::base_structures::memory_query::MemoryQueryWitness; +pub type MemoryQueueStateWitnesses = + Vec<(MemoryQueryWitness, [F; FULL_SPONGE_QUEUE_STATE_WIDTH])>; \ No newline at end of file diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 914cdab8..3828f26b 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -824,7 +824,7 @@ use crate::witness::artifacts::DecommitmentArtifactsForMainVM; use crate::witness::artifacts::LogQueueStates; use crate::zkevm_circuits::demux_log_queue::NUM_DEMUX_OUTPUTS; -use circuit_definitions::encodings::memory_query::MemoryQueueState; +use circuit_definitions::encodings::memory_query::{MemoryQueueState, MemoryQueueStateWitnesses}; fn simulate_memory_queue( geometry: GeometryConfig, @@ -1389,8 +1389,6 @@ fn process_memory_related_circuits( ) } -pub type MemoryQueueWitness = - Vec<(MemoryQueryWitness, [F; FULL_SPONGE_QUEUE_STATE_WIDTH])>; pub enum WitnessGenerationArtifact { BaseLayerCircuit(ZkSyncBaseLayerCircuit), RecursionQueue( @@ -1400,8 +1398,8 @@ pub enum WitnessGenerationArtifact { Vec>, ), ), - SortedMemoryQueueWitness(MemoryQueueWitness), - UnsortedMemoryQueueWitness(MemoryQueueWitness), + SortedMemoryQueueWitness(MemoryQueueStateWitnesses), + UnsortedMemoryQueueWitness(MemoryQueueStateWitnesses), } /// Make basic circuits instances and witnesses, From de4b511b9f5e71ae6e7b1f963d37059efc0ceb9f Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Wed, 14 Aug 2024 11:46:23 +0200 Subject: [PATCH 29/33] Simplify WitnessGenerationArtifact enum --- src/tests/complex_tests/mod.rs | 11 ++++++----- src/tests/run_manually.rs | 11 ++++++----- src/witness/oracle.rs | 13 +++++++------ 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/tests/complex_tests/mod.rs b/src/tests/complex_tests/mod.rs index 758a616c..932846e9 100644 --- a/src/tests/complex_tests/mod.rs +++ b/src/tests/complex_tests/mod.rs @@ -269,11 +269,12 @@ pub(crate) fn generate_base_layer( .map(|x| ZkSyncBaseLayerStorage::from_inner(a as u8, x)) .collect(), )), - WitnessGenerationArtifact::UnsortedMemoryQueueWitness(witnesses) => { - unsorted_memory_queue_witnesses.push(witnesses) - } - WitnessGenerationArtifact::SortedMemoryQueueWitness(witnesses) => { - sorted_memory_queue_witnesses.push(witnesses) + WitnessGenerationArtifact::MemoryQueueWitness((witnesses, sorted)) => { + if sorted { + sorted_memory_queue_witnesses.push(witnesses); + } else { + unsorted_memory_queue_witnesses.push(witnesses); + } } }; diff --git a/src/tests/run_manually.rs b/src/tests/run_manually.rs index b814988f..2cc04a2a 100644 --- a/src/tests/run_manually.rs +++ b/src/tests/run_manually.rs @@ -271,11 +271,12 @@ pub(crate) fn run_with_options(entry_point_bytecode: Vec<[u8; 32]>, options: Opt let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { WitnessGenerationArtifact::BaseLayerCircuit(circuit) => basic_block_circuits.push(circuit), - WitnessGenerationArtifact::UnsortedMemoryQueueWitness(witnesses) => { - unsorted_memory_queue_witnesses.push(witnesses) - } - WitnessGenerationArtifact::SortedMemoryQueueWitness(witnesses) => { - sorted_memory_queue_witnesses.push(witnesses) + WitnessGenerationArtifact::MemoryQueueWitness((witnesses, sorted)) => { + if sorted { + sorted_memory_queue_witnesses.push(witnesses) + } else { + unsorted_memory_queue_witnesses.push(witnesses) + } } _ => {} }; diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 3828f26b..6e0192ba 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -892,9 +892,10 @@ fn simulate_memory_queue( // send to storage channel_sender - .send(WitnessGenerationArtifact::UnsortedMemoryQueueWitness( + .send(WitnessGenerationArtifact::MemoryQueueWitness(( unsorted_witnesses_for_circuit, - )) + false, // unsorted + ))) .unwrap(); } @@ -1063,9 +1064,10 @@ fn simulate_sorted_memory_queue( // send to storage channel_sender - .send(WitnessGenerationArtifact::SortedMemoryQueueWitness( + .send(WitnessGenerationArtifact::MemoryQueueWitness(( sorted_witnesses_for_circuit, - )) + true, // sorted + ))) .unwrap(); } } @@ -1398,8 +1400,7 @@ pub enum WitnessGenerationArtifact { Vec>, ), ), - SortedMemoryQueueWitness(MemoryQueueStateWitnesses), - UnsortedMemoryQueueWitness(MemoryQueueStateWitnesses), + MemoryQueueWitness((MemoryQueueStateWitnesses, bool)), // sorted/unsorted } /// Make basic circuits instances and witnesses, From 9d5e200a80def44fb11ad4ce57bd5d13f618d2ae Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Wed, 14 Aug 2024 12:10:12 +0200 Subject: [PATCH 30/33] Remove RAM profiling --- Cargo.toml | 1 - src/utils.rs | 94 ------------------------------------------- src/witness/oracle.rs | 17 -------- 3 files changed, 112 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3c1416f0..fc1776d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,6 @@ smallvec = "1.13" structopt = "0.3.26" codegen = "0.2.0" regex = { version = "1.10.6", features = ["pattern"] } -peak_alloc = "*" [dev-dependencies] rand = "0.4" diff --git a/src/utils.rs b/src/utils.rs index 2005de17..ecc29038 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,100 +4,6 @@ use crate::witness::tree::BinaryHasher; use circuit_definitions::encodings::BytesSerializable; -use peak_alloc::PeakAlloc; -use std::sync::{Mutex, OnceLock}; -use std::time::Duration; -use std::time::Instant; - -#[global_allocator] -static PEAK_ALLOC: PeakAlloc = PeakAlloc; - -fn mem_array() -> &'static Mutex> { - static MEM_ARRAY: OnceLock>> = OnceLock::new(); - MEM_ARRAY.get_or_init(|| Mutex::new(vec![])) -} - -fn time_instant() -> &'static Mutex { - static TIME_INSTANT: OnceLock> = OnceLock::new(); - TIME_INSTANT.get_or_init(|| Mutex::new(Instant::now())) -} - -fn time_array() -> &'static Mutex> { - static TIME_ARRAY: OnceLock>> = OnceLock::new(); - TIME_ARRAY.get_or_init(|| Mutex::new(vec![])) -} - -fn peak_mem_array() -> &'static Mutex> { - static PEAK_MEM_ARRAY: OnceLock>> = OnceLock::new(); - PEAK_MEM_ARRAY.get_or_init(|| Mutex::new(vec![])) -} - -fn peak_snapshot_mem(label: &str) { - let peak_mem = PEAK_ALLOC.peak_usage_as_mb(); - println!("PEAK MEM: {}: {}", label.to_owned(), peak_mem); - peak_mem_array() - .lock() - .unwrap() - .push((label.to_owned(), peak_mem)); -} - -pub fn print_mem_snapshots() { - let mem = mem_array().lock().unwrap(); - println!("MEMORY SNAPSHOTS"); - for snapshot in mem.clone() { - println!("{}: {}", snapshot.0, snapshot.1); - } - println!(); -} - -pub fn print_peak_mem_snapshots() { - let mem = peak_mem_array().lock().unwrap(); - println!("PEAK MEMORY SNAPSHOTS"); - for snapshot in mem.clone() { - println!("{}: {}", snapshot.0, snapshot.1); - } - println!(); -} - -pub fn print_time_snapshots() { - let time = time_array().lock().unwrap(); - println!("TIME SNAPSHOTS"); - for snapshot in time.clone() { - println!("{}: {:.2?}", snapshot.0, snapshot.1); - } - println!(); -} - -pub fn snapshot_mem(label: &str) { - let current_mem = PEAK_ALLOC.current_usage_as_mb(); - println!("MEM: {}: {}", label.to_owned(), current_mem); - mem_array() - .lock() - .unwrap() - .push((label.to_owned(), current_mem)); -} - -pub fn snapshot_prof(label: &str) { - snapshot_mem(label); - peak_snapshot_mem(label); - snapshot_time(label); - reset_peak_snapshot_mem(); -} - -pub fn snapshot_time(label: &str) { - let mut instant = time_instant().lock().unwrap(); - println!("TIME: {}: {:.2?}", label.to_owned(), instant.elapsed()); - time_array() - .lock() - .unwrap() - .push((label.to_owned(), instant.elapsed())); - *instant = Instant::now(); -} - -pub fn reset_peak_snapshot_mem() { - PEAK_ALLOC.reset_peak_usage(); -} - pub fn u64_as_u32_le(value: u64) -> [u32; 2] { [value as u32, (value >> 32) as u32] } diff --git a/src/witness/oracle.rs b/src/witness/oracle.rs index 6e0192ba..5ada14e7 100644 --- a/src/witness/oracle.rs +++ b/src/witness/oracle.rs @@ -14,7 +14,6 @@ use crate::boojum::field::SmallField; use crate::boojum::gadgets::queue::QueueState; use crate::boojum::gadgets::traits::allocatable::CSAllocatable; use crate::ethereum_types::U256; -use crate::snapshot_prof; use crate::toolset::GeometryConfig; use crate::witness::artifacts::{DemuxedLogQueries, MemoryArtifacts, MemoryCircuitsArtifacts}; use crate::witness::aux_data_structs::one_per_circuit_accumulator::{ @@ -1172,8 +1171,6 @@ fn process_memory_related_circuits( - 1) / geometry.cycles_per_ram_permutation; - snapshot_prof("BEFORE QUEUES SIMULATION"); - // Memory queues simulation is a slowest part in basic witness generation. // Each queue simulation is sequential single-threaded computation of hashes. // We will simulate unsorted and sorted queues in separate threads. @@ -1258,8 +1255,6 @@ fn process_memory_related_circuits( implicit_memory_states.amount_of_states() ); - snapshot_prof("AFTER QUEUES SIMULATION"); - use crate::witness::individual_circuits::memory_related::decommit_code::compute_decommitter_circuit_snapshots; tracing::debug!("Running code code decommitter simulation"); @@ -1362,8 +1357,6 @@ fn process_memory_related_circuits( tracing::debug!("Running RAM permutation simulation"); - snapshot_prof("BEFORE RAM"); - let (ram_permutation_circuits, ram_permutation_circuits_compact_forms_witnesses) = compute_ram_circuit_snapshots( amount_of_memory_queries, @@ -1380,8 +1373,6 @@ fn process_memory_related_circuits( &mut artifacts_callback, ); - snapshot_prof("AFTER RAM"); - ( circuits_data, memory_artifacts_for_main_vm, @@ -1468,8 +1459,6 @@ pub(crate) fn create_artifacts_from_tracer std::mem::take(&mut callstack_with_aux_data.flat_new_frames_history); drop(callstack_with_aux_data); - snapshot_prof("BEFORE MUX LOG QUEUE CIRCUITS"); - tracing::debug!("Running multiplexed log queue simulation"); // We have all log queries in one multiplexed queue. We need to simulate this queue, @@ -1487,8 +1476,6 @@ pub(crate) fn create_artifacts_from_tracer *round_function, ); - snapshot_prof("AFTER MUX LOG QUEUE CIRCUITS"); - use std::thread; let callstack_handle = { let log_rollback_tails_for_frames = log_rollback_tails_for_frames.clone(); @@ -1532,8 +1519,6 @@ pub(crate) fn create_artifacts_from_tracer tracing::debug!("Processing log circuits"); - snapshot_prof("AFTER LOG DEMUX CIRCUITS"); - // Process part of log circuits that do not use memory (I/O-like). // Precompiles will be processed in process_memory_related_circuits. // Also makes storage application circuits and compact form witnesses. @@ -1547,8 +1532,6 @@ pub(crate) fn create_artifacts_from_tracer &mut artifacts_callback, ); - snapshot_prof("AFTER LOG CIRCUITS"); - tracing::debug!("Processing memory-related circuits"); let precompiles_data = PrecompilesInputData { From ee759ebee1ea722f903de80133c64213cf40458a Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Wed, 14 Aug 2024 12:11:22 +0200 Subject: [PATCH 31/33] Cargo fmt --- circuit_encodings/src/memory_query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuit_encodings/src/memory_query.rs b/circuit_encodings/src/memory_query.rs index a9faef5e..19a6994f 100644 --- a/circuit_encodings/src/memory_query.rs +++ b/circuit_encodings/src/memory_query.rs @@ -160,4 +160,4 @@ impl CircuitEquivalentReflection for MemoryQuery { use zkevm_circuits::base_structures::memory_query::MemoryQueryWitness; pub type MemoryQueueStateWitnesses = - Vec<(MemoryQueryWitness, [F; FULL_SPONGE_QUEUE_STATE_WIDTH])>; \ No newline at end of file + Vec<(MemoryQueryWitness, [F; FULL_SPONGE_QUEUE_STATE_WIDTH])>; From ba9ce6252771d3c015e683b7d229043878d3bdb3 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Wed, 14 Aug 2024 12:21:02 +0200 Subject: [PATCH 32/33] Remove unnecessary structs from ram permutation processing --- .../memory_related/ram_permutation.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index c16998b9..d2a0411c 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -224,16 +224,6 @@ pub(crate) fn compute_ram_circuit_snapshots Date: Thu, 15 Aug 2024 10:59:04 +0200 Subject: [PATCH 33/33] Add comment in ram_permutation --- .../individual_circuits/memory_related/ram_permutation.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/witness/individual_circuits/memory_related/ram_permutation.rs b/src/witness/individual_circuits/memory_related/ram_permutation.rs index d2a0411c..eec69c2d 100644 --- a/src/witness/individual_circuits/memory_related/ram_permutation.rs +++ b/src/witness/individual_circuits/memory_related/ram_permutation.rs @@ -319,6 +319,9 @@ pub(crate) fn compute_ram_circuit_snapshots