Skip to content

Commit

Permalink
Merge branch 'matter-labs:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@Saladerl
Saladerl authored Dec 4, 2023
2 parents d131187 + aebb70d commit fd0dbe2
Show file tree
Hide file tree
Showing 61 changed files with 725 additions and 288 deletions.
74 changes: 74 additions & 0 deletions core/lib/dal/sqlx-data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2744,6 +2744,24 @@
},
"query": "UPDATE transactions\n SET in_mempool = TRUE\n FROM (\n SELECT hash FROM (\n SELECT hash\n FROM transactions\n WHERE miniblock_number IS NULL AND in_mempool = FALSE AND error IS NULL\n AND (is_priority = TRUE OR (max_fee_per_gas >= $2 and gas_per_pubdata_limit >= $3))\n AND tx_format != $4\n ORDER BY is_priority DESC, priority_op_id, received_at\n LIMIT $1\n ) as subquery1\n ORDER BY hash\n ) as subquery2\n WHERE transactions.hash = subquery2.hash\n RETURNING transactions.*"
},
"2cc57497090a97bcb453036f7b5e2139b590699aa1a2df4d6fd2b19e27e06251": {
"describe": {
"columns": [
{
"name": "l1_batch_number",
"ordinal": 0,
"type_info": "Int8"
}
],
"nullable": [
false
],
"parameters": {
"Left": []
}
},
"query": "SELECT l1_batch_number FROM proof_generation_details WHERE status <> 'generated' ORDER BY l1_batch_number ASC LIMIT 1"
},
"2e3f116ca05ae70b7c83ac550302194c91f57b69902ff8e42140fde732ae5e6a": {
"describe": {
"columns": [],
Expand Down Expand Up @@ -3971,6 +3989,24 @@
},
"query": "VACUUM storage_logs"
},
"4860c1118485da8673963a260ded76eb8e13989936f9ab17e23687a1103132cb": {
"describe": {
"columns": [
{
"name": "l1_batch_number",
"ordinal": 0,
"type_info": "Int8"
}
],
"nullable": [
false
],
"parameters": {
"Left": []
}
},
"query": "SELECT l1_batch_number FROM proof_generation_details WHERE status = 'ready_to_be_proven' ORDER BY l1_batch_number ASC LIMIT 1"
},
"4ab8a25620b5400d836e1b847320d4e176629a27e1a6cb0666ab02bb55371769": {
"describe": {
"columns": [
Expand Down Expand Up @@ -8814,6 +8850,26 @@
},
"query": "SELECT MAX(operation_number) as \"max?\" FROM storage_logs WHERE miniblock_number = $1"
},
"a7c7e8f036404d24dc6bfa184a84b92d8f73ca034970481af34b6163e66dc59a": {
"describe": {
"columns": [
{
"name": "l1_batch_number",
"ordinal": 0,
"type_info": "Int8"
}
],
"nullable": [
false
],
"parameters": {
"Left": [
"Int2"
]
}
},
"query": "\n SELECT l1_batch_number \n FROM prover_jobs_fri \n WHERE status <> 'skipped'\n AND status <> 'successful'\n AND aggregation_round = $1 \n ORDER BY l1_batch_number ASC \n LIMIT 1\n "
},
"a8b32073a67ad77caab11e73a5cac5aa5b5382648ff95d6787a309eb3f64d434": {
"describe": {
"columns": [],
Expand Down Expand Up @@ -11541,6 +11597,24 @@
},
"query": "\n UPDATE node_aggregation_witness_jobs\n SET status='queued'\n WHERE l1_batch_number IN\n (SELECT prover_jobs.l1_batch_number\n FROM prover_jobs\n JOIN node_aggregation_witness_jobs nawj ON prover_jobs.l1_batch_number = nawj.l1_batch_number\n WHERE nawj.status = 'waiting_for_proofs'\n AND prover_jobs.status = 'successful'\n AND prover_jobs.aggregation_round = 1\n GROUP BY prover_jobs.l1_batch_number, nawj.number_of_leaf_circuits\n HAVING COUNT(*) = nawj.number_of_leaf_circuits)\n RETURNING l1_batch_number;\n "
},
"f15f0848cfd830ec5d5b479fdcdd36c6a4439495b7680614ac1b0e4d73fb992f": {
"describe": {
"columns": [
{
"name": "l1_batch_number",
"ordinal": 0,
"type_info": "Int8"
}
],
"nullable": [
false
],
"parameters": {
"Left": []
}
},
"query": "SELECT l1_batch_number FROM proof_compression_jobs_fri WHERE status <> 'successful' ORDER BY l1_batch_number ASC LIMIT 1"
},
"f1defa140e20b9c250d3212602dc259c0a35598c2e69d1c42746a8fab6dd8d3e": {
"describe": {
"columns": [],
Expand Down
16 changes: 16 additions & 0 deletions core/lib/dal/src/fri_proof_compressor_dal.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,22 @@ impl FriProofCompressorDal<'_, '_> {
}
}

pub async fn get_oldest_not_compressed_batch(&mut self) -> Option<L1BatchNumber> {
let result: Option<L1BatchNumber> = sqlx::query!(
"SELECT l1_batch_number \
FROM proof_compression_jobs_fri \
WHERE status <> 'successful' \
ORDER BY l1_batch_number ASC \
LIMIT 1",
)
.fetch_optional(self.storage.conn())
.await
.unwrap()
.map(|row| L1BatchNumber(row.l1_batch_number as u32));

result
}

pub async fn requeue_stuck_jobs(
&mut self,
processing_timeout: Duration,
Expand Down
22 changes: 22 additions & 0 deletions core/lib/dal/src/fri_prover_dal.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,28 @@ impl FriProverDal<'_, '_> {
}
}

pub async fn min_unproved_l1_batch_number_for_aggregation_round(
&mut self,
aggregation_round: AggregationRound,
) -> Option<L1BatchNumber> {
sqlx::query!(
r#"
SELECT l1_batch_number
FROM prover_jobs_fri
WHERE status <> 'skipped'
AND status <> 'successful'
AND aggregation_round = $1
ORDER BY l1_batch_number ASC
LIMIT 1
"#,
aggregation_round as i16
)
.fetch_optional(self.storage.conn())
.await
.unwrap()
.map(|row| L1BatchNumber(row.l1_batch_number as u32))
}

pub async fn update_status(&mut self, id: u32, status: &str) {
sqlx::query!(
"UPDATE prover_jobs_fri \
Expand Down
32 changes: 32 additions & 0 deletions core/lib/dal/src/proof_generation_dal.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,4 +109,36 @@ impl ProofGenerationDal<'_, '_> {
.then_some(())
.ok_or(sqlx::Error::RowNotFound)
}

pub async fn get_oldest_unprocessed_batch(&mut self) -> Option<L1BatchNumber> {
let result: Option<L1BatchNumber> = sqlx::query!(
"SELECT l1_batch_number \
FROM proof_generation_details \
WHERE status = 'ready_to_be_proven' \
ORDER BY l1_batch_number ASC \
LIMIT 1",
)
.fetch_optional(self.storage.conn())
.await
.unwrap()
.map(|row| L1BatchNumber(row.l1_batch_number as u32));

result
}

pub async fn get_oldest_not_generated_batch(&mut self) -> Option<L1BatchNumber> {
let result: Option<L1BatchNumber> = sqlx::query!(
"SELECT l1_batch_number \
FROM proof_generation_details \
WHERE status <> 'generated' \
ORDER BY l1_batch_number ASC \
LIMIT 1",
)
.fetch_optional(self.storage.conn())
.await
.unwrap()
.map(|row| L1BatchNumber(row.l1_batch_number as u32));

result
}
}
15 changes: 15 additions & 0 deletions core/lib/multivm/src/tracers/call_tracer/metrics.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
use vise::{Buckets, Histogram, Metrics};

#[derive(Debug, Metrics)]
#[metrics(prefix = "vm_call_tracer")]
pub struct CallMetrics {
/// Maximum call stack depth during the execution of the transaction.
#[metrics(buckets = Buckets::exponential(1.0..=64.0, 2.0))]
pub call_stack_depth: Histogram<usize>,
/// Maximum number of near calls during the execution of the transaction.
#[metrics(buckets = Buckets::exponential(1.0..=64.0, 2.0))]
pub max_near_calls: Histogram<usize>,
}

#[vise::register]
pub static CALL_METRICS: vise::Global<CallMetrics> = vise::Global::new();
46 changes: 46 additions & 0 deletions core/lib/multivm/src/tracers/call_tracer/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
use crate::tracers::call_tracer::metrics::CALL_METRICS;
use once_cell::sync::OnceCell;
use std::sync::Arc;
use zksync_types::vm_trace::Call;

mod metrics;
pub mod vm_latest;
pub mod vm_refunds_enhancement;
pub mod vm_virtual_blocks;
Expand All @@ -10,19 +12,32 @@ pub mod vm_virtual_blocks;
pub struct CallTracer {
stack: Vec<FarcallAndNearCallCount>,
result: Arc<OnceCell<Vec<Call>>>,

max_stack_depth: usize,
max_near_calls: usize,
}

#[derive(Debug, Clone)]
struct FarcallAndNearCallCount {
farcall: Call,
near_calls_after: usize,
stack_depth_on_prefix: usize,
}

impl Drop for CallTracer {
fn drop(&mut self) {
CALL_METRICS.call_stack_depth.observe(self.max_stack_depth);
CALL_METRICS.max_near_calls.observe(self.max_near_calls);
}
}

impl CallTracer {
pub fn new(result: Arc<OnceCell<Vec<Call>>>) -> Self {
Self {
stack: vec![],
result,
max_stack_depth: 0,
max_near_calls: 0,
}
}

Expand All @@ -38,4 +53,35 @@ impl CallTracer {
let cell = self.result.as_ref();
cell.set(result).unwrap();
}

fn push_call_and_update_stats(&mut self, farcall: Call, near_calls_after: usize) {
let stack_depth = self
.stack
.last()
.map(|x| x.stack_depth_on_prefix)
.unwrap_or(0);

let depth_on_prefix = stack_depth + 1 + near_calls_after;

let call = FarcallAndNearCallCount {
farcall,
near_calls_after,
stack_depth_on_prefix: depth_on_prefix,
};

self.stack.push(call);

self.max_stack_depth = self.max_stack_depth.max(depth_on_prefix);
self.max_near_calls = self.max_near_calls.max(near_calls_after);
}

fn increase_near_call_count(&mut self) {
if let Some(last) = self.stack.last_mut() {
last.near_calls_after += 1;
last.stack_depth_on_prefix += 1;

self.max_near_calls = self.max_near_calls.max(last.near_calls_after);
self.max_stack_depth = self.max_stack_depth.max(last.stack_depth_on_prefix);
}
}
}
15 changes: 5 additions & 10 deletions core/lib/multivm/src/tracers/call_tracer/vm_latest/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use crate::interface::{
tracer::VmExecutionStopReason, traits::tracers::dyn_tracers::vm_1_4_0::DynTracer,
VmRevertReason,
};
use crate::tracers::call_tracer::{CallTracer, FarcallAndNearCallCount};
use crate::tracers::call_tracer::CallTracer;
use crate::vm_latest::{BootloaderState, HistoryMode, SimpleMemory, VmTracer, ZkSyncVmState};

impl<S, H: HistoryMode> DynTracer<S, SimpleMemory<H>> for CallTracer {
Expand All @@ -28,9 +28,7 @@ impl<S, H: HistoryMode> DynTracer<S, SimpleMemory<H>> for CallTracer {
) {
match data.opcode.variant.opcode {
Opcode::NearCall(_) => {
if let Some(last) = self.stack.last_mut() {
last.near_calls_after += 1;
}
self.increase_near_call_count();
}
Opcode::FarCall(far_call) => {
// We use parent gas for properly calculating gas used in the trace.
Expand All @@ -51,10 +49,7 @@ impl<S, H: HistoryMode> DynTracer<S, SimpleMemory<H>> for CallTracer {
};

self.handle_far_call_op_code_latest(state, memory, &mut current_call);
self.stack.push(FarcallAndNearCallCount {
farcall: current_call,
near_calls_after: 0,
});
self.push_call_and_update_stats(current_call, 0);
}
Opcode::Ret(ret_code) => {
self.handle_ret_op_code_latest(state, memory, ret_code);
Expand Down Expand Up @@ -187,7 +182,7 @@ impl CallTracer {

if current_call.near_calls_after > 0 {
current_call.near_calls_after -= 1;
self.stack.push(current_call);
self.push_call_and_update_stats(current_call.farcall, current_call.near_calls_after);
return;
}

Expand All @@ -203,7 +198,7 @@ impl CallTracer {
if let Some(parent_call) = self.stack.last_mut() {
parent_call.farcall.calls.push(current_call.farcall);
} else {
self.stack.push(current_call);
self.push_call_and_update_stats(current_call.farcall, current_call.near_calls_after);
}
}
}
16 changes: 6 additions & 10 deletions core/lib/multivm/src/tracers/call_tracer/vm_refunds_enhancement/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use crate::interface::{
tracer::VmExecutionStopReason, traits::tracers::dyn_tracers::vm_1_3_3::DynTracer,
VmRevertReason,
};
use crate::tracers::call_tracer::{CallTracer, FarcallAndNearCallCount};
use crate::tracers::call_tracer::CallTracer;
use crate::vm_refunds_enhancement::{
BootloaderState, HistoryMode, SimpleMemory, VmTracer, ZkSyncVmState,
};
Expand All @@ -30,9 +30,7 @@ impl<S, H: HistoryMode> DynTracer<S, SimpleMemory<H>> for CallTracer {
) {
match data.opcode.variant.opcode {
Opcode::NearCall(_) => {
if let Some(last) = self.stack.last_mut() {
last.near_calls_after += 1;
}
self.increase_near_call_count();
}
Opcode::FarCall(far_call) => {
// We use parent gas for properly calculating gas used in the trace.
Expand All @@ -53,10 +51,8 @@ impl<S, H: HistoryMode> DynTracer<S, SimpleMemory<H>> for CallTracer {
};

self.handle_far_call_op_code_refunds_enhancement(state, memory, &mut current_call);
self.stack.push(FarcallAndNearCallCount {
farcall: current_call,
near_calls_after: 0,
});

self.push_call_and_update_stats(current_call, 0);
}
Opcode::Ret(ret_code) => {
self.handle_ret_op_code_refunds_enhancement(state, memory, ret_code);
Expand Down Expand Up @@ -189,7 +185,7 @@ impl CallTracer {

if current_call.near_calls_after > 0 {
current_call.near_calls_after -= 1;
self.stack.push(current_call);
self.push_call_and_update_stats(current_call.farcall, current_call.near_calls_after);
return;
}

Expand All @@ -205,7 +201,7 @@ impl CallTracer {
if let Some(parent_call) = self.stack.last_mut() {
parent_call.farcall.calls.push(current_call.farcall);
} else {
self.stack.push(current_call);
self.push_call_and_update_stats(current_call.farcall, current_call.near_calls_after);
}
}
}
Loading

0 comments on commit fd0dbe2

Please sign in to comment.