diff --git a/Cargo.lock b/Cargo.lock index dc00c768..d5104e2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2061,7 +2061,6 @@ dependencies = [ "test-generator", "thiserror 1.0.69", "try_from_iterator", - "typenum", "types", ] @@ -2657,7 +2656,6 @@ dependencies = [ "thiserror 1.0.69", "transition_functions", "tynm", - "typenum", "types", "unwrap_none", ] @@ -2694,7 +2692,6 @@ dependencies = [ "thiserror 1.0.69", "transition_functions", "tynm", - "typenum", "types", "unwrap_none", ] diff --git a/eip_7594/Cargo.toml b/eip_7594/Cargo.toml index 31d0be74..eb4c269c 100644 --- a/eip_7594/Cargo.toml +++ b/eip_7594/Cargo.toml @@ -17,7 +17,6 @@ ssz = { workspace = true} sha2 = { workspace = true } thiserror = { workspace = true } try_from_iterator = { workspace = true } -typenum = { workspace = true } types = { workspace = true } prometheus_metrics = { workspace = true, optional = true } diff --git a/eip_7594/src/lib.rs b/eip_7594/src/lib.rs index 74891a24..150d6d9c 100644 --- a/eip_7594/src/lib.rs +++ b/eip_7594/src/lib.rs @@ -11,13 +11,11 @@ use num_traits::One as _; use sha2::{Digest as _, Sha256}; use ssz::{ByteVector, ContiguousList, ContiguousVector, SszHash as _, Uint256}; use try_from_iterator::TryFromIterator as _; -use typenum::Unsigned as _; use types::{ combined::SignedBeaconBlock, config::Config, deneb::primitives::{Blob, KzgProof}, fulu::{ - consts::NumberOfColumns, containers::{DataColumnSidecar, MatrixEntry}, primitives::{Cell, ColumnIndex, CustodyIndex}, }, @@ -35,6 +33,10 @@ mod trusted_setup; #[cfg(test)] mod tests; +type ColumnCells = [Cell; CELLS_PER_EXT_BLOB]; +type ColumnProofs = [KzgProof; CELLS_PER_EXT_BLOB]; +type CellsAndKzgProofs = Vec<(ColumnCells, ColumnProofs)>; + #[cfg(feature = "metrics")] use prometheus_metrics::METRICS; @@ -98,9 +100,8 @@ pub fn compute_columns_for_custody_group( }, ); - let columns_per_custody_group = NumberOfColumns::U64 / number_of_custody_groups; let mut columns = Vec::new(); - for i in 0..columns_per_custody_group { + for i in 0..config.columns_per_group() { columns.push(ColumnIndex::from( number_of_custody_groups * i + custody_group, )); @@ -139,7 +140,10 @@ pub fn compute_subnets_for_node( } /// Verify if the data column sidecar is valid. -pub fn verify_data_column_sidecar(data_column_sidecar: &DataColumnSidecar

) -> bool { +pub fn verify_data_column_sidecar( + data_column_sidecar: &DataColumnSidecar

, + config: &Config, +) -> bool { let DataColumnSidecar { index, column, @@ -149,7 +153,7 @@ pub fn verify_data_column_sidecar(data_column_sidecar: &DataColumnSid } = data_column_sidecar; // The sidecar index must be within the valid range - if *index >= NumberOfColumns::U64 { + if *index >= config.number_of_columns { return false; } @@ -255,7 +259,7 @@ pub fn compute_matrix(blobs: &[CKzgBlob]) -> Result> { let (cells, proofs) = CKzgCell::compute_cells_and_kzg_proofs(blob, kzg_settings)?; for (cell_index, (cell, proof)) in cells.into_iter().zip(proofs.into_iter()).enumerate() { matrix.push(MatrixEntry { - cell: try_convert_ckzg_cell_to_cell(&cell)?, + cell: try_convert_to_cell(&cell)?, kzg_proof: KzgProof::from(proof.to_bytes().into_inner()), row_index: blob_index as u64, column_index: cell_index as u64, @@ -303,7 +307,7 @@ pub fn recover_matrix( .enumerate() { matrix.push(MatrixEntry { - cell: try_convert_ckzg_cell_to_cell(&cell)?, + cell: try_convert_to_cell(&cell)?, kzg_proof: KzgProof::from(proof.to_bytes().into_inner()), row_index: blob_index as u64, column_index: cell_index as u64, @@ -314,10 +318,10 @@ pub fn recover_matrix( Ok(matrix) } -// TODO(peerdas-fulu): refactor params pub fn construct_data_column_sidecars( signed_block: &SignedBeaconBlock

, - cells_and_kzg_proofs: &[([Cell; CELLS_PER_EXT_BLOB], [KzgProof; CELLS_PER_EXT_BLOB])], + cells_and_kzg_proofs: &CellsAndKzgProofs, + config: &Config, ) -> Result>> { let signed_block_header = signed_block.to_header(); @@ -341,7 +345,7 @@ pub fn construct_data_column_sidecars( let kzg_commitments_inclusion_proof = misc::kzg_commitments_inclusion_proof(post_electra_beacon_block_body); - for column_index in 0..NumberOfColumns::USIZE { + for column_index in 0..config.number_of_columns() { let column = ContiguousList::try_from_iter( (0..blob_count) .map(|row_index| cells_and_kzg_proofs[row_index].0[column_index].clone()), @@ -366,7 +370,7 @@ pub fn construct_data_column_sidecars( pub fn try_convert_to_cells_and_kzg_proofs( blobs: impl Iterator>, -) -> Result> { +) -> Result { let kzg_settings = kzg_settings(); let cells_and_kzg_proofs = blobs .map(|blob| { @@ -375,51 +379,46 @@ pub fn try_convert_to_cells_and_kzg_proofs( }) .collect::>>()?; - let mut result: Vec<([Cell; CELLS_PER_EXT_BLOB], [KzgProof; CELLS_PER_EXT_BLOB])> = vec![]; - for (column_cells, column_proofs) in cells_and_kzg_proofs { - let cells = column_cells + let mut result = vec![]; + for (cells, proofs) in cells_and_kzg_proofs { + let cells = cells .iter() - .map(try_convert_ckzg_cell_to_cell) - .collect::>>()?; + .map(try_convert_to_cell) + .collect::>>()?; + let column_cells = cells + .try_into() + .expect("column cells should have length of CELLS_PER_EXT_BLOB"); - let proofs = column_proofs + let proofs = proofs .iter() .map(|proof| KzgProof::from(proof.to_bytes().into_inner())) - .collect::>(); - - let column: [Cell; CELLS_PER_EXT_BLOB] = cells - .try_into() - .expect("cells should not be more than number of columns"); - let kzg_proofs: [KzgProof; CELLS_PER_EXT_BLOB] = proofs + .collect::>(); + let column_proofs = proofs .try_into() - .expect("kzg_proofs should not be more than number of columns"); - result.push((column, kzg_proofs)); + .expect("column proofs should have length of CELLS_PER_EXT_BLOB"); + + result.push((column_cells, column_proofs)); } Ok(result) } -fn try_convert_ckzg_cell_to_cell(cell: &CKzgCell) -> Result { - let bytes = cell.to_bytes(); - - ContiguousVector::try_from_iter(bytes) +fn try_convert_to_cell(cell: &CKzgCell) -> Result { + ContiguousVector::try_from_iter(cell.to_bytes()) .map(ByteVector::from) - .map(Box::new) + .map(Cell::from) .map_err(Into::into) } pub fn construct_cells_and_kzg_proofs( full_matrix: Vec, blob_count: usize, -) -> Result> { +) -> Result { let default_cell = Cell::default(); - let mut cells_and_kzg_proofs: Vec<( - [Cell; NumberOfColumns::USIZE], - [KzgProof; NumberOfColumns::USIZE], - )> = vec![ + let mut cells_and_kzg_proofs = vec![ ( core::array::from_fn(|_| default_cell.clone()), - [KzgProof::repeat_byte(u8::MAX); NumberOfColumns::USIZE], + [KzgProof::zero(); CELLS_PER_EXT_BLOB], ); blob_count ]; diff --git a/fork_choice_control/Cargo.toml b/fork_choice_control/Cargo.toml index ed983b82..3d197bd3 100644 --- a/fork_choice_control/Cargo.toml +++ b/fork_choice_control/Cargo.toml @@ -46,7 +46,6 @@ tap = { workspace = true } thiserror = { workspace = true } transition_functions = { workspace = true } tynm = { workspace = true } -typenum = { workspace = true } types = { workspace = true } [dev-dependencies] diff --git a/fork_choice_control/src/mutator.rs b/fork_choice_control/src/mutator.rs index 6a7b82fc..4c1e8116 100644 --- a/fork_choice_control/src/mutator.rs +++ b/fork_choice_control/src/mutator.rs @@ -46,12 +46,10 @@ use num_traits::identities::Zero as _; use prometheus_metrics::Metrics; use ssz::SszHash as _; use std_ext::ArcExt as _; -use typenum::Unsigned as _; use types::{ combined::{BeaconState, ExecutionPayloadParams, SignedBeaconBlock}, deneb::containers::{BlobIdentifier, BlobSidecar}, fulu::{ - consts::NumberOfColumns, containers::{DataColumnIdentifier, DataColumnSidecar, MatrixEntry}, primitives::ColumnIndex, }, @@ -555,7 +553,8 @@ where let missing_column_indices = self.store.indices_of_missing_data_columns(&parent.block); - if missing_column_indices.len() * 2 < NumberOfColumns::USIZE + let number_of_columns = self.store.chain_config().number_of_columns(); + if missing_column_indices.len() * 2 < number_of_columns || !self.store.is_forward_synced() { let body = parent @@ -572,7 +571,7 @@ where // TODO(feature/fulu): random delay reconstruction as stated in the [specs] // (https://github.com/ethereum/consensus-specs/blob/8696fbf75387fb37a32fc08a6b934653198c6c0c/specs/fulu/das-core.md?plain=1#L257) - if available_columns.len() > NumberOfColumns::USIZE / 2 + if available_columns.len() > number_of_columns / 2 && !self .store .has_reconstructed_data_column_sidecars(parent.block_root) @@ -1456,15 +1455,18 @@ where blob_count: usize, full_matrix: Vec, ) -> Result<()> { + let config = self.store.chain_config(); let chain_link = self.store.chain_link(block_root).expect( "block must be available in the store during data column sidecars reconstruction", ); let cells_and_kzg_proofs = eip_7594::construct_cells_and_kzg_proofs(full_matrix, blob_count)?; - for data_column_sidecar in - eip_7594::construct_data_column_sidecars(&chain_link.block, &cells_and_kzg_proofs)? - { + for data_column_sidecar in eip_7594::construct_data_column_sidecars( + &chain_link.block, + &cells_and_kzg_proofs, + config, + )? { let data_column_sidecar = Arc::new(data_column_sidecar); let data_column_identifier: DataColumnIdentifier = data_column_sidecar.as_ref().into(); diff --git a/fork_choice_control/src/queries.rs b/fork_choice_control/src/queries.rs index 872eb64f..5cd3096d 100644 --- a/fork_choice_control/src/queries.rs +++ b/fork_choice_control/src/queries.rs @@ -11,14 +11,12 @@ use fork_choice_store::{ use helper_functions::misc; use itertools::Itertools as _; use serde::Serialize; -use ssz::ContiguousList; use std_ext::ArcExt; use thiserror::Error; use types::{ combined::{BeaconState, SignedAggregateAndProof, SignedBeaconBlock}, deneb::containers::{BlobIdentifier, BlobSidecar}, fulu::{ - consts::NumberOfColumns, containers::{DataColumnIdentifier, DataColumnSidecar}, primitives::ColumnIndex, }, @@ -509,7 +507,7 @@ where pub fn data_column_sidecars_by_range( &self, range: Range, - columns: &ContiguousList, + columns: &[ColumnIndex], max_request_data_column_sidecars: usize, ) -> Result>>> { let canonical_chain_blocks = self.blocks_by_range(range)?; @@ -518,8 +516,8 @@ where .iter() .filter_map(|BlockWithRoot { block, root }| { block.message().body().post_deneb().map(|_| { - columns.iter().map(|index| DataColumnIdentifier { - index: *index, + columns.iter().copied().map(|index| DataColumnIdentifier { + index, block_root: *root, }) }) diff --git a/fork_choice_control/src/tasks.rs b/fork_choice_control/src/tasks.rs index bd198884..4ccd0c1c 100644 --- a/fork_choice_control/src/tasks.rs +++ b/fork_choice_control/src/tasks.rs @@ -20,11 +20,10 @@ use helper_functions::{ }; use log::{debug, warn}; use prometheus_metrics::Metrics; -use typenum::Unsigned as _; use types::{ combined::{AttesterSlashing, SignedAggregateAndProof, SignedBeaconBlock}, deneb::containers::BlobSidecar, - fulu::{consts::NumberOfColumns, containers::DataColumnSidecar}, + fulu::containers::DataColumnSidecar, nonstandard::{RelativeEpoch, ValidationOutcome}, phase0::{ containers::Checkpoint, @@ -509,27 +508,23 @@ impl Run for ReconstructDataColumnSidecarsTask { let available_columns = store_snapshot.available_columns_at_block(block_root); - if available_columns.len() < NumberOfColumns::USIZE { - let partial_matrix = available_columns - .into_iter() - .flat_map(|sidecar| misc::compute_matrix_for_data_column_sidecar(&sidecar)) - .collect::>(); - - match eip_7594::recover_matrix(&partial_matrix, blob_count) { - Ok(full_matrix) => { - MutatorMessage::ReconstructedMissingColumns { - block_root, - blob_count, - full_matrix, - } - .send(&mutator_tx); - } - Err(error) => { - warn!("failed to reconstruct missing data column sidecars: {error:?}"); + let partial_matrix = available_columns + .into_iter() + .flat_map(|sidecar| misc::compute_matrix_for_data_column_sidecar(&sidecar)) + .collect::>(); + + match eip_7594::recover_matrix(&partial_matrix, blob_count) { + Ok(full_matrix) => { + MutatorMessage::ReconstructedMissingColumns { + block_root, + blob_count, + full_matrix, } + .send(&mutator_tx); + } + Err(error) => { + warn!("failed to reconstruct missing data column sidecars: {error:?}"); } - } else { - debug!("all required sample columns are available, dismiss recontruction"); } } } diff --git a/fork_choice_store/Cargo.toml b/fork_choice_store/Cargo.toml index 725b7797..73419c05 100644 --- a/fork_choice_store/Cargo.toml +++ b/fork_choice_store/Cargo.toml @@ -35,6 +35,5 @@ tap = { workspace = true } thiserror = { workspace = true } transition_functions = { workspace = true } tynm = { workspace = true } -typenum = { workspace = true } types = { workspace = true } unwrap_none = { workspace = true } diff --git a/fork_choice_store/src/store.rs b/fork_choice_store/src/store.rs index b8f399d3..0167ab5a 100644 --- a/fork_choice_store/src/store.rs +++ b/fork_choice_store/src/store.rs @@ -31,7 +31,6 @@ use transition_functions::{ combined, unphased::{self, ProcessSlots, StateRootPolicy}, }; -use typenum::Unsigned as _; use types::{ combined::{ Attestation, AttesterSlashing, AttestingIndices, BeaconState, SignedAggregateAndProof, @@ -43,7 +42,6 @@ use types::{ primitives::{BlobIndex, KzgCommitment}, }, fulu::{ - consts::NumberOfColumns, containers::{DataColumnIdentifier, DataColumnSidecar}, primitives::ColumnIndex, }, @@ -1118,7 +1116,9 @@ impl Store

{ if state.is_post_fulu() { let missing_indices = self.indices_of_missing_data_columns(&parent.block); - if missing_indices.len() * 2 >= NumberOfColumns::USIZE && self.is_forward_synced() { + if missing_indices.len() * 2 >= self.chain_config.number_of_columns() + && self.is_forward_synced() + { return Ok(BlockAction::DelayUntilBlobs(block.clone_arc())); } } else if !self.indices_of_missing_blobs(block).is_empty() { @@ -1915,7 +1915,7 @@ impl Store

{ // [REJECT] The sidecar is valid as verified by `verify_data_column_sidecar(sidecar). ensure!( - verify_data_column_sidecar(&data_column_sidecar), + verify_data_column_sidecar(&data_column_sidecar, &self.chain_config), Error::DataColumnSidecarInvalid { data_column_sidecar }, @@ -3385,11 +3385,11 @@ impl Store

{ } pub fn is_supernode(&self) -> bool { - self.sampling_columns.len() == NumberOfColumns::USIZE + self.sampling_columns.len() == self.chain_config.number_of_columns() } pub fn available_columns_at_block(&self, block_root: H256) -> Vec>> { - (0..NumberOfColumns::U64) + (0..self.chain_config.number_of_columns) .filter_map(|index| { self.data_column_cache .get(DataColumnIdentifier { block_root, index }) diff --git a/http_api/src/standard.rs b/http_api/src/standard.rs index ec6412d5..343cfa52 100644 --- a/http_api/src/standard.rs +++ b/http_api/src/standard.rs @@ -987,8 +987,11 @@ pub async fn publish_block( if controller.chain_config().phase_at_slot::

(slot) >= Phase::Fulu { let cells_and_kzg_proofs = eip_7594::try_convert_to_cells_and_kzg_proofs::

(blobs.into_iter())?; - let data_column_sidecars = - eip_7594::construct_data_column_sidecars(&signed_beacon_block, &cells_and_kzg_proofs)?; + let data_column_sidecars = eip_7594::construct_data_column_sidecars( + &signed_beacon_block, + &cells_and_kzg_proofs, + controller.chain_config(), + )?; publish_signed_block_with_data_column_sidecar( Arc::new(signed_beacon_block), @@ -1046,8 +1049,11 @@ pub async fn publish_blinded_block( let cells_and_kzg_proofs = eip_7594::try_convert_to_cells_and_kzg_proofs::

( blobs.unwrap_or_default().into_iter(), )?; - let data_column_sidecars = - eip_7594::construct_data_column_sidecars(&signed_beacon_block, &cells_and_kzg_proofs)?; + let data_column_sidecars = eip_7594::construct_data_column_sidecars( + &signed_beacon_block, + &cells_and_kzg_proofs, + controller.chain_config(), + )?; publish_signed_block_with_data_column_sidecar( signed_beacon_block, @@ -1129,8 +1135,11 @@ pub async fn publish_block_v2( if controller.chain_config().phase_at_slot::

(slot) >= Phase::Fulu { let cells_and_kzg_proofs = eip_7594::try_convert_to_cells_and_kzg_proofs::

(blobs.into_iter())?; - let data_column_sidecars = - eip_7594::construct_data_column_sidecars(&signed_beacon_block, &cells_and_kzg_proofs)?; + let data_column_sidecars = eip_7594::construct_data_column_sidecars( + &signed_beacon_block, + &cells_and_kzg_proofs, + controller.chain_config(), + )?; publish_signed_block_v2_with_data_column_sidecar( Arc::new(signed_beacon_block), diff --git a/types/src/config.rs b/types/src/config.rs index 573f6e3a..2274ff1e 100644 --- a/types/src/config.rs +++ b/types/src/config.rs @@ -166,9 +166,11 @@ pub struct Config { #[serde(with = "serde_utils::string_or_native")] pub custody_requirement: u64, #[serde(with = "serde_utils::string_or_native")] - pub samples_per_slot: u64, + pub number_of_columns: u64, #[serde(with = "serde_utils::string_or_native")] pub number_of_custody_groups: u64, + #[serde(with = "serde_utils::string_or_native")] + pub samples_per_slot: u64, // Later phases and other unknown variables // @@ -268,8 +270,9 @@ impl Default for Config { // Custody custody_requirement: 4, - samples_per_slot: 8, + number_of_columns: 128, number_of_custody_groups: 128, + samples_per_slot: 8, // Later phases and other unknown variables unknown: BTreeMap::new(), @@ -782,7 +785,21 @@ impl Config { #[must_use] pub fn sampling_size(&self, custody_group_count: u64) -> u64 { - core::cmp::max(custody_group_count, self.samples_per_slot) + core::cmp::max( + custody_group_count.saturating_mul(self.columns_per_group()), + self.samples_per_slot, + ) + } + + #[must_use] + pub const fn columns_per_group(&self) -> u64 { + self.number_of_columns + .saturating_div(self.number_of_custody_groups) + } + + #[must_use] + pub fn number_of_columns(&self) -> usize { + usize::try_from(self.number_of_columns).expect("should be able to parse number_of_columns") } fn fork_slots(&self) -> impl Iterator)> + '_ { diff --git a/validator/src/validator.rs b/validator/src/validator.rs index 0e1e467c..9dd01e78 100644 --- a/validator/src/validator.rs +++ b/validator/src/validator.rs @@ -875,9 +875,11 @@ impl Validator { if self.chain_config.phase_at_slot::

(slot_head.slot()) >= Phase::Fulu { let cells_and_kzg_proofs = eip_7594::try_convert_to_cells_and_kzg_proofs::

(blobs.into_iter())?; - for data_column_sidecar in - eip_7594::construct_data_column_sidecars(&block, &cells_and_kzg_proofs)? - { + for data_column_sidecar in eip_7594::construct_data_column_sidecars( + &block, + &cells_and_kzg_proofs, + &self.chain_config, + )? { let data_column_sidecar = Arc::new(data_column_sidecar); self.controller.on_own_data_column_sidecar(