From 5fad38a5e7ab3a1f7964b3c1c727daed71dbeb50 Mon Sep 17 00:00:00 2001 From: Andrea Cerone <22031682+acerone85@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:31:06 +0100 Subject: [PATCH] Gas price simulation data retriever (#2533) ## Linked Issues/PRs - #2469 ## Description - Adds a data retrieval service that fetches L2 and L1 cost data for gas price v1 simulations ## Checklist - [x] Breaking changes are clearly marked as such in the PR description and changelog - [x] New behavior is reflected in tests - [x] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [x] I have reviewed the code myself - [x] I have created follow-up issues caused by this PR and linked them here ### After merging, notify other teams [Add or remove entries as needed] - [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/) - [ ] [Sway compiler](https://github.com/FuelLabs/sway/) - [ ] [Platform documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+) (for out-of-organization contributors, the person merging the PR will do this) - [ ] Someone else? --------- Co-authored-by: rymnc <43716372+rymnc@users.noreply.github.com> Co-authored-by: Mitchell Turner --- .../gas-price-data-fetcher/.gitignore | 2 + .../gas-price-data-fetcher/Cargo.toml | 33 +++ .../gas-price-data-fetcher/README.md | 10 + .../gas-price-data-fetcher/build.rs | 14 ++ .../gas-price-data-fetcher/src/client_ext.rs | 236 ++++++++++++++++++ .../gas-price-data-fetcher/src/layer1.rs | 94 +++++++ .../gas-price-data-fetcher/src/layer2.rs | 214 ++++++++++++++++ .../gas-price-data-fetcher/src/main.rs | 170 +++++++++++++ .../gas-price-data-fetcher/src/summary.rs | 73 ++++++ .../gas-price-data-fetcher/src/types.rs | 103 ++++++++ .../gas-price-data-reader/.gitignore | 2 + .../gas-price-data-reader/Cargo.toml | 10 + .../gas-price-data-reader/src/main.rs | 84 +++++++ 13 files changed, 1045 insertions(+) create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/.gitignore create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/Cargo.toml create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/README.md create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/build.rs create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/client_ext.rs create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer1.rs create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer2.rs create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/main.rs create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/summary.rs create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/types.rs create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-reader/.gitignore create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-reader/Cargo.toml create mode 100644 crates/fuel-gas-price-algorithm/gas-price-data-reader/src/main.rs diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/.gitignore b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/.gitignore new file mode 100644 index 00000000000..a3253ee3df3 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/.gitignore @@ -0,0 +1,2 @@ +Cargo.lock +data diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/Cargo.toml b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/Cargo.toml new file mode 100644 index 00000000000..88119f5f5f5 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "fuel-gas-price-data-fetcher" +version = "0.0.1" +edition = "2021" +publish = false + +[workspace] + +[dependencies] +anyhow = "1.0.86" +clap = { version = "4.5.16", features = ["derive"] } +csv = "1.3.0" +fuel-gas-price-algorithm = { path = ".." } +futures = "0.3.30" +plotters = "0.3.5" +rand = "0.8.5" +rand_distr = "0.4.3" +serde = { version = "1.0.209", features = ["derive"] } +tokio = { version = "1.40.0", features = ["macros", "rt", "rt-multi-thread"] } +reqwest = { version = "0.12.11", features = ["json"] } +serde_json = { version = "1.0.134" } +fuel-core-client = { version = "0.40.2" } # locked to whatever version you're supposed to be fetching data from +fuel-core-types = { version = "0.40.2" } +postcard = { version = "1.0" } +tracing = { version = "0.1.41" } +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } + +async-trait = "0.1" +cynic = { version = "2.2", features = ["http-reqwest"] } +itertools = { version = "0.13" } + +[build-dependencies] +fuel-core-client = { version = "0.40.2" } diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/README.md b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/README.md new file mode 100644 index 00000000000..935451cb9f9 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/README.md @@ -0,0 +1,10 @@ +# Gas Price Analysis Data Fetcher + +Binary allowing retrieveing the L1 blob and L2 block data needed by the gas price simulation binary. +It requires being able to connect to the block committer, and either being able to connect to a sentry node or access to the database of a mainnet synched node. + +## Usage + +``` +cargo run -- --block-committer-endpoint ${BLOCK_COMMITTER_URL} --block-range 0 1000 --db-path ${FUEL_MAINNET_DB_PATH} +``` diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/build.rs b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/build.rs new file mode 100644 index 00000000000..f215fc0766e --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/build.rs @@ -0,0 +1,14 @@ +#![deny(clippy::arithmetic_side_effects)] +#![deny(clippy::cast_possible_truncation)] +#![deny(unused_crate_dependencies)] +#![deny(warnings)] + +use std::fs; + +fn main() { + fs::create_dir_all("target").expect("Unable to create target directory"); + fs::write("target/schema.sdl", fuel_core_client::SCHEMA_SDL) + .expect("Unable to write schema file"); + + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/client_ext.rs b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/client_ext.rs new file mode 100644 index 00000000000..dc8aa8ee78f --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/client_ext.rs @@ -0,0 +1,236 @@ +#![deny(clippy::arithmetic_side_effects)] +#![deny(clippy::cast_possible_truncation)] +#![deny(warnings)] + +// This was copied from https://github.com/FuelLabs/fuel-core-client-ext/blob/b792ef76cbcf82eda45a944b15433682fe094fee/src/lib.rs + +use cynic::QueryBuilder; +use fuel_core_client::{ + client, + client::{ + pagination::{ + PaginatedResult, + PaginationRequest, + }, + schema::{ + block::{ + BlockByHeightArgs, + Consensus, + Header, + }, + schema, + tx::OpaqueTransactionWithStatus, + ConnectionArgs, + PageInfo, + }, + types::{ + TransactionResponse, + TransactionStatus, + }, + FuelClient, + }, +}; +use fuel_core_types::{ + blockchain::{ + self, + block::Block, + header::{ + ApplicationHeader, + ConsensusHeader, + PartialBlockHeader, + }, + SealedBlock, + }, + fuel_tx::{ + Bytes32, + Receipt, + }, +}; +use itertools::Itertools; + +#[derive(cynic::QueryFragment, Debug)] +#[cynic( + schema_path = "./target/schema.sdl", + graphql_type = "Query", + variables = "ConnectionArgs" +)] +pub struct FullBlocksQuery { + #[arguments(after: $after, before: $before, first: $first, last: $last)] + pub blocks: FullBlockConnection, +} + +#[derive(cynic::QueryFragment, Debug)] +#[cynic(schema_path = "./target/schema.sdl", graphql_type = "BlockConnection")] +pub struct FullBlockConnection { + pub edges: Vec, + pub page_info: PageInfo, +} + +#[derive(cynic::QueryFragment, Debug)] +#[cynic(schema_path = "./target/schema.sdl", graphql_type = "BlockEdge")] +pub struct FullBlockEdge { + pub cursor: String, + pub node: FullBlock, +} + +#[derive(cynic::QueryFragment, Debug)] +#[cynic( + schema_path = "./target/schema.sdl", + graphql_type = "Query", + variables = "BlockByHeightArgs" +)] +pub struct FullBlockByHeightQuery { + #[arguments(height: $height)] + pub block: Option, +} + +#[derive(cynic::QueryFragment, Debug)] +#[cynic(schema_path = "./target/schema.sdl", graphql_type = "Block")] +pub struct FullBlock { + pub header: Header, + pub consensus: Consensus, + pub transactions: Vec, +} + +impl From for PaginatedResult { + fn from(conn: FullBlockConnection) -> Self { + PaginatedResult { + cursor: conn.page_info.end_cursor, + has_next_page: conn.page_info.has_next_page, + has_previous_page: conn.page_info.has_previous_page, + results: conn.edges.into_iter().map(|e| e.node).collect(), + } + } +} + +#[async_trait::async_trait] +pub trait ClientExt { + async fn full_blocks( + &self, + request: PaginationRequest, + ) -> std::io::Result>; +} + +#[async_trait::async_trait] +impl ClientExt for FuelClient { + async fn full_blocks( + &self, + request: PaginationRequest, + ) -> std::io::Result> { + let query = FullBlocksQuery::build(request.into()); + let blocks = self.query(query).await?.blocks.into(); + Ok(blocks) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SealedBlockWithMetadata { + pub block: SealedBlock, + pub receipts: Vec>>, +} + +impl TryFrom for SealedBlockWithMetadata { + type Error = anyhow::Error; + + fn try_from(full_block: FullBlock) -> Result { + let transactions: Vec = full_block + .transactions + .into_iter() + .map(TryInto::try_into) + .try_collect()?; + + let receipts = transactions + .iter() + .map(|tx| &tx.status) + .map(|status| match status { + TransactionStatus::Success { receipts, .. } => Some(receipts.clone()), + _ => None, + }) + .collect_vec(); + + let messages = receipts + .iter() + .flatten() + .flat_map(|receipt| receipt.iter().filter_map(|r| r.message_id())) + .collect_vec(); + + let transactions = transactions + .into_iter() + .map(|tx| tx.transaction) + .collect_vec(); + + let partial_header = PartialBlockHeader { + application: ApplicationHeader { + da_height: full_block.header.da_height.0.into(), + consensus_parameters_version: full_block + .header + .consensus_parameters_version + .into(), + state_transition_bytecode_version: full_block + .header + .state_transition_bytecode_version + .into(), + generated: Default::default(), + }, + consensus: ConsensusHeader { + prev_root: full_block.header.prev_root.into(), + height: full_block.header.height.into(), + time: full_block.header.time.into(), + generated: Default::default(), + }, + }; + + let header = partial_header + .generate( + &transactions, + &messages, + full_block.header.event_inbox_root.into(), + ) + .map_err(|e| anyhow::anyhow!(e))?; + + let actual_id: Bytes32 = full_block.header.id.into(); + let expected_id: Bytes32 = header.id().into(); + if expected_id != actual_id { + return Err(anyhow::anyhow!("Header id mismatch")); + } + + let block = Block::try_from_executed(header, transactions) + .ok_or(anyhow::anyhow!("Failed to create block from transactions"))?; + + let consensus: client::types::Consensus = full_block.consensus.into(); + + let consensus = match consensus { + client::types::Consensus::Genesis(genesis) => { + use blockchain::consensus as core_consensus; + core_consensus::Consensus::Genesis(core_consensus::Genesis { + chain_config_hash: genesis.chain_config_hash, + coins_root: genesis.coins_root, + contracts_root: genesis.contracts_root, + messages_root: genesis.messages_root, + transactions_root: genesis.transactions_root, + }) + } + client::types::Consensus::PoAConsensus(poa) => { + use blockchain::consensus as core_consensus; + core_consensus::Consensus::PoA(core_consensus::poa::PoAConsensus { + signature: poa.signature, + }) + } + client::types::Consensus::Unknown => { + return Err(anyhow::anyhow!("Unknown consensus type")); + } + }; + + let sealed = SealedBlock { + entity: block, + consensus, + }; + + let sealed = SealedBlockWithMetadata { + block: sealed, + receipts, + }; + + Ok(sealed) + } +} diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer1.rs b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer1.rs new file mode 100644 index 00000000000..cbb618c3281 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer1.rs @@ -0,0 +1,94 @@ +use std::ops::Range; + +use fuel_core_types::fuel_types::BlockHeight; +use reqwest::{ + header::{ + HeaderMap, + CONTENT_TYPE, + }, + Url, +}; + +use crate::types::BlockCommitterCosts; + +pub struct BlockCommitterDataFetcher { + client: reqwest::Client, + endpoint: Url, + num_responses: usize, +} + +impl BlockCommitterDataFetcher { + pub fn new(endpoint: Url, num_responses: usize) -> anyhow::Result { + let mut content_type_json_header = HeaderMap::new(); + content_type_json_header.insert( + CONTENT_TYPE, + "application/json" + .parse() + .expect("Content-Type header value is valid"), + ); + let client = reqwest::ClientBuilder::new() + .default_headers(content_type_json_header) + .build()?; + Ok(Self { + client, + endpoint, + num_responses, + }) + } + + // TODO: Better error type; qed + async fn fetch_blob_data( + &self, + from_height: u64, + ) -> anyhow::Result> { + let query = self.endpoint.join("v1/costs")?.join(&format!( + "?variant=specific&value={}&limit={}", + from_height, self.num_responses + ))?; + + tracing::debug!("Query: {}", query.as_str()); + + let response = self.client.get(query).send().await?; + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Failed to fetch data from block committer: {}", + response.status(), + ) + .into()); + } + + let block_committer_costs = response.json::>().await?; + Ok(block_committer_costs) + } + + pub async fn fetch_l1_block_costs( + &self, + blocks: Range, + ) -> Result, anyhow::Error> { + let mut block_costs = vec![]; + let mut current_block_height = blocks.start; + while current_block_height < blocks.end { + let Ok(mut costs) = + self.fetch_blob_data((*current_block_height).into()).await + else { + Err(anyhow::anyhow!( + "Could not fetch data for block {}", + current_block_height + ))? + }; + + if costs.is_empty() { + // Might be that the block committer doesn't have data for the block, in which case we return prematurely. + // If this happens, we should increase the value of results returned by the block committer in the query. + break; + } + + // Block committer will return the data for the block in the next batch, hence we don't increment the height of the last + // block. + current_block_height = (*costs.last().unwrap().end_height).into(); + block_costs.append(&mut costs); + } + + Ok(block_costs) + } +} diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer2.rs b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer2.rs new file mode 100644 index 00000000000..754ca64af38 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer2.rs @@ -0,0 +1,214 @@ +use std::{ + collections::{ + HashMap, + HashSet, + }, + ops::Range, +}; + +use crate::types::{ + BytesSize, + GasUnits, + Layer2BlockData, +}; + +use super::client_ext::{ + ClientExt, + SealedBlockWithMetadata, +}; +use fuel_core_client::client::{ + pagination::{ + PageDirection, + PaginationRequest, + }, + FuelClient, +}; +use fuel_core_types::{ + fuel_tx::{ + Chargeable, + ConsensusParameters, + }, + fuel_types::BlockHeight, +}; +use itertools::Itertools; + +#[derive(Clone)] +pub struct BlockFetcher { + client: FuelClient, +} + +impl BlockFetcher { + pub fn new(url: impl AsRef) -> anyhow::Result { + let client = FuelClient::new(url)?; + Ok(Self { client }) + } +} + +impl BlockFetcher { + pub async fn blocks_for( + &self, + range: Range, + ) -> anyhow::Result> { + if range.is_empty() { + return Ok(vec![]); + } + + let start = range.start.saturating_sub(1); + let size = i32::try_from(range.len()).expect("Should be a valid i32"); + + let request = PaginationRequest { + cursor: Some(start.to_string()), + results: size, + direction: PageDirection::Forward, + }; + let response = self.client.full_blocks(request).await?; + let blocks = response + .results + .into_iter() + .map(TryInto::try_into) + .try_collect()?; + Ok(blocks) + } + + pub async fn get_l2_block_data( + &self, + range: Range, + num_results: usize, + ) -> anyhow::Result> { + let range: Range = range.start.try_into()?..range.end.try_into()?; + let mut ranges: Vec> = + Vec::with_capacity(range.len().saturating_div(num_results)); + + for start in range.clone().step_by(num_results) { + let end = start.saturating_add(num_results as u32).min(range.end); + ranges.push(start..end); + } + + let mut blocks = Vec::with_capacity(range.len()); + for range in ranges { + tracing::info!("Fetching blocks for range {:?}", range); + let blocks_for_range = self.blocks_for(range).await?; + blocks.extend(blocks_for_range); + } + + let consensus_parameters_versions = blocks + .iter() + .map(|b| b.block.entity.header().consensus_parameters_version) + .collect::>(); + + tracing::debug!( + "Consensus parameter versions: {:?}", + consensus_parameters_versions + ); + + let mut consensus_parameters: HashMap = HashMap::new(); + for consensus_parameters_version in consensus_parameters_versions { + let cp = self + .client + .consensus_parameters(consensus_parameters_version.try_into()?) + .await?; + + if let Some(cp) = cp { + tracing::debug!( + "Found consensus parameters for version {}: {:?}", + consensus_parameters_version, + cp + ); + consensus_parameters.insert(consensus_parameters_version, cp); + } + } + + let mut block_data = HashMap::with_capacity(range.len()); + + for b in blocks { + let block_height = height(&b); + let consensus_parameters = consensus_parameters + .get(&b.block.entity.header().consensus_parameters_version) + .ok_or(anyhow::anyhow!( + "Consensus parameters not found for block {}", + block_height + ))?; + let block_size = + BytesSize(postcard::to_allocvec(&b.block)?.len().try_into()?); + + let gas_consumed = total_gas_consumed(&b, consensus_parameters)?; + let capacity = GasUnits(consensus_parameters.block_gas_limit()); + let bytes_capacity = + BytesSize(consensus_parameters.block_transaction_size_limit()); + let transactions_count = b.block.entity.transactions().len(); + + block_data.insert( + block_height, + Layer2BlockData { + block_height, + block_size, + gas_consumed, + capacity, + bytes_capacity, + transactions_count, + }, + ); + } + + Ok(block_data) + } +} + +fn height(block: &SealedBlockWithMetadata) -> BlockHeight { + *block.block.entity.header().height() +} + +fn total_gas_consumed( + block: &SealedBlockWithMetadata, + consensus_parameters: &ConsensusParameters, +) -> Result { + let min_gas: u64 = block + .block + .entity + .transactions() + .iter() + .filter_map(|tx| match tx { + fuel_core_types::fuel_tx::Transaction::Script(chargeable_transaction) => { + Some(chargeable_transaction.min_gas( + consensus_parameters.gas_costs(), + consensus_parameters.fee_params(), + )) + } + fuel_core_types::fuel_tx::Transaction::Create(chargeable_transaction) => { + Some(chargeable_transaction.min_gas( + consensus_parameters.gas_costs(), + consensus_parameters.fee_params(), + )) + } + fuel_core_types::fuel_tx::Transaction::Mint(_mint) => None, + fuel_core_types::fuel_tx::Transaction::Upgrade(chargeable_transaction) => { + Some(chargeable_transaction.min_gas( + consensus_parameters.gas_costs(), + consensus_parameters.fee_params(), + )) + } + fuel_core_types::fuel_tx::Transaction::Upload(chargeable_transaction) => { + Some(chargeable_transaction.min_gas( + consensus_parameters.gas_costs(), + consensus_parameters.fee_params(), + )) + } + fuel_core_types::fuel_tx::Transaction::Blob(chargeable_transaction) => { + Some(chargeable_transaction.min_gas( + consensus_parameters.gas_costs(), + consensus_parameters.fee_params(), + )) + } + }) + .sum(); + let gas_consumed = block + .receipts + .iter() + .flatten() + .map(|r| r.iter().filter_map(|r| r.gas_used()).sum::()) + .sum(); + let total_gas = min_gas + .checked_add(gas_consumed) + .ok_or(anyhow::anyhow!("Gas overflow"))?; + Ok(GasUnits(total_gas)) +} diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/main.rs b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/main.rs new file mode 100644 index 00000000000..e8abaeefd28 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/main.rs @@ -0,0 +1,170 @@ +use std::{ + env, + path::PathBuf, +}; + +use fuel_core_types::fuel_types::BlockHeight; +use layer1::BlockCommitterDataFetcher; +use reqwest::Url; +use tracing_subscriber::EnvFilter; +use types::Layer2BlockData; + +use clap::{ + Args, + Parser, +}; + +use tracing_subscriber::prelude::*; + +pub mod client_ext; +mod layer1; +mod layer2; +mod summary; +mod types; + +// const SENTRY_NODE_GRAPHQL_RESULTS_PER_QUERY: usize = 5_000; +const SENTRY_NODE_GRAPHQL_RESULTS_PER_QUERY: usize = 40; + +#[derive(Parser)] +#[command(version, about, long_about = None)] +struct Arg { + #[clap(flatten)] + l2_block_data_source: L2BlockDataSource, + #[arg(short, long)] + /// Endpoint of the block committer to fetch L1 blobs data + block_committer_endpoint: Url, + #[arg( + short = 'r', + long, + num_args = 2..=2, + required = true + )] + /// Range of blocks to fetch the data for. Lower bound included, Upper bound excluded. + block_range: Vec, + + #[arg(required = true)] + /// The output CSV file where Block and Blob data will be written to + output_file: PathBuf, +} + +#[derive(Debug, Args)] +#[group(required = true, multiple = false)] +struct L2BlockDataSource { + #[arg(short, long)] + /// Path of the database stored by a fuel-node to retrieve L2 block data. Alternatively, the endpoint of a sentry node can be provided using --sentry-node-endpoint. + db_path: Option, + #[arg(short, long)] + /// Endpoint of the sentry node to fetch L2 block data. Alternatively, the path of the database stored by a fuel-node can be provided using --db-path. + sentry_node_endpoint: Option, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let Arg { + block_committer_endpoint, + l2_block_data_source, + block_range, + output_file, + } = Arg::parse(); + + let filter = match env::var_os("RUST_LOG") { + Some(_) => { + EnvFilter::try_from_default_env().expect("Invalid `RUST_LOG` provided") + } + None => EnvFilter::new("info"), + }; + + let fmt = tracing_subscriber::fmt::Layer::default() + .with_level(true) + .boxed(); + + tracing_subscriber::registry().with(fmt).with(filter).init(); + + // Safety: The block range is always a vector of length 2 + let start_block_included = BlockHeight::from(block_range[0]); + // When requested a set of results, the block committer will fetch the data for the next blob which + // might not include the current height. Each blob contains 3_600 blocks, hence we subtract + // this amount from the block height we use for the first request. + let end_block_excluded = BlockHeight::from(block_range[1]); + let block_range = start_block_included..end_block_excluded; + + if end_block_excluded < start_block_included { + return Err(anyhow::anyhow!( + "Invalid block range - start block must be lower than end block: {}..{}", + start_block_included, + end_block_excluded + )); + } + + let block_committer_data_fetcher = + BlockCommitterDataFetcher::new(block_committer_endpoint, 10)?; + + let block_costs = block_committer_data_fetcher + .fetch_l1_block_costs(block_range) + .await?; + + tracing::debug!("{:?}", block_costs); + match l2_block_data_source { + L2BlockDataSource { + db_path: Some(_db_path), + sentry_node_endpoint: None, + } => { + todo!(); + } + L2BlockDataSource { + db_path: None, + sentry_node_endpoint: Some(sentry_node_endpoint), + } => { + tracing::info!( + "Retrieving L2 data from sentry node: {}", + sentry_node_endpoint + ); + let sentry_node_client = layer2::BlockFetcher::new(sentry_node_endpoint)?; + let blocks_range = start_block_included..end_block_excluded; + + tracing::info!( + "Retrieving L2 data for blocks: {}..{}", + start_block_included, + end_block_excluded + ); + let res = sentry_node_client + .get_l2_block_data(blocks_range, SENTRY_NODE_GRAPHQL_RESULTS_PER_QUERY) + .await; + tracing::info!("results: {:?}", res); + + let blocks_with_gas_consumed = res?; + for ( + _block_height, + Layer2BlockData { + block_height, + block_size, + gas_consumed, + capacity, + bytes_capacity, + transactions_count, + }, + ) in &blocks_with_gas_consumed + { + tracing::debug!( + "Block Height: {}, Block Size: {}, Gas Consumed: {}, Capacity: {}, Bytes Capacity: {}, Transactions count: {}", + block_height, **block_size, **gas_consumed, **capacity, **bytes_capacity, transactions_count + ); + } + summary::summarise_available_data( + &output_file, + &block_costs, + &blocks_with_gas_consumed, + ) + .inspect_err(|e| { + tracing::error!("Failed to write to CSV file: {:?}, {:?}", output_file, e) + })?; + } + _ => { + return Err(anyhow::anyhow!( + "Either db-path or sentry-node-endpoint must be provided" + )); + } + }; + + Ok(()) +} diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/summary.rs b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/summary.rs new file mode 100644 index 00000000000..bdc1ba7ccd9 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/summary.rs @@ -0,0 +1,73 @@ +use std::{ + collections::HashMap, + fs::File, + path::Path, +}; + +use fuel_core_types::fuel_types::BlockHeight; + +use crate::types::{ + BlockCommitterCosts, + Layer2BlockData, +}; + +#[derive(Debug, serde::Serialize)] +struct BlockSummary { + l1_block_number: u64, + l1_blob_fee_wei: u128, + l2_block_number: u32, + l2_gas_fullness: u64, + l2_gas_capacity: u64, + l2_byte_size: u64, + l2_byte_capacity: u64, + l2_block_transactions_count: usize, +} + +fn summarise_data_for_block_committer_costs( + costs: &BlockCommitterCosts, + l2_data: &HashMap, +) -> Vec { + tracing::info!("Building summary for: {:?}", costs); + let block_range_len: usize = costs.len().try_into().unwrap_or_default(); + let mut summaries = Vec::with_capacity(block_range_len); + for block_height in costs.iter() { + let Ok(block_height): Result = block_height.try_into() else { + continue + }; + let l2_data = l2_data.get(&block_height); + if let Some(l2_data) = l2_data { + summaries.push(BlockSummary { + l1_block_number: *costs.da_block_height, + l1_blob_fee_wei: *costs.cost, + l2_block_number: block_height, + l2_gas_fullness: *l2_data.gas_consumed, + l2_gas_capacity: *l2_data.capacity, + l2_byte_size: *l2_data.block_size, + l2_byte_capacity: *l2_data.bytes_capacity, + l2_block_transactions_count: l2_data.transactions_count, + }); + } + } + summaries +} + +pub fn summarise_available_data( + output_file_path: &Path, + costs: &[BlockCommitterCosts], + l2_data: &HashMap, +) -> Result<(), anyhow::Error> { + let file_writer = File::create(output_file_path)?; + let mut writer = csv::WriterBuilder::new() + .has_headers(true) + .from_writer(file_writer); + for block_costs_entry in costs { + let summaries = + summarise_data_for_block_committer_costs(block_costs_entry, l2_data); + for summary in summaries { + tracing::debug!("Serializing record: {:?}", summary); + writer.serialize(summary)?; + } + } + writer.flush()?; + Ok(()) +} diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/types.rs b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/types.rs new file mode 100644 index 00000000000..ecbc1642373 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/types.rs @@ -0,0 +1,103 @@ +use std::{ + iter::Map, + ops::{ + Deref, + RangeInclusive, + }, +}; + +use fuel_core_types::{ + blockchain::primitives::DaBlockHeight, + fuel_types::BlockHeight, +}; +use serde::{ + Deserialize, + Serialize, +}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Wei(u128); + +impl Deref for Wei { + type Target = u128; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(transparent)] +pub struct GasUnits(pub u64); + +impl Deref for GasUnits { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct BytesSize(pub u64); + +impl Deref for BytesSize { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct BlockCommitterCosts { + /// The cost of the block, supposedly in Wei but need to check + pub cost: Wei, + pub size: BytesSize, + pub da_block_height: DaBlockHeight, + pub start_height: BlockHeight, + pub end_height: BlockHeight, +} + +impl BlockCommitterCosts { + pub fn iter(&self) -> impl Iterator { + let start_height = *self.start_height; + let end_height = *self.end_height; + (start_height..=end_height) + .map(|raw_block_height| BlockHeight::from(raw_block_height)) + } + + pub fn len(&self) -> u32 { + // Remove 1 from end height because range is inclusive. + self.end_height + .saturating_sub(1) + .saturating_sub(*self.start_height) + } +} + +impl IntoIterator for BlockCommitterCosts { + type Item = BlockHeight; + + // `impl Trait` in associated types is unstable + // see issue #63063 for more information + // We must specify the concrete type here. + type IntoIter = Map, fn(u32) -> BlockHeight>; + + fn into_iter(self) -> Self::IntoIter { + let start_height = *self.start_height; + let end_height = *self.end_height; + (start_height..=end_height) + .map(|raw_block_height| BlockHeight::from(raw_block_height)) + } +} + +#[derive(Debug)] +pub struct Layer2BlockData { + pub block_height: BlockHeight, + pub block_size: BytesSize, + pub gas_consumed: GasUnits, + pub capacity: GasUnits, + pub bytes_capacity: BytesSize, + pub transactions_count: usize, +} diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-reader/.gitignore b/crates/fuel-gas-price-algorithm/gas-price-data-reader/.gitignore new file mode 100644 index 00000000000..ae872b4d900 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-reader/.gitignore @@ -0,0 +1,2 @@ +Cargo.lock +data \ No newline at end of file diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-reader/Cargo.toml b/crates/fuel-gas-price-algorithm/gas-price-data-reader/Cargo.toml new file mode 100644 index 00000000000..baa9e89a078 --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-reader/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "gas-price-data-reader" +version = "0.1.0" +edition = "2021" + +[dependencies] +csv = "1.3.1" +serde = { version = "1.0.217", features = ["derive"] } + +[workspace] diff --git a/crates/fuel-gas-price-algorithm/gas-price-data-reader/src/main.rs b/crates/fuel-gas-price-algorithm/gas-price-data-reader/src/main.rs new file mode 100644 index 00000000000..5e9accdeafc --- /dev/null +++ b/crates/fuel-gas-price-algorithm/gas-price-data-reader/src/main.rs @@ -0,0 +1,84 @@ +use std::collections::HashMap; + +const WEI_PER_ETH: f64 = 1_000_000_000_000_000_000.; +// l1_block_number,l1_blob_fee_wei,l2_block_number,l2_gas_fullness,l2_gas_capacity,l2_byte_size,l2_byte_capacity +// 21403864,509018984154240,9099900,0,30000000,488,260096 +// 21403864,509018984154240,9099901,1073531,30000000,3943,260096 +// 21403864,509018984154240,9099902,0,30000000,488,260096 +// parse data +#[derive(Debug, serde::Deserialize, Eq, PartialEq, Hash)] +struct Record { + l1_block_number: u64, + l1_blob_fee_wei: u128, + l2_block_number: u64, + l2_gas_fullness: u64, + l2_gas_capacity: u64, + l2_byte_size: u64, + l2_byte_capacity: u64, +} +fn get_records_from_csv_file(file_path: &str) -> Vec { + let mut rdr = csv::ReaderBuilder::new() + .has_headers(true) + .from_path(file_path) + .unwrap(); + let headers = csv::StringRecord::from(vec![ + "l1_block_number", + "l1_blob_fee_wei", + "l2_block_number", + "l2_gas_fullness", + "l2_gas_capacity", + "l2_byte_size", + "l2_byte_capacity", + ]); + let records = rdr + .records() + .skip(1) + .map(|r| r.unwrap().deserialize(Some(&headers)).unwrap()) + .collect::>(); + records +} + +const ENV_VAR_NAME: &str = "BLOCK_HISTORY_FILE"; +fn get_path_to_file() -> String { + if let Some(path) = std::env::var_os(ENV_VAR_NAME) { + return path.to_str().unwrap().to_string(); + } else { + let maybe_path = std::env::args().nth(1); + if let Some(path) = maybe_path { + return path; + } else { + panic!("Please provide a path to the file or set the {ENV_VAR_NAME} environment variable"); + } + } +} + +fn main() { + let path = get_path_to_file(); + let records = get_records_from_csv_file(&path); + let length = records.len(); + let costs = records + .iter() + .map(|r| (r.l1_block_number, r.l1_blob_fee_wei)) + .collect::>(); + let total_costs: u128 = costs.values().sum(); + let total_l2_gas = records.iter().map(|r| r.l2_gas_fullness).sum::(); + + // println!("Average cost: {}", average); + println!("Length: {}", length); + println!("Total cost: {}", total_costs); + println!("Total cost (ETH): {}", total_costs as f64 / WEI_PER_ETH); + println!( + "Average cost per l2 block: {}", + total_costs / length as u128 + ); + println!( + "Average cost per l2 block (ETH): {}", + (total_costs as f64 / length as f64) / WEI_PER_ETH + ); + // get cost per l2 gas fullness + let average_cost_per_l2_gas_fullness = total_costs / total_l2_gas as u128; + println!( + "Average cost per l2 gas: {}", + average_cost_per_l2_gas_fullness + ); +}