Gas price simulation data retriever

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/.gitignore

@@ -0,0 +1 @@
+Cargo.lock

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "fuel-gas-price-data-fetcher"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[workspace]
+
+[dependencies]
+anyhow = "1.0.86"
+clap = { version = "4.5.16", features = ["derive"] }
+csv = "1.3.0"
+fuel-gas-price-algorithm = { path = ".." }
+futures = "0.3.30"
+plotters = "0.3.5"
+rand = "0.8.5"
+rand_distr = "0.4.3"
+serde = { version = "1.0.209", features = ["derive"] }
+tokio = { version = "1.40.0", features = ["macros", "rt", "rt-multi-thread"] }
+reqwest = { version = "0.12.11", features = ["json"] }
+serde_json = { version = "1.0.134" }
+fuel-core-client = { version = "0.40.2" }
+fuel-core-types = { version = "0.40.2" }
+postcard = { version = "1.0" }
+tracing = { version = "0.1.41" }
+tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
+
+async-trait = "0.1"
+cynic = { version = "2.2", features = ["http-reqwest"] }
+itertools = { version = "0.13" }
+
+[build-dependencies]
+fuel-core-client = { version = "0.40.2" }

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/README.md

@@ -0,0 +1,10 @@
+# Gas Price Analysis Data Fetcher
+
+Binary allowing retrieveing the L1 blob and L2 block data needed by the gas price simulation binary. 
+It requires being able to connect to the block committer, and either being able to connect to a sentry node or access to the database of a mainnet synched node.
+
+## Usage
+
+```
+cargo run -- --block-committer-endpoint ${BLOCK_COMMITTER_URL} --block-range 0 1000 --db-path ${FUEL_MAINNET_DB_PATH}
+``` 

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/build.rs

@@ -0,0 +1,14 @@
+#![deny(clippy::arithmetic_side_effects)]
+#![deny(clippy::cast_possible_truncation)]
+#![deny(unused_crate_dependencies)]
+#![deny(warnings)]
+
+use std::fs;
+
+fn main() {
+    fs::create_dir_all("target").expect("Unable to create target directory");
+    fs::write("target/schema.sdl", fuel_core_client::SCHEMA_SDL)
+        .expect("Unable to write schema file");
+
+    println!("cargo:rerun-if-changed=build.rs");
+}

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/client_ext.rs

@@ -0,0 +1,266 @@
+#![deny(clippy::arithmetic_side_effects)]
+#![deny(clippy::cast_possible_truncation)]
+#![deny(warnings)]
+
+// TODO: This was copy pasted from
+
+use cynic::QueryBuilder;
+use fuel_core_client::{
+    client,
+    client::{
+        pagination::{
+            PaginatedResult,
+            PaginationRequest,
+        },
+        schema::{
+            block::{
+                BlockByHeightArgs,
+                Consensus,
+                Header,
+            },
+            schema,
+            tx::OpaqueTransactionWithStatus,
+            ConnectionArgs,
+            PageInfo,
+        },
+        types::{
+            TransactionResponse,
+            TransactionStatus,
+        },
+        FuelClient,
+    },
+};
+use fuel_core_types::{
+    blockchain::{
+        self,
+        block::Block,
+        header::{
+            ApplicationHeader,
+            ConsensusHeader,
+            PartialBlockHeader,
+        },
+        SealedBlock,
+    },
+    fuel_tx::{
+        Bytes32,
+        Receipt,
+    },
+};
+use itertools::Itertools;
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(
+    schema_path = "./target/schema.sdl",
+    graphql_type = "Query",
+    variables = "ConnectionArgs"
+)]
+pub struct FullBlocksQuery {
+    #[arguments(after: $after, before: $before, first: $first, last: $last)]
+    pub blocks: FullBlockConnection,
+}
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(schema_path = "./target/schema.sdl", graphql_type = "BlockConnection")]
+pub struct FullBlockConnection {
+    pub edges: Vec<FullBlockEdge>,
+    pub page_info: PageInfo,
+}
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(schema_path = "./target/schema.sdl", graphql_type = "BlockEdge")]
+pub struct FullBlockEdge {
+    pub cursor: String,
+    pub node: FullBlock,
+}
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(
+    schema_path = "./target/schema.sdl",
+    graphql_type = "Query",
+    variables = "BlockByHeightArgs"
+)]
+pub struct FullBlockByHeightQuery {
+    #[arguments(height: $height)]
+    pub block: Option<FullBlock>,
+}
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(schema_path = "./target/schema.sdl", graphql_type = "Block")]
+pub struct FullBlock {
+    pub header: Header,
+    pub consensus: Consensus,
+    pub transactions: Vec<OpaqueTransactionWithStatus>,
+}
+
+impl From<FullBlockConnection> for PaginatedResult<FullBlock, String> {
+    fn from(conn: FullBlockConnection) -> Self {
+        PaginatedResult {
+            cursor: conn.page_info.end_cursor,
+            has_next_page: conn.page_info.has_next_page,
+            has_previous_page: conn.page_info.has_previous_page,
+            results: conn.edges.into_iter().map(|e| e.node).collect(),
+        }
+    }
+}
+
+#[async_trait::async_trait]
+pub trait ClientExt {
+    async fn full_blocks(
+        &self,
+        request: PaginationRequest<String>,
+    ) -> std::io::Result<PaginatedResult<FullBlock, String>>;
+}
+
+#[async_trait::async_trait]
+impl ClientExt for FuelClient {
+    async fn full_blocks(
+        &self,
+        request: PaginationRequest<String>,
+    ) -> std::io::Result<PaginatedResult<FullBlock, String>> {
+        let query = FullBlocksQuery::build(request.into());
+        let blocks = self.query(query).await?.blocks.into();
+        Ok(blocks)
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SealedBlockWithMetadata {
+    pub block: SealedBlock,
+    pub receipts: Vec<Option<Vec<Receipt>>>,
+}
+
+impl TryFrom<FullBlock> for SealedBlockWithMetadata {
+    type Error = anyhow::Error;
+
+    fn try_from(full_block: FullBlock) -> Result<Self, Self::Error> {
+        let transactions: Vec<TransactionResponse> = full_block
+            .transactions
+            .into_iter()
+            .map(TryInto::try_into)
+            .try_collect()?;
+
+        let receipts = transactions
+            .iter()
+            .map(|tx| &tx.status)
+            .map(|status| match status {
+                TransactionStatus::Success { receipts, .. } => Some(receipts.clone()),
+                _ => None,
+            })
+            .collect_vec();
+
+        let messages = receipts
+            .iter()
+            .flatten()
+            .flat_map(|receipt| receipt.iter().filter_map(|r| r.message_id()))
+            .collect_vec();
+
+        let transactions = transactions
+            .into_iter()
+            .map(|tx| tx.transaction)
+            .collect_vec();
+
+        let partial_header = PartialBlockHeader {
+            application: ApplicationHeader {
+                da_height: full_block.header.da_height.0.into(),
+                consensus_parameters_version: full_block
+                    .header
+                    .consensus_parameters_version
+                    .into(),
+                state_transition_bytecode_version: full_block
+                    .header
+                    .state_transition_bytecode_version
+                    .into(),
+                generated: Default::default(),
+            },
+            consensus: ConsensusHeader {
+                prev_root: full_block.header.prev_root.into(),
+                height: full_block.header.height.into(),
+                time: full_block.header.time.into(),
+                generated: Default::default(),
+            },
+        };
+
+        let header = partial_header
+            .generate(
+                &transactions,
+                &messages,
+                full_block.header.event_inbox_root.into(),
+            )
+            .map_err(|e| anyhow::anyhow!(e))?;
+
+        let actual_id: Bytes32 = full_block.header.id.into();
+        let expected_id: Bytes32 = header.id().into();
+        if expected_id != actual_id {
+            return Err(anyhow::anyhow!("Header id mismatch"));
+        }
+
+        let block = Block::try_from_executed(header, transactions)
+            .ok_or(anyhow::anyhow!("Failed to create block from transactions"))?;
+
+        let consensus: client::types::Consensus = full_block.consensus.into();
+
+        let consensus = match consensus {
+            client::types::Consensus::Genesis(genesis) => {
+                use blockchain::consensus as core_consensus;
+                core_consensus::Consensus::Genesis(core_consensus::Genesis {
+                    chain_config_hash: genesis.chain_config_hash,
+                    coins_root: genesis.coins_root,
+                    contracts_root: genesis.contracts_root,
+                    messages_root: genesis.messages_root,
+                    transactions_root: genesis.transactions_root,
+                })
+            }
+            client::types::Consensus::PoAConsensus(poa) => {
+                use blockchain::consensus as core_consensus;
+                core_consensus::Consensus::PoA(core_consensus::poa::PoAConsensus {
+                    signature: poa.signature,
+                })
+            }
+            client::types::Consensus::Unknown => {
+                return Err(anyhow::anyhow!("Unknown consensus type"));
+            }
+        };
+
+        let sealed = SealedBlock {
+            entity: block,
+            consensus,
+        };
+
+        let sealed = SealedBlockWithMetadata {
+            block: sealed,
+            receipts,
+        };
+
+        Ok(sealed)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use fuel_core_client::client::pagination::PageDirection;
+
+    #[tokio::test]
+    async fn testnet_works() {
+        let client = FuelClient::new("https://testnet.fuel.network")
+            .expect("Should connect to the beta 5 network");
+
+        let request = PaginationRequest {
+            cursor: None,
+            results: 1,
+            direction: PageDirection::Backward,
+        };
+        let full_blocks = client
+            .full_blocks(request)
+            .await
+            .expect("Should get a blocks");
+
+        let full_block = full_blocks
+            .results
+            .into_iter()
+            .next()
+            .expect("Should have a block");
+        let result: anyhow::Result<SealedBlockWithMetadata> = full_block.try_into();
+        assert!(result.is_ok(), "{result:?}");
+    }
+}