Gas price simulation data retriever (#2533)

## Linked Issues/PRs
<!-- List of related issues/PRs -->
- #2469

## Description
<!-- List of detailed changes -->
- Adds a data retrieval service that fetches L2 and L1 cost data for gas
price v1 simulations

## Checklist
- [x] Breaking changes are clearly marked as such in the PR description
and changelog
- [x] New behavior is reflected in tests
- [x] [The specification](https://github.com/FuelLabs/fuel-specs/)
matches the implemented behavior (link update PR if changes are needed)

### Before requesting review
- [x] I have reviewed the code myself
- [x] I have created follow-up issues caused by this PR and linked them
here

### After merging, notify other teams

[Add or remove entries as needed]

- [ ] [Rust SDK](https://github.com/FuelLabs/fuels-rs/)
- [ ] [Sway compiler](https://github.com/FuelLabs/sway/)
- [ ] [Platform
documentation](https://github.com/FuelLabs/devrel-requests/issues/new?assignees=&labels=new+request&projects=&template=NEW-REQUEST.yml&title=%5BRequest%5D%3A+)
(for out-of-organization contributors, the person merging the PR will do
this)
- [ ] Someone else?

---------

Co-authored-by: rymnc <43716372+rymnc@users.noreply.github.com>
Co-authored-by: Mitchell Turner <james.mitchell.turner@gmail.com>

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/.gitignore

@@ -0,0 +1,2 @@
+Cargo.lock
+data

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "fuel-gas-price-data-fetcher"
+version = "0.0.1"
+edition = "2021"
+publish = false
+
+[workspace]
+
+[dependencies]
+anyhow = "1.0.86"
+clap = { version = "4.5.16", features = ["derive"] }
+csv = "1.3.0"
+fuel-gas-price-algorithm = { path = ".." }
+futures = "0.3.30"
+plotters = "0.3.5"
+rand = "0.8.5"
+rand_distr = "0.4.3"
+serde = { version = "1.0.209", features = ["derive"] }
+tokio = { version = "1.40.0", features = ["macros", "rt", "rt-multi-thread"] }
+reqwest = { version = "0.12.11", features = ["json"] }
+serde_json = { version = "1.0.134" }
+fuel-core-client = { version = "0.40.2" }                                      # locked to whatever version you're supposed to be fetching data from
+fuel-core-types = { version = "0.40.2" }
+postcard = { version = "1.0" }
+tracing = { version = "0.1.41" }
+tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
+
+async-trait = "0.1"
+cynic = { version = "2.2", features = ["http-reqwest"] }
+itertools = { version = "0.13" }
+
+[build-dependencies]
+fuel-core-client = { version = "0.40.2" }

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/README.md

@@ -0,0 +1,10 @@
+# Gas Price Analysis Data Fetcher
+
+Binary allowing retrieveing the L1 blob and L2 block data needed by the gas price simulation binary. 
+It requires being able to connect to the block committer, and either being able to connect to a sentry node or access to the database of a mainnet synched node.
+
+## Usage
+
+```
+cargo run -- --block-committer-endpoint ${BLOCK_COMMITTER_URL} --block-range 0 1000 --db-path ${FUEL_MAINNET_DB_PATH}
+``` 

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/build.rs

@@ -0,0 +1,14 @@
+#![deny(clippy::arithmetic_side_effects)]
+#![deny(clippy::cast_possible_truncation)]
+#![deny(unused_crate_dependencies)]
+#![deny(warnings)]
+
+use std::fs;
+
+fn main() {
+    fs::create_dir_all("target").expect("Unable to create target directory");
+    fs::write("target/schema.sdl", fuel_core_client::SCHEMA_SDL)
+        .expect("Unable to write schema file");
+
+    println!("cargo:rerun-if-changed=build.rs");
+}

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/client_ext.rs

@@ -0,0 +1,236 @@
+#![deny(clippy::arithmetic_side_effects)]
+#![deny(clippy::cast_possible_truncation)]
+#![deny(warnings)]
+
+// This was copied from https://github.com/FuelLabs/fuel-core-client-ext/blob/b792ef76cbcf82eda45a944b15433682fe094fee/src/lib.rs
+
+use cynic::QueryBuilder;
+use fuel_core_client::{
+    client,
+    client::{
+        pagination::{
+            PaginatedResult,
+            PaginationRequest,
+        },
+        schema::{
+            block::{
+                BlockByHeightArgs,
+                Consensus,
+                Header,
+            },
+            schema,
+            tx::OpaqueTransactionWithStatus,
+            ConnectionArgs,
+            PageInfo,
+        },
+        types::{
+            TransactionResponse,
+            TransactionStatus,
+        },
+        FuelClient,
+    },
+};
+use fuel_core_types::{
+    blockchain::{
+        self,
+        block::Block,
+        header::{
+            ApplicationHeader,
+            ConsensusHeader,
+            PartialBlockHeader,
+        },
+        SealedBlock,
+    },
+    fuel_tx::{
+        Bytes32,
+        Receipt,
+    },
+};
+use itertools::Itertools;
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(
+    schema_path = "./target/schema.sdl",
+    graphql_type = "Query",
+    variables = "ConnectionArgs"
+)]
+pub struct FullBlocksQuery {
+    #[arguments(after: $after, before: $before, first: $first, last: $last)]
+    pub blocks: FullBlockConnection,
+}
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(schema_path = "./target/schema.sdl", graphql_type = "BlockConnection")]
+pub struct FullBlockConnection {
+    pub edges: Vec<FullBlockEdge>,
+    pub page_info: PageInfo,
+}
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(schema_path = "./target/schema.sdl", graphql_type = "BlockEdge")]
+pub struct FullBlockEdge {
+    pub cursor: String,
+    pub node: FullBlock,
+}
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(
+    schema_path = "./target/schema.sdl",
+    graphql_type = "Query",
+    variables = "BlockByHeightArgs"
+)]
+pub struct FullBlockByHeightQuery {
+    #[arguments(height: $height)]
+    pub block: Option<FullBlock>,
+}
+
+#[derive(cynic::QueryFragment, Debug)]
+#[cynic(schema_path = "./target/schema.sdl", graphql_type = "Block")]
+pub struct FullBlock {
+    pub header: Header,
+    pub consensus: Consensus,
+    pub transactions: Vec<OpaqueTransactionWithStatus>,
+}
+
+impl From<FullBlockConnection> for PaginatedResult<FullBlock, String> {
+    fn from(conn: FullBlockConnection) -> Self {
+        PaginatedResult {
+            cursor: conn.page_info.end_cursor,
+            has_next_page: conn.page_info.has_next_page,
+            has_previous_page: conn.page_info.has_previous_page,
+            results: conn.edges.into_iter().map(|e| e.node).collect(),
+        }
+    }
+}
+
+#[async_trait::async_trait]
+pub trait ClientExt {
+    async fn full_blocks(
+        &self,
+        request: PaginationRequest<String>,
+    ) -> std::io::Result<PaginatedResult<FullBlock, String>>;
+}
+
+#[async_trait::async_trait]
+impl ClientExt for FuelClient {
+    async fn full_blocks(
+        &self,
+        request: PaginationRequest<String>,
+    ) -> std::io::Result<PaginatedResult<FullBlock, String>> {
+        let query = FullBlocksQuery::build(request.into());
+        let blocks = self.query(query).await?.blocks.into();
+        Ok(blocks)
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SealedBlockWithMetadata {
+    pub block: SealedBlock,
+    pub receipts: Vec<Option<Vec<Receipt>>>,
+}
+
+impl TryFrom<FullBlock> for SealedBlockWithMetadata {
+    type Error = anyhow::Error;
+
+    fn try_from(full_block: FullBlock) -> Result<Self, Self::Error> {
+        let transactions: Vec<TransactionResponse> = full_block
+            .transactions
+            .into_iter()
+            .map(TryInto::try_into)
+            .try_collect()?;
+
+        let receipts = transactions
+            .iter()
+            .map(|tx| &tx.status)
+            .map(|status| match status {
+                TransactionStatus::Success { receipts, .. } => Some(receipts.clone()),
+                _ => None,
+            })
+            .collect_vec();
+
+        let messages = receipts
+            .iter()
+            .flatten()
+            .flat_map(|receipt| receipt.iter().filter_map(|r| r.message_id()))
+            .collect_vec();
+
+        let transactions = transactions
+            .into_iter()
+            .map(|tx| tx.transaction)
+            .collect_vec();
+
+        let partial_header = PartialBlockHeader {
+            application: ApplicationHeader {
+                da_height: full_block.header.da_height.0.into(),
+                consensus_parameters_version: full_block
+                    .header
+                    .consensus_parameters_version
+                    .into(),
+                state_transition_bytecode_version: full_block
+                    .header
+                    .state_transition_bytecode_version
+                    .into(),
+                generated: Default::default(),
+            },
+            consensus: ConsensusHeader {
+                prev_root: full_block.header.prev_root.into(),
+                height: full_block.header.height.into(),
+                time: full_block.header.time.into(),
+                generated: Default::default(),
+            },
+        };
+
+        let header = partial_header
+            .generate(
+                &transactions,
+                &messages,
+                full_block.header.event_inbox_root.into(),
+            )
+            .map_err(|e| anyhow::anyhow!(e))?;
+
+        let actual_id: Bytes32 = full_block.header.id.into();
+        let expected_id: Bytes32 = header.id().into();
+        if expected_id != actual_id {
+            return Err(anyhow::anyhow!("Header id mismatch"));
+        }
+
+        let block = Block::try_from_executed(header, transactions)
+            .ok_or(anyhow::anyhow!("Failed to create block from transactions"))?;
+
+        let consensus: client::types::Consensus = full_block.consensus.into();
+
+        let consensus = match consensus {
+            client::types::Consensus::Genesis(genesis) => {
+                use blockchain::consensus as core_consensus;
+                core_consensus::Consensus::Genesis(core_consensus::Genesis {
+                    chain_config_hash: genesis.chain_config_hash,
+                    coins_root: genesis.coins_root,
+                    contracts_root: genesis.contracts_root,
+                    messages_root: genesis.messages_root,
+                    transactions_root: genesis.transactions_root,
+                })
+            }
+            client::types::Consensus::PoAConsensus(poa) => {
+                use blockchain::consensus as core_consensus;
+                core_consensus::Consensus::PoA(core_consensus::poa::PoAConsensus {
+                    signature: poa.signature,
+                })
+            }
+            client::types::Consensus::Unknown => {
+                return Err(anyhow::anyhow!("Unknown consensus type"));
+            }
+        };
+
+        let sealed = SealedBlock {
+            entity: block,
+            consensus,
+        };
+
+        let sealed = SealedBlockWithMetadata {
+            block: sealed,
+            receipts,
+        };
+
+        Ok(sealed)
+    }
+}

crates/fuel-gas-price-algorithm/gas-price-data-fetcher/src/layer1.rs

@@ -0,0 +1,94 @@
+use std::ops::Range;
+
+use fuel_core_types::fuel_types::BlockHeight;
+use reqwest::{
+    header::{
+        HeaderMap,
+        CONTENT_TYPE,
+    },
+    Url,
+};
+
+use crate::types::BlockCommitterCosts;
+
+pub struct BlockCommitterDataFetcher {
+    client: reqwest::Client,
+    endpoint: Url,
+    num_responses: usize,
+}
+
+impl BlockCommitterDataFetcher {
+    pub fn new(endpoint: Url, num_responses: usize) -> anyhow::Result<Self> {
+        let mut content_type_json_header = HeaderMap::new();
+        content_type_json_header.insert(
+            CONTENT_TYPE,
+            "application/json"
+                .parse()
+                .expect("Content-Type header value is valid"),
+        );
+        let client = reqwest::ClientBuilder::new()
+            .default_headers(content_type_json_header)
+            .build()?;
+        Ok(Self {
+            client,
+            endpoint,
+            num_responses,
+        })
+    }
+
+    // TODO: Better error type; qed
+    async fn fetch_blob_data(
+        &self,
+        from_height: u64,
+    ) -> anyhow::Result<Vec<BlockCommitterCosts>> {
+        let query = self.endpoint.join("v1/costs")?.join(&format!(
+            "?variant=specific&value={}&limit={}",
+            from_height, self.num_responses
+        ))?;
+
+        tracing::debug!("Query: {}", query.as_str());
+
+        let response = self.client.get(query).send().await?;
+        if !response.status().is_success() {
+            return Err(anyhow::anyhow!(
+                "Failed to fetch data from block committer: {}",
+                response.status(),
+            )
+            .into());
+        }
+
+        let block_committer_costs = response.json::<Vec<BlockCommitterCosts>>().await?;
+        Ok(block_committer_costs)
+    }
+
+    pub async fn fetch_l1_block_costs(
+        &self,
+        blocks: Range<BlockHeight>,
+    ) -> Result<Vec<BlockCommitterCosts>, anyhow::Error> {
+        let mut block_costs = vec![];
+        let mut current_block_height = blocks.start;
+        while current_block_height < blocks.end {
+            let Ok(mut costs) =
+                self.fetch_blob_data((*current_block_height).into()).await
+            else {
+                Err(anyhow::anyhow!(
+                    "Could not fetch data for block {}",
+                    current_block_height
+                ))?
+            };
+
+            if costs.is_empty() {
+                // Might be that the block committer doesn't have data for the block, in which case we return prematurely.
+                // If this happens, we should increase the value of results returned by the block committer in the query.
+                break;
+            }
+
+            // Block committer will return the data for the block in the next batch, hence we don't increment the height of the last
+            // block.
+            current_block_height = (*costs.last().unwrap().end_height).into();
+            block_costs.append(&mut costs);
+        }
+
+        Ok(block_costs)
+    }
+}