diff --git a/Cargo.lock b/Cargo.lock index 1c05dafd1..2e4c17f29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3137,6 +3137,7 @@ dependencies = [ "itertools 0.12.1", "jf-merkle-tree", "jf-vid", + "lazy_static", "log", "portpicker", "prometheus", diff --git a/Cargo.toml b/Cargo.toml index e279aa462..a8fac5522 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,7 @@ jf-vid = { version = "0.1.0", git = "https://github.com/EspressoSystems/jellyfis "std", "parallel", ] } +lazy_static = "1" prometheus = "0.13" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/src/data_source/storage/sql/queries/explorer.rs b/src/data_source/storage/sql/queries/explorer.rs index 4b3b22ada..14fb489a0 100644 --- a/src/data_source/storage/sql/queries/explorer.rs +++ b/src/data_source/storage/sql/queries/explorer.rs @@ -14,7 +14,7 @@ use super::{ super::transaction::{query, Transaction, TransactionMode}, - Database, Db, DecodeError, QueryBuilder, BLOCK_COLUMNS, + Database, Db, DecodeError, BLOCK_COLUMNS, }; use crate::{ availability::{BlockQueryData, QueryableHeader, QueryablePayload, TransactionIndex}, @@ -39,7 +39,7 @@ use futures::stream::{self, StreamExt, TryStreamExt}; use hotshot_types::traits::node_implementation::NodeType; use itertools::Itertools; use sqlx::{types::Json, FromRow, Row}; -use std::num::NonZeroUsize; +use std::{collections::VecDeque, num::NonZeroUsize}; use tagged_base64::{Tagged, TaggedBase64}; impl From for GetExplorerSummaryError { @@ -105,6 +105,164 @@ where } } +lazy_static::lazy_static! { + static ref GET_BLOCK_SUMMARIES_QUERY_FOR_LATEST: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + ORDER BY h.height DESC + LIMIT $1" + ) + }; + + static ref GET_BLOCK_SUMMARIES_QUERY_FOR_HEIGHT: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.height <= $1 + ORDER BY h.height DESC + LIMIT $2" + ) + }; + + // We want to match the blocks starting with the given hash, and working backwards + // until we have returned up to the number of requested blocks. The hash for a + // block should be unique, so we should just need to start with identifying the + // block height with the given hash, and return all blocks with a height less than + // or equal to that height, up to the number of requested blocks. + static ref GET_BLOCK_SUMMARIES_QUERY_FOR_HASH: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.height <= (SELECT h1.height FROM header AS h1 WHERE h1.hash = $1) + ORDER BY h.height DESC + LIMIT $2", + ) + }; + + static ref GET_BLOCK_DETAIL_QUERY_FOR_LATEST: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + ORDER BY h.height DESC + LIMIT 1" + ) + }; + + static ref GET_BLOCK_DETAIL_QUERY_FOR_HEIGHT: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.height = $1 + ORDER BY h.height DESC + LIMIT 1" + ) + }; + + static ref GET_BLOCK_DETAIL_QUERY_FOR_HASH: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.hash = $1 + ORDER BY h.height DESC + LIMIT 1" + ) + }; + + + static ref GET_TRANSACTION_SUMMARIES_QUERY_FOR_NO_FILTER: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.height IN ( + SELECT t.block_height + FROM transactions AS t + WHERE + (t.block_height = $1 AND t.idx <= $2) + OR t.block_height < $1 + ORDER BY t.block_height DESC, t.idx DESC + LIMIT $3 + ) + ORDER BY h.height DESC" + ) + }; + + static ref GET_TRANSACTION_SUMMARIES_QUERY_FOR_BLOCK: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.height = $1 + ORDER BY h.height DESC" + ) + }; + + static ref GET_TRANSACTION_DETAIL_QUERY_FOR_LATEST: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.height = ( + SELECT MAX(t1.block_height) + FROM transactions AS t1 + ) + ORDER BY h.height DESC" + ) + }; + + static ref GET_TRANSACTION_DETAIL_QUERY_FOR_HEIGHT_AND_OFFSET: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.height = ( + SELECT t1.block_height + FROM transactions AS t1 + WHERE t1.block_height = $1 + ORDER BY t1.block_height, t1.idx + OFFSET $2 + LIMIT 1 + ) + ORDER BY h.height DESC", + ) + }; + + static ref GET_TRANSACTION_DETAIL_QUERY_FOR_HASH: String = { + format!( + "SELECT {BLOCK_COLUMNS} + FROM header AS h + JOIN payload AS p ON h.height = p.height + WHERE h.height = ( + SELECT t1.block_height + FROM transactions AS t1 + WHERE t1.hash = $1 + ORDER BY t1.block_height DESC, t1.idx DESC + LIMIT 1 + ) + ORDER BY h.height DESC" + ) + }; +} + +/// [EXPLORER_SUMMARY_HISTOGRAM_NUM_ENTRIES] is the number of entries we want +/// to return in our histogram summary. +const EXPLORER_SUMMARY_HISTOGRAM_NUM_ENTRIES: usize = 50; + +/// [EXPLORER_SUMMARY_NUM_BLOCKS] is the number of blocks we want to return in +/// our explorer summary. +const EXPLORER_SUMMARY_NUM_BLOCKS: usize = 10; + +/// [EXPLORER_SUMMARY_NUM_TRANSACTIONS] is the number of transactions we want +/// to return in our explorer summary. +const EXPLORER_SUMMARY_NUM_TRANSACTIONS: usize = 10; + #[async_trait] impl ExplorerStorage for Transaction where @@ -121,46 +279,19 @@ where ) -> Result>, GetBlockSummariesError> { let request = &request.0; - let mut query = QueryBuilder::default(); - let sql = match request.target { - BlockIdentifier::Latest => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - ORDER BY h.height DESC - LIMIT {}", - query.bind(request.num_blocks.get() as i64)?, - ), - BlockIdentifier::Height(height) => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.height <= {} - ORDER BY h.height DESC - LIMIT {}", - query.bind(height as i64)?, - query.bind(request.num_blocks.get() as i64)?, - ), - BlockIdentifier::Hash(hash) => { - // We want to match the blocks starting with the given hash, and working backwards - // until we have returned up to the number of requested blocks. The hash for a - // block should be unique, so we should just need to start with identifying the - // block height with the given hash, and return all blocks with a height less than - // or equal to that height, up to the number of requested blocks. - format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.height <= (SELECT h1.height FROM header AS h1 WHERE h1.hash = {}) - ORDER BY h.height DESC - LIMIT {}", - query.bind(hash.to_string())?, - query.bind(request.num_blocks.get() as i64)?, - ) + let query_stmt = match request.target { + BlockIdentifier::Latest => { + query(&GET_BLOCK_SUMMARIES_QUERY_FOR_LATEST).bind(request.num_blocks.get() as i64) } + BlockIdentifier::Height(height) => query(&GET_BLOCK_SUMMARIES_QUERY_FOR_HEIGHT) + .bind(height as i64) + .bind(request.num_blocks.get() as i64), + BlockIdentifier::Hash(hash) => query(&GET_BLOCK_SUMMARIES_QUERY_FOR_HASH) + .bind(hash.to_string()) + .bind(request.num_blocks.get() as i64), }; - let row_stream = query.query(&sql).fetch(self.as_mut()); + let row_stream = query_stmt.fetch(self.as_mut()); let result = row_stream.map(|row| BlockSummary::from_row(&row?)); Ok(result.try_collect().await?) @@ -170,36 +301,17 @@ where &mut self, request: BlockIdentifier, ) -> Result, GetBlockDetailError> { - let mut query = QueryBuilder::default(); - let sql = match request { - BlockIdentifier::Latest => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - ORDER BY h.height DESC - LIMIT 1" - ), - BlockIdentifier::Height(height) => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.height = {} - ORDER BY h.height DESC - LIMIT 1", - query.bind(height as i64)?, - ), - BlockIdentifier::Hash(hash) => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.hash = {} - ORDER BY h.height DESC - LIMIT 1", - query.bind(hash.to_string())?, - ), + let query_stmt = match request { + BlockIdentifier::Latest => query(&GET_BLOCK_DETAIL_QUERY_FOR_LATEST), + BlockIdentifier::Height(height) => { + query(&GET_BLOCK_DETAIL_QUERY_FOR_HEIGHT).bind(height as i64) + } + BlockIdentifier::Hash(hash) => { + query(&GET_BLOCK_DETAIL_QUERY_FOR_HASH).bind(hash.to_string()) + } }; - let query_result = query.query(&sql).fetch_one(self.as_mut()).await?; + let query_result = query_stmt.fetch_one(self.as_mut()).await?; let block = BlockDetail::from_row(&query_result)?; Ok(block) @@ -253,38 +365,20 @@ where // transactions from that point. We then grab only the blocks for those // identified transactions, as only those blocks are needed to pull all // of the relevant transactions. - let mut query = QueryBuilder::default(); - let sql = match filter { + let query_stmt = match filter { TransactionSummaryFilter::RollUp(_) => return Ok(vec![]), - TransactionSummaryFilter::None => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.height IN ( - SELECT t.block_height - FROM transactions AS t - WHERE (t.block_height, t.idx) <= ({}, {}) - ORDER BY t.block_height DESC, t.idx DESC - LIMIT {} - ) - ORDER BY h.height DESC", - query.bind(block_height as i64)?, - query.bind(transaction_index)?, - query.bind((range.num_transactions.get() + offset) as i64)?, - ), + TransactionSummaryFilter::None => query(&GET_TRANSACTION_SUMMARIES_QUERY_FOR_NO_FILTER) + .bind(block_height as i64) + .bind(transaction_index) + .bind((range.num_transactions.get() + offset) as i64), - TransactionSummaryFilter::Block(block) => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.height = {} - ORDER BY h.height DESC", - query.bind(*block as i64)?, - ), + TransactionSummaryFilter::Block(block) => { + query(&GET_TRANSACTION_SUMMARIES_QUERY_FOR_BLOCK).bind(*block as i64) + } }; - let block_stream = query - .query(&sql) + + let block_stream = query_stmt .fetch(self.as_mut()) .map(|row| BlockQueryData::from_row(&row?)); @@ -322,6 +416,7 @@ where false } }) + .take(range.num_transactions.get()) .collect::>>()) } @@ -331,51 +426,19 @@ where ) -> Result, GetTransactionDetailError> { let target = request; - let mut query = QueryBuilder::default(); - let sql = match target { - TransactionIdentifier::Latest => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.height = ( - SELECT MAX(t1.block_height) - FROM transactions AS t1 - ) - ORDER BY h.height DESC" - ), - TransactionIdentifier::HeightAndOffset(height, offset) => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.height = ( - SELECT t1.block_height - FROM transactions AS t1 - WHERE t1.block_height = {} - ORDER BY t1.block_height, t1.idx - OFFSET {} - LIMIT 1 - ) - ORDER BY h.height DESC", - query.bind(height as i64)?, - query.bind(offset as i64)?, - ), - TransactionIdentifier::Hash(hash) => format!( - "SELECT {BLOCK_COLUMNS} - FROM header AS h - JOIN payload AS p ON h.height = p.height - WHERE h.height = ( - SELECT t1.block_height - FROM transactions AS t1 - WHERE t1.hash = {} - ORDER BY t1.block_height DESC, t1.idx DESC - LIMIT 1 - ) - ORDER BY h.height DESC", - query.bind(hash.to_string())?, - ), + let query_stmt = match target { + TransactionIdentifier::Latest => query(&GET_TRANSACTION_DETAIL_QUERY_FOR_LATEST), + TransactionIdentifier::HeightAndOffset(height, offset) => { + query(&GET_TRANSACTION_DETAIL_QUERY_FOR_HEIGHT_AND_OFFSET) + .bind(height as i64) + .bind(offset as i64) + } + TransactionIdentifier::Hash(hash) => { + query(&GET_TRANSACTION_DETAIL_QUERY_FOR_HASH).bind(hash.to_string()) + } }; - let query_row = query.query(&sql).fetch_one(self.as_mut()).await?; + let query_row = query_stmt.fetch_one(self.as_mut()).await?; let block = BlockQueryData::::from_row(&query_row)?; let txns = block.enumerate().map(|(_, txn)| txn).collect::>(); @@ -409,7 +472,7 @@ where &mut self, ) -> Result, GetExplorerSummaryError> { let histograms = { - let historgram_query_result = query( + let histogram_query_result = query( "SELECT h.height AS height, h.timestamp AS timestamp, @@ -420,13 +483,14 @@ where JOIN payload AS p ON p.height = h.height WHERE - h.height IN (SELECT height FROM header ORDER BY height DESC LIMIT 50) - ORDER BY h.height + h.height IN (SELECT height FROM header ORDER BY height DESC LIMIT $1) + ORDER BY h.height ", ) + .bind((EXPLORER_SUMMARY_HISTOGRAM_NUM_ENTRIES + 1) as i64) .fetch(self.as_mut()); - let histograms: Result = historgram_query_result + let mut histograms: ExplorerHistograms = histogram_query_result .map(|row_stream| { row_stream.map(|row| { let height: i64 = row.try_get("height")?; @@ -440,24 +504,32 @@ where }) .try_fold( ExplorerHistograms { - block_time: Vec::with_capacity(50), - block_size: Vec::with_capacity(50), - block_transactions: Vec::with_capacity(50), - block_heights: Vec::with_capacity(50), + block_time: VecDeque::with_capacity(EXPLORER_SUMMARY_HISTOGRAM_NUM_ENTRIES), + block_size: VecDeque::with_capacity(EXPLORER_SUMMARY_HISTOGRAM_NUM_ENTRIES), + block_transactions: VecDeque::with_capacity(EXPLORER_SUMMARY_HISTOGRAM_NUM_ENTRIES), + block_heights: VecDeque::with_capacity(EXPLORER_SUMMARY_HISTOGRAM_NUM_ENTRIES), }, |mut histograms: ExplorerHistograms, row: sqlx::Result<(i64, i64, Option, Option, i32)>| async { let (height, _timestamp, time, size, num_transactions) = row?; - histograms.block_time.push(time.map(|i| i as u64)); - histograms.block_size.push(size.map(|i| i as u64)); - histograms.block_transactions.push(num_transactions as u64); - histograms.block_heights.push(height as u64); + + histograms.block_time.push_back(time.map(|i| i as u64)); + histograms.block_size.push_back(size.map(|i| i as u64)); + histograms.block_transactions.push_back(num_transactions as u64); + histograms.block_heights.push_back(height as u64); Ok(histograms) }, ) - .await; + .await?; + + while histograms.block_time.len() > EXPLORER_SUMMARY_HISTOGRAM_NUM_ENTRIES { + histograms.block_time.pop_front(); + histograms.block_size.pop_front(); + histograms.block_transactions.pop_front(); + histograms.block_heights.pop_front(); + } - histograms? + histograms }; let genesis_overview = { @@ -477,7 +549,7 @@ where let latest_blocks: Vec> = self .get_block_summaries(GetBlockSummariesRequest(BlockRange { target: BlockIdentifier::Latest, - num_blocks: NonZeroUsize::new(10).unwrap(), + num_blocks: NonZeroUsize::new(EXPLORER_SUMMARY_NUM_BLOCKS).unwrap(), })) .await?; @@ -485,7 +557,7 @@ where .get_transaction_summaries(GetTransactionSummariesRequest { range: TransactionRange { target: TransactionIdentifier::Latest, - num_transactions: NonZeroUsize::new(10).unwrap(), + num_transactions: NonZeroUsize::new(EXPLORER_SUMMARY_NUM_TRANSACTIONS).unwrap(), }, filter: TransactionSummaryFilter::None, }) diff --git a/src/explorer/query_data.rs b/src/explorer/query_data.rs index 2ac8551f7..f4469e964 100644 --- a/src/explorer/query_data.rs +++ b/src/explorer/query_data.rs @@ -23,6 +23,7 @@ use crate::{node::BlockHash, types::HeightIndexed}; use hotshot_types::traits::node_implementation::NodeType; use serde::{Deserialize, Serialize}; use std::{ + collections::VecDeque, fmt::{Debug, Display}, num::{NonZeroUsize, TryFromIntError}, }; @@ -380,6 +381,7 @@ where BlockQueryData: HeightIndexed, Payload: QueryablePayload, Header: QueryableHeader + ExplorerHeader, + ::Transaction: ExplorerTransaction, { type Error = TimestampConversionError; @@ -399,7 +401,7 @@ where block_confirmed: true, offset: offset as u64, num_transactions: block.num_transactions, - size: block.size, + size: transaction.payload_size(), time: Timestamp(time::OffsetDateTime::from_unix_timestamp(seconds)?), sequencing_fees: vec![], fee_details: vec![], @@ -468,10 +470,10 @@ pub struct GenesisOverview { /// `block_transactions` for those `block_heights`. #[derive(Debug, Serialize, Deserialize)] pub struct ExplorerHistograms { - pub block_time: Vec>, - pub block_size: Vec>, - pub block_transactions: Vec, - pub block_heights: Vec, + pub block_time: VecDeque>, + pub block_size: VecDeque>, + pub block_transactions: VecDeque, + pub block_heights: VecDeque, } /// [ExplorerSummary] is a struct that represents an at-a-glance snapshot of diff --git a/src/explorer/traits.rs b/src/explorer/traits.rs index a6d7c489a..7b44e9a0a 100644 --- a/src/explorer/traits.rs +++ b/src/explorer/traits.rs @@ -66,5 +66,9 @@ pub trait ExplorerTransaction { /// a representation of it that adheres to the trait restrictions specified. type NamespaceId: Clone + Debug + Serialize + DeserializeOwned + Send + Sync + PartialEq + Eq; + /// namespace_id returns the namespace id of the individual transaction. fn namespace_id(&self) -> Self::NamespaceId; + + /// payload_size returns the size of the payload of the transaction. + fn payload_size(&self) -> u64; } diff --git a/src/testing/mocks.rs b/src/testing/mocks.rs index a5e51559e..5c271ca1f 100644 --- a/src/testing/mocks.rs +++ b/src/testing/mocks.rs @@ -89,6 +89,10 @@ impl ExplorerTransaction for MockTransaction { fn namespace_id(&self) -> Self::NamespaceId { 0 } + + fn payload_size(&self) -> u64 { + self.bytes().len() as u64 + } } impl HeightIndexed for MockHeader {