diff --git a/api/availability.toml b/api/availability.toml index 62b94173..2a5823b6 100644 --- a/api/availability.toml +++ b/api/availability.toml @@ -72,6 +72,9 @@ PATH = ["leaf/:from/:until"] DOC = """ Get the leaves based on their position in the ledger, the leaves are taken starting from the given :from up until the given :until. + +The allowable length of the requested range may be restricted by an implementation-defined limit +(see `/limits`). Requests for ranges exceeding these limits will fail with a 400 status code. """ [route.stream_leaves] @@ -102,6 +105,9 @@ PATH = ["header/:from/:until"] DOC = """ Get the headers based on their position in the ledger, the headers are taken starting from the given :from up until the given :until. + +The allowable length of the requested range may be restricted by an implementation-defined limit +(see `/limits`). Requests for ranges exceeding these limits will fail with a 400 status code. """ [route.stream_headers] @@ -147,6 +153,9 @@ PATH = ["block/:from/:until"] DOC = """ Get the blocks based on their position in the ledger, the blocks are taken starting from the given :from up until the given :until. + +The allowable length of the requested range may be restricted by an implementation-defined limit +(see `/limits`). Requests for ranges exceeding these limits will fail with a 400 status code. """ [route.stream_blocks] @@ -175,6 +184,9 @@ PATH = ["payload/:from/:until"] DOC = """ Get the payloads of blocks based on their position in the ledger, the payloads are taken starting from the given :from up until the given :until. + +The allowable length of the requested range may be restricted by an implementation-defined limit +(see `/limits`). Requests for ranges exceeding these limits will fail with a 400 status code. """ [route.stream_payloads] @@ -276,4 +288,37 @@ PATH = ["block/summaries/:from/:until"] DOC = """ Get the Block Summary entries for blocks based on their position in the ledger, the blocks are taken starting from the given :from up until the given :until. + +The allowable length of the requested range may be restricted by an implementation-defined limit +(see `/limits`). Requests for ranges exceeding these limits will fail with a 400 status code. +""" + +[route.get_limits] +PATH = ["limits"] +DOC = """ +Get implementation-defined limits restricting certain requests. + +* `small_object_range_limit`: the maximum number of small objects which can be loaded in a single + range query. + + Currently small objects include leaves only. In the future this limit will also apply to headers, + block summaries, and VID common, however + - loading of headers and block summaries is currently implemented by loading the entire block + - imperfect VID parameter tuning means that VID common can be much larger than it should + +* `large_object_range_limit`: the maximum number of large objects which can be loaded in a single + range query. + + Large objects include anything that _might_ contain a full payload or an object proportional in + size to a payload. Note that this limit applies to the entire class of objects: we do not check + the size of objects while loading to determine which limit to apply. If an object belongs to a + class which might contain a large payload, the large object limit always applies. + +Returns +``` +{ + "large_object_range_limit": integer, + "small_object_range_limit": integer +} +``` """ diff --git a/api/node.toml b/api/node.toml index 1a2247ee..119936dd 100644 --- a/api/node.toml +++ b/api/node.toml @@ -118,6 +118,12 @@ indicate that the response is not complete. The client can then use one of the ` this endpoint to fetch the remaining blocks from where the first response left off, once they become available. If no blocks are available, not even `prev`, this endpoint will return an error. +It is also possible that the number of blocks returned may be restricted by an implementation- +defined limit (see `/limits`), even if subsequent blocks within the window are currently available. +In this case, `next` will be `null` and the client should call again using one of the `/from/` forms +to get the next page of results, exactly as in the case where some blocks in the window have yet to +be produced. + Returns ```json @@ -130,3 +136,19 @@ Returns All timestamps are denominated in an integer number of seconds. """ + +[route.get_limits] +PATH = ["limits"] +DOC = """ +Get implementation-defined limits restricting certain requests. + +* `window_limit`: the maximum number of headers which can be loaded in a single `header/window` + query. + +Returns +``` +{ + "window_limit": integer +} +``` +""" diff --git a/src/availability.rs b/src/availability.rs index df75baf2..d105e195 100644 --- a/src/availability.rs +++ b/src/availability.rs @@ -43,6 +43,7 @@ pub use data_source::*; pub use fetch::Fetch; pub use query_data::*; +#[derive(Debug)] pub struct Options { pub api_path: Option, @@ -58,6 +59,24 @@ pub struct Options { /// These optional files may contain route definitions for application-specific routes that have /// been added as extensions to the basic availability API. pub extensions: Vec, + + /// The maximum number of small objects which can be loaded in a single range query. + /// + /// Currently small objects include leaves only. In the future this limit will also apply to + /// headers, block summaries, and VID common, however + /// * loading of headers and block summaries is currently implemented by loading the entire + /// block + /// * imperfect VID parameter tuning means that VID common can be much larger than it should + pub small_object_range_limit: usize, + + /// The maximum number of large objects which can be loaded in a single range query. + /// + /// Large objects include anything that _might_ contain a full payload or an object proportional + /// in size to a payload. Note that this limit applies to the entire class of objects: we do not + /// check the size of objects while loading to determine which limit to apply. If an object + /// belongs to a class which might contain a large payload, the large object limit always + /// applies. + pub large_object_range_limit: usize, } impl Default for Options { @@ -66,6 +85,8 @@ impl Default for Options { api_path: None, fetch_timeout: Duration::from_millis(500), extensions: vec![], + large_object_range_limit: 100, + small_object_range_limit: 500, } } } @@ -97,6 +118,13 @@ pub enum Error { height: u64, index: u64, }, + #[snafu(display("request for range {from}..{until} exceeds limit {limit}"))] + #[from(ignore)] + RangeLimit { + from: usize, + until: usize, + limit: usize, + }, Custom { message: String, status: StatusCode, @@ -113,7 +141,7 @@ impl Error { pub fn status(&self) -> StatusCode { match self { - Self::Request { .. } => StatusCode::BAD_REQUEST, + Self::Request { .. } | Self::RangeLimit { .. } => StatusCode::BAD_REQUEST, Self::FetchLeaf { .. } | Self::FetchBlock { .. } | Self::FetchTransaction { .. } => { StatusCode::NOT_FOUND } @@ -138,6 +166,8 @@ where options.extensions.clone(), )?; let timeout = options.fetch_timeout; + let small_object_range_limit = options.small_object_range_limit; + let large_object_range_limit = options.large_object_range_limit; api.with_version("0.0.1".parse().unwrap()) .at("get_leaf", move |req, state| { @@ -157,6 +187,7 @@ where async move { let from = req.integer_param::<_, usize>("from")?; let until = req.integer_param("until")?; + enforce_range_limit(from, until, small_object_range_limit)?; let leaves = state .read(|state| state.get_leaf_range(from..until).boxed()) @@ -210,6 +241,7 @@ where async move { let from = req.integer_param::<_, usize>("from")?; let until = req.integer_param::<_, usize>("until")?; + enforce_range_limit(from, until, large_object_range_limit)?; let headers = state .read(|state| state.get_block_range(from..until).boxed()) @@ -265,6 +297,7 @@ where async move { let from = req.integer_param::<_, usize>("from")?; let until = req.integer_param("until")?; + enforce_range_limit(from, until, large_object_range_limit)?; let blocks = state .read(|state| state.get_block_range(from..until).boxed()) @@ -313,6 +346,7 @@ where async move { let from = req.integer_param::<_, usize>("from")?; let until = req.integer_param("until")?; + enforce_range_limit(from, until, large_object_range_limit)?; let payloads = state .read(|state| state.get_payload_range(from..until).boxed()) @@ -422,6 +456,7 @@ where async move { let from: usize = req.integer_param("from")?; let until: usize = req.integer_param("until")?; + enforce_range_limit(from, until, large_object_range_limit)?; let blocks = state .read(|state| state.get_block_range(from..until).boxed()) @@ -440,10 +475,26 @@ where Ok(result) } .boxed() + })? + .at("get_limits", move |_req, _state| { + async move { + Ok(Limits { + small_object_range_limit, + large_object_range_limit, + }) + } + .boxed() })?; Ok(api) } +fn enforce_range_limit(from: usize, until: usize, limit: usize) -> Result<(), Error> { + if until.saturating_sub(from) > limit { + return Err(Error::RangeLimit { from, until, limit }); + } + Ok(()) +} + #[cfg(test)] mod test { use super::*; @@ -464,8 +515,9 @@ mod test { use futures::future::FutureExt; use hotshot_types::{data::Leaf, simple_certificate::QuorumCertificate}; use portpicker::pick_unused_port; - use std::time::Duration; - use surf_disco::Client; + use serde::de::DeserializeOwned; + use std::{fmt::Debug, time::Duration}; + use surf_disco::{Client, Error as _}; use tempfile::TempDir; use tide_disco::App; use toml::toml; @@ -947,4 +999,98 @@ mod test { 0 ); } + + #[tokio::test(flavor = "multi_thread")] + async fn test_range_limit() { + setup_test(); + + let large_object_range_limit = 2; + let small_object_range_limit = 3; + + // Create the consensus network. + let mut network = MockNetwork::::init().await; + network.start().await; + + // Start the web server. + let port = pick_unused_port().unwrap(); + let mut app = App::<_, Error>::with_state(ApiState::from(network.data_source())); + app.register_module( + "availability", + define_api( + &Options { + large_object_range_limit, + small_object_range_limit, + ..Default::default() + }, + MockBase::instance(), + ) + .unwrap(), + ) + .unwrap(); + network.spawn( + "server", + app.serve(format!("0.0.0.0:{}", port), MockBase::instance()), + ); + + // Start a client. + let client = Client::::new( + format!("http://localhost:{}/availability", port) + .parse() + .unwrap(), + ); + assert!(client.connect(Some(Duration::from_secs(60))).await); + + // Check reported limits. + assert_eq!( + client.get::("limits").send().await.unwrap(), + Limits { + small_object_range_limit, + large_object_range_limit + } + ); + + // Wait for enough blocks to be produced. + client + .socket("stream/blocks/0") + .subscribe::>() + .await + .unwrap() + .take(small_object_range_limit + 1) + .try_collect::>() + .await + .unwrap(); + + async fn check_limit( + client: &Client, + req: &str, + limit: usize, + ) { + let range: Vec = client + .get(&format!("{req}/0/{limit}")) + .send() + .await + .unwrap(); + assert_eq!(range.len(), limit); + let err = client + .get::>(&format!("{req}/0/{}", limit + 1)) + .send() + .await + .unwrap_err(); + assert_eq!(err.status(), StatusCode::BAD_REQUEST); + } + + check_limit::>(&client, "leaf", small_object_range_limit).await; + check_limit::>(&client, "header", large_object_range_limit).await; + check_limit::>(&client, "block", large_object_range_limit).await; + check_limit::>(&client, "payload", large_object_range_limit) + .await; + check_limit::>( + &client, + "block/summaries", + large_object_range_limit, + ) + .await; + + network.shut_down().await; + } } diff --git a/src/availability/query_data.rs b/src/availability/query_data.rs index 265bbee3..3aa660ab 100644 --- a/src/availability/query_data.rs +++ b/src/availability/query_data.rs @@ -717,3 +717,9 @@ where } } } + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct Limits { + pub small_object_range_limit: usize, + pub large_object_range_limit: usize, +} diff --git a/src/data_source.rs b/src/data_source.rs index 80ac1cfc..a4c614ad 100644 --- a/src/data_source.rs +++ b/src/data_source.rs @@ -1113,10 +1113,7 @@ pub mod node_tests { } #[tokio::test(flavor = "multi_thread")] - pub async fn test_timestamp_window() - where - for<'a> D::ReadOnly<'a>: NodeStorage, - { + pub async fn test_timestamp_window() { setup_test(); let mut network = MockNetwork::::init().await; @@ -1179,7 +1176,7 @@ pub mod node_tests { let ds = ds.clone(); async move { let window = ds - .get_header_window(WindowStart::Time(start), end) + .get_header_window(WindowStart::Time(start), end, i64::MAX as usize) .await .unwrap(); tracing::info!("window for timestamp range {start}-{end}: {window:#?}"); @@ -1217,14 +1214,10 @@ pub mod node_tests { // previously (ie fetch a slightly overlapping window) to ensure there is at least one block // in the new window. let from = test_blocks.iter().flatten().count() - 1; - let more = { - ds.read() - .await - .unwrap() - .get_header_window(WindowStart::Height(from as u64), end) - .await - .unwrap() - }; + let more = ds + .get_header_window(WindowStart::Height(from as u64), end, i64::MAX as usize) + .await + .unwrap(); check_invariants(&more, start, end, false); assert_eq!( more.prev.as_ref().unwrap(), @@ -1236,14 +1229,14 @@ pub mod node_tests { ); assert_eq!(res.next, None); // We should get the same result whether we query by block height or hash. - let more2 = { - ds.read() - .await - .unwrap() - .get_header_window(test_blocks[2].last().unwrap().commit(), end) - .await - .unwrap() - }; + let more2 = ds + .get_header_window( + test_blocks[2].last().unwrap().commit(), + end, + i64::MAX as usize, + ) + .await + .unwrap(); check_invariants(&more2, start, end, false); assert_eq!(more2.from().unwrap(), more.from().unwrap()); assert_eq!(more2.prev, more.prev); @@ -1258,15 +1251,48 @@ pub mod node_tests { assert_eq!(res.next.unwrap(), test_blocks[1][0]); assert_eq!(res.window, vec![]); - // Case 5: no relevant blocks are available yet. - { - ds.read() - .await - .unwrap() - .get_header_window(WindowStart::Time((i64::MAX - 1) as u64), i64::MAX as u64) - .await - .unwrap_err(); - } + // Case 4: no relevant blocks are available yet. + ds.get_header_window( + WindowStart::Time((i64::MAX - 1) as u64), + i64::MAX as u64, + i64::MAX as usize, + ) + .await + .unwrap_err(); + + // Case 5: limits. + let blocks = [test_blocks[0].clone(), test_blocks[1].clone()] + .into_iter() + .flatten() + .collect::>(); + // Make a query that would return everything, but gets limited. + let start = blocks[0].timestamp(); + let end = test_blocks[2][0].timestamp(); + let res = ds + .get_header_window(WindowStart::Time(start), end, 1) + .await + .unwrap(); + assert_eq!(res.prev, None); + assert_eq!(res.window, [blocks[0].clone()]); + assert_eq!(res.next, None); + // Query the next page of results, get limited again. + let res = ds + .get_header_window(WindowStart::Height(blocks[0].height() + 1), end, 1) + .await + .unwrap(); + assert_eq!(res.window, [blocks[1].clone()]); + assert_eq!(res.next, None); + // Get the rest of the results. + let res = ds + .get_header_window( + WindowStart::Height(blocks[1].height() + 1), + end, + blocks.len() - 1, + ) + .await + .unwrap(); + assert_eq!(res.window, blocks[2..].to_vec()); + assert_eq!(res.next, Some(test_blocks[2][0].clone())); } } diff --git a/src/data_source/extension.rs b/src/data_source/extension.rs index dc33cbfd..6831930c 100644 --- a/src/data_source/extension.rs +++ b/src/data_source/extension.rs @@ -289,8 +289,9 @@ where &self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>> { - self.data_source.get_header_window(start, end).await + self.data_source.get_header_window(start, end, limit).await } } diff --git a/src/data_source/fetching.rs b/src/data_source/fetching.rs index f00bc796..61deed94 100644 --- a/src/data_source/fetching.rs +++ b/src/data_source/fetching.rs @@ -1666,11 +1666,12 @@ where &self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>> { let mut tx = self.read().await.map_err(|err| QueryError::Error { message: err.to_string(), })?; - tx.get_header_window(start, end).await + tx.get_header_window(start, end, limit).await } } diff --git a/src/data_source/storage.rs b/src/data_source/storage.rs index f5ea20bc..49ce6181 100644 --- a/src/data_source/storage.rs +++ b/src/data_source/storage.rs @@ -217,6 +217,7 @@ pub trait NodeStorage { &mut self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>>; /// Search the database for missing objects and generate a report. diff --git a/src/data_source/storage/fail_storage.rs b/src/data_source/storage/fail_storage.rs index 8064ca90..c52010c7 100644 --- a/src/data_source/storage/fail_storage.rs +++ b/src/data_source/storage/fail_storage.rs @@ -522,9 +522,10 @@ where &mut self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>> { self.maybe_fail_read(FailableAction::Any).await?; - self.inner.get_header_window(start, end).await + self.inner.get_header_window(start, end, limit).await } } diff --git a/src/data_source/storage/fs.rs b/src/data_source/storage/fs.rs index b9638f73..bfb1122d 100644 --- a/src/data_source/storage/fs.rs +++ b/src/data_source/storage/fs.rs @@ -734,6 +734,7 @@ where &mut self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>> { let first_block = match start.into() { WindowStart::Height(h) => h, @@ -772,6 +773,9 @@ where break; } res.window.push(header); + if res.window.len() >= limit { + break; + } } Ok(res) diff --git a/src/data_source/storage/no_storage.rs b/src/data_source/storage/no_storage.rs index a7a77d46..3f67055d 100644 --- a/src/data_source/storage/no_storage.rs +++ b/src/data_source/storage/no_storage.rs @@ -277,6 +277,7 @@ where &mut self, _start: impl Into> + Send + Sync, _end: u64, + _limit: usize, ) -> QueryResult>> { Err(QueryError::Missing) } @@ -770,10 +771,11 @@ pub mod testing { &mut self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>> { match self { - Transaction::Sql(tx) => tx.get_header_window(start, end).await, - Transaction::NoStorage(tx) => tx.get_header_window(start, end).await, + Transaction::Sql(tx) => tx.get_header_window(start, end, limit).await, + Transaction::NoStorage(tx) => tx.get_header_window(start, end, limit).await, } } } @@ -855,11 +857,14 @@ pub mod testing { &self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>> { match self { - DataSource::Sql(data_source) => data_source.get_header_window(start, end).await, + DataSource::Sql(data_source) => { + data_source.get_header_window(start, end, limit).await + } DataSource::NoStorage(data_source) => { - data_source.get_header_window(start, end).await + data_source.get_header_window(start, end, limit).await } } } diff --git a/src/data_source/storage/sql/queries/node.rs b/src/data_source/storage/sql/queries/node.rs index 34e78546..70147e84 100644 --- a/src/data_source/storage/sql/queries/node.rs +++ b/src/data_source/storage/sql/queries/node.rs @@ -203,6 +203,7 @@ where &mut self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>> { // Find the specific block that starts the requested window. let first_block = match start.into() { @@ -211,7 +212,7 @@ where // use a different method to find the window, as detecting whether we have // sufficient data to answer the query is not as simple as just trying `load_header` // for a specific block ID. - return self.time_window::(t, end).await; + return self.time_window::(t, end, limit).await; } WindowStart::Height(h) => h, WindowStart::Hash(h) => self.load_header::(h).await?.block_number(), @@ -224,15 +225,17 @@ where "SELECT {HEADER_COLUMNS} FROM header AS h WHERE h.height >= $1 AND h.timestamp < $2 - ORDER BY h.height" + ORDER BY h.height + LIMIT $3" ); let rows = query(&sql) .bind(first_block as i64) .bind(end as i64) + .bind(limit as i64) .fetch(self.as_mut()); let window = rows .map(|row| parse_header::(row?)) - .try_collect() + .try_collect::>() .await?; // Find the block just before the window. @@ -242,26 +245,35 @@ where None }; - // Find the block just after the window. We order by timestamp _then_ height, because the - // timestamp order allows the query planner to use the index on timestamp to also - // efficiently solve the WHERE clause, but this process may turn up multiple results, due to - // the 1-second resolution of block timestamps. The final sort by height guarantees us a - // unique, deterministic result (the first block with a given timestamp). This sort may not - // be able to use an index, but it shouldn't be too expensive, since there will never be - // more than a handful of blocks with the same timestamp. - let sql = format!( - "SELECT {HEADER_COLUMNS} + let next = if window.len() < limit { + // If we are not limited, complete the window by finding the block just after the + // window. We order by timestamp _then_ height, because the timestamp order allows the + // query planner to use the index on timestamp to also efficiently solve the WHERE + // clause, but this process may turn up multiple results, due to the 1-second resolution + // of block timestamps. The final sort by height guarantees us a unique, deterministic + // result (the first block with a given timestamp). This sort may not be able to use an + // index, but it shouldn't be too expensive, since there will never be more than a + // handful of blocks with the same timestamp. + let sql = format!( + "SELECT {HEADER_COLUMNS} FROM header AS h WHERE h.timestamp >= $1 ORDER BY h.timestamp, h.height LIMIT 1" - ); - let next = query(&sql) - .bind(end as i64) - .fetch_optional(self.as_mut()) - .await? - .map(parse_header::) - .transpose()?; + ); + query(&sql) + .bind(end as i64) + .fetch_optional(self.as_mut()) + .await? + .map(parse_header::) + .transpose()? + } else { + // If we have been limited, return a `null` next block indicating an incomplete window. + // The client will have to query again with an adjusted starting point to get subsequent + // results. + tracing::debug!(limit, "cutting off header window request due to limit"); + None + }; Ok(TimeWindowQueryData { window, prev, next }) } @@ -333,6 +345,7 @@ impl Transaction { &mut self, start: u64, end: u64, + limit: usize, ) -> QueryResult>> { // Find all blocks whose timestamps fall within the window [start, end). Block timestamps // are monotonically increasing, so this query is guaranteed to return a contiguous range of @@ -349,31 +362,41 @@ impl Transaction { "SELECT {HEADER_COLUMNS} FROM header AS h WHERE h.timestamp >= $1 AND h.timestamp < $2 - ORDER BY h.timestamp, h.height" + ORDER BY h.timestamp, h.height + LIMIT $3" ); let rows = query(&sql) .bind(start as i64) .bind(end as i64) + .bind(limit as i64) .fetch(self.as_mut()); let window: Vec<_> = rows .map(|row| parse_header::(row?)) .try_collect() .await?; - // Find the block just after the window. - let sql = format!( - "SELECT {HEADER_COLUMNS} + let next = if window.len() < limit { + // If we are not limited, complete the window by finding the block just after. + let sql = format!( + "SELECT {HEADER_COLUMNS} FROM header AS h WHERE h.timestamp >= $1 ORDER BY h.timestamp, h.height LIMIT 1" - ); - let next = query(&sql) - .bind(end as i64) - .fetch_optional(self.as_mut()) - .await? - .map(parse_header::) - .transpose()?; + ); + query(&sql) + .bind(end as i64) + .fetch_optional(self.as_mut()) + .await? + .map(parse_header::) + .transpose()? + } else { + // If we have been limited, return a `null` next block indicating an incomplete window. + // The client will have to query again with an adjusted starting point to get subsequent + // results. + tracing::debug!(limit, "cutting off header window request due to limit"); + None + }; // If the `next` block exists, _or_ if any block in the window exists, we know we have // enough information to definitively say at least where the window starts (we may or may diff --git a/src/lib.rs b/src/lib.rs index 0370d435..7bf465fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -382,8 +382,9 @@ //! &self, //! start: impl Into> + Send + Sync, //! end: u64, +//! limit: usize, //! ) -> QueryResult>> { -//! self.hotshot_qs.get_header_window(start, end).await +//! self.hotshot_qs.get_header_window(start, end, limit).await //! } //! } //! @@ -777,8 +778,9 @@ mod test { &self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>> { - self.hotshot_qs.get_header_window(start, end).await + self.hotshot_qs.get_header_window(start, end, limit).await } } diff --git a/src/node.rs b/src/node.rs index f1fd0c87..d0347c00 100644 --- a/src/node.rs +++ b/src/node.rs @@ -35,7 +35,7 @@ pub(crate) mod query_data; pub use data_source::*; pub use query_data::*; -#[derive(Default)] +#[derive(Debug)] pub struct Options { pub api_path: Option, @@ -44,6 +44,19 @@ pub struct Options { /// These optional files may contain route definitions for application-specific routes that have /// been added as extensions to the basic node API. pub extensions: Vec, + + /// The maximum number of headers which can be loaded in a single `header/window` query. + pub window_limit: usize, +} + +impl Default for Options { + fn default() -> Self { + Self { + api_path: None, + extensions: vec![], + window_limit: 500, + } + } } #[derive(Clone, Debug, From, Snafu, Deserialize, Serialize)] @@ -109,6 +122,7 @@ where include_str!("../api/node.toml"), options.extensions.clone(), )?; + let window_limit = options.window_limit; api.with_version("0.0.1".parse().unwrap()) .get("block_height", |_req, state| { async move { state.block_height().await.context(QuerySnafu) }.boxed() @@ -159,7 +173,7 @@ where .get("sync_status", |_req, state| { async move { state.sync_status().await.context(QuerySnafu) }.boxed() })? - .get("get_header_window", |req, state| { + .get("get_header_window", move |req, state| { async move { let start = if let Some(height) = req.opt_integer_param("height")? { WindowStart::Height(height) @@ -170,7 +184,7 @@ where }; let end = req.integer_param("end")?; state - .get_header_window(start, end) + .get_header_window(start, end, window_limit) .await .context(QueryWindowSnafu { start: format!("{start:?}"), @@ -178,6 +192,9 @@ where }) } .boxed() + })? + .get("get_limits", move |_req, _state| { + async move { Ok(Limits { window_limit }) }.boxed() })?; Ok(api) } @@ -217,6 +234,8 @@ mod test { async fn test_api() { setup_test(); + let window_limit = 78; + // Create the consensus network. let mut network = MockNetwork::::init().await; let mut events = network.handle().event_stream(); @@ -227,7 +246,14 @@ mod test { let mut app = App::<_, Error>::with_state(ApiState::from(network.data_source())); app.register_module( "node", - define_api(&Default::default(), MockBase::instance()).unwrap(), + define_api( + &Options { + window_limit, + ..Default::default() + }, + MockBase::instance(), + ) + .unwrap(), ) .unwrap(); network.spawn( @@ -241,6 +267,12 @@ mod test { ); assert!(client.connect(Some(Duration::from_secs(60))).await); + // Check limits endpoint. + assert_eq!( + client.get::("limits").send().await.unwrap(), + Limits { window_limit } + ); + // Wait until a few blocks have been sequenced. let block_height = loop { let block_height = client.get::("block-height").send().await.unwrap(); diff --git a/src/node/data_source.rs b/src/node/data_source.rs index 5081fdc8..952a3661 100644 --- a/src/node/data_source.rs +++ b/src/node/data_source.rs @@ -66,6 +66,7 @@ pub trait NodeDataSource { &self, start: impl Into> + Send + Sync, end: u64, + limit: usize, ) -> QueryResult>>; /// Search the database for missing objects and generate a report. diff --git a/src/node/query_data.rs b/src/node/query_data.rs index f8502e1d..f1805bc1 100644 --- a/src/node/query_data.rs +++ b/src/node/query_data.rs @@ -61,3 +61,8 @@ impl TimeWindowQueryData { .map(|t| t.height()) } } + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct Limits { + pub window_limit: usize, +}