From c0e3fa8d6a799e8a37ecd3e55e57a34b08814282 Mon Sep 17 00:00:00 2001 From: AurelienFT <32803821+AurelienFT@users.noreply.github.com> Date: Mon, 6 Jan 2025 10:10:11 +0100 Subject: [PATCH] Make the rocksdb cache optional in config and add policy for column opening (#2526) ## Description Cache is now an option and can be enable or disabled for each rocksdb instances. A new rocksdb config has been added to choose between the mode that open columns on database in a lazy way (useful for tests) and the mode that open all columns at once at start (useful for production and benches). Benches now use the `rocksdb-production` feature of fuel-core. In order to be more future-proof on the database configuration options, a `DatabaseConfig` structure has been created and is used across the code. ## Breaking change Before, if `max-database-cache-size` was unspecified then `DEFAULT_DATABASE_CACHE_SIZE` was used. Now it make the cache disabled. ## Checklist - [x] Breaking changes are clearly marked as such in the PR description and changelog - [x] New behavior is reflected in tests - [x] [The specification](https://github.com/FuelLabs/fuel-specs/) matches the implemented behavior (link update PR if changes are needed) ### Before requesting review - [x] I have reviewed the code myself - [x] I have created follow-up issues caused by this PR and linked them here --------- Co-authored-by: Aaryamann Challani <43716372+rymnc@users.noreply.github.com> --- CHANGELOG.md | 2 + benches/Cargo.toml | 2 +- benches/benches/block_target_gas.rs | 19 +- benches/benches/state.rs | 27 ++- benches/benches/transaction_throughput.rs | 17 +- benches/benches/vm_set/blockchain.rs | 15 +- benches/src/db_lookup_times_utils/utils.rs | 15 +- bin/fuel-core/src/cli/rollback.rs | 12 +- bin/fuel-core/src/cli/run.rs | 26 +-- bin/fuel-core/src/cli/snapshot.rs | 15 +- crates/fuel-core/src/combined_database.rs | 72 ++++--- crates/fuel-core/src/database.rs | 37 ++-- .../fuel-core/src/graphql_api/indexation.rs | 55 ++++-- crates/fuel-core/src/service/config.rs | 13 +- .../fuel-core/src/state/historical_rocksdb.rs | 8 +- crates/fuel-core/src/state/rocks_db.rs | 175 ++++++++++++------ tests/test-helpers/src/builder.rs | 9 +- tests/tests/aws_kms.rs | 13 +- tests/tests/health.rs | 38 ++-- tests/tests/relayer.rs | 20 +- 20 files changed, 415 insertions(+), 175 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adcff97b9ed..f30e2200ddd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2416](https://github.com/FuelLabs/fuel-core/issues/2416): Define the `GasPriceServiceV1` task. - [2033](https://github.com/FuelLabs/fuel-core/pull/2033): Remove `Option` in favor of `BlockHeightQuery` where applicable. - [2472](https://github.com/FuelLabs/fuel-core/pull/2472): Added the `amountU128` field to the `Balance` GraphQL schema, providing the total balance as a `U128`. The existing `amount` field clamps any balance exceeding `U64` to `u64::MAX`. +- [2526](https://github.com/FuelLabs/fuel-core/pull/2526): Add possibility to not have any cache set for RocksDB. Add an option to either load the RocksDB columns families on creation of the database or when the column is used. ### Fixed - [2365](https://github.com/FuelLabs/fuel-core/pull/2365): Fixed the error during dry run in the case of race condition. @@ -57,6 +58,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2154](https://github.com/FuelLabs/fuel-core/pull/2154): Transaction graphql endpoints use `TransactionType` instead of `fuel_tx::Transaction`. - [2446](https://github.com/FuelLabs/fuel-core/pull/2446): Use graphiql instead of graphql-playground due to known vulnerability and stale development. - [2379](https://github.com/FuelLabs/fuel-core/issues/2379): Change `kv_store::Value` to be `Arc<[u8]>` instead of `Arc>`. +- [2526](https://github.com/FuelLabs/fuel-core/pull/2526): By default the cache of RocksDB is now disabled instead of being `1024 * 1024 * 1024`. ## [Version 0.40.2] diff --git a/benches/Cargo.toml b/benches/Cargo.toml index e4ee7db927b..b59fb0e458f 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -63,7 +63,7 @@ harness = false name = "vm" [features] -default = ["fuel-core/rocksdb"] +default = ["fuel-core/rocksdb", "fuel-core/rocksdb-production"] [[bench]] harness = false diff --git a/benches/benches/block_target_gas.rs b/benches/benches/block_target_gas.rs index fddd14a24c9..e1384e65a81 100644 --- a/benches/benches/block_target_gas.rs +++ b/benches/benches/block_target_gas.rs @@ -27,7 +27,13 @@ use fuel_core::{ Config, FuelService, }, - state::historical_rocksdb::StateRewindPolicy, + state::{ + historical_rocksdb::StateRewindPolicy, + rocks_db::{ + ColumnsPolicy, + DatabaseConfig, + }, + }, }; use fuel_core_benches::{ default_gas_costs::default_gas_costs, @@ -266,8 +272,15 @@ fn service_with_many_contracts( .build() .unwrap(); let _drop = rt.enter(); - let mut database = Database::rocksdb_temp(StateRewindPolicy::NoRewind) - .expect("Failed to create database"); + let mut database = Database::rocksdb_temp( + StateRewindPolicy::NoRewind, + DatabaseConfig { + cache_capacity: Some(16 * 1024 * 1024 * 1024), + max_fds: -1, + columns_policy: ColumnsPolicy::OnCreation, + }, + ) + .expect("Failed to create database"); let mut chain_config = ChainConfig::local_testnet(); diff --git a/benches/benches/state.rs b/benches/benches/state.rs index d08ecf2104c..e661ee9ded2 100644 --- a/benches/benches/state.rs +++ b/benches/benches/state.rs @@ -5,10 +5,19 @@ use criterion::{ BenchmarkGroup, Criterion, }; -use fuel_core::database::{ - database_description::on_chain::OnChain, - state::StateInitializer, - Database, +use fuel_core::{ + database::{ + database_description::on_chain::OnChain, + state::StateInitializer, + Database, + }, + state::{ + historical_rocksdb::StateRewindPolicy, + rocks_db::{ + ColumnsPolicy, + DatabaseConfig, + }, + }, }; use fuel_core_storage::{ transactional::{ @@ -71,7 +80,15 @@ fn insert_state_single_contract_database(c: &mut Criterion) { let mut bench_state = |group: &mut BenchmarkGroup, name: &str, n: usize| { group.bench_function(name, |b| { - let mut db = Database::::default(); + let mut db = Database::::rocksdb_temp( + StateRewindPolicy::NoRewind, + DatabaseConfig { + cache_capacity: Some(16 * 1024 * 1024 * 1024), + max_fds: -1, + columns_policy: ColumnsPolicy::OnCreation, + }, + ) + .unwrap(); let contract: ContractId = rng.gen(); setup(&mut db, &contract, n); let outer = db.write_transaction(); diff --git a/benches/benches/transaction_throughput.rs b/benches/benches/transaction_throughput.rs index 5d78818e8da..5e37c3af6a5 100644 --- a/benches/benches/transaction_throughput.rs +++ b/benches/benches/transaction_throughput.rs @@ -9,7 +9,16 @@ use criterion::{ SamplingMode, }; use ed25519_dalek::Signer; -use fuel_core::service::config::Trigger; +use fuel_core::{ + service::{ + config::Trigger, + DbType, + }, + state::rocks_db::{ + ColumnsPolicy, + DatabaseConfig, + }, +}; use fuel_core_benches::*; use fuel_core_storage::transactional::AtomicView; use fuel_core_types::{ @@ -90,6 +99,12 @@ where test_builder.utxo_validation = true; test_builder.gas_limit = Some(10_000_000_000); test_builder.block_size_limit = Some(1_000_000_000_000); + test_builder.database_type = DbType::RocksDb; + test_builder.database_config = DatabaseConfig { + cache_capacity: Some(16 * 1024 * 1024 * 1024), + max_fds: -1, + columns_policy: ColumnsPolicy::OnCreation, + }; // spin up node let transactions: Vec = diff --git a/benches/benches/vm_set/blockchain.rs b/benches/benches/vm_set/blockchain.rs index cd51bf6dbc9..86d638a06bd 100644 --- a/benches/benches/vm_set/blockchain.rs +++ b/benches/benches/vm_set/blockchain.rs @@ -20,7 +20,13 @@ use fuel_core::{ GenesisDatabase, }, service::Config, - state::historical_rocksdb::HistoricalRocksDB, + state::{ + historical_rocksdb::HistoricalRocksDB, + rocks_db::{ + ColumnsPolicy, + DatabaseConfig, + }, + }, }; use fuel_core_benches::*; use fuel_core_storage::{ @@ -73,9 +79,12 @@ impl BenchDb { let db = HistoricalRocksDB::::default_open( tmp_dir.path(), - None, Default::default(), - -1, + DatabaseConfig { + cache_capacity: None, + max_fds: -1, + columns_policy: ColumnsPolicy::OnCreation, + }, ) .unwrap(); let db = Arc::new(db); diff --git a/benches/src/db_lookup_times_utils/utils.rs b/benches/src/db_lookup_times_utils/utils.rs index 72f376e1e7e..1cbd2b0f7f7 100644 --- a/benches/src/db_lookup_times_utils/utils.rs +++ b/benches/src/db_lookup_times_utils/utils.rs @@ -5,7 +5,11 @@ use crate::db_lookup_times_utils::full_block_table::{ use anyhow::anyhow; use fuel_core::{ database::database_description::DatabaseDescription, - state::rocks_db::RocksDb, + state::rocks_db::{ + ColumnsPolicy, + DatabaseConfig, + RocksDb, + }, }; use fuel_core_storage::kv_store::{ KeyValueInspect, @@ -39,7 +43,14 @@ pub fn get_random_block_height( pub fn open_rocks_db( path: &Path, ) -> Result> { - let db = RocksDb::default_open(path, None, -1)?; + let db = RocksDb::default_open( + path, + DatabaseConfig { + cache_capacity: Some(16 * 1024 * 1024 * 1024), + max_fds: -1, + columns_policy: ColumnsPolicy::OnCreation, + }, + )?; Ok(db) } diff --git a/bin/fuel-core/src/cli/rollback.rs b/bin/fuel-core/src/cli/rollback.rs index 12e18a0547b..52b7ea3b2b8 100644 --- a/bin/fuel-core/src/cli/rollback.rs +++ b/bin/fuel-core/src/cli/rollback.rs @@ -3,7 +3,10 @@ use anyhow::Context; use clap::Parser; use fuel_core::{ combined_database::CombinedDatabase, - state::historical_rocksdb::StateRewindPolicy, + state::{ + historical_rocksdb::StateRewindPolicy, + rocks_db::DatabaseConfig, + }, }; use rlimit::{ getrlimit, @@ -51,9 +54,12 @@ pub async fn exec(command: Command) -> anyhow::Result<()> { let path = command.database_path.as_path(); let db = CombinedDatabase::open( path, - 64 * 1024 * 1024, StateRewindPolicy::RewindFullRange, - command.rocksdb_max_fds, + DatabaseConfig { + cache_capacity: Some(64 * 1024 * 1024), + max_fds: command.rocksdb_max_fds, + columns_policy: Default::default(), + }, ) .map_err(Into::::into) .context(format!("failed to open combined database at path {path:?}"))?; diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 06d00219b50..0b1b3964b59 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -34,6 +34,10 @@ use fuel_core::{ RelayerConsensusConfig, VMConfig, }, + state::rocks_db::{ + ColumnsPolicy, + DatabaseConfig, + }, txpool::config::{ BlackList, Config as TxPoolConfig, @@ -85,8 +89,6 @@ use tracing::{ #[cfg(feature = "rocksdb")] use fuel_core::state::historical_rocksdb::StateRewindPolicy; -use super::DEFAULT_DATABASE_CACHE_SIZE; - #[cfg(feature = "p2p")] mod p2p; @@ -105,12 +107,8 @@ pub struct Command { pub service_name: String, /// The maximum database cache size in bytes. - #[arg( - long = "max-database-cache-size", - default_value_t = DEFAULT_DATABASE_CACHE_SIZE, - env - )] - pub max_database_cache_size: usize, + #[arg(long = "max-database-cache-size", env)] + pub max_database_cache_size: Option, #[clap( name = "DB_PATH", @@ -456,11 +454,17 @@ impl Command { let combined_db_config = CombinedDatabaseConfig { database_path, database_type, - max_database_cache_size, #[cfg(feature = "rocksdb")] - state_rewind_policy, + database_config: DatabaseConfig { + max_fds: rocksdb_max_fds, + cache_capacity: max_database_cache_size, + #[cfg(feature = "production")] + columns_policy: ColumnsPolicy::OnCreation, + #[cfg(not(feature = "production"))] + columns_policy: ColumnsPolicy::Lazy, + }, #[cfg(feature = "rocksdb")] - max_fds: rocksdb_max_fds, + state_rewind_policy, }; let block_importer = fuel_core::service::config::fuel_core_importer::Config::new( diff --git a/bin/fuel-core/src/cli/snapshot.rs b/bin/fuel-core/src/cli/snapshot.rs index 3f562e38733..060dd9a87a2 100644 --- a/bin/fuel-core/src/cli/snapshot.rs +++ b/bin/fuel-core/src/cli/snapshot.rs @@ -6,7 +6,13 @@ use clap::{ }; use fuel_core::{ combined_database::CombinedDatabase, - state::historical_rocksdb::StateRewindPolicy, + state::{ + historical_rocksdb::StateRewindPolicy, + rocks_db::{ + ColumnsPolicy, + DatabaseConfig, + }, + }, types::fuel_types::ContractId, }; use fuel_core_chain_config::ChainConfig; @@ -209,9 +215,12 @@ fn open_db( ) -> anyhow::Result { CombinedDatabase::open( path, - capacity.unwrap_or(1024 * 1024 * 1024), StateRewindPolicy::NoRewind, - max_fds, + DatabaseConfig { + cache_capacity: Some(capacity.unwrap_or(1024 * 1024 * 1024)), + max_fds, + columns_policy: ColumnsPolicy::OnCreation, + }, ) .map_err(Into::::into) .context(format!("failed to open combined database at path {path:?}",)) diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index c0b6d291af1..b8ff591e274 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -1,5 +1,9 @@ #[cfg(feature = "rocksdb")] -use crate::state::historical_rocksdb::StateRewindPolicy; +use crate::state::{ + historical_rocksdb::StateRewindPolicy, + rocks_db::DatabaseConfig, +}; + use crate::{ database::{ database_description::{ @@ -36,11 +40,9 @@ use std::path::PathBuf; pub struct CombinedDatabaseConfig { pub database_path: PathBuf, pub database_type: DbType, - pub max_database_cache_size: usize, #[cfg(feature = "rocksdb")] + pub database_config: DatabaseConfig, pub state_rewind_policy: StateRewindPolicy, - #[cfg(feature = "rocksdb")] - pub max_fds: i32, } /// A database that combines the on-chain, off-chain and relayer databases into one entity. @@ -79,24 +81,49 @@ impl CombinedDatabase { #[cfg(feature = "rocksdb")] pub fn open( path: &std::path::Path, - capacity: usize, state_rewind_policy: StateRewindPolicy, - max_fds: i32, + database_config: DatabaseConfig, ) -> crate::database::Result { // Split the fds in equitable manner between the databases - let max_fds = match max_fds { + + let max_fds = match database_config.max_fds { -1 => -1, - _ => max_fds.saturating_div(4), + _ => database_config.max_fds.saturating_div(4), }; + // TODO: Use different cache sizes for different databases - let on_chain = - Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?; - let off_chain = - Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?; - let relayer = - Database::open_rocksdb(path, capacity, StateRewindPolicy::NoRewind, max_fds)?; - let gas_price = - Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?; + let on_chain = Database::open_rocksdb( + path, + state_rewind_policy, + DatabaseConfig { + max_fds, + ..database_config + }, + )?; + let off_chain = Database::open_rocksdb( + path, + state_rewind_policy, + DatabaseConfig { + max_fds, + ..database_config + }, + )?; + let relayer = Database::open_rocksdb( + path, + StateRewindPolicy::NoRewind, + DatabaseConfig { + max_fds, + ..database_config + }, + )?; + let gas_price = Database::open_rocksdb( + path, + state_rewind_policy, + DatabaseConfig { + max_fds, + ..database_config + }, + )?; Ok(Self { on_chain, off_chain, @@ -109,10 +136,11 @@ impl CombinedDatabase { #[cfg(feature = "rocksdb")] pub fn temp_database_with_state_rewind_policy( state_rewind_policy: StateRewindPolicy, + database_config: DatabaseConfig, ) -> DatabaseResult { Ok(Self { - on_chain: Database::rocksdb_temp(state_rewind_policy)?, - off_chain: Database::rocksdb_temp(state_rewind_policy)?, + on_chain: Database::rocksdb_temp(state_rewind_policy, database_config)?, + off_chain: Database::rocksdb_temp(state_rewind_policy, database_config)?, relayer: Default::default(), gas_price: Default::default(), }) @@ -129,19 +157,19 @@ impl CombinedDatabase { ); CombinedDatabase::temp_database_with_state_rewind_policy( config.state_rewind_policy, + config.database_config, )? } else { tracing::info!( - "Opening database {:?} with cache size \"{}\" and state rewind policy \"{:?}\"", + "Opening database {:?} with cache size \"{:?}\" and state rewind policy \"{:?}\"", config.database_path, - config.max_database_cache_size, + config.database_config.cache_capacity, config.state_rewind_policy, ); CombinedDatabase::open( &config.database_path, - config.max_database_cache_size, config.state_rewind_policy, - config.max_fds, + config.database_config, )? } } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 41d0451f221..7def87dd3ff 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -78,7 +78,10 @@ use crate::state::{ HistoricalRocksDB, StateRewindPolicy, }, - rocks_db::RocksDb, + rocks_db::{ + DatabaseConfig, + RocksDb, + }, }; #[cfg(feature = "rocksdb")] use std::path::Path; @@ -201,16 +204,15 @@ where #[cfg(feature = "rocksdb")] pub fn open_rocksdb( path: &Path, - capacity: impl Into>, state_rewind_policy: StateRewindPolicy, - max_fds: i32, + database_config: DatabaseConfig, ) -> Result { use anyhow::Context; + let db = HistoricalRocksDB::::default_open( path, - capacity.into(), state_rewind_policy, - max_fds, + database_config, ) .map_err(Into::::into) .with_context(|| { @@ -255,9 +257,14 @@ where } #[cfg(feature = "rocksdb")] - pub fn rocksdb_temp(rewind_policy: StateRewindPolicy) -> Result { - let db = RocksDb::>::default_open_temp(None)?; - let historical_db = HistoricalRocksDB::new(db, rewind_policy)?; + pub fn rocksdb_temp( + state_rewind_policy: StateRewindPolicy, + database_config: DatabaseConfig, + ) -> Result { + let db = RocksDb::>::default_open_temp_with_params( + database_config, + )?; + let historical_db = HistoricalRocksDB::new(db, state_rewind_policy)?; let data = Arc::new(historical_db); Ok(Self::from_storage(DataSource::new(data, Stage::default()))) } @@ -278,8 +285,15 @@ where } #[cfg(feature = "rocksdb")] { - Self::rocksdb_temp(StateRewindPolicy::NoRewind) - .expect("Failed to create a temporary database") + Self::rocksdb_temp( + StateRewindPolicy::NoRewind, + DatabaseConfig { + cache_capacity: None, + max_fds: 512, + columns_policy: Default::default(), + }, + ) + .expect("Failed to create a temporary database") } } } @@ -1107,9 +1121,8 @@ mod tests { let db = Database::::open_rocksdb( temp_dir.path(), - 1024 * 1024 * 1024, Default::default(), - 512, + DatabaseConfig::config_for_tests(), ) .unwrap(); // rocks db fails diff --git a/crates/fuel-core/src/graphql_api/indexation.rs b/crates/fuel-core/src/graphql_api/indexation.rs index 1b490c2f13d..d515504863c 100644 --- a/crates/fuel-core/src/graphql_api/indexation.rs +++ b/crates/fuel-core/src/graphql_api/indexation.rs @@ -238,6 +238,7 @@ mod tests { MessageBalances, }, }, + state::rocks_db::DatabaseConfig, }; impl PartialEq for IndexationError { @@ -352,9 +353,12 @@ mod tests { fn balances_enabled_flag_is_respected() { use tempfile::TempDir; let tmp_dir = TempDir::new().unwrap(); - let mut db: Database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) - .unwrap(); + let mut db: Database = Database::open_rocksdb( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let mut tx = db.write_transaction(); const BALANCES_ARE_DISABLED: bool = false; @@ -409,9 +413,12 @@ mod tests { fn coins() { use tempfile::TempDir; let tmp_dir = TempDir::new().unwrap(); - let mut db: Database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) - .unwrap(); + let mut db: Database = Database::open_rocksdb( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let mut tx = db.write_transaction(); const BALANCES_ARE_ENABLED: bool = true; @@ -481,9 +488,12 @@ mod tests { fn messages() { use tempfile::TempDir; let tmp_dir = TempDir::new().unwrap(); - let mut db: Database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) - .unwrap(); + let mut db: Database = Database::open_rocksdb( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let mut tx = db.write_transaction(); const BALANCES_ARE_ENABLED: bool = true; @@ -589,9 +599,12 @@ mod tests { fn coin_balance_overflow_does_not_error() { use tempfile::TempDir; let tmp_dir = TempDir::new().unwrap(); - let mut db: Database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) - .unwrap(); + let mut db: Database = Database::open_rocksdb( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let mut tx = db.write_transaction(); const BALANCES_ARE_ENABLED: bool = true; @@ -623,9 +636,12 @@ mod tests { fn message_balance_overflow_does_not_error() { use tempfile::TempDir; let tmp_dir = TempDir::new().unwrap(); - let mut db: Database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) - .unwrap(); + let mut db: Database = Database::open_rocksdb( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let mut tx = db.write_transaction(); const BALANCES_ARE_ENABLED: bool = true; @@ -661,9 +677,12 @@ mod tests { fn coin_balance_underflow_causes_error() { use tempfile::TempDir; let tmp_dir = TempDir::new().unwrap(); - let mut db: Database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) - .unwrap(); + let mut db: Database = Database::open_rocksdb( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let mut tx = db.write_transaction(); const BALANCES_ARE_ENABLED: bool = true; diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index 0e093c2b8b6..1756ead3b32 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -104,6 +104,8 @@ impl Config { #[cfg(feature = "test-helpers")] pub fn local_node_with_reader(snapshot_reader: SnapshotReader) -> Self { + use crate::state::rocks_db::DatabaseConfig; + let block_importer = fuel_core_importer::Config::new(false); let latest_block = snapshot_reader.last_block_config(); // In tests, we always want to use the native executor as a default configuration. @@ -117,7 +119,12 @@ impl Config { let combined_db_config = CombinedDatabaseConfig { // Set the cache for tests = 10MB - max_database_cache_size: 10 * 1024 * 1024, + #[cfg(feature = "rocksdb")] + database_config: DatabaseConfig { + cache_capacity: Some(10 * 1024 * 1024), + columns_policy: Default::default(), + max_fds: 512, + }, database_path: Default::default(), #[cfg(feature = "rocksdb")] database_type: DbType::RocksDb, @@ -126,8 +133,6 @@ impl Config { #[cfg(feature = "rocksdb")] state_rewind_policy: crate::state::historical_rocksdb::StateRewindPolicy::RewindFullRange, - #[cfg(feature = "rocksdb")] - max_fds: 512, }; let starting_gas_price = 0; let gas_price_change_percent = 0; @@ -234,7 +239,7 @@ pub struct VMConfig { } #[derive( - Clone, Debug, Display, Eq, PartialEq, EnumString, EnumVariantNames, ValueEnum, + Clone, Copy, Debug, Display, Eq, PartialEq, EnumString, EnumVariantNames, ValueEnum, )] #[strum(serialize_all = "kebab_case")] pub enum DbType { diff --git a/crates/fuel-core/src/state/historical_rocksdb.rs b/crates/fuel-core/src/state/historical_rocksdb.rs index 059fb35f12b..371ca99cce0 100644 --- a/crates/fuel-core/src/state/historical_rocksdb.rs +++ b/crates/fuel-core/src/state/historical_rocksdb.rs @@ -64,6 +64,8 @@ use std::{ path::Path, }; +use super::rocks_db::DatabaseConfig; + pub mod description; pub mod modifications_history; pub mod view_at_height; @@ -106,12 +108,10 @@ where pub fn default_open>( path: P, - capacity: Option, state_rewind_policy: StateRewindPolicy, - max_fds: i32, + database_config: DatabaseConfig, ) -> DatabaseResult { - let db = - RocksDb::>::default_open(path, capacity, max_fds)?; + let db = RocksDb::>::default_open(path, database_config)?; Ok(Self { state_rewind_policy, db, diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 3df5c7127e5..518ff6590c5 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -96,10 +96,40 @@ impl Drop for DropResources { } } +#[derive(Clone, Copy, Default, Debug, PartialEq, Eq)] +/// Defined behaviour for opening the columns of the database. +pub enum ColumnsPolicy { + #[cfg_attr(not(feature = "rocksdb-production"), default)] + // Open a new column only when a database interaction is done with it. + Lazy, + #[cfg_attr(feature = "rocksdb-production", default)] + // Open all columns on creation on the service. + OnCreation, +} + +/// Configuration to create a database +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DatabaseConfig { + pub cache_capacity: Option, + pub max_fds: i32, + pub columns_policy: ColumnsPolicy, +} + +#[cfg(feature = "test-helpers")] +impl DatabaseConfig { + pub fn config_for_tests() -> Self { + Self { + cache_capacity: None, + max_fds: 512, + columns_policy: ColumnsPolicy::Lazy, + } + } +} + pub struct RocksDb { read_options: ReadOptions, db: Arc, - create_family: Arc>>, + create_family: Option>>>, snapshot: Option>, metrics: Arc, // used for RAII @@ -126,13 +156,22 @@ where Description: DatabaseDescription, { pub fn default_open_temp(capacity: Option) -> DatabaseResult { + Self::default_open_temp_with_params(DatabaseConfig { + cache_capacity: capacity, + max_fds: 512, + columns_policy: Default::default(), + }) + } + + pub fn default_open_temp_with_params( + database_config: DatabaseConfig, + ) -> DatabaseResult { let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path(); let result = Self::open( path, enum_iterator::all::().collect::>(), - capacity, - 512, + database_config, ); let mut db = result?; @@ -151,14 +190,12 @@ where pub fn default_open>( path: P, - capacity: Option, - max_fds: i32, + database_config: DatabaseConfig, ) -> DatabaseResult { Self::open( path, enum_iterator::all::().collect::>(), - capacity, - max_fds, + database_config, ) } @@ -172,18 +209,16 @@ where pub fn open>( path: P, columns: Vec, - capacity: Option, - max_fds: i32, + database_config: DatabaseConfig, ) -> DatabaseResult { - Self::open_with(DB::open_cf_descriptors, path, columns, capacity, max_fds) + Self::open_with(DB::open_cf_descriptors, path, columns, database_config) } pub fn open_read_only>( path: P, columns: Vec, - capacity: Option, error_if_log_file_exist: bool, - max_fds: i32, + database_config: DatabaseConfig, ) -> DatabaseResult { Self::open_with( |options, primary_path, cfs| { @@ -196,8 +231,7 @@ where }, path, columns, - capacity, - max_fds, + database_config, ) } @@ -205,8 +239,7 @@ where path: PrimaryPath, secondary_path: SecondaryPath, columns: Vec, - capacity: Option, - max_fds: i32, + database_config: DatabaseConfig, ) -> DatabaseResult where PrimaryPath: AsRef, @@ -223,8 +256,7 @@ where }, path, columns, - capacity, - max_fds, + database_config, ) } @@ -232,8 +264,7 @@ where opener: F, path: P, columns: Vec, - capacity: Option, - max_fds: i32, + database_config: DatabaseConfig, ) -> DatabaseResult where F: Fn( @@ -257,7 +288,7 @@ where // See https://github.com/facebook/rocksdb/blob/a1523efcdf2f0e8133b9a9f6e170a0dad49f928f/include/rocksdb/table.h#L246-L271 for details on what the format versions are/do. block_opts.set_format_version(5); - if let Some(capacity) = capacity { + if let Some(capacity) = database_config.cache_capacity { // Set cache size 1/3 of the capacity as recommended by // https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#block-cache-size let block_cache_size = capacity / 3; @@ -281,7 +312,7 @@ where let cpu_number = i32::try_from(num_cpus::get()).expect("The number of CPU can't exceed `i32`"); opts.increase_parallelism(cmp::max(1, cpu_number / 2)); - if let Some(capacity) = capacity { + if let Some(capacity) = database_config.cache_capacity { // Set cache size 1/3 of the capacity. Another 1/3 is // used by block cache and the last 1 / 3 remains for other purposes: // @@ -292,7 +323,7 @@ where } opts.set_max_background_jobs(6); opts.set_bytes_per_sync(1048576); - opts.set_max_open_files(max_fds); + opts.set_max_open_files(database_config.max_fds); let existing_column_families = DB::list_cf(&opts, &path).unwrap_or_default(); @@ -308,7 +339,10 @@ where } } - if cf_descriptors_to_open.is_empty() { + if database_config.columns_policy == ColumnsPolicy::OnCreation + || (database_config.columns_policy == ColumnsPolicy::Lazy + && cf_descriptors_to_open.is_empty()) + { opts.create_if_missing(true); } @@ -349,8 +383,17 @@ where } .map_err(|e| DatabaseError::Other(e.into()))?; + let create_family = match database_config.columns_policy { + ColumnsPolicy::OnCreation => { + for (name, opt) in cf_descriptors_to_create { + db.create_cf(name, &opt) + .map_err(|e| DatabaseError::Other(e.into()))?; + } + None + } + ColumnsPolicy::Lazy => Some(Arc::new(Mutex::new(cf_descriptors_to_create))), + }; let db = Arc::new(db); - let create_family = Arc::new(Mutex::new(cf_descriptors_to_create)); let rocks_db = RocksDb { read_options: Self::generate_read_options(&None), @@ -424,26 +467,29 @@ where match family { None => { - let mut lock = self - .create_family - .lock() - .expect("The create family lock should be available"); - - let name = Self::col_name(column); - let Some(family) = lock.remove(&name) else { - return self - .db - .cf_handle(&Self::col_name(column)) - .expect("No column family found"); - }; - - self.db - .create_cf(&name, &family) - .expect("Couldn't create column family"); - - let family = self.db.cf_handle(&name).expect("invalid column state"); - - family + if let Some(create_family) = &self.create_family { + let mut lock = create_family + .lock() + .expect("The create family lock should be available"); + + let name = Self::col_name(column); + let Some(family) = lock.remove(&name) else { + return self + .db + .cf_handle(&Self::col_name(column)) + .expect("No column family found"); + }; + + self.db + .create_cf(&name, &family) + .expect("Couldn't create column family"); + + let family = self.db.cf_handle(&name).expect("invalid column state"); + + family + } else { + panic!("Columns in the DB should have been created on DB opening"); + } } Some(family) => family, } @@ -926,7 +972,8 @@ mod tests { fn create_db() -> (RocksDb, TempDir) { let tmp_dir = TempDir::new().unwrap(); ( - RocksDb::default_open(tmp_dir.path(), None, 512).unwrap(), + RocksDb::default_open(tmp_dir.path(), DatabaseConfig::config_for_tests()) + .unwrap(), tmp_dir, ) } @@ -938,17 +985,24 @@ mod tests { // Given let old_columns = vec![Column::Coins, Column::Messages, Column::UploadedBytecodes]; - let database_with_old_columns = - RocksDb::::open(tmp_dir.path(), old_columns.clone(), None, 512) - .expect("Failed to open database with old columns"); + let database_with_old_columns = RocksDb::::open( + tmp_dir.path(), + old_columns.clone(), + DatabaseConfig::config_for_tests(), + ) + .expect("Failed to open database with old columns"); drop(database_with_old_columns); // When let mut new_columns = old_columns; new_columns.push(Column::ContractsAssets); new_columns.push(Column::Metadata); - let database_with_new_columns = - RocksDb::::open(tmp_dir.path(), new_columns, None, 512).map(|_| ()); + let database_with_new_columns = RocksDb::::open( + tmp_dir.path(), + new_columns, + DatabaseConfig::config_for_tests(), + ) + .map(|_| ()); // Then assert_eq!(Ok(()), database_with_new_columns); @@ -1117,7 +1171,11 @@ mod tests { // When let columns = enum_iterator::all::<::Column>() .collect::>(); - let result = RocksDb::::open(tmp_dir.path(), columns, None, 512); + let result = RocksDb::::open( + tmp_dir.path(), + columns, + DatabaseConfig::config_for_tests(), + ); // Then assert!(result.is_err()); @@ -1134,9 +1192,8 @@ mod tests { let result = RocksDb::::open_read_only( tmp_dir.path(), old_columns.clone(), - None, false, - 512, + DatabaseConfig::config_for_tests(), ) .map(|_| ()); @@ -1157,8 +1214,7 @@ mod tests { tmp_dir.path(), secondary_temp.path(), old_columns.clone(), - None, - 512, + DatabaseConfig::config_for_tests(), ) .map(|_| ()); @@ -1250,8 +1306,11 @@ mod tests { enum_iterator::all::<::Column>() .skip(1) .collect::>(); - let open_with_part_of_columns = - RocksDb::::open(tmp_dir.path(), part_of_columns, None, 512); + let open_with_part_of_columns = RocksDb::::open( + tmp_dir.path(), + part_of_columns, + DatabaseConfig::config_for_tests(), + ); // Then let _ = open_with_part_of_columns diff --git a/tests/test-helpers/src/builder.rs b/tests/test-helpers/src/builder.rs index 0d16bc48dc6..fa23f8608a1 100644 --- a/tests/test-helpers/src/builder.rs +++ b/tests/test-helpers/src/builder.rs @@ -13,6 +13,7 @@ use fuel_core::{ DbType, FuelService, }, + state::rocks_db::DatabaseConfig, }; use fuel_core_client::client::FuelClient; use fuel_core_poa::Trigger; @@ -99,6 +100,8 @@ pub struct TestSetupBuilder { pub privileged_address: Address, pub base_asset_id: AssetId, pub trigger: Trigger, + pub database_type: DbType, + pub database_config: DatabaseConfig, } impl TestSetupBuilder { @@ -231,14 +234,14 @@ impl TestSetupBuilder { ..StateConfig::default() }; - let config = Config { + let mut config = Config { utxo_validation: self.utxo_validation, txpool: fuel_core_txpool::config::Config::default(), block_production: self.trigger, starting_gas_price: self.starting_gas_price, ..Config::local_node_with_configs(chain_conf, state) }; - assert_eq!(config.combined_db_config.database_type, DbType::RocksDb); + config.combined_db_config.database_config = self.database_config; let srv = FuelService::new_node(config).await.unwrap(); let client = FuelClient::from(srv.bound_address); @@ -265,6 +268,8 @@ impl Default for TestSetupBuilder { privileged_address: Default::default(), base_asset_id: AssetId::BASE, trigger: Trigger::Instant, + database_type: DbType::RocksDb, + database_config: DatabaseConfig::config_for_tests(), } } } diff --git a/tests/tests/aws_kms.rs b/tests/tests/aws_kms.rs index 31dcb8e0448..e832a4afc2c 100644 --- a/tests/tests/aws_kms.rs +++ b/tests/tests/aws_kms.rs @@ -1,4 +1,7 @@ -use fuel_core::combined_database::CombinedDatabase; +use fuel_core::{ + combined_database::CombinedDatabase, + state::rocks_db::DatabaseConfig, +}; use fuel_core_storage::transactional::AtomicView; use fuel_core_types::blockchain::consensus::Consensus; use test_helpers::fuel_core_driver::FuelCoreDriver; @@ -47,8 +50,12 @@ async fn can_get_sealed_block_from_poa_produced_block_when_signing_with_kms() { // stop the node and just grab the database let db_path = driver.kill().await; - let db = CombinedDatabase::open(db_path.path(), 1024 * 1024, Default::default(), 512) - .unwrap(); + let db = CombinedDatabase::open( + db_path.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let view = db.on_chain().latest_view().unwrap(); diff --git a/tests/tests/health.rs b/tests/tests/health.rs index d0b241fcb5a..231628493f1 100644 --- a/tests/tests/health.rs +++ b/tests/tests/health.rs @@ -5,6 +5,7 @@ use fuel_core::{ Config, FuelService, }, + state::rocks_db::DatabaseConfig, types::fuel_tx::Transaction, }; use fuel_core_client::client::FuelClient; @@ -28,9 +29,12 @@ async fn can_restart_node() { // start node once { - let database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) - .unwrap(); + let database = Database::open_rocksdb( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let first_startup = FuelService::from_database(database, Config::local_node()) .await .unwrap(); @@ -41,9 +45,12 @@ async fn can_restart_node() { } { - let database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) - .unwrap(); + let database = Database::open_rocksdb( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let _second_startup = FuelService::from_database(database, Config::local_node()) .await .unwrap(); @@ -52,14 +59,16 @@ async fn can_restart_node() { #[tokio::test] async fn can_restart_node_with_transactions() { - let capacity = 1024 * 1024; let tmp_dir = tempfile::TempDir::new().unwrap(); { // Given - let database = - CombinedDatabase::open(tmp_dir.path(), capacity, Default::default(), 512) - .unwrap(); + let database = CombinedDatabase::open( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let service = FuelService::from_combined_database(database, Config::local_node()) .await .unwrap(); @@ -76,9 +85,12 @@ async fn can_restart_node_with_transactions() { { // When - let database = - CombinedDatabase::open(tmp_dir.path(), capacity, Default::default(), 512) - .unwrap(); + let database = CombinedDatabase::open( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let service = FuelService::from_combined_database(database, Config::local_node()) .await .unwrap(); diff --git a/tests/tests/relayer.rs b/tests/tests/relayer.rs index e92e335b3fc..1e9c2dd542f 100644 --- a/tests/tests/relayer.rs +++ b/tests/tests/relayer.rs @@ -15,6 +15,7 @@ use fuel_core::{ Config, FuelService, }, + state::rocks_db::DatabaseConfig, }; use fuel_core_client::client::{ pagination::{ @@ -332,14 +333,16 @@ async fn can_restart_node_with_relayer_data() { .try_into() .unwrap()]); - let capacity = 1024 * 1024; let tmp_dir = tempfile::TempDir::new().unwrap(); { // Given - let database = - CombinedDatabase::open(tmp_dir.path(), capacity, Default::default(), 512) - .unwrap(); + let database = CombinedDatabase::open( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let service = FuelService::from_combined_database(database, config.clone()) .await @@ -357,9 +360,12 @@ async fn can_restart_node_with_relayer_data() { { // When - let database = - CombinedDatabase::open(tmp_dir.path(), capacity, Default::default(), 512) - .unwrap(); + let database = CombinedDatabase::open( + tmp_dir.path(), + Default::default(), + DatabaseConfig::config_for_tests(), + ) + .unwrap(); let service = FuelService::from_combined_database(database, config) .await .unwrap();