Merge branch 'master' into cargo_update

CHANGELOG.md

@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - [2416](https://github.com/FuelLabs/fuel-core/issues/2416): Define the `GasPriceServiceV1` task.
 - [2033](https://github.com/FuelLabs/fuel-core/pull/2033): Remove `Option<BlockHeight>` in favor of `BlockHeightQuery` where applicable.
 - [2472](https://github.com/FuelLabs/fuel-core/pull/2472): Added the `amountU128` field to the `Balance` GraphQL schema, providing the total balance as a `U128`. The existing `amount` field clamps any balance exceeding `U64` to `u64::MAX`.
+- [2526](https://github.com/FuelLabs/fuel-core/pull/2526): Add possibility to not have any cache set for RocksDB. Add an option to either load the RocksDB columns families on creation of the database or when the column is used.
 
 ### Fixed
 - [2365](https://github.com/FuelLabs/fuel-core/pull/2365): Fixed the error during dry run in the case of race condition.
@@ -57,6 +58,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - [2154](https://github.com/FuelLabs/fuel-core/pull/2154): Transaction graphql endpoints use `TransactionType` instead of `fuel_tx::Transaction`.
 - [2446](https://github.com/FuelLabs/fuel-core/pull/2446): Use graphiql instead of graphql-playground due to known vulnerability and stale development.
 - [2379](https://github.com/FuelLabs/fuel-core/issues/2379): Change `kv_store::Value` to be `Arc<[u8]>` instead of `Arc<Vec<u8>>`.
+- [2526](https://github.com/FuelLabs/fuel-core/pull/2526): By default the cache of RocksDB is now disabled instead of being `1024 * 1024 * 1024`.
 
 ## [Version 0.40.2]
 

benches/Cargo.toml

@@ -63,7 +63,7 @@ harness = false
 name = "vm"
 
 [features]
-default = ["fuel-core/rocksdb"]
+default = ["fuel-core/rocksdb", "fuel-core/rocksdb-production"]
 
 [[bench]]
 harness = false

benches/benches/block_target_gas.rs

@@ -27,7 +27,13 @@ use fuel_core::{
         Config,
         FuelService,
     },
-    state::historical_rocksdb::StateRewindPolicy,
+    state::{
+        historical_rocksdb::StateRewindPolicy,
+        rocks_db::{
+            ColumnsPolicy,
+            DatabaseConfig,
+        },
+    },
 };
 use fuel_core_benches::{
     default_gas_costs::default_gas_costs,
@@ -266,8 +272,15 @@ fn service_with_many_contracts(
         .build()
         .unwrap();
     let _drop = rt.enter();
-    let mut database = Database::rocksdb_temp(StateRewindPolicy::NoRewind)
-        .expect("Failed to create database");
+    let mut database = Database::rocksdb_temp(
+        StateRewindPolicy::NoRewind,
+        DatabaseConfig {
+            cache_capacity: Some(16 * 1024 * 1024 * 1024),
+            max_fds: -1,
+            columns_policy: ColumnsPolicy::OnCreation,
+        },
+    )
+    .expect("Failed to create database");
 
     let mut chain_config = ChainConfig::local_testnet();
 

benches/benches/state.rs

@@ -5,10 +5,19 @@ use criterion::{
     BenchmarkGroup,
     Criterion,
 };
-use fuel_core::database::{
-    database_description::on_chain::OnChain,
-    state::StateInitializer,
-    Database,
+use fuel_core::{
+    database::{
+        database_description::on_chain::OnChain,
+        state::StateInitializer,
+        Database,
+    },
+    state::{
+        historical_rocksdb::StateRewindPolicy,
+        rocks_db::{
+            ColumnsPolicy,
+            DatabaseConfig,
+        },
+    },
 };
 use fuel_core_storage::{
     transactional::{
@@ -71,7 +80,15 @@ fn insert_state_single_contract_database(c: &mut Criterion) {
 
     let mut bench_state = |group: &mut BenchmarkGroup<WallTime>, name: &str, n: usize| {
         group.bench_function(name, |b| {
-            let mut db = Database::<OnChain>::default();
+            let mut db = Database::<OnChain>::rocksdb_temp(
+                StateRewindPolicy::NoRewind,
+                DatabaseConfig {
+                    cache_capacity: Some(16 * 1024 * 1024 * 1024),
+                    max_fds: -1,
+                    columns_policy: ColumnsPolicy::OnCreation,
+                },
+            )
+            .unwrap();
             let contract: ContractId = rng.gen();
             setup(&mut db, &contract, n);
             let outer = db.write_transaction();

benches/benches/transaction_throughput.rs

@@ -9,7 +9,16 @@ use criterion::{
     SamplingMode,
 };
 use ed25519_dalek::Signer;
-use fuel_core::service::config::Trigger;
+use fuel_core::{
+    service::{
+        config::Trigger,
+        DbType,
+    },
+    state::rocks_db::{
+        ColumnsPolicy,
+        DatabaseConfig,
+    },
+};
 use fuel_core_benches::*;
 use fuel_core_storage::transactional::AtomicView;
 use fuel_core_types::{
@@ -90,6 +99,12 @@ where
             test_builder.utxo_validation = true;
             test_builder.gas_limit = Some(10_000_000_000);
             test_builder.block_size_limit = Some(1_000_000_000_000);
+            test_builder.database_type = DbType::RocksDb;
+            test_builder.database_config = DatabaseConfig {
+                cache_capacity: Some(16 * 1024 * 1024 * 1024),
+                max_fds: -1,
+                columns_policy: ColumnsPolicy::OnCreation,
+            };
 
             // spin up node
             let transactions: Vec<Transaction> =

benches/benches/vm_set/blockchain.rs

@@ -20,7 +20,13 @@ use fuel_core::{
         GenesisDatabase,
     },
     service::Config,
-    state::historical_rocksdb::HistoricalRocksDB,
+    state::{
+        historical_rocksdb::HistoricalRocksDB,
+        rocks_db::{
+            ColumnsPolicy,
+            DatabaseConfig,
+        },
+    },
 };
 use fuel_core_benches::*;
 use fuel_core_storage::{
@@ -73,9 +79,12 @@ impl BenchDb {
 
         let db = HistoricalRocksDB::<OnChain>::default_open(
             tmp_dir.path(),
-            None,
             Default::default(),
-            -1,
+            DatabaseConfig {
+                cache_capacity: None,
+                max_fds: -1,
+                columns_policy: ColumnsPolicy::OnCreation,
+            },
         )
         .unwrap();
         let db = Arc::new(db);

benches/src/db_lookup_times_utils/utils.rs

@@ -5,7 +5,11 @@ use crate::db_lookup_times_utils::full_block_table::{
 use anyhow::anyhow;
 use fuel_core::{
     database::database_description::DatabaseDescription,
-    state::rocks_db::RocksDb,
+    state::rocks_db::{
+        ColumnsPolicy,
+        DatabaseConfig,
+        RocksDb,
+    },
 };
 use fuel_core_storage::kv_store::{
     KeyValueInspect,
@@ -39,7 +43,14 @@ pub fn get_random_block_height(
 pub fn open_rocks_db<Description: DatabaseDescription>(
     path: &Path,
 ) -> Result<RocksDb<Description>> {
-    let db = RocksDb::default_open(path, None, -1)?;
+    let db = RocksDb::default_open(
+        path,
+        DatabaseConfig {
+            cache_capacity: Some(16 * 1024 * 1024 * 1024),
+            max_fds: -1,
+            columns_policy: ColumnsPolicy::OnCreation,
+        },
+    )?;
     Ok(db)
 }
 

bin/fuel-core/src/cli/rollback.rs

@@ -3,7 +3,10 @@ use anyhow::Context;
 use clap::Parser;
 use fuel_core::{
     combined_database::CombinedDatabase,
-    state::historical_rocksdb::StateRewindPolicy,
+    state::{
+        historical_rocksdb::StateRewindPolicy,
+        rocks_db::DatabaseConfig,
+    },
 };
 use rlimit::{
     getrlimit,
@@ -51,9 +54,12 @@ pub async fn exec(command: Command) -> anyhow::Result<()> {
     let path = command.database_path.as_path();
     let db = CombinedDatabase::open(
         path,
-        64 * 1024 * 1024,
         StateRewindPolicy::RewindFullRange,
-        command.rocksdb_max_fds,
+        DatabaseConfig {
+            cache_capacity: Some(64 * 1024 * 1024),
+            max_fds: command.rocksdb_max_fds,
+            columns_policy: Default::default(),
+        },
     )
     .map_err(Into::<anyhow::Error>::into)
     .context(format!("failed to open combined database at path {path:?}"))?;

bin/fuel-core/src/cli/run.rs

@@ -34,6 +34,10 @@ use fuel_core::{
         RelayerConsensusConfig,
         VMConfig,
     },
+    state::rocks_db::{
+        ColumnsPolicy,
+        DatabaseConfig,
+    },
     txpool::config::{
         BlackList,
         Config as TxPoolConfig,
@@ -85,8 +89,6 @@ use tracing::{
 #[cfg(feature = "rocksdb")]
 use fuel_core::state::historical_rocksdb::StateRewindPolicy;
 
-use super::DEFAULT_DATABASE_CACHE_SIZE;
-
 #[cfg(feature = "p2p")]
 mod p2p;
 
@@ -105,12 +107,8 @@ pub struct Command {
     pub service_name: String,
 
     /// The maximum database cache size in bytes.
-    #[arg(
-        long = "max-database-cache-size",
-        default_value_t = DEFAULT_DATABASE_CACHE_SIZE,
-        env
-    )]
-    pub max_database_cache_size: usize,
+    #[arg(long = "max-database-cache-size", env)]
+    pub max_database_cache_size: Option<usize>,
 
     #[clap(
         name = "DB_PATH",
@@ -456,11 +454,17 @@ impl Command {
         let combined_db_config = CombinedDatabaseConfig {
             database_path,
             database_type,
-            max_database_cache_size,
             #[cfg(feature = "rocksdb")]
-            state_rewind_policy,
+            database_config: DatabaseConfig {
+                max_fds: rocksdb_max_fds,
+                cache_capacity: max_database_cache_size,
+                #[cfg(feature = "production")]
+                columns_policy: ColumnsPolicy::OnCreation,
+                #[cfg(not(feature = "production"))]
+                columns_policy: ColumnsPolicy::Lazy,
+            },
             #[cfg(feature = "rocksdb")]
-            max_fds: rocksdb_max_fds,
+            state_rewind_policy,
         };
 
         let block_importer = fuel_core::service::config::fuel_core_importer::Config::new(

bin/fuel-core/src/cli/snapshot.rs

@@ -6,7 +6,13 @@ use clap::{
 };
 use fuel_core::{
     combined_database::CombinedDatabase,
-    state::historical_rocksdb::StateRewindPolicy,
+    state::{
+        historical_rocksdb::StateRewindPolicy,
+        rocks_db::{
+            ColumnsPolicy,
+            DatabaseConfig,
+        },
+    },
     types::fuel_types::ContractId,
 };
 use fuel_core_chain_config::ChainConfig;
@@ -209,9 +215,12 @@ fn open_db(
 ) -> anyhow::Result<CombinedDatabase> {
     CombinedDatabase::open(
         path,
-        capacity.unwrap_or(1024 * 1024 * 1024),
         StateRewindPolicy::NoRewind,
-        max_fds,
+        DatabaseConfig {
+            cache_capacity: Some(capacity.unwrap_or(1024 * 1024 * 1024)),
+            max_fds,
+            columns_policy: ColumnsPolicy::OnCreation,
+        },
     )
     .map_err(Into::<anyhow::Error>::into)
     .context(format!("failed to open combined database at path {path:?}",))