diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 1ae04c3976..56784bc6ce 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -168,15 +168,13 @@ namespace golos { while (itr.first.block_num() != last_block_num) { auto cur_block_num = itr.first.block_num(); if (cur_block_num % 100000 == 0) { - std::cerr << " " << double(cur_block_num * 100) / - last_block_num << "% " - << cur_block_num << " of " - << last_block_num << - " (" - << (get_free_memory() / (1024 * 1024)) - << "M free)\n"; + std::cerr + << " " << double(cur_block_num * 100) / last_block_num << "% " + << cur_block_num << " of " << last_block_num + << " (" << (free_memory() / (1024 * 1024)) << "M free)\n"; } apply_block(itr.first, skip_flags); + check_free_memory(true, itr.first.block_num()); itr = _block_log.read_block(itr.second); } @@ -196,6 +194,56 @@ namespace golos { } + void database::min_free_shared_memory_size(size_t value) { + _min_free_shared_memory_size = value; + } + + void database::inc_shared_memory_size(size_t value) { + _inc_shared_memory_size = value; + } + + void database::block_num_check_free_size(uint32_t value) { + _block_num_check_free_memory = value; + } + + void database::check_free_memory(bool skip_print, uint32_t current_block_num) { + if (0 != current_block_num % _block_num_check_free_memory) { + return; + } + + uint64_t free_mem = free_memory(); + uint64_t max_mem = max_memory(); + + if (_inc_shared_memory_size != 0 && _min_free_shared_memory_size != 0 && + free_mem < _min_free_shared_memory_size + ) { + size_t new_max = max_mem + _inc_shared_memory_size; + wlog( + "Memory is almost full on block ${block}, increasing to ${mem}M", + ("block", current_block_num)("mem", new_max / (1024 * 1024))); + resize(new_max); + free_mem = free_memory(); + uint32_t free_mb = uint32_t(free_mem / (1024 * 1024)); + wlog("Free memory is now ${free}M", ("free", free_mb)); + _last_free_gb_printed = free_mb / 1024; + } else if (!skip_print && _inc_shared_memory_size == 0 && _min_free_shared_memory_size == 0) { + uint32_t free_gb = uint32_t(free_mem / (1024 * 1024 * 1024)); + if ((free_gb < _last_free_gb_printed) || (free_gb > _last_free_gb_printed + 1)) { + ilog( + "Free memory is now ${n}G. Current block number: ${block}", + ("n", free_gb)("block", current_block_num)); + _last_free_gb_printed = free_gb; + } + + if (free_gb == 0) { + uint32_t free_mb = uint32_t(free_mem / (1024 * 1024)); + if (free_mb <= 500 && current_block_num % 10 == 0) { + elog("Free memory is now ${n}M. Increase shared file size immediately!", ("n", free_mb)); + } + } + } + } + void database::wipe(const fc::path &data_dir, const fc::path &shared_mem_dir, bool include_blocks) { close(); chainbase::database::wipe(shared_mem_dir); @@ -637,6 +685,8 @@ namespace golos { FC_CAPTURE_AND_RETHROW((new_block)) }); }); + + check_free_memory(false, new_block.block_num()); }); //fc::time_point end_time = fc::time_point::now(); @@ -689,7 +739,7 @@ namespace golos { // ilog( "pushing blocks from fork ${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->data.id()) ); optional except; try { - auto session = start_undo_session(true); + auto session = start_undo_session(); apply_block((*ritr)->data, skip); session.push(); } @@ -715,7 +765,7 @@ namespace golos { for (auto ritr = branches.second.rbegin(); ritr != branches.second.rend(); ++ritr) { - auto session = start_undo_session(true); + auto session = start_undo_session(); apply_block((*ritr)->data, skip); session.push(); } @@ -730,7 +780,7 @@ namespace golos { } try { - auto session = start_undo_session(true); + auto session = start_undo_session(); apply_block(new_block, skip); session.push(); } @@ -771,7 +821,7 @@ namespace golos { // If this is the first transaction pushed after applying a block, start a new undo session. // This allows us to quickly rewind to the clean state of the head block, in case a new block arrives. if (!_pending_tx_session.valid()) { - _pending_tx_session = start_undo_session(true); + _pending_tx_session = start_undo_session(); } // Create a temporary undo session as a child of _pending_tx_session. @@ -779,7 +829,7 @@ namespace golos { // _apply_transaction fails. If we make it to merge(), we // apply the changes. - auto temp_session = start_undo_session(true); + auto temp_session = start_undo_session(); _apply_transaction(trx); _pending_tx.push_back(trx); @@ -843,7 +893,7 @@ namespace golos { // re-apply pending transactions in this method. // _pending_tx_session.reset(); - _pending_tx_session = start_undo_session(true); + _pending_tx_session = start_undo_session(); uint64_t postponed_tx_count = 0; // pop pending state (reset to head block state) @@ -865,7 +915,7 @@ namespace golos { } try { - auto temp_session = start_undo_session(true); + auto temp_session = start_undo_session(); _apply_transaction(tx); temp_session.squash(); @@ -2931,7 +2981,7 @@ namespace golos { void database::validate_transaction(const signed_transaction &trx) { database::with_weak_write_lock([&]() { - auto session = start_undo_session(true); + auto session = start_undo_session(); _apply_transaction(trx); session.undo(); }); @@ -3036,14 +3086,6 @@ namespace golos { } } - uint32_t free_gb = uint32_t( - get_free_memory() / (1024 * 1024 * 1024)); - if ((free_gb < _last_free_gb_printed) || - (free_gb > _last_free_gb_printed + 1)) { - ilog("Free memory is now ${n}G", ("n", free_gb)); - _last_free_gb_printed = free_gb; - } - } FC_CAPTURE_AND_RETHROW((next_block)) } diff --git a/libraries/chain/include/golos/chain/database.hpp b/libraries/chain/include/golos/chain/database.hpp index 9ec7d8a60e..26fb661435 100644 --- a/libraries/chain/include/golos/chain/database.hpp +++ b/libraries/chain/include/golos/chain/database.hpp @@ -97,6 +97,11 @@ namespace golos { void reindex(const fc::path &data_dir, const fc::path &shared_mem_dir, uint64_t shared_file_size = ( 1024l * 1024l * 1024l * 8l)); + void min_free_shared_memory_size(size_t); + void inc_shared_memory_size(size_t); + void block_num_check_free_size(uint32_t); + void check_free_memory(bool skip_print, uint32_t current_block_num); + /** * @brief wipe Delete database from disk, and potentially the raw chain as well. * @param include_blocks If true, delete the raw chain as well as the database. @@ -121,7 +126,7 @@ namespace golos { uint32_t get_pow_summary_target() const; - block_id_type get_block_id_for_num( uint32_t block_num )const; + block_id_type get_block_id_for_num( uint32_t block_num )const; block_id_type find_block_id_for_num(uint32_t block_num) const; @@ -592,6 +597,11 @@ namespace golos { uint32_t _last_free_gb_printed = 0; + size_t _inc_shared_memory_size = 0; + size_t _min_free_shared_memory_size = 0; + + uint32_t _block_num_check_free_memory = 1000; + flat_map> _custom_operation_interpreters; std::string _json_schema; }; diff --git a/libraries/chain/include/golos/chain/generic_custom_operation_interpreter.hpp b/libraries/chain/include/golos/chain/generic_custom_operation_interpreter.hpp index c81d656ed9..05b53f56dc 100644 --- a/libraries/chain/include/golos/chain/generic_custom_operation_interpreter.hpp +++ b/libraries/chain/include/golos/chain/generic_custom_operation_interpreter.hpp @@ -28,7 +28,7 @@ namespace golos { } void apply_operations(const vector &custom_operations, const operation &outer_o) { - auto plugin_session = this->_db.start_undo_session(true); + auto plugin_session = this->_db.start_undo_session(); flat_set outer_active; flat_set outer_owner; diff --git a/plugins/chain/plugin.cpp b/plugins/chain/plugin.cpp index a55b284141..036cbaeaa8 100644 --- a/plugins/chain/plugin.cpp +++ b/plugins/chain/plugin.cpp @@ -38,6 +38,11 @@ namespace chain { uint64_t write_wait_micro; uint32_t max_write_wait_retries; + size_t inc_shared_memory_size; + size_t min_free_shared_memory_size; + + uint32_t block_num_check_free_size = 0; + golos::chain::database db; bool single_write_thread = false; @@ -141,69 +146,54 @@ namespace chain { boost::program_options::options_description &cfg) { cfg.add_options() ( - "shared-file-dir", - boost::program_options::value()->default_value("blockchain"), + "shared-file-dir", boost::program_options::value()->default_value("blockchain"), "the location of the chain shared memory files (absolute path or relative to application data dir)" - ) - ( - "shared-file-size", - boost::program_options::value()->default_value("64G"), - "Size of the shared memory file. Default: 54G" - ) - ( - "checkpoint,c", - boost::program_options::value>()->composing(), + ) ( + "shared-file-size", boost::program_options::value()->default_value("2G"), + "Start size of the shared memory file. Default: 2G" + ) ( + "inc-shared-file-size", boost::program_options::value()->default_value("2G"), + "Increasing size on reaching limit of free space in shared memory file (see min-free-shared-file-size). Default: 2G" + ) ( + "min-free-shared-file-size", boost::program_options::value()->default_value("500M"), + "Minimum free space in shared memory file (see inc-shared-file-size). Default: 500M" + ) ( + "block-num-check-free-size", boost::program_options::value()->default_value(1000), + "Check free space in shared memory each N blocks. Default: 1000 (each 3000 seconds)." + ) ( + "checkpoint,c", boost::program_options::value>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints." - ) - ( - "flush-state-interval", - boost::program_options::value(), + ) ( + "flush-state-interval", boost::program_options::value(), "flush shared memory changes to disk every N blocks" - ) - ( - "read-wait-micro", - boost::program_options::value(), + ) ( + "read-wait-micro", boost::program_options::value(), "maximum microseconds for trying to get read lock" - ) - ( - "max-read-wait-retries", - boost::program_options::value(), + ) ( + "max-read-wait-retries", boost::program_options::value(), "maximum number of retries to get read lock" - ) - ( - "write-wait-micro", - boost::program_options::value(), + ) ( + "write-wait-micro", boost::program_options::value(), "maximum microseconds for trying to get write lock" - ) - ( - "max-write-wait-retries", - boost::program_options::value(), + ) ( + "max-write-wait-retries", boost::program_options::value(), "maximum number of retries to get write lock" - ) - ( - "single-write-thread", - boost::program_options::value()->default_value(false), + ) ( + "single-write-thread", boost::program_options::value()->default_value(false), "push blocks and transactions from one thread" ); cli.add_options() ( - "replay-blockchain", - boost::program_options::bool_switch()->default_value(false), + "replay-blockchain", boost::program_options::bool_switch()->default_value(false), "clear chain database and replay all blocks" - ) - ( - "resync-blockchain", - boost::program_options::bool_switch()->default_value(false), + ) ( + "resync-blockchain", boost::program_options::bool_switch()->default_value(false), "clear chain database and block log" - ) - ( - "check-locks", - boost::program_options::bool_switch()->default_value(false), + ) ( + "check-locks", boost::program_options::bool_switch()->default_value(false), "Check correctness of chainbase locking" - ) - ( - "validate-database-invariants", - boost::program_options::bool_switch()->default_value(false), + ) ( + "validate-database-invariants", boost::program_options::bool_switch()->default_value(false), "Validate all supply invariants check out" ); } @@ -241,6 +231,12 @@ namespace chain { my->single_write_thread = options.at("single-write-thread").as(); my->shared_memory_size = fc::parse_size(options.at("shared-file-size").as()); + my->inc_shared_memory_size = fc::parse_size(options.at("inc-shared-file-size").as()); + my->min_free_shared_memory_size = fc::parse_size(options.at("min-free-shared-file-size").as()); + + if (options.count("block-num-check-free-size")) { + my->block_num_check_free_size = options.at("block-num-check-free-size").as(); + } my->replay = options.at("replay-blockchain").as(); my->resync = options.at("resync-blockchain").as(); @@ -279,6 +275,13 @@ namespace chain { my->db.write_wait_micro(my->write_wait_micro); my->db.max_write_wait_retries(my->max_write_wait_retries); + my->db.inc_shared_memory_size(my->inc_shared_memory_size); + my->db.min_free_shared_memory_size(my->min_free_shared_memory_size); + + if (my->block_num_check_free_size) { + my->db.block_num_check_free_size(my->block_num_check_free_size); + } + if (my->replay) { ilog("Replaying blockchain on user request."); my->db.reindex(appbase::app().data_dir() / "blockchain", my->shared_memory_dir, my->shared_memory_size); diff --git a/share/golosd/config/config.ini b/share/golosd/config/config.ini index 5ad2b742f4..9bf0169e34 100644 --- a/share/golosd/config/config.ini +++ b/share/golosd/config/config.ini @@ -41,7 +41,10 @@ max-write-wait-retries = 3 single-write-thread = true -shared-file-size = 128G +shared-file-size = 2G +inc-shared-file-size = 2G +min-free-shared-file-size = 500M +block-num-check-free-size = 1000 # each 3000 seconds plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network market_history account_by_key account_history chain_stats block_info raw_block diff --git a/share/golosd/config/config_debug.ini b/share/golosd/config/config_debug.ini index cac53fa493..cfbfbdcf1e 100644 --- a/share/golosd/config/config_debug.ini +++ b/share/golosd/config/config_debug.ini @@ -33,7 +33,10 @@ webserver-http-endpoint = 0.0.0.0:8090 webserver-ws-endpoint = 0.0.0.0:8091 rpc-endpoint = 0.0.0.0:2001 -shared-file-size = 64G +shared-file-size = 100M +inc-shared-file-size = 100M +min-free-shared-file-size = 50M +block-num-check-free-size = 100 # each 300 seconds read-wait-micro = 500000 max-read-wait-retries = 2 diff --git a/share/golosd/config/config_stock_exchange.ini b/share/golosd/config/config_stock_exchange.ini index 42a8950ffb..0b8b8a4444 100644 --- a/share/golosd/config/config_stock_exchange.ini +++ b/share/golosd/config/config_stock_exchange.ini @@ -33,7 +33,10 @@ webserver-http-endpoint = 0.0.0.0:8090 webserver-ws-endpoint = 0.0.0.0:8091 rpc-endpoint = 0.0.0.0:2001 -shared-file-size = 128G +shared-file-size = 2G +inc-shared-file-size = 2G +min-free-shared-file-size = 500M +block-num-check-free-size = 1000 # each 3000 seconds read-wait-micro = 500000 max-read-wait-retries = 3 diff --git a/share/golosd/config/config_witness.ini b/share/golosd/config/config_witness.ini index 355e6b79a7..febc340271 100644 --- a/share/golosd/config/config_witness.ini +++ b/share/golosd/config/config_witness.ini @@ -41,7 +41,10 @@ max-write-wait-retries = 3 single-write-thread = true -shared-file-size = 128G +shared-file-size = 2G +inc-shared-file-size = 2G +min-free-shared-file-size = 500M +block-num-check-free-size = 1000 # each 3000 seconds plugin = chain p2p json_rpc webserver network_broadcast_api witness database_api diff --git a/thirdparty/chainbase b/thirdparty/chainbase index 6290139793..0c8e4987b3 160000 --- a/thirdparty/chainbase +++ b/thirdparty/chainbase @@ -1 +1 @@ -Subproject commit 629013979367e839c0def405fe41738b912bf426 +Subproject commit 0c8e4987b3923503dc5b23ae64f5d3f98085dd6f