Skip to content
This repository has been archived by the owner on Oct 4, 2019. It is now read-only.

Commit

Permalink
Merge pull request #503 from GolosChain/golos-v0.17.1
Browse files Browse the repository at this point in the history 
Golos v0.17.1
  • Loading branch information
@kotbegemot
kotbegemot committed Apr 3, 2018
2 parents a556538 + 0bb80f9 commit 64b96ef
Show file tree
Hide file tree
Showing 9 changed files with 146 additions and 79 deletions.
88 changes: 65 additions & 23 deletions libraries/chain/database.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,15 +168,13 @@ namespace golos {
while (itr.first.block_num() != last_block_num) {
auto cur_block_num = itr.first.block_num();
if (cur_block_num % 100000 == 0) {
std::cerr << " " << double(cur_block_num * 100) /
last_block_num << "% "
<< cur_block_num << " of "
<< last_block_num <<
" ("
<< (get_free_memory() / (1024 * 1024))
<< "M free)\n";
std::cerr
<< " " << double(cur_block_num * 100) / last_block_num << "% "
<< cur_block_num << " of " << last_block_num
<< " (" << (free_memory() / (1024 * 1024)) << "M free)\n";
}
apply_block(itr.first, skip_flags);
check_free_memory(true, itr.first.block_num());
itr = _block_log.read_block(itr.second);
}

Expand All @@ -196,6 +194,56 @@ namespace golos {

}

void database::min_free_shared_memory_size(size_t value) {
_min_free_shared_memory_size = value;
}

void database::inc_shared_memory_size(size_t value) {
_inc_shared_memory_size = value;
}

void database::block_num_check_free_size(uint32_t value) {
_block_num_check_free_memory = value;
}

void database::check_free_memory(bool skip_print, uint32_t current_block_num) {
if (0 != current_block_num % _block_num_check_free_memory) {
return;
}

uint64_t free_mem = free_memory();
uint64_t max_mem = max_memory();

if (_inc_shared_memory_size != 0 && _min_free_shared_memory_size != 0 &&
free_mem < _min_free_shared_memory_size
) {
size_t new_max = max_mem + _inc_shared_memory_size;
wlog(
"Memory is almost full on block ${block}, increasing to ${mem}M",
("block", current_block_num)("mem", new_max / (1024 * 1024)));
resize(new_max);
free_mem = free_memory();
uint32_t free_mb = uint32_t(free_mem / (1024 * 1024));
wlog("Free memory is now ${free}M", ("free", free_mb));
_last_free_gb_printed = free_mb / 1024;
} else if (!skip_print && _inc_shared_memory_size == 0 && _min_free_shared_memory_size == 0) {
uint32_t free_gb = uint32_t(free_mem / (1024 * 1024 * 1024));
if ((free_gb < _last_free_gb_printed) || (free_gb > _last_free_gb_printed + 1)) {
ilog(
"Free memory is now ${n}G. Current block number: ${block}",
("n", free_gb)("block", current_block_num));
_last_free_gb_printed = free_gb;
}

if (free_gb == 0) {
uint32_t free_mb = uint32_t(free_mem / (1024 * 1024));
if (free_mb <= 500 && current_block_num % 10 == 0) {
elog("Free memory is now ${n}M. Increase shared file size immediately!", ("n", free_mb));
}
}
}
}

void database::wipe(const fc::path &data_dir, const fc::path &shared_mem_dir, bool include_blocks) {
close();
chainbase::database::wipe(shared_mem_dir);
Expand Down Expand Up @@ -637,6 +685,8 @@ namespace golos {
FC_CAPTURE_AND_RETHROW((new_block))
});
});

check_free_memory(false, new_block.block_num());
});

//fc::time_point end_time = fc::time_point::now();
Expand Down Expand Up @@ -689,7 +739,7 @@ namespace golos {
// ilog( "pushing blocks from fork ${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->data.id()) );
optional<fc::exception> except;
try {
auto session = start_undo_session(true);
auto session = start_undo_session();
apply_block((*ritr)->data, skip);
session.push();
}
Expand All @@ -715,7 +765,7 @@ namespace golos {
for (auto ritr = branches.second.rbegin();
ritr !=
branches.second.rend(); ++ritr) {
auto session = start_undo_session(true);
auto session = start_undo_session();
apply_block((*ritr)->data, skip);
session.push();
}
Expand All @@ -730,7 +780,7 @@ namespace golos {
}

try {
auto session = start_undo_session(true);
auto session = start_undo_session();
apply_block(new_block, skip);
session.push();
}
Expand Down Expand Up @@ -771,15 +821,15 @@ namespace golos {
// If this is the first transaction pushed after applying a block, start a new undo session.
// This allows us to quickly rewind to the clean state of the head block, in case a new block arrives.
if (!_pending_tx_session.valid()) {
_pending_tx_session = start_undo_session(true);
_pending_tx_session = start_undo_session();
}

// Create a temporary undo session as a child of _pending_tx_session.
// The temporary session will be discarded by the destructor if
// _apply_transaction fails. If we make it to merge(), we
// apply the changes.

auto temp_session = start_undo_session(true);
auto temp_session = start_undo_session();
_apply_transaction(trx);
_pending_tx.push_back(trx);

Expand Down Expand Up @@ -843,7 +893,7 @@ namespace golos {
// re-apply pending transactions in this method.
//
_pending_tx_session.reset();
_pending_tx_session = start_undo_session(true);
_pending_tx_session = start_undo_session();

uint64_t postponed_tx_count = 0;
// pop pending state (reset to head block state)
Expand All @@ -865,7 +915,7 @@ namespace golos {
}

try {
auto temp_session = start_undo_session(true);
auto temp_session = start_undo_session();
_apply_transaction(tx);
temp_session.squash();

Expand Down Expand Up @@ -2931,7 +2981,7 @@ namespace golos {

void database::validate_transaction(const signed_transaction &trx) {
database::with_weak_write_lock([&]() {
auto session = start_undo_session(true);
auto session = start_undo_session();
_apply_transaction(trx);
session.undo();
});
Expand Down Expand Up @@ -3036,14 +3086,6 @@ namespace golos {
}
}

uint32_t free_gb = uint32_t(
get_free_memory() / (1024 * 1024 * 1024));
if ((free_gb < _last_free_gb_printed) ||
(free_gb > _last_free_gb_printed + 1)) {
ilog("Free memory is now ${n}G", ("n", free_gb));
_last_free_gb_printed = free_gb;
}

} FC_CAPTURE_AND_RETHROW((next_block))
}

Expand Down
12 changes: 11 additions & 1 deletion libraries/chain/include/golos/chain/database.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,11 @@ namespace golos {
void reindex(const fc::path &data_dir, const fc::path &shared_mem_dir, uint64_t shared_file_size = (
1024l * 1024l * 1024l * 8l));

void min_free_shared_memory_size(size_t);
void inc_shared_memory_size(size_t);
void block_num_check_free_size(uint32_t);
void check_free_memory(bool skip_print, uint32_t current_block_num);

/**
* @brief wipe Delete database from disk, and potentially the raw chain as well.
* @param include_blocks If true, delete the raw chain as well as the database.
Expand All @@ -121,7 +126,7 @@ namespace golos {

uint32_t get_pow_summary_target() const;

block_id_type get_block_id_for_num( uint32_t block_num )const;
block_id_type get_block_id_for_num( uint32_t block_num )const;

block_id_type find_block_id_for_num(uint32_t block_num) const;

Expand Down Expand Up @@ -592,6 +597,11 @@ namespace golos {

uint32_t _last_free_gb_printed = 0;

size_t _inc_shared_memory_size = 0;
size_t _min_free_shared_memory_size = 0;

uint32_t _block_num_check_free_memory = 1000;

flat_map<std::string, std::shared_ptr<custom_operation_interpreter>> _custom_operation_interpreters;
std::string _json_schema;
};
Expand Down
2 changes: 1 addition & 1 deletion libraries/chain/include/golos/chain/generic_custom_operation_interpreter.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ namespace golos {
}

void apply_operations(const vector<CustomOperationType> &custom_operations, const operation &outer_o) {
auto plugin_session = this->_db.start_undo_session(true);
auto plugin_session = this->_db.start_undo_session();

flat_set<account_name_type> outer_active;
flat_set<account_name_type> outer_owner;
Expand Down
101 changes: 52 additions & 49 deletions plugins/chain/plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ namespace chain {
uint64_t write_wait_micro;
uint32_t max_write_wait_retries;

size_t inc_shared_memory_size;
size_t min_free_shared_memory_size;

uint32_t block_num_check_free_size = 0;

golos::chain::database db;

bool single_write_thread = false;
Expand Down Expand Up @@ -141,69 +146,54 @@ namespace chain {
boost::program_options::options_description &cfg) {
cfg.add_options()
(
"shared-file-dir",
boost::program_options::value<boost::filesystem::path>()->default_value("blockchain"),
"shared-file-dir", boost::program_options::value<boost::filesystem::path>()->default_value("blockchain"),
"the location of the chain shared memory files (absolute path or relative to application data dir)"
)
(
"shared-file-size",
boost::program_options::value<std::string>()->default_value("64G"),
"Size of the shared memory file. Default: 54G"
)
(
"checkpoint,c",
boost::program_options::value<std::vector<std::string>>()->composing(),
) (
"shared-file-size", boost::program_options::value<std::string>()->default_value("2G"),
"Start size of the shared memory file. Default: 2G"
) (
"inc-shared-file-size", boost::program_options::value<std::string>()->default_value("2G"),
"Increasing size on reaching limit of free space in shared memory file (see min-free-shared-file-size). Default: 2G"
) (
"min-free-shared-file-size", boost::program_options::value<std::string>()->default_value("500M"),
"Minimum free space in shared memory file (see inc-shared-file-size). Default: 500M"
) (
"block-num-check-free-size", boost::program_options::value<uint32_t>()->default_value(1000),
"Check free space in shared memory each N blocks. Default: 1000 (each 3000 seconds)."
) (
"checkpoint,c", boost::program_options::value<std::vector<std::string>>()->composing(),
"Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints."
)
(
"flush-state-interval",
boost::program_options::value<uint32_t>(),
) (
"flush-state-interval", boost::program_options::value<uint32_t>(),
"flush shared memory changes to disk every N blocks"
)
(
"read-wait-micro",
boost::program_options::value<uint64_t>(),
) (
"read-wait-micro", boost::program_options::value<uint64_t>(),
"maximum microseconds for trying to get read lock"
)
(
"max-read-wait-retries",
boost::program_options::value<uint32_t>(),
) (
"max-read-wait-retries", boost::program_options::value<uint32_t>(),
"maximum number of retries to get read lock"
)
(
"write-wait-micro",
boost::program_options::value<uint64_t>(),
) (
"write-wait-micro", boost::program_options::value<uint64_t>(),
"maximum microseconds for trying to get write lock"
)
(
"max-write-wait-retries",
boost::program_options::value<uint32_t>(),
) (
"max-write-wait-retries", boost::program_options::value<uint32_t>(),
"maximum number of retries to get write lock"
)
(
"single-write-thread",
boost::program_options::value<bool>()->default_value(false),
) (
"single-write-thread", boost::program_options::value<bool>()->default_value(false),
"push blocks and transactions from one thread"
);
cli.add_options()
(
"replay-blockchain",
boost::program_options::bool_switch()->default_value(false),
"replay-blockchain", boost::program_options::bool_switch()->default_value(false),
"clear chain database and replay all blocks"
)
(
"resync-blockchain",
boost::program_options::bool_switch()->default_value(false),
) (
"resync-blockchain", boost::program_options::bool_switch()->default_value(false),
"clear chain database and block log"
)
(
"check-locks",
boost::program_options::bool_switch()->default_value(false),
) (
"check-locks", boost::program_options::bool_switch()->default_value(false),
"Check correctness of chainbase locking"
)
(
"validate-database-invariants",
boost::program_options::bool_switch()->default_value(false),
) (
"validate-database-invariants", boost::program_options::bool_switch()->default_value(false),
"Validate all supply invariants check out"
);
}
Expand Down Expand Up @@ -241,6 +231,12 @@ namespace chain {
my->single_write_thread = options.at("single-write-thread").as<bool>();

my->shared_memory_size = fc::parse_size(options.at("shared-file-size").as<std::string>());
my->inc_shared_memory_size = fc::parse_size(options.at("inc-shared-file-size").as<std::string>());
my->min_free_shared_memory_size = fc::parse_size(options.at("min-free-shared-file-size").as<std::string>());

if (options.count("block-num-check-free-size")) {
my->block_num_check_free_size = options.at("block-num-check-free-size").as<uint32_t>();
}

my->replay = options.at("replay-blockchain").as<bool>();
my->resync = options.at("resync-blockchain").as<bool>();
Expand Down Expand Up @@ -279,6 +275,13 @@ namespace chain {
my->db.write_wait_micro(my->write_wait_micro);
my->db.max_write_wait_retries(my->max_write_wait_retries);

my->db.inc_shared_memory_size(my->inc_shared_memory_size);
my->db.min_free_shared_memory_size(my->min_free_shared_memory_size);

if (my->block_num_check_free_size) {
my->db.block_num_check_free_size(my->block_num_check_free_size);
}

if (my->replay) {
ilog("Replaying blockchain on user request.");
my->db.reindex(appbase::app().data_dir() / "blockchain", my->shared_memory_dir, my->shared_memory_size);
Expand Down
5 changes: 4 additions & 1 deletion share/golosd/config/config.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@ max-write-wait-retries = 3

single-write-thread = true

shared-file-size = 128G
shared-file-size = 2G
inc-shared-file-size = 2G
min-free-shared-file-size = 500M
block-num-check-free-size = 1000 # each 3000 seconds

plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network market_history account_by_key account_history chain_stats block_info raw_block

Expand Down
5 changes: 4 additions & 1 deletion share/golosd/config/config_debug.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ webserver-http-endpoint = 0.0.0.0:8090
webserver-ws-endpoint = 0.0.0.0:8091
rpc-endpoint = 0.0.0.0:2001

shared-file-size = 64G
shared-file-size = 100M
inc-shared-file-size = 100M
min-free-shared-file-size = 50M
block-num-check-free-size = 100 # each 300 seconds

read-wait-micro = 500000
max-read-wait-retries = 2
Expand Down
5 changes: 4 additions & 1 deletion share/golosd/config/config_stock_exchange.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ webserver-http-endpoint = 0.0.0.0:8090
webserver-ws-endpoint = 0.0.0.0:8091
rpc-endpoint = 0.0.0.0:2001

shared-file-size = 128G
shared-file-size = 2G
inc-shared-file-size = 2G
min-free-shared-file-size = 500M
block-num-check-free-size = 1000 # each 3000 seconds

read-wait-micro = 500000
max-read-wait-retries = 3
Expand Down
Loading

0 comments on commit 64b96ef

Please sign in to comment.