From fc9b53a927d37dfc89bd9dbf5a0ee1bc4ff0b7e4 Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Tue, 27 Feb 2018 15:00:16 +0300 Subject: [PATCH 01/22] debug_node plugin: added support to push blocks from json file + testnet can load accounts snapshot --- libraries/chain/database.cpp | 15 +++- .../golos/plugins/debug_node/plugin.hpp | 4 + plugins/debug_node/plugin.cpp | 76 +++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 1f61e8e27f..f8f587b040 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -2905,14 +2905,21 @@ namespace golos { p.maximum_block_size = STEEMIT_MAX_BLOCK_SIZE; }); -#ifndef STEEMIT_BUILD_TESTNET auto snapshot_path = string("./snapshot5392323.json"); auto snapshot_file = fc::path(snapshot_path); - FC_ASSERT(fc::exists(snapshot_file), "Snapshot file '${file}' was not found.", ("file", snapshot_file)); + auto snapshot_exists = fc::exists(snapshot_file); + +#ifdef STEEMIT_BUILD_TESTNET + // Note: snapshot is not exist (not copied) in default TESTNET config + if (snapshot_exists) { +#else + FC_ASSERT(snapshot_exists, "Snapshot file '${file}' was not found.", ("file", snapshot_file)); +#endif std::cout << "Initializing state from snapshot file: " << snapshot_file.generic_string() << "\n"; +#ifndef STEEMIT_BUILD_TESTNET unsigned char digest[MD5_DIGEST_LENGTH]; char snapshot_checksum[] = "081b0149f0b2a570ae76b663090cfb0c"; char md5hash[33]; @@ -2923,6 +2930,7 @@ namespace golos { } FC_ASSERT(memcmp(md5hash, snapshot_checksum, 32) == 0, "Checksum of snapshot [${h}] is not equal [${s}]", ("h", md5hash)("s", snapshot_checksum)); +#endif snapshot_state snapshot = fc::json::from_file(snapshot_file).as(); for (account_summary &account : snapshot.accounts) { @@ -2944,6 +2952,9 @@ namespace golos { std::cout << "Imported " << snapshot.accounts.size() << " accounts from " << snapshot_file.generic_string() << ".\n"; + +#ifdef STEEMIT_BUILD_TESTNET + } #endif // Nothing to do diff --git a/plugins/debug_node/include/golos/plugins/debug_node/plugin.hpp b/plugins/debug_node/include/golos/plugins/debug_node/plugin.hpp index 8fd9e1abd3..95dad413ae 100644 --- a/plugins/debug_node/include/golos/plugins/debug_node/plugin.hpp +++ b/plugins/debug_node/include/golos/plugins/debug_node/plugin.hpp @@ -27,6 +27,7 @@ using golos::chain::witness_schedule_object; DEFINE_API_ARGS ( debug_generate_blocks, msg_pack, uint32_t ) DEFINE_API_ARGS ( debug_generate_blocks_until, msg_pack, uint32_t ) DEFINE_API_ARGS ( debug_push_blocks, msg_pack, uint32_t ) +DEFINE_API_ARGS ( debug_push_json_blocks, msg_pack, uint32_t ) DEFINE_API_ARGS ( debug_pop_block, msg_pack, fc::optional< protocol::signed_block > ) DEFINE_API_ARGS ( debug_get_witness_schedule, msg_pack, witness_schedule_object ) // DEFINE_API_ARGS ( debug_get_hardfork_property_object, msg_pack, debug_get_hardfork_property_object_r ) @@ -65,6 +66,9 @@ class plugin final : public appbase::plugin { */ (debug_push_blocks) + /* Push blocks from json file (array of signed blocks)*/ + (debug_push_json_blocks) + /** * Generate blocks locally. */ diff --git a/plugins/debug_node/plugin.cpp b/plugins/debug_node/plugin.cpp index 365713931b..fe6331c9cf 100644 --- a/plugins/debug_node/plugin.cpp +++ b/plugins/debug_node/plugin.cpp @@ -42,6 +42,12 @@ struct plugin::plugin_impl { bool skip_validate_invariants = false ); + uint32_t debug_push_json_blocks( + std::string json_filename, + uint32_t count, + uint32_t skip_flags = golos::chain::database::skip_nothing + ); + uint32_t debug_generate_blocks_until( std::string debug_key, fc::time_point_sec head_block_time, @@ -371,6 +377,54 @@ uint32_t plugin::plugin_impl::debug_push_blocks( return 0; } +uint32_t plugin::plugin_impl::debug_push_json_blocks( + std::string json_filename, + uint32_t count, + uint32_t skip_flags +) { + uint32_t pushed = 0; + + if (count == 0) + return pushed; + fc::path src_path = fc::path(json_filename); + if (fc::exists(src_path) && !fc::is_directory(src_path)) { + ilog("Loading ${n} JSON blocks from ${f}", ("n", count)("f", json_filename)); + idump((json_filename)(count)(skip_flags)); + + vector blocks; + try { + blocks = fc::json::from_file(src_path).as>(); + } + catch (const fc::exception &e) { + elog("Failed to load JSON blocks from ${f}", ("f", json_filename)); + elog("Exception backtrace: ${bt}", ("bt", e.to_detail_string())); + return pushed; + } + + uint32_t n_blocks = blocks.size(); + for (uint32_t i = 0; i < count; i++) { + if (i >= n_blocks) { + wlog("File ${f} only contained ${i} of ${n} requested blocks", + ("i", i)("n", count)("f", json_filename)); + break; + } + + signed_block blk = blocks[i]; + try { + database().push_block(blk, skip_flags); + pushed++; + } + catch (const fc::exception& e) { + elog("Got exception pushing block ${bn} : ${bid} (${i} of ${n})", + ("bn", blk.block_num())("bid", blk.id())("i", i)("n", count)); + elog("Exception backtrace: ${bt}", ("bt", e.to_detail_string())); + } + } + ilog("Completed loading json blocks, pushed ${ok} of ${n}", ("ok", pushed)("n", n_blocks plugin::plugin_impl::debug_pop_block() { auto & db = database(); return db.fetch_block_by_number( db.head_block_num() ); @@ -481,6 +535,28 @@ DEFINE_API ( plugin, debug_push_blocks ) { }); } +DEFINE_API ( plugin, debug_push_json_blocks ) { + std::string json_filename; + uint32_t count; + uint32_t skip_flags = golos::chain::database::skip_nothing; + // `skip_flags` can be set to 577 if loading mainnet blocks + // 577 = skip_witness_signature | skip_authority_check | skip_witness_schedule_check + + FC_ASSERT(args.args.valid(), "Invalid parameters" ); + auto args_count = args.args->size() ; + FC_ASSERT(args_count > 0 && args_count < 4, "Wrong parameters number, given ${n}", ("n", args_count)); + auto args_vector = *(args.args); + json_filename = args_vector[0].as_string(); + count = args_vector[1].as_int64(); + if (args_count > 2) { + skip_flags = args_vector[2].as_int64(); + } + auto &db = my->database(); + return db.with_read_lock([&]() { + return my->debug_push_json_blocks(json_filename, count, skip_flags); + }); +} + DEFINE_API ( plugin, debug_generate_blocks_until ) { std::string debug_key; fc::time_point_sec head_block_time; From 88cfd50227a1bb3be1e277cab01937ad67f0bc20 Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Wed, 28 Feb 2018 16:38:06 +0300 Subject: [PATCH 02/22] debug_node plugin: remove arguments boilerplate of DEFINE_API methods --- .../golos/plugins/debug_node/api_helper.hpp | 107 +++++++++++++ plugins/debug_node/plugin.cpp | 145 ++++++------------ 2 files changed, 150 insertions(+), 102 deletions(-) create mode 100644 plugins/debug_node/include/golos/plugins/debug_node/api_helper.hpp diff --git a/plugins/debug_node/include/golos/plugins/debug_node/api_helper.hpp b/plugins/debug_node/include/golos/plugins/debug_node/api_helper.hpp new file mode 100644 index 0000000000..b2041a7de2 --- /dev/null +++ b/plugins/debug_node/include/golos/plugins/debug_node/api_helper.hpp @@ -0,0 +1,107 @@ +#pragma once + +// Provides PLUGIN_API_VALIDATE_ARGS macro usable in DEFINE_API methods +// to generate arguments processing boilerplate. + +//Usage: +// PLUGIN_API_VALIDATE_ARGS( +// (std::string, json_filename) +// ) +//Expands to: +// std::string json_filename; +// FC_ASSERT(args.args.valid(), "Invalid parameters"); +// auto n_args = args.args->size(); +// FC_ASSERT(n_args == 1, "Expected 1 parameter, received ${n}", ("n", n_args)); +// json_filename = args.args->at(0).as(); +// +//And this: +// PLUGIN_API_VALIDATE_ARGS( +// (std::string, json_filename) +// (uint32_t, count) +// (uint32_t, skip_flags, golos::chain::database::skip_nothing) +// ) +//Expands to: +// std::string json_filename; +// uint32_t count; +// uint32_t skip_flags = golos::chain::database::skip_nothing; +// FC_ASSERT(args.args.valid(), "Invalid parameters"); +// auto n_args = args.args->size(); +// FC_ASSERT(n_args >= 2 && n_args <= 3, "Expected at least 2 and up to 3 parameters, received ${n}", ("n", n_args)); +// json_filename = args.args->at(0).as(); +// count = args.args->at(1).as(); +// if (n_args > 2) { +// skip_flags = args.args->at(2).as(); +// } + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REMOVE_PARENTHESES(...) __VA_ARGS__ + +// prepare +#define PLUGIN_API_VALIDATE_ARGS(ARGS) PLUGIN_API_VALIDATE_ARGS_I(BOOST_PP_VARIADIC_SEQ_TO_SEQ(ARGS)) +#define PLUGIN_API_VALIDATE_ARGS_I(ARGS) PLUGIN_API_VALIDATE_ARGS_II(ARGS, COUNT_REQUIRED_ARGS(ARGS), BOOST_PP_SEQ_SIZE(ARGS)) + +// main +#define PLUGIN_API_VALIDATE_ARGS_II(ARGS, N_REQ, N_ARGS) \ + PLUGIN_API_DECLARE_ARG_VARS(ARGS) \ + FC_ASSERT(args.args.valid(), "Invalid parameters"); /* is it possible to get invalid?*/ \ + auto n_args = args.args->size(); \ + PLUGIN_API_ARGS_NUM_ASSERT(N_REQ, N_ARGS) \ + PLUGIN_API_LOAD_AND_CHECK_ARGS(ARGS, N_REQ) + +// helper +#define COUNT_REQUIRED_ARGS(args) COUNT_REQUIRED_ARGS_I(args, BOOST_PP_SEQ_SIZE(args)) +#define COUNT_REQUIRED_ARGS_I(args, sz) BOOST_PP_SEQ_ELEM(2, BOOST_PP_WHILE(COUNT_PR, COUNT_OP, (args(end))(sz)(0))) +#define STATE_OP(op, state) op(BOOST_PP_SEQ_ELEM(0, state), BOOST_PP_SEQ_ELEM(1, state), BOOST_PP_SEQ_ELEM(2, state)) +#define COUNT_OP(d, state) STATE_OP(COUNT_OP_I, state) +#define COUNT_PR(d, state) STATE_OP(COUNT_PR_I, state) +#define COUNT_OP_I(args, sz, i) (args)(sz)(BOOST_PP_INC(i)) +#define COUNT_PR_I(args, sz, i) BOOST_PP_AND(HAVE_NEXT_ARG(sz, i), GOT_REQUIRED(args, i)) +#define HAVE_NEXT_ARG(sz, i) BOOST_PP_LESS(i, sz) +#define GOT_REQUIRED(args, i) BOOST_PP_LESS(ARG_SIZE(BOOST_PP_SEQ_ELEM(i, args)), 3) +#define ARG_SIZE(arg) BOOST_PP_VARIADIC_SIZE(REMOVE_PARENTHESES arg) + +// steps: +#define PLUGIN_API_DECLARE_ARG_VARS(ARGS) BOOST_PP_SEQ_FOR_EACH(PLUGIN_API_DECLARE_ARG_VAR, _, ARGS) +#define PLUGIN_API_DECLARE_ARG_VAR(r, data, arg) \ + ARG_TYPE(arg) ARG_NAME(arg) ARG_OPT_VALUE(arg); + +#define ARG_ELEM(arg, n) BOOST_PP_VARIADIC_ELEM(n, REMOVE_PARENTHESES arg, nop) +#define ARG_TYPE(arg) ARG_ELEM(arg, 0) +#define ARG_NAME(arg) ARG_ELEM(arg, 1) +#define ARG_VALUE(arg) ARG_ELEM(arg, 2) +#define ARG_HAVE_VALUE(arg) BOOST_PP_GREATER(ARG_SIZE(arg), 2) +#define ARG_OPT_VALUE(arg) BOOST_PP_IF(ARG_HAVE_VALUE(arg), = ARG_VALUE(arg), BOOST_PP_EMPTY()) + +#define PLUGIN_API_ARGS_NUM_ASSERT(req, all) PLUGIN_API_ARGS_NUM_ASSERT_I(BOOST_PP_LESS(req, all), req, all) +#define PLUGIN_API_ARGS_NUM_ASSERT_I(less, req, all) \ + PLUGIN_API_ARGS_NUM_ASSERT_II(AN_ASSERT_COND(less, req, all), AN_ASSERT_MSG(less, req, all)) +#define PLUGIN_API_ARGS_NUM_ASSERT_II(cond, msg) \ + FC_ASSERT(cond, msg, ("n", n_args)); + +#define AN_ASSERT_COND(less, req, all) BOOST_PP_IF(less, n_args >= req && n_args <= all, n_args == all) +#define AN_ASSERT_MSG(LESS, REQ, ALL) \ + "Expected " BOOST_PP_STRINGIZE( BOOST_PP_IF(LESS, at least REQ and up to ALL, REQ) ) \ + " parameter" PLURAL_FINAL(ALL) ", received ${n}" +#define PLURAL_FINAL(n) BOOST_PP_IF(BOOST_PP_LESS(n, 2), "", "s") + +#define PLUGIN_API_LOAD_AND_CHECK_ARGS(ARGS, n_req) BOOST_PP_SEQ_FOR_EACH_I(LOAD_AND_CHECK_ARG, n_req, ARGS) +#define LOAD_AND_CHECK_ARG(r, n_req, i, arg) LOAD_AND_CHECK_ARG_I(arg, i, BOOST_PP_LESS(i, n_req)) +#define LOAD_AND_CHECK_ARG_I(arg, i, req) BOOST_PP_IF(req, LOAD_ARG, LOAD_OPTIONAL_ARG)(arg, i) +#define LOAD_ARG(arg, i) ARG_NAME(arg) = args.args->at(i).as(); +#define LOAD_OPTIONAL_ARG(arg, i) \ + if (n_args > i) { \ + LOAD_ARG(arg, i) \ + } diff --git a/plugins/debug_node/plugin.cpp b/plugins/debug_node/plugin.cpp index fe6331c9cf..387dd2c58a 100644 --- a/plugins/debug_node/plugin.cpp +++ b/plugins/debug_node/plugin.cpp @@ -478,32 +478,19 @@ bool plugin::plugin_impl::debug_has_hardfork( uint32_t hardfork_id ) { // // -DEFINE_API ( plugin, debug_generate_blocks ) { - std::string debug_key; - uint32_t count = 0; - uint32_t skip = golos::chain::database::skip_nothing; - uint32_t miss_blocks = 0; - bool edit_if_needed = true; - - FC_ASSERT(args.args.valid(), "Invalid parameters" ); - - auto args_count = args.args->size() ; - - FC_ASSERT( args_count > 0 && args_count < 6, "Wrong parameters number, given ${n}", ("n", args_count) ); - auto args_vector = *(args.args); - debug_key = args_vector[0].as_string(); - if (args_count > 1) { - count = args_vector[1].as_int64(); - } - if (args_count > 2) { - skip = args_vector[2].as_int64(); - } - if (args_count > 3) { - miss_blocks = args_vector[3].as_int64(); - } - if (args_count > 4) { - edit_if_needed = args_vector[4].as_bool(); - } + +#include + +#define DEFINE_PLUGIN_API(name) DEFINE_API(plugin, name) + +DEFINE_PLUGIN_API ( debug_generate_blocks ) { + PLUGIN_API_VALIDATE_ARGS( + (std::string, debug_key) + (uint32_t, count, 0) + (uint32_t, skip, golos::chain::database::skip_nothing) + (uint32_t, miss_blocks, 0) + (bool, edit_if_needed, true) + ) auto &db = my->database(); return db.with_read_lock([&]() { @@ -511,23 +498,12 @@ DEFINE_API ( plugin, debug_generate_blocks ) { }); } -DEFINE_API ( plugin, debug_push_blocks ) { - std::string src_filename; - uint32_t count; - bool skip_validate_invariants = false; - - FC_ASSERT(args.args.valid(), "Invalid parameters" ); // is it possible to get invalid? - - auto args_count = args.args->size() ; - - FC_ASSERT( args_count > 0 && args_count < 4, "Wrong parameters number, given ${n}", ("n", args_count) ); - auto args_vector = *(args.args); - - src_filename = args_vector[0].as_string(); - count = args_vector[1].as_int64(); - if (args_count > 2) { - skip_validate_invariants = args_vector[2].as_bool(); - } +DEFINE_PLUGIN_API ( debug_push_blocks ) { + PLUGIN_API_VALIDATE_ARGS( + (std::string, src_filename) + (uint32_t, count) + (bool, skip_validate_invariants, false) + ) auto &db = my->database(); return db.with_read_lock([&]() { @@ -535,51 +511,28 @@ DEFINE_API ( plugin, debug_push_blocks ) { }); } -DEFINE_API ( plugin, debug_push_json_blocks ) { - std::string json_filename; - uint32_t count; - uint32_t skip_flags = golos::chain::database::skip_nothing; +DEFINE_PLUGIN_API ( debug_push_json_blocks ) { + PLUGIN_API_VALIDATE_ARGS( + (std::string, json_filename) + (uint32_t, count) + (uint32_t, skip_flags, golos::chain::database::skip_nothing) + ) // `skip_flags` can be set to 577 if loading mainnet blocks // 577 = skip_witness_signature | skip_authority_check | skip_witness_schedule_check - - FC_ASSERT(args.args.valid(), "Invalid parameters" ); - auto args_count = args.args->size() ; - FC_ASSERT(args_count > 0 && args_count < 4, "Wrong parameters number, given ${n}", ("n", args_count)); - auto args_vector = *(args.args); - json_filename = args_vector[0].as_string(); - count = args_vector[1].as_int64(); - if (args_count > 2) { - skip_flags = args_vector[2].as_int64(); - } + auto &db = my->database(); return db.with_read_lock([&]() { return my->debug_push_json_blocks(json_filename, count, skip_flags); }); } -DEFINE_API ( plugin, debug_generate_blocks_until ) { - std::string debug_key; - fc::time_point_sec head_block_time; - bool generate_sparsely = true; - uint32_t skip = golos::chain::database::skip_nothing; - - FC_ASSERT(args.args.valid(), "Invalid parameters" ); - - auto args_count = args.args->size() ; - - FC_ASSERT( args_count > 0 && args_count < 5, "Wrong parameters number, given ${n}", ("n", args_count) ); - auto args_vector = *(args.args); - - debug_key = args_vector[0].as_string(); - head_block_time = fc::time_point_sec::from_iso_string( args_vector[1].as_string() ); - - if (args_count > 2) { - generate_sparsely = args_vector[2].as_bool(); - } - - if (args_count > 3) { - skip = args_vector[3].as_int64(); - } +DEFINE_PLUGIN_API ( debug_generate_blocks_until ) { + PLUGIN_API_VALIDATE_ARGS( + (std::string, debug_key) + (fc::time_point_sec,head_block_time) + (bool, generate_sparsely, true) + (uint32_t, skip, golos::chain::database::skip_nothing) + ) auto &db = my->database(); return db.with_read_lock([&]() { @@ -587,14 +540,14 @@ DEFINE_API ( plugin, debug_generate_blocks_until ) { }); } -DEFINE_API ( plugin, debug_pop_block ) { +DEFINE_PLUGIN_API ( debug_pop_block ) { auto &db = my->database(); return db.with_read_lock([&]() { return my->debug_pop_block(); }); } -DEFINE_API ( plugin, debug_get_witness_schedule ) { +DEFINE_PLUGIN_API ( debug_get_witness_schedule ) { auto &db = my->database(); return db.with_read_lock([&]() { return my->debug_get_witness_schedule(); @@ -609,16 +562,10 @@ DEFINE_API ( plugin, debug_get_witness_schedule ) { // }); // } -DEFINE_API ( plugin, debug_set_hardfork ) { - uint32_t hardfork_id; - - FC_ASSERT(args.args.valid(), "Invalid parameters" ) ; - - auto args_count = args.args->size() ; - - FC_ASSERT( args_count == 1, "Wrong parameters number, given ${n}", ("n", args_count) ); - auto args_vector = *(args.args); - hardfork_id = args_vector[0].as_int64(); +DEFINE_PLUGIN_API ( debug_set_hardfork ) { + PLUGIN_API_VALIDATE_ARGS( + (uint32_t, hardfork_id) + ) auto &db = my->database(); return db.with_read_lock([&]() { @@ -627,16 +574,10 @@ DEFINE_API ( plugin, debug_set_hardfork ) { }); } -DEFINE_API ( plugin, debug_has_hardfork ) { - uint32_t hardfork_id; - - FC_ASSERT(args.args.valid(), "Invalid parameters" ) ; - - auto args_count = args.args->size() ; - - FC_ASSERT( args_count == 1, "Wrong parameters number, given ${n}", ("n", args_count) ); - auto args_vector = *(args.args); - hardfork_id = args_vector[0].as_int64(); +DEFINE_PLUGIN_API ( debug_has_hardfork ) { + PLUGIN_API_VALIDATE_ARGS( + (uint32_t, hardfork_id) + ) auto &db = my->database(); return db.with_read_lock([&]() { From cb180529eb51995fb8c02ac010fbb77f9ae17b6f Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Wed, 30 May 2018 23:37:31 +0700 Subject: [PATCH 03/22] Fix tests after merging of update debug_node plugin. --- plugins/debug_node/plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/debug_node/plugin.cpp b/plugins/debug_node/plugin.cpp index ea21d4edd1..187aa07a84 100644 --- a/plugins/debug_node/plugin.cpp +++ b/plugins/debug_node/plugin.cpp @@ -515,7 +515,7 @@ DEFINE_PLUGIN_API ( debug_push_json_blocks ) { // 577 = skip_witness_signature | skip_authority_check | skip_witness_schedule_check auto &db = my->database(); - return db.with_read_lock([&]() { + return db.with_weak_read_lock([&]() { return my->debug_push_json_blocks(json_filename, count, skip_flags); }); } @@ -542,7 +542,7 @@ DEFINE_API ( plugin, debug_get_witness_schedule ) { // DEFINE_API ( plugin, debug_get_hardfork_property_object ) { // auto tmp = args.args->at(0).as(); // auto &db = my->database(); -// return db.with_read_lock([&]() { +// return db.with_weak_read_lock([&]() { // return my->debug_get_hardfork_property_object(tmp); // }); // } From 97fbbfb2d09ca1179030bb0d757d2a95652a2cea Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Wed, 30 May 2018 23:38:46 +0700 Subject: [PATCH 04/22] Fix publishing of docker images in travis configuration. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3977775406..ce5e40c328 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ matrix: script: - if [ "$TRAVIS_BRANCH" == "master" ]; then export DOCKERNAME="latest""$DOCKERNAME"; - export EXPORTNAME="latest""$DOCKERNAME"; + export EXPORTNAME="$DOCKERNAME"; elif [ -n "$TRAVIS_TAG" ]; then export DOCKERNAME="$TRAVIS_TAG""$DOCKERNAME"; - export EXPORTNAME="$TRAVIS_TAG""$DOCKERNAME"; + export EXPORTNAME="$DOCKERNAME"; else export DOCKERNAME=develop"$DOCKERNAME"; fi From eccab9328ccdf4d82e749bb37a206241076a5714 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Mon, 4 Jun 2018 12:59:29 +0700 Subject: [PATCH 05/22] Add limits on nested proposals. #687 --- .../include/golos/chain/steem_evaluator.hpp | 26 +++++++++++- libraries/chain/proposal_evaluator.cpp | 40 +++++++++++++++++++ .../include/golos/protocol/config.hpp | 2 + 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/golos/chain/steem_evaluator.hpp b/libraries/chain/include/golos/chain/steem_evaluator.hpp index 6dae8a45c6..8a698df583 100644 --- a/libraries/chain/include/golos/chain/steem_evaluator.hpp +++ b/libraries/chain/include/golos/chain/steem_evaluator.hpp @@ -52,9 +52,31 @@ namespace golos { namespace chain { DEFINE_EVALUATOR(reset_account) DEFINE_EVALUATOR(set_reset_account) DEFINE_EVALUATOR(delegate_vesting_shares) - DEFINE_EVALUATOR(proposal_create) - DEFINE_EVALUATOR(proposal_update) DEFINE_EVALUATOR(proposal_delete) DEFINE_EVALUATOR(chain_properties_update) + class proposal_create_evaluator: public evaluator_impl { + public: + using operation_type = proposal_create_operation; + + proposal_create_evaluator(database& db); + + void do_apply(const operation_type& o); + + protected: + int depth_ = 0; + }; + + class proposal_update_evaluator: public evaluator_impl { + public: + using operation_type = proposal_update_operation; + + proposal_update_evaluator(database& db); + + void do_apply(const operation_type& o); + + protected: + int depth_ = 0; + }; + } } // golos::chain diff --git a/libraries/chain/proposal_evaluator.cpp b/libraries/chain/proposal_evaluator.cpp index a712526f1b..2b411ac955 100644 --- a/libraries/chain/proposal_evaluator.cpp +++ b/libraries/chain/proposal_evaluator.cpp @@ -77,11 +77,37 @@ namespace golos { namespace chain { } } } + + struct safe_int_increment { + safe_int_increment(int& value) + : value_(value) { + value_++; + } + + ~safe_int_increment() { + value_--; + } + + int& value_; + }; + } + + proposal_create_evaluator::proposal_create_evaluator(database& db) + : evaluator_impl(db) { } void proposal_create_evaluator::do_apply(const proposal_create_operation& o) { try { ASSERT_REQ_HF(STEEMIT_HARDFORK_0_18__542, "Proposal transaction creating"); // remove after hf + safe_int_increment depth_increment(depth_); + + if (db().is_producing()) { + FC_ASSERT( + depth_ <= STEEMIT_MAX_PROPOSAL_DEPTH, + "You can't create more than ${depth} nested proposals", + ("depth", STEEMIT_MAX_PROPOSAL_DEPTH)); + } + FC_ASSERT(nullptr == db().find_proposal(o.author, o.title), "Proposal already exists."); const auto now = db().head_block_time(); @@ -173,8 +199,22 @@ namespace golos { namespace chain { } } FC_CAPTURE_AND_RETHROW((o)) } + proposal_update_evaluator::proposal_update_evaluator(database& db) + : evaluator_impl(db) { + } + void proposal_update_evaluator::do_apply(const proposal_update_operation& o) { try { ASSERT_REQ_HF(STEEMIT_HARDFORK_0_18__542, "Proposal transaction updating"); // remove after hf + + safe_int_increment depth_increment(depth_); + + if (db().is_producing()) { + FC_ASSERT( + depth_ <= STEEMIT_MAX_PROPOSAL_DEPTH, + "You can't create more than ${depth} nested proposals", + ("depth", STEEMIT_MAX_PROPOSAL_DEPTH)); + } + auto& proposal = db().get_proposal(o.author, o.title); const auto now = db().head_block_time(); diff --git a/libraries/protocol/include/golos/protocol/config.hpp b/libraries/protocol/include/golos/protocol/config.hpp index 62e9d71d4a..24cf4498dd 100644 --- a/libraries/protocol/include/golos/protocol/config.hpp +++ b/libraries/protocol/include/golos/protocol/config.hpp @@ -31,6 +31,7 @@ #define STEEMIT_VOTE_CHANGE_LOCKOUT_PERIOD (60*10) /// 10 minutes #define STEEMIT_MAX_PROPOSAL_LIFETIME_SEC (60*60*12) /// 12 hours +#define STEEMIT_MAX_PROPOSAL_DEPTH (2) #define STEEMIT_ORIGINAL_MIN_ACCOUNT_CREATION_FEE 0 #define STEEMIT_MIN_ACCOUNT_CREATION_FEE 0 @@ -240,6 +241,7 @@ #define STEEMIT_VOTE_CHANGE_LOCKOUT_PERIOD (60*60*2) /// 2 hours #define STEEMIT_MAX_PROPOSAL_LIFETIME_SEC (60*60*24*7*4) /// 4 weeks +#define STEEMIT_MAX_PROPOSAL_DEPTH (2) #define STEEMIT_ORIGINAL_MIN_ACCOUNT_CREATION_FEE 100000 #define STEEMIT_MIN_ACCOUNT_CREATION_FEE 1 From d6fb715a561bf9607d560c1b10076ebc3d3d17e9 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Mon, 4 Jun 2018 13:00:17 +0700 Subject: [PATCH 06/22] Replace db() with _db. #687 --- libraries/chain/proposal_evaluator.cpp | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/libraries/chain/proposal_evaluator.cpp b/libraries/chain/proposal_evaluator.cpp index 2b411ac955..8486462e18 100644 --- a/libraries/chain/proposal_evaluator.cpp +++ b/libraries/chain/proposal_evaluator.cpp @@ -101,16 +101,16 @@ namespace golos { namespace chain { safe_int_increment depth_increment(depth_); - if (db().is_producing()) { + if (_db.is_producing()) { FC_ASSERT( depth_ <= STEEMIT_MAX_PROPOSAL_DEPTH, "You can't create more than ${depth} nested proposals", ("depth", STEEMIT_MAX_PROPOSAL_DEPTH)); } - FC_ASSERT(nullptr == db().find_proposal(o.author, o.title), "Proposal already exists."); + FC_ASSERT(nullptr == _db.find_proposal(o.author, o.title), "Proposal already exists."); - const auto now = db().head_block_time(); + const auto now = _db.head_block_time(); FC_ASSERT( o.expiration_time > now, "Proposal has already expired on creation."); @@ -152,7 +152,7 @@ namespace golos { namespace chain { // because it will be never approved. for (const auto& account: required_total) { FC_ASSERT( - nullptr != db().find_account(account), + nullptr != _db.find_account(account), "Account '${account}' for proposed operation doesn't exist", ("account", account)); } @@ -162,7 +162,7 @@ namespace golos { namespace chain { for (const auto& op : o.proposed_operations) { trx.operations.push_back(op.op); } - trx.set_expiration(db().head_block_time() + STEEMIT_MAX_TIME_UNTIL_EXPIRATION); + trx.set_expiration(_db.head_block_time() + STEEMIT_MAX_TIME_UNTIL_EXPIRATION); const uint32_t skip_steps = golos::chain::database::skip_authority_check | @@ -170,11 +170,11 @@ namespace golos { namespace chain { golos::chain::database::skip_tapos_check | golos::chain::database::skip_database_locking; - db().validate_transaction(trx, skip_steps); + _db.validate_transaction(trx, skip_steps); auto ops_size = fc::raw::pack_size(trx.operations); - const auto& proposal = db().create([&](proposal_object& p){ + const auto& proposal = _db.create([&](proposal_object& p){ p.author = o.author; from_string(p.title, o.title); from_string(p.memo, o.memo); @@ -192,7 +192,7 @@ namespace golos { namespace chain { }); for (const auto& account: required_total) { - db().create([&](required_approval_object& o){ + _db.create([&](required_approval_object& o){ o.account = account; o.proposal = proposal.id; }); @@ -208,15 +208,15 @@ namespace golos { namespace chain { safe_int_increment depth_increment(depth_); - if (db().is_producing()) { + if (_db.is_producing()) { FC_ASSERT( depth_ <= STEEMIT_MAX_PROPOSAL_DEPTH, "You can't create more than ${depth} nested proposals", ("depth", STEEMIT_MAX_PROPOSAL_DEPTH)); } - auto& proposal = db().get_proposal(o.author, o.title); - const auto now = db().head_block_time(); + auto& proposal = _db.get_proposal(o.author, o.title); + const auto now = _db.head_block_time(); if (proposal.review_period_time && now >= *proposal.review_period_time) { FC_ASSERT( @@ -249,7 +249,7 @@ namespace golos { namespace chain { check_duplicate(o.posting_approvals_to_add, proposal.available_posting_approvals); check_duplicate(o.key_approvals_to_add, proposal.available_key_approvals); - db().modify(proposal, [&](proposal_object &p){ + _db.modify(proposal, [&](proposal_object &p){ p.available_active_approvals.insert(o.active_approvals_to_add.begin(), o.active_approvals_to_add.end()); p.available_owner_approvals.insert(o.owner_approvals_to_add.begin(), o.owner_approvals_to_add.end()); p.available_posting_approvals.insert(o.posting_approvals_to_add.begin(), o.posting_approvals_to_add.end()); @@ -269,17 +269,17 @@ namespace golos { namespace chain { proposal.available_posting_approvals.empty() && proposal.available_key_approvals.empty() ) { - db().remove(proposal); + _db.remove(proposal); } return; } - assert_irrelevant_proposal_authority(db(), proposal, o); + assert_irrelevant_proposal_authority(_db, proposal, o); - if (proposal.is_authorized_to_execute(db())) { + if (proposal.is_authorized_to_execute(_db)) { // All required approvals are satisfied. Execute! try { - db().push_proposal(proposal); + _db.push_proposal(proposal); } catch (fc::exception &e) { wlog( "Proposed transaction ${author}::${title} failed to apply once approved with exception:\n" @@ -292,7 +292,7 @@ namespace golos { namespace chain { void proposal_delete_evaluator::do_apply(const proposal_delete_operation& o) { try { ASSERT_REQ_HF(STEEMIT_HARDFORK_0_18__542, "Proposal transaction deleting"); // remove after hf - const auto& proposal = db().get_proposal(o.author, o.title); + const auto& proposal = _db.get_proposal(o.author, o.title); FC_ASSERT( proposal.author == o.requester || @@ -302,7 +302,7 @@ namespace golos { namespace chain { "Provided authority is not authoritative for this proposal.", ("author", o.author)("title", o.title)("requester", o.requester)); - db().remove(proposal); + _db.remove(proposal); } FC_CAPTURE_AND_RETHROW((o)) } From 7f8e668b0890ab78be9179fd9a3e87865741150f Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Mon, 4 Jun 2018 13:10:37 +0700 Subject: [PATCH 07/22] Add update_chain_properties to cli_wallet. #693 --- .../chain/chain_properties_evaluators.cpp | 27 ++++++- libraries/chain/database.cpp | 76 +++++-------------- .../wallet/include/golos/wallet/wallet.hpp | 26 +++++-- libraries/wallet/wallet.cpp | 61 ++++++++++----- 4 files changed, 108 insertions(+), 82 deletions(-) diff --git a/libraries/chain/chain_properties_evaluators.cpp b/libraries/chain/chain_properties_evaluators.cpp index 89c8333697..233b4f425e 100644 --- a/libraries/chain/chain_properties_evaluators.cpp +++ b/libraries/chain/chain_properties_evaluators.cpp @@ -18,8 +18,22 @@ namespace golos { namespace chain { FC_ASSERT(o.props.account_creation_fee.symbol == STEEM_SYMBOL); } else if (o.props.account_creation_fee.symbol != STEEM_SYMBOL) { // after HF, above check can be moved to validate() if reindex doesn't show this warning - wlog("Wrong fee symbol in block ${b}", ("b", - db().head_block_num() + 1)); + wlog("Wrong fee symbol in block ${b}", ("b", db().head_block_num() + 1)); + } + + const bool has_hf18 = db().has_hardfork(STEEMIT_HARDFORK_0_18__673); + + // TODO: remove this after HF 18 + if (has_hf18) { + if (o.props.account_creation_fee.amount.value != STEEMIT_MIN_ACCOUNT_CREATION_FEE) { + wlog("The chain_properties_update_operation should be used to update account_creation_fee"); + } + if (o.props.sbd_interest_rate != STEEMIT_DEFAULT_SBD_INTEREST_RATE) { + wlog("The chain_properties_update_operation should be used to update sbd_interest_rate"); + } + if (o.props.maximum_block_size != STEEMIT_MIN_BLOCK_SIZE_LIMIT * 2) { + wlog("The chain_properties_update_operation should be used to update maximum_block_size"); + } } const auto &idx = db().get_index().indices().get(); @@ -28,7 +42,9 @@ namespace golos { namespace chain { db().modify(*itr, [&](witness_object& w) { from_string(w.url, o.url); w.signing_key = o.block_signing_key; - w.props = o.props; + if (!has_hf18) { + w.props = o.props; + } }); } else { db().create([&](witness_object& w) { @@ -36,7 +52,9 @@ namespace golos { namespace chain { from_string(w.url, o.url); w.signing_key = o.block_signing_key; w.created = db().head_block_time(); - w.props = o.props; + if (!has_hf18) { + w.props = o.props; + } }); } } @@ -70,4 +88,5 @@ namespace golos { namespace chain { }); } } + } } // golos::chain \ No newline at end of file diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 4b07b837a7..f373100394 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -1718,67 +1718,33 @@ namespace golos { namespace chain { active.push_back(&get_witness(wso.current_shuffled_witnesses[i])); } - /// sort them by account_creation_fee - std::sort(active.begin(), active.end(), [&](const witness_object *a, const witness_object *b) { - return a->props.account_creation_fee.amount < - b->props.account_creation_fee.amount; - }); - asset median_account_creation_fee = active[active.size() / 2]->props.account_creation_fee; - - /// sort them by create_account_with_golos_modifier - std::sort(active.begin(), active.end(), [&](const witness_object *a, const witness_object *b) { - return a->props.create_account_with_golos_modifier < - b->props.create_account_with_golos_modifier; - }); - auto median_with_golos_modifier = active[active.size() / 2]->props.create_account_with_golos_modifier; - - /// sort them by create_account_delegation_ratio - std::sort(active.begin(), active.end(), [&](const witness_object *a, const witness_object *b) { - return a->props.create_account_delegation_ratio < - b->props.create_account_delegation_ratio; - }); - auto median_delegation_ratio = active[active.size() / 2]->props.create_account_delegation_ratio; - - /// sort them by create_account_delegation_time - std::sort(active.begin(), active.end(), [&](const witness_object *a, const witness_object *b) { - return a->props.create_account_delegation_time < - b->props.create_account_delegation_time; - }); - auto median_delegation_time = active[active.size() / 2]->props.create_account_delegation_time; + chain_properties_18 median_props; - /// sort them by min_delegation_multiplier - std::sort(active.begin(), active.end(), [&](const witness_object *a, const witness_object *b) { - return a->props.min_delegation_multiplier < - b->props.min_delegation_multiplier; - }); - auto median_delegation_multiplier = active[active.size() / 2]->props.min_delegation_multiplier; - - /// sort them by maximum_block_size - std::sort(active.begin(), active.end(), [&](const witness_object *a, const witness_object *b) { - return a->props.maximum_block_size < - b->props.maximum_block_size; - }); - uint32_t median_maximum_block_size = active[active.size() / 2]->props.maximum_block_size; - - /// sort them by sbd_interest_rate - std::sort(active.begin(), active.end(), [&](const witness_object *a, const witness_object *b) { - return a->props.sbd_interest_rate < b->props.sbd_interest_rate; - }); - uint16_t median_sbd_interest_rate = active[active.size() / 2]->props.sbd_interest_rate; + auto calc_median = [&](auto&& param) { + std::nth_element( + active.begin(), active.begin() + active.size() / 2, active.end(), + [&](const auto* a, const auto* b) { + return a->props.*param < b->props.*param; + } + ); + median_props.*param = active[active.size() / 2]->props.*param; + }; + + calc_median(&chain_properties_17::account_creation_fee); + calc_median(&chain_properties_17::maximum_block_size); + calc_median(&chain_properties_17::sbd_interest_rate); + calc_median(&chain_properties_18::create_account_with_golos_modifier); + calc_median(&chain_properties_18::create_account_delegation_ratio); + calc_median(&chain_properties_18::create_account_delegation_time); + calc_median(&chain_properties_18::min_delegation_multiplier); modify(wso, [&](witness_schedule_object &_wso) { - _wso.median_props.account_creation_fee = median_account_creation_fee; - _wso.median_props.maximum_block_size = median_maximum_block_size; - _wso.median_props.sbd_interest_rate = median_sbd_interest_rate; - _wso.median_props.create_account_with_golos_modifier = median_with_golos_modifier; - _wso.median_props.create_account_delegation_ratio = median_delegation_ratio; - _wso.median_props.create_account_delegation_time = median_delegation_time; - _wso.median_props.min_delegation_multiplier = median_delegation_multiplier; + _wso.median_props = median_props; }); modify(get_dynamic_global_properties(), [&](dynamic_global_property_object &_dgpo) { - _dgpo.maximum_block_size = median_maximum_block_size; - _dgpo.sbd_interest_rate = median_sbd_interest_rate; + _dgpo.maximum_block_size = median_props.maximum_block_size; + _dgpo.sbd_interest_rate = median_props.sbd_interest_rate; }); } diff --git a/libraries/wallet/include/golos/wallet/wallet.hpp b/libraries/wallet/include/golos/wallet/wallet.hpp index bf3c257126..cdec75f236 100644 --- a/libraries/wallet/include/golos/wallet/wallet.hpp +++ b/libraries/wallet/include/golos/wallet/wallet.hpp @@ -661,11 +661,26 @@ namespace golos { namespace wallet { * @param props The chain properties the witness is voting on. * @param broadcast true if you wish to broadcast the transaction. */ - annotated_signed_transaction update_witness(string witness_name, - string url, - public_key_type block_signing_key, - const chain_properties& props, - bool broadcast = false); + annotated_signed_transaction update_witness( + string witness_name, + string url, + public_key_type block_signing_key, + optional props, + bool broadcast = false + ); + + /** + * Vote for the chain properties + * + * @param witness_name The name of the witness account. + * @param props The chain properties the witness is voting on. + * @param broadcast true if you wish to broadcast the transaction. + */ + annotated_signed_transaction update_chain_properties( + string witness_name, + const chain_properties& props, + bool broadcast = false + ); /** Set the voting proxy for an account. * @@ -1134,6 +1149,7 @@ FC_API( golos::wallet::wallet_api, (update_account_memo_key) (delegate_vesting_shares) (update_witness) + (update_chain_properties) (set_voting_proxy) (vote_for_witness) //(follow) diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 3b25fc9b1c..d99ee9645f 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -1850,46 +1850,71 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st * will be controlable by this wallet. */ - annotated_signed_transaction wallet_api::update_witness( string witness_account_name, - string url, - public_key_type block_signing_key, - const chain_properties& props, - bool broadcast ) - { - FC_ASSERT( !is_locked() ); + annotated_signed_transaction wallet_api::update_witness( + string witness_account_name, + string url, + public_key_type block_signing_key, + optional props, + bool broadcast + ) { + FC_ASSERT(!is_locked()); + + const auto hf = my->_remote_database_api->get_hardfork_version(); + const auto has_hf18 = hf >= hardfork_version(0, STEEMIT_HARDFORK_0_18__673); + signed_transaction tx; witness_update_operation op; - optional< witness_api::witness_api_object > wit = my->_remote_witness_api->get_witness_by_account( witness_account_name ); - if( !wit.valid() ) { + auto wit = my->_remote_witness_api->get_witness_by_account(witness_account_name); + if (!wit.valid()) { op.url = url; } else { - FC_ASSERT( wit->owner == witness_account_name ); - if( url != "" ) + FC_ASSERT(wit->owner == witness_account_name); + if (!url.empty()) { op.url = url; - else + } else { op.url = wit->url; + } } + op.owner = witness_account_name; op.block_signing_key = block_signing_key; - op.props = props; - signed_transaction tx; + if (!has_hf18 && props.valid()) { + op.props = *props; + } + tx.operations.push_back(op); - auto hf = my->_remote_database_api->get_hardfork_version(); - if (hf >= hardfork_version(0, STEEMIT_HARDFORK_0_18)) { + if (has_hf18 && props.valid()) { chain_properties_update_operation chain_op; chain_op.owner = witness_account_name; - chain_op.props = props; + chain_op.props = *props; tx.operations.push_back(chain_op); } tx.validate(); + return my->sign_transaction(tx, broadcast); + } + annotated_signed_transaction wallet_api::update_chain_properties( + string witness_account_name, + const chain_properties& props, + bool broadcast + ) { + FC_ASSERT(!is_locked()); - return my->sign_transaction( tx, broadcast ); + signed_transaction tx; + chain_properties_update_operation op; + + op.owner = witness_account_name; + op.props = props; + tx.operations.push_back(op); + + tx.validate(); + + return my->sign_transaction(tx, broadcast); } annotated_signed_transaction wallet_api::vote_for_witness(string voting_account, string witness_to_vote_for, bool approve, bool broadcast ) From 4ba03eea1bc8fdbe689ab4d3be043190c22540d2 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Mon, 4 Jun 2018 13:16:31 +0700 Subject: [PATCH 08/22] Replace db() with _db. #693 --- .../chain/chain_properties_evaluators.cpp | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/libraries/chain/chain_properties_evaluators.cpp b/libraries/chain/chain_properties_evaluators.cpp index 233b4f425e..073505ad62 100644 --- a/libraries/chain/chain_properties_evaluators.cpp +++ b/libraries/chain/chain_properties_evaluators.cpp @@ -5,23 +5,23 @@ namespace golos { namespace chain { void witness_update_evaluator::do_apply(const witness_update_operation& o) { - db().get_account(o.owner); // verify owner exists + _db.get_account(o.owner); // verify owner exists - if (db().has_hardfork(STEEMIT_HARDFORK_0_1)) { + if (_db.has_hardfork(STEEMIT_HARDFORK_0_1)) { FC_ASSERT(o.url.size() <= STEEMIT_MAX_WITNESS_URL_LENGTH, "URL is too long"); } else if (o.url.size() > STEEMIT_MAX_WITNESS_URL_LENGTH) { // after HF, above check can be moved to validate() if reindex doesn't show this warning - wlog("URL is too long in block ${b}", ("b", db().head_block_num() + 1)); + wlog("URL is too long in block ${b}", ("b", _db.head_block_num() + 1)); } - if (db().has_hardfork(STEEMIT_HARDFORK_0_14__410)) { + if (_db.has_hardfork(STEEMIT_HARDFORK_0_14__410)) { FC_ASSERT(o.props.account_creation_fee.symbol == STEEM_SYMBOL); } else if (o.props.account_creation_fee.symbol != STEEM_SYMBOL) { // after HF, above check can be moved to validate() if reindex doesn't show this warning - wlog("Wrong fee symbol in block ${b}", ("b", db().head_block_num() + 1)); + wlog("Wrong fee symbol in block ${b}", ("b", _db.head_block_num() + 1)); } - const bool has_hf18 = db().has_hardfork(STEEMIT_HARDFORK_0_18__673); + const bool has_hf18 = _db.has_hardfork(STEEMIT_HARDFORK_0_18__673); // TODO: remove this after HF 18 if (has_hf18) { @@ -36,10 +36,10 @@ namespace golos { namespace chain { } } - const auto &idx = db().get_index().indices().get(); + const auto &idx = _db.get_index().indices().get(); auto itr = idx.find(o.owner); if (itr != idx.end()) { - db().modify(*itr, [&](witness_object& w) { + _db.modify(*itr, [&](witness_object& w) { from_string(w.url, o.url); w.signing_key = o.block_signing_key; if (!has_hf18) { @@ -47,11 +47,11 @@ namespace golos { namespace chain { } }); } else { - db().create([&](witness_object& w) { + _db.create([&](witness_object& w) { w.owner = o.owner; from_string(w.url, o.url); w.signing_key = o.block_signing_key; - w.created = db().head_block_time(); + w.created = _db.head_block_time(); if (!has_hf18) { w.props = o.props; } @@ -72,18 +72,18 @@ namespace golos { namespace chain { void chain_properties_update_evaluator::do_apply(const chain_properties_update_operation& o) { ASSERT_REQ_HF(STEEMIT_HARDFORK_0_18__673, "Chain properties"); // remove after hf - db().get_account(o.owner); // verify owner exists + _db.get_account(o.owner); // verify owner exists - const auto &idx = db().get_index().indices().get(); + const auto &idx = _db.get_index().indices().get(); auto itr = idx.find(o.owner); if (itr != idx.end()) { - db().modify(*itr, [&](witness_object& w) { + _db.modify(*itr, [&](witness_object& w) { w.props = o.props.visit(chain_properties_convert()); }); } else { - db().create([&](witness_object& w) { + _db.create([&](witness_object& w) { w.owner = o.owner; - w.created = db().head_block_time(); + w.created = _db.head_block_time(); w.props = o.props.visit(chain_properties_convert()); }); } From 66d972d4cc32edf6d6a204ba5fcf46ca4bf7b034 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Mon, 4 Jun 2018 14:02:16 +0700 Subject: [PATCH 09/22] Change default value for GOLOS_CREATE_ACCOUNT_WITH_GOLOS_MODIFIER to 1. #697 --- libraries/protocol/include/golos/protocol/config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/protocol/include/golos/protocol/config.hpp b/libraries/protocol/include/golos/protocol/config.hpp index 62e9d71d4a..2b12fd1742 100644 --- a/libraries/protocol/include/golos/protocol/config.hpp +++ b/libraries/protocol/include/golos/protocol/config.hpp @@ -106,7 +106,7 @@ #define STEEMIT_SOFT_MAX_COMMENT_DEPTH 0xff // 255 #define STEEMIT_MAX_RESERVE_RATIO (20000) -#define GOLOS_CREATE_ACCOUNT_WITH_GOLOS_MODIFIER 30 +#define GOLOS_CREATE_ACCOUNT_WITH_GOLOS_MODIFIER 1 #define GOLOS_CREATE_ACCOUNT_DELEGATION_RATIO 5 #define GOLOS_CREATE_ACCOUNT_DELEGATION_TIME (fc::days(1)) #define GOLOS_MIN_DELEGATION_MULTIPLIER 10 From a2a53a939ed9db3296fde6c2f55c20c5e5f3c0ca Mon Sep 17 00:00:00 2001 From: maslenitsa93 Date: Mon, 4 Jun 2018 11:01:41 +0300 Subject: [PATCH 10/22] Mongo_db #421, #608, #609, #635, #643 --- .travis.yml | 1 + CMakeLists.txt | 8 + Dockerfile | 1 + plugins/mongo_db/CMakeLists.txt | 81 ++ .../plugins/mongo_db/mongo_db_operations.hpp | 85 ++ .../plugins/mongo_db/mongo_db_plugin.hpp | 50 ++ .../golos/plugins/mongo_db/mongo_db_state.hpp | 102 +++ .../golos/plugins/mongo_db/mongo_db_types.hpp | 128 +++ .../plugins/mongo_db/mongo_db_writer.hpp | 92 +++ plugins/mongo_db/mongo_db_operations.cpp | 730 ++++++++++++++++++ plugins/mongo_db/mongo_db_plugin.cpp | 127 +++ plugins/mongo_db/mongo_db_state.cpp | 555 +++++++++++++ plugins/mongo_db/mongo_db_types.cpp | 17 + plugins/mongo_db/mongo_db_writer.cpp | 296 +++++++ programs/golosd/CMakeLists.txt | 1 + programs/golosd/main.cpp | 6 + share/golosd/config/config_debug_mongo.ini | 149 ++++ share/golosd/config/config_mongo.ini | 149 ++++ share/golosd/docker/Dockerfile-lowmem | 1 + share/golosd/docker/Dockerfile-lowmem-small | 1 + share/golosd/docker/Dockerfile-mongo | 159 ++++ share/golosd/docker/Dockerfile-small | 1 + share/golosd/docker/Dockerfile-test | 2 + share/golosd/docker/Dockerfile-testnet | 1 + share/golosd/docker/Dockerfile-testnet-mongo | 156 ++++ 25 files changed, 2899 insertions(+) create mode 100644 plugins/mongo_db/CMakeLists.txt create mode 100644 plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp create mode 100644 plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_plugin.hpp create mode 100644 plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp create mode 100644 plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_types.hpp create mode 100644 plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_writer.hpp create mode 100644 plugins/mongo_db/mongo_db_operations.cpp create mode 100644 plugins/mongo_db/mongo_db_plugin.cpp create mode 100644 plugins/mongo_db/mongo_db_state.cpp create mode 100644 plugins/mongo_db/mongo_db_types.cpp create mode 100644 plugins/mongo_db/mongo_db_writer.cpp create mode 100644 share/golosd/config/config_debug_mongo.ini create mode 100644 share/golosd/config/config_mongo.ini create mode 100644 share/golosd/docker/Dockerfile-mongo create mode 100644 share/golosd/docker/Dockerfile-testnet-mongo diff --git a/.travis.yml b/.travis.yml index ce5e40c328..aace6349b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - DOCKERFILE=share/golosd/docker/Dockerfile-test DOCKERNAME="-test" - DOCKERFILE=share/golosd/docker/Dockerfile-testnet DOCKERNAME="-testnet" - DOCKERFILE=share/golosd/docker/Dockerfile-lowmem DOCKERNAME="-lowmem" + - DOCKERFILE=share/golosd/docker/Dockerfile-mongo DOCKERNAME="-mongo" matrix: fast_finish: true diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b63b35db7..e7d54dc2a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,14 @@ if(CHAINBASE_CHECK_LOCKING) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DCHAINBASE_CHECK_LOCKING") endif() +option(ENABLE_MONGO_PLUGIN "Build with mongodb plugin" FALSE) +if(ENABLE_MONGO_PLUGIN) + set(MONGO_LIB golos::mongo_db) + + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMONGODB_PLUGIN_BUILT") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DMONGODB_PLUGIN_BUILT") +endif() + if(WIN32) set(BOOST_ROOT $ENV{BOOST_ROOT}) set(Boost_USE_MULTITHREADED ON) diff --git a/Dockerfile b/Dockerfile index f00ffa58b9..2c0fe271b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,7 @@ RUN \ -DBUILD_SHARED_LIBRARIES=FALSE \ -DLOW_MEMORY_NODE=FALSE \ -DCHAINBASE_CHECK_LOCKING=FALSE \ + -DENABLE_MONGO_PLUGIN=FALSE \ .. \ && \ make -j$(nproc) && \ diff --git a/plugins/mongo_db/CMakeLists.txt b/plugins/mongo_db/CMakeLists.txt new file mode 100644 index 0000000000..a090dbd24f --- /dev/null +++ b/plugins/mongo_db/CMakeLists.txt @@ -0,0 +1,81 @@ + +if(ENABLE_MONGO_PLUGIN) + + # Golos has no direct dependencies on libmongoc but its shared libraries + # will need to be present at runtime for the C++ libraries we use: + # libbsoncxx & libmongocxx (both from github.com/mongodb/mongo-cxx-driver) + + # The *.cmake package files provided by mongo-cxx-driver don't give us the + # absolute path to the libraries, which is needed whenever they are not + # installed in system-known locations. CMake requires the absolute paths + # in target_link_libraries() since we are builiding an archive and the + # link step for all executables using this archive must include the + # mongo-cxx-driver libraries libmongocxx and libbsoncxx. + + find_package(libmongoc-1.0 1.8) + + if (libmongoc-1.0_FOUND) + + message("MongoDB found, building plugin...") + + find_package(libbsoncxx REQUIRED) + find_package(libmongocxx REQUIRED) + + set(CURRENT_TARGET mongo_db) + + list(APPEND CURRENT_TARGET_HEADERS + include/golos/plugins/mongo_db/mongo_db_plugin.hpp + include/golos/plugins/mongo_db/mongo_db_writer.hpp + include/golos/plugins/mongo_db/mongo_db_operations.hpp + include/golos/plugins/mongo_db/mongo_db_state.hpp + include/golos/plugins/mongo_db/mongo_db_types.hpp + ) + + list(APPEND CURRENT_TARGET_SOURCES + mongo_db_plugin.cpp + mongo_db_writer.cpp + mongo_db_operations.cpp + mongo_db_state.cpp + mongo_db_types.cpp + ) + + if(BUILD_SHARED_LIBRARIES) + add_library(golos_${CURRENT_TARGET} SHARED + ${CURRENT_TARGET_HEADERS} + ${CURRENT_TARGET_SOURCES} + ) + else() + add_library(golos_${CURRENT_TARGET} STATIC + ${CURRENT_TARGET_HEADERS} + ${CURRENT_TARGET_SOURCES} + ) + endif() + + add_library(golos::${CURRENT_TARGET} ALIAS golos_${CURRENT_TARGET}) + set_property(TARGET golos_${CURRENT_TARGET} PROPERTY EXPORT_NAME ${CURRENT_TARGET}) + + target_link_libraries( + golos_${CURRENT_TARGET} + golos::json_rpc + golos_chain + golos::chain_plugin + golos::follow + appbase + fc + ${LIBBSONCXX_LIBRARIES} + ${LIBMONGOCXX_LIBRARIES} + ) + target_include_directories(golos_${CURRENT_TARGET} PUBLIC "include" ${LIBMONGOCXX_INCLUDE_DIRS} ${LIBBSONCXX_INCLUDE_DIRS}) + + install(TARGETS + golos_${CURRENT_TARGET} + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + ) + install( FILES ${HEADERS} DESTINATION "include/golos/plugins/mongo_db" ) + else() + message("Mongo drivers NOT found. Use proper Dockerfile") + endif() +endif() \ No newline at end of file diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp new file mode 100644 index 0000000000..cb91c15420 --- /dev/null +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include +#include + +#include + + +namespace golos { +namespace plugins { +namespace mongo_db { + + class operation_writer { + public: + using result_type = document; + + operation_writer(); + + result_type operator()(const vote_operation& op); + result_type operator()(const comment_operation& op); + result_type operator()(const transfer_operation& op); + result_type operator()(const transfer_to_vesting_operation& op); + result_type operator()(const withdraw_vesting_operation& op); + result_type operator()(const limit_order_create_operation& op); + result_type operator()(const limit_order_cancel_operation& op); + result_type operator()(const feed_publish_operation& op); + result_type operator()(const convert_operation& op); + result_type operator()(const account_create_operation& op); + result_type operator()(const account_update_operation& op); + result_type operator()(const witness_update_operation& op); + result_type operator()(const account_witness_vote_operation& op); + result_type operator()(const account_witness_proxy_operation& op); + result_type operator()(const pow_operation& op); + result_type operator()(const custom_operation& op); + result_type operator()(const report_over_production_operation& op); + result_type operator()(const delete_comment_operation& op); + result_type operator()(const custom_json_operation& op); + result_type operator()(const comment_options_operation& op); + result_type operator()(const set_withdraw_vesting_route_operation& op); + result_type operator()(const limit_order_create2_operation& op); + result_type operator()(const challenge_authority_operation& op); + result_type operator()(const prove_authority_operation& op); + result_type operator()(const request_account_recovery_operation& op); + result_type operator()(const recover_account_operation& op); + result_type operator()(const change_recovery_account_operation& op); + result_type operator()(const escrow_transfer_operation& op); + result_type operator()(const escrow_dispute_operation& op); + result_type operator()(const escrow_release_operation&op); + result_type operator()(const pow2_operation& op); + result_type operator()(const escrow_approve_operation& op); + result_type operator()(const transfer_to_savings_operation& op); + result_type operator()(const transfer_from_savings_operation& op); + result_type operator()(const cancel_transfer_from_savings_operation&op); + result_type operator()(const custom_binary_operation& op); + result_type operator()(const decline_voting_rights_operation& op); + result_type operator()(const reset_account_operation& op); + result_type operator()(const set_reset_account_operation& op); +// + result_type operator()(const delegate_vesting_shares_operation& op); + result_type operator()(const account_create_with_delegation_operation& op); + result_type operator()(const account_metadata_operation& op); + result_type operator()(const proposal_create_operation& op); + result_type operator()(const proposal_update_operation& op); + result_type operator()(const proposal_delete_operation& op); +// + result_type operator()(const fill_convert_request_operation& op); + result_type operator()(const author_reward_operation& op); + result_type operator()(const curation_reward_operation& op); + result_type operator()(const comment_reward_operation& op); + result_type operator()(const liquidity_reward_operation& op); + result_type operator()(const interest_operation& op); + result_type operator()(const fill_vesting_withdraw_operation& op); + result_type operator()(const fill_order_operation& op); + result_type operator()(const shutdown_witness_operation& op); + result_type operator()(const fill_transfer_from_savings_operation& op); + result_type operator()(const hardfork_operation& op); + result_type operator()(const comment_payout_update_operation& op); + result_type operator()(const comment_benefactor_reward_operation& op); +// + result_type operator()(const return_vesting_delegation_operation& op); +// + result_type operator()(const chain_properties_update_operation& op); + }; + +}}} // golos::plugins::mongo_db diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_plugin.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_plugin.hpp new file mode 100644 index 0000000000..c9ea28fb24 --- /dev/null +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_plugin.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace golos { +namespace plugins { +namespace mongo_db { + + class mongo_db_plugin final : public appbase::plugin { + public: + + APPBASE_PLUGIN_REQUIRES( + (chain::plugin) + ) + + mongo_db_plugin(); + + virtual ~mongo_db_plugin(); + + void set_program_options( + boost::program_options::options_description& cli, + boost::program_options::options_description& cfg + ) override; + + void plugin_initialize(const boost::program_options::variables_map& options) override; + + void plugin_startup() override; + + void plugin_shutdown() override; + + constexpr const static char *plugin_name = "mongo_db"; + + static const std::string& name() { + static std::string name = plugin_name; + return name; + } + + private: + class mongo_db_plugin_impl; + + std::unique_ptr pimpl_; + }; + +}}} // namespace golos::plugins::mongo_db + diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp new file mode 100644 index 0000000000..cfb5eb7e2f --- /dev/null +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace golos { +namespace plugins { +namespace mongo_db { + + class state_writer { + public: + using result_type = void; + + state_writer(db_map& bmi_to_add, const signed_block& block); + + result_type operator()(const vote_operation& op); + result_type operator()(const comment_operation& op); + result_type operator()(const transfer_operation& op); + result_type operator()(const transfer_to_vesting_operation& op); + result_type operator()(const withdraw_vesting_operation& op); + result_type operator()(const limit_order_create_operation& op); + result_type operator()(const limit_order_cancel_operation& op); + result_type operator()(const feed_publish_operation& op); + result_type operator()(const convert_operation& op); + result_type operator()(const account_create_operation& op); + result_type operator()(const account_update_operation& op); + result_type operator()(const witness_update_operation& op); + result_type operator()(const account_witness_vote_operation& op); + result_type operator()(const account_witness_proxy_operation& op); + result_type operator()(const pow_operation& op); + result_type operator()(const custom_operation& op); + result_type operator()(const report_over_production_operation& op); + result_type operator()(const delete_comment_operation& op); + result_type operator()(const custom_json_operation& op); + result_type operator()(const comment_options_operation& op); + result_type operator()(const set_withdraw_vesting_route_operation& op); + result_type operator()(const limit_order_create2_operation& op); + result_type operator()(const challenge_authority_operation& op); + result_type operator()(const prove_authority_operation& op); + result_type operator()(const request_account_recovery_operation& op); + result_type operator()(const recover_account_operation& op); + result_type operator()(const change_recovery_account_operation& op); + result_type operator()(const escrow_transfer_operation& op); + result_type operator()(const escrow_dispute_operation& op); + result_type operator()(const escrow_release_operation&op); + result_type operator()(const pow2_operation& op); + result_type operator()(const escrow_approve_operation& op); + result_type operator()(const transfer_to_savings_operation& op); + result_type operator()(const transfer_from_savings_operation& op); + result_type operator()(const cancel_transfer_from_savings_operation&op); + result_type operator()(const custom_binary_operation& op); + result_type operator()(const decline_voting_rights_operation& op); + result_type operator()(const reset_account_operation& op); + result_type operator()(const set_reset_account_operation& op); + result_type operator()(const delegate_vesting_shares_operation& op); + result_type operator()(const account_create_with_delegation_operation& op); + result_type operator()(const account_metadata_operation& op); + result_type operator()(const proposal_create_operation& op); + result_type operator()(const proposal_update_operation& op); + result_type operator()(const proposal_delete_operation& op); + result_type operator()(const fill_convert_request_operation& op); + result_type operator()(const author_reward_operation& op); + result_type operator()(const curation_reward_operation& op); + result_type operator()(const comment_reward_operation& op); + result_type operator()(const liquidity_reward_operation& op); + result_type operator()(const interest_operation& op); + result_type operator()(const fill_vesting_withdraw_operation& op); + result_type operator()(const fill_order_operation& op); + result_type operator()(const shutdown_witness_operation& op); + result_type operator()(const fill_transfer_from_savings_operation& op); + result_type operator()(const hardfork_operation& op); + result_type operator()(const comment_payout_update_operation& op); + result_type operator()(const comment_benefactor_reward_operation& op); + result_type operator()(const return_vesting_delegation_operation& op); + result_type operator()(const chain_properties_update_operation& op); + + private: + database &db_; + + const signed_block &state_block; + + db_map &all_docs; + + void format_comment(const std::string& auth, const std::string& perm); + + named_document create_document(const std::string& name, + const std::string& key, const std::string& keyval); + + named_document create_removal_document(const std::string& name, + const std::string& key, const std::string& keyval); + }; + +}}} // golos::plugins::mongo_db diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_types.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_types.hpp new file mode 100644 index 0000000000..6d8c93c0f9 --- /dev/null +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_types.hpp @@ -0,0 +1,128 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace golos { +namespace plugins { +namespace mongo_db { + + using namespace golos::chain; + using namespace golos::protocol; + using golos::chain::to_string; + using golos::chain::shared_string; + using bsoncxx::builder::stream::document; + + struct named_document { + std::string collection_name; + document doc; + bool is_removal; + //bool is_virtual; + std::vector indexes_to_create; + std::string key; + std::string keyval; + }; + + struct hashed_idx; + + typedef boost::multi_index_container< + named_document, + boost::multi_index::indexed_by< + boost::multi_index::random_access< + >, + boost::multi_index::hashed_unique< + boost::multi_index::tag, + boost::multi_index::composite_key< + named_document, + boost::multi_index::member, + boost::multi_index::member, + boost::multi_index::member, + boost::multi_index::member + > + > + > + > db_map; + + void bmi_insert_or_replace(db_map& bmi, named_document doc); + + using named_document_ptr = std::unique_ptr; + + inline void format_oid(document& doc, const std::string& name, const std::string& value) { + auto oid = fc::sha1::hash(value).str().substr(0, 24); + doc << name << bsoncxx::oid(oid); + } + + inline void format_oid(document& doc, const std::string& value) { + format_oid(doc, "_id", value); + } + + // Helper functions + inline void format_value(document& doc, const std::string& name, const asset& value) { + doc << name + "_value" << value.to_real(); + doc << name + "_symbol" << value.symbol_name(); + } + + inline void format_value(document& doc, const std::string& name, const std::string& value) { + doc << name << value; + } + + inline void format_value(document& doc, const std::string& name, const bool value) { + doc << name << value; + } + + inline void format_value(document& doc, const std::string& name, const double value) { + doc << name << value; + } + + inline void format_value(document& doc, const std::string& name, const fc::uint128_t& value) { + doc << name << static_cast(value.lo); + } + + inline void format_value(document& doc, const std::string& name, const fc::time_point_sec& value) { + doc << name << value.to_iso_string(); + } + + inline void format_value(document& doc, const std::string& name, const shared_string& value) { + doc << name << to_string(value); + } + + template + inline void format_value(document& doc, const std::string& name, const fc::fixed_string& value) { + doc << name << static_cast(value); + } + + template + inline void format_value(document& doc, const std::string& name, const T& value) { + doc << name << static_cast(value); + } + + template + inline void format_value(document& doc, const std::string& name, const fc::safe& value) { + doc << name << static_cast(value.value); + } + +}}} // golos::plugins::mongo_db diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_writer.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_writer.hpp new file mode 100644 index 0000000000..f0885b224d --- /dev/null +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_writer.hpp @@ -0,0 +1,92 @@ +#pragma once +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include + + +namespace golos { +namespace plugins { +namespace mongo_db { + + using golos::protocol::signed_block; + using golos::protocol::signed_transaction; + using bsoncxx::builder::stream::document; + using bsoncxx::builder::stream::open_document; + using golos::chain::operation_notification; + using namespace golos::protocol; + + using bulk_ptr = std::unique_ptr; + + class mongo_db_writer final { + public: + mongo_db_writer(); + ~mongo_db_writer(); + + bool initialize(const std::string& uri_str, const bool write_raw, const std::vector& op); + + void on_block(const signed_block& block); + void on_operation(const golos::chain::operation_notification& note); + + private: + using operations = std::vector; + + void write_blocks(); + void write_raw_block(const signed_block& block, const operations&); + void write_block_operations(state_writer& st_writer, const signed_block& block, const operations&); + void write_document(named_document const& named_doc); + void remove_document(named_document const& named_doc); + + void format_block_info(const signed_block& block, document& doc); + void format_transaction_info(const signed_transaction& tran, document& doc); + + void write_data(); + + uint64_t processed_blocks = 0; + + std::string db_name; + + // Key = Block num, Value = block + uint32_t last_irreversible_block_num; + std::map blocks; + std::map virtual_ops; + // Table name, bulk write + std::map formatted_blocks; + + bool write_raw_blocks; + flat_set write_operations; + + // Mongo connection members + mongocxx::instance mongo_inst; + mongocxx::database mongo_database; + mongocxx::uri uri; + mongocxx::client mongo_conn; + mongocxx::options::bulk_write bulk_opts; + + std::unordered_map indexes; // Prevent repeative create_index() calls. Only in current session + + golos::chain::database &_db; + }; +}}} + diff --git a/plugins/mongo_db/mongo_db_operations.cpp b/plugins/mongo_db/mongo_db_operations.cpp new file mode 100644 index 0000000000..4e4bd6a324 --- /dev/null +++ b/plugins/mongo_db/mongo_db_operations.cpp @@ -0,0 +1,730 @@ +#include + +#include +#include +#include +#include + + +namespace golos { +namespace plugins { +namespace mongo_db { + + using bsoncxx::builder::stream::array; + using bsoncxx::builder::stream::document; + + document format_authority(const authority &auth) { + array account_auths_arr; + for (const auto &iter : auth.account_auths) { + document temp; + temp << "public_key_type" << (std::string) iter.first + << "weight_type" << iter.second; + account_auths_arr << temp; + } + array key_auths_arr; + for (auto &iter : auth.key_auths) { + document temp; + temp << "public_key_type" << (std::string) iter.first + << "weight_type" << iter.second; + key_auths_arr << temp; + } + + document authority_doc; + authority_doc << "owner" << auth.owner; + format_value(authority_doc, "weight_threshold", auth.weight_threshold); + authority_doc << "account_auths" << account_auths_arr; + authority_doc << "key_auths" << key_auths_arr; + + return authority_doc; + } + + void format_authority(document &doc, const std::string &name, const authority &auth) { + doc << name << format_authority(auth); + } + + std::string to_string(const signature_type& signature) { + std::string retVal; + + for (auto& iter : signature) { + retVal += iter; + } + return retVal; + } + + void format_chain_properties_17(document& doc, const chain_properties_17& props) { + format_value(doc, "account_creation_fee", props.account_creation_fee); + format_value(doc, "maximum_block_size", props.maximum_block_size); + format_value(doc, "sbd_interest_rate", props.sbd_interest_rate); + } + + void format_chain_properties_v(document& doc, const versioned_chain_properties& props) { + //TODO + } + + ///////////////////////////////////////////////// + + operation_writer::operation_writer() { + } + + auto operation_writer::operator()(const vote_operation& op) -> result_type { + result_type body; + + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "voter", op.voter); + format_value(body, "weight", op.weight); + + return body; + } + + auto operation_writer::operator()(const comment_operation& op) -> result_type { + result_type body; + + format_value(body, "parent_author", op.parent_author); + format_value(body, "parent_permlink", op.parent_permlink); + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "title", op.title); + format_value(body, "body", op.body); + format_value(body, "json_metadata", op.json_metadata); + + return body; + } + + auto operation_writer::operator()(const transfer_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "amount", op.amount); + format_value(body, "memo", op.memo); + + return body; + } + + auto operation_writer::operator()(const transfer_to_vesting_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "amount", op.amount); + + return body; + } + + auto operation_writer::operator()(const withdraw_vesting_operation& op) -> result_type { + result_type body; + + format_value(body, "account", op.account); + format_value(body, "vesting_shares", op.vesting_shares); + + return body; + } + + auto operation_writer::operator()(const limit_order_create_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + format_value(body, "orderid", op.orderid); + format_value(body, "amount_to_sell", op.amount_to_sell); + format_value(body, "min_to_receive", op.min_to_receive); + format_value(body, "expiration", op.expiration); + + return body; + } + + auto operation_writer::operator()(const limit_order_cancel_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + format_value(body, "orderid", op.orderid); + + return body; + } + + auto operation_writer::operator()(const feed_publish_operation& op) -> result_type { + result_type body; + + format_value(body, "publisher", op.publisher); + format_value(body, "exchange_rate", op.exchange_rate.to_real()); + + return body; + } + + auto operation_writer::operator()(const convert_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + format_value(body, "requestid", op.requestid); + format_value(body, "amount", op.amount); + + return body; + } + + auto operation_writer::operator()(const account_create_operation& op) -> result_type { + result_type body; + + format_value(body, "fee", op.fee); + format_value(body, "creator", op.creator); + format_value(body, "new_account_name", op.new_account_name); + format_authority(body, "owner", op.owner); + format_value(body, "json_metadata", op.json_metadata); + format_value(body, "memo_key", (std::string)op.memo_key); + format_authority(body, "posting", op.posting); + + return body; + } + + auto operation_writer::operator()(const account_update_operation& op) -> result_type { + result_type body; + + if (op.owner) { + format_authority(body, "owner", *op.owner); + } + + document posting_owner_doc; + if (op.posting) { + format_authority(body, "posting", *op.posting); + } + + document active_owner_doc; + if (op.active) { + format_authority(body, "active", *op.active); + } + + format_value(body, "account", op.account); + format_value(body, "json_metadata", op.json_metadata); + format_value(body, "memo_key", (std::string)op.memo_key); + + return body; + } + + auto operation_writer::operator()(const witness_update_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + format_value(body, "fee", op.fee); + format_value(body, "url", op.url); + format_value(body, "block_signing_key", (std::string)op.block_signing_key); + format_chain_properties_17(body, op.props); + + return body; + } + + auto operation_writer::operator()(const account_witness_vote_operation& op) -> result_type { + result_type body; + + format_value(body, "account", op.account); + format_value(body, "witness", op.witness); + format_value(body, "approve", (op.approve ? std::string("true") : std::string("false"))); + + return body; + } + + auto operation_writer::operator()(const account_witness_proxy_operation& op) -> result_type { + result_type body; + + format_value(body, "account", op.account); + format_value(body, "proxy", op.proxy); + + return body; + } + + auto operation_writer::operator()(const pow_operation& op) -> result_type { + result_type body; + + format_value(body, "worker", (std::string)op.work.worker); + format_value(body, "input", op.work.input.str()); + format_value(body, "signature", to_string(op.work.signature)); + format_value(body, "work", op.work.work.str()); + + format_value(body, "block_id", op.block_id.str()); + format_value(body, "worker_account", op.worker_account); + format_value(body, "nonce", op.nonce); + + format_chain_properties_17(body, op.props); + + return body; + } + + auto operation_writer::operator()(const custom_operation& op) -> result_type { + result_type body; + + format_value(body, "id", op.id); + if (!op.required_auths.empty()) { + array auths; + for (auto& iter : op.required_auths) { + auths << iter; + } + body << "required_auths" << auths; + } + + return body; + } + + auto operation_writer::operator()(const report_over_production_operation& op) -> result_type { + result_type body; + + document doc1; + format_value(doc1, "id", op.first_block.id().str()); + format_value(doc1, "timestamp", op.first_block.timestamp); + format_value(doc1, "witness", op.first_block.witness); + + document doc2; + format_value(doc2, "id", op.second_block.id().str()); + format_value(doc2, "timestamp", op.first_block.timestamp); + format_value(doc2, "witness", op.second_block.witness); + + format_value(body, "reporter", op.reporter); + + body << "first_block" << doc1; + body << "second_block" << doc2; + + return body; + } + + auto operation_writer::operator()(const delete_comment_operation& op) -> result_type { + result_type body; + + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + + return body; + } + + auto operation_writer::operator()(const custom_json_operation& op) -> result_type { + result_type body; + + format_value(body, "id", op.id); + format_value(body, "json", op.json); + + return body; + } + + auto operation_writer::operator()(const comment_options_operation& op) -> result_type { + result_type body; + + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "max_accepted_payout", op.max_accepted_payout); + format_value(body, "percent_steem_dollars", op.percent_steem_dollars); + format_value(body, "allow_votes", op.allow_votes); + format_value(body, "allow_curation_rewards", op.allow_curation_rewards); + // comment_options_extensions_type extensions; + + return body; + } + + auto operation_writer::operator()(const set_withdraw_vesting_route_operation& op) -> result_type { + result_type body; + + format_value(body, "from_account", op.from_account); + format_value(body, "to_account", op.to_account); + format_value(body, "percent", op.percent); + format_value(body, "auto_vest", op.auto_vest); + + return body; + } + + auto operation_writer::operator()(const limit_order_create2_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + format_value(body, "orderid", op.orderid); + format_value(body, "amount_to_sell", op.amount_to_sell); + format_value(body, "fill_or_kill", op.fill_or_kill); + format_value(body, "exchange_rate", op.exchange_rate.to_real()); + format_value(body, "expiration", op.expiration); + + return body; + } + + auto operation_writer::operator()(const challenge_authority_operation& op) -> result_type { + result_type body; + + format_value(body, "challenger", op.challenger); + format_value(body, "challenged", op.challenged); + format_value(body, "require_owner", op.require_owner); + + return body; + } + + auto operation_writer::operator()(const prove_authority_operation& op) -> result_type { + result_type body; + + format_value(body, "challenged", op.challenged); + format_value(body, "require_owner", op.require_owner); + + return body; + } + + auto operation_writer::operator()(const request_account_recovery_operation& op) -> result_type { + result_type body; + + format_value(body, "recovery_account", op.recovery_account); + format_value(body, "account_to_recover", op.account_to_recover); + format_authority(body, "new_owner_authority", op.new_owner_authority); + + return body; + } + + auto operation_writer::operator()(const recover_account_operation& op) -> result_type { + result_type body; + + format_value(body, "account_to_recover", op.account_to_recover); + format_authority(body, "new_owner_authority", op.new_owner_authority); + format_authority(body, "recent_owner_authority", op.recent_owner_authority); + + return body; + } + + auto operation_writer::operator()(const change_recovery_account_operation& op) -> result_type { + result_type body; + + format_value(body, "account_to_recover", op.account_to_recover); + format_value(body, "new_recovery_account", op.new_recovery_account); + + return body; + } + + auto operation_writer::operator()(const escrow_transfer_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "agent", op.agent); + format_value(body, "escrow_id", op.escrow_id); + + format_value(body, "sbd_amount", op.sbd_amount); + format_value(body, "steem_amount ", op.steem_amount); + format_value(body, "fee", op.fee); + format_value(body, "json_meta", op.json_meta); + + return body; + } + + auto operation_writer::operator()(const escrow_dispute_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "agent", op.agent); + format_value(body, "who", op.who); + format_value(body, "escrow_id", op.escrow_id); + + return body; + } + + auto operation_writer::operator()(const escrow_release_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "agent", op.agent); + format_value(body, "who", op.who); + format_value(body, "receiver", op.receiver); + format_value(body, "escrow_id", op.escrow_id); + + format_value(body, "sbd_amount", op.sbd_amount); + format_value(body, "steem_amount", op.steem_amount); + + return body; + } + + auto operation_writer::operator()(const pow2_operation& op) -> result_type { + result_type body; + + format_chain_properties_17(body, op.props); + if (op.new_owner_key) { + format_value(body, "new_owner_key", (std::string)(*op.new_owner_key)); + } + + return body; + } + + auto operation_writer::operator()(const escrow_approve_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "agent", op.agent); + format_value(body, "who", op.who); + format_value(body, "escrow_id", op.escrow_id); + format_value(body, "approve", op.approve); + + return body; + } + + auto operation_writer::operator()(const transfer_to_savings_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "amount", op.amount); + format_value(body, "memo", op.memo); + + return body; + } + + auto operation_writer::operator()(const transfer_from_savings_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "amount", op.amount); + format_value(body, "memo", op.memo); + format_value(body, "request_id", op.request_id); + + return body; + } + + auto operation_writer::operator()(const cancel_transfer_from_savings_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "request_id", op.request_id); + + return body; + } + + auto operation_writer::operator()(const custom_binary_operation& op) -> result_type { + result_type body; + + array required_owner_auths_arr; + for (auto& iter : op.required_owner_auths) { + required_owner_auths_arr << iter; + } + + array required_active_auths_arr; + for (auto& iter : op.required_active_auths) { + required_active_auths_arr << iter; + } + + array required_posting_auths_arr; + for (auto& iter : op.required_posting_auths) { + required_posting_auths_arr << iter; + } + + array auths; + for (auto& iter : op.required_auths) { + auths << format_authority(iter); + } + + format_value(body, "id", op.id); + body << "required_owner_auths" << required_owner_auths_arr; + body << "required_active_auths" << required_active_auths_arr; + body << "required_posting_auths" << required_posting_auths_arr; + body << "required_auths" << auths; + + return body; + } + + auto operation_writer::operator()(const decline_voting_rights_operation& op) -> result_type { + result_type body; + + format_value(body, "account", op.account); + format_value(body, "decline", op.decline); + + return body; + } + + auto operation_writer::operator()(const reset_account_operation& op) -> result_type { + result_type body; + + format_value(body, "reset_account", op.reset_account); + format_value(body, "account_to_reset", op.account_to_reset); + format_authority(body, "new_owner_authority", op.new_owner_authority); + + return body; + } + + auto operation_writer::operator()(const set_reset_account_operation& op) -> result_type { + result_type body; + + format_value(body, "account", op.account); + format_value(body, "current_reset_account", op.current_reset_account); + format_value(body, "reset_account", op.reset_account); + + return body; + } + +// + auto operation_writer::operator()(const delegate_vesting_shares_operation& op) -> result_type { + result_type body; + + return body; + } + auto operation_writer::operator()(const account_create_with_delegation_operation& op) -> result_type { + result_type body; + + return body; + } + auto operation_writer::operator()(const account_metadata_operation& op) -> result_type { + result_type body; + + return body; + } + auto operation_writer::operator()(const proposal_create_operation& op) -> result_type { + result_type body; + + return body; + } + auto operation_writer::operator()(const proposal_update_operation& op) -> result_type { + result_type body; + + return body; + } + auto operation_writer::operator()(const proposal_delete_operation& op) -> result_type { + result_type body; + + return body; + } +// + auto operation_writer::operator()(const fill_convert_request_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + format_value(body, "requestid", op.requestid); + format_value(body, "amount_in", op.amount_in); + format_value(body, "amount_out", op.amount_out); + + return body; + } + + auto operation_writer::operator()(const author_reward_operation& op) -> result_type { + result_type body; + + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "sbd_payout", op.sbd_payout); + format_value(body, "steem_payout", op.steem_payout); + format_value(body, "vesting_payout", op.vesting_payout); + + return body; + } + + auto operation_writer::operator()(const curation_reward_operation& op) -> result_type { + result_type body; + + format_value(body, "curator", op.curator); + format_value(body, "reward", op.reward); + format_value(body, "comment_author", op.comment_author); + format_value(body, "comment_permlink", op.comment_permlink); + + return body; + } + + auto operation_writer::operator()(const comment_reward_operation& op) -> result_type { + result_type body; + + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "payout", op.payout); + + return body; + } + + auto operation_writer::operator()(const liquidity_reward_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + format_value(body, "payout", op.payout); + + return body; + } + + auto operation_writer::operator()(const interest_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + format_value(body, "interest", op.interest); + + return body; + } + + auto operation_writer::operator()(const fill_vesting_withdraw_operation& op) -> result_type { + result_type body; + + format_value(body, "from_account", op.from_account); + format_value(body, "to_account", op.to_account); + format_value(body, "withdrawn", op.withdrawn); + format_value(body, "deposited", op.deposited); + + return body; + } + + auto operation_writer::operator()(const fill_order_operation& op) -> result_type { + result_type body; + + format_value(body, "current_owner", op.current_owner); + format_value(body, "current_orderid", op.current_orderid); + format_value(body, "current_pays", op.current_pays); + format_value(body, "open_owner", op.open_owner); + format_value(body, "open_orderid", op.open_orderid); + format_value(body, "open_pays", op.open_pays); + + return body; + } + + auto operation_writer::operator()(const shutdown_witness_operation& op) -> result_type { + result_type body; + + format_value(body, "owner", op.owner); + + return body; + } + + auto operation_writer::operator()(const fill_transfer_from_savings_operation& op) -> result_type { + result_type body; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "amount", op.amount); + format_value(body, "request_id", op.request_id); + format_value(body, "memo", op.memo); + + return body; + } + + auto operation_writer::operator()(const hardfork_operation& op) -> result_type { + result_type body; + + format_value(body, "hardfork_id", op.hardfork_id); + + return body; + } + + auto operation_writer::operator()(const comment_payout_update_operation& op) -> result_type { + result_type body; + + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + + return body; + } + + auto operation_writer::operator()(const comment_benefactor_reward_operation& op) -> result_type { + result_type body; + + format_value(body, "benefactor", op.benefactor); + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "reward", op.reward); + + return body; + } + + auto operation_writer::operator()(const return_vesting_delegation_operation& op) -> result_type { + result_type body; + + return body; + } + + auto operation_writer::operator()(const chain_properties_update_operation& op) -> result_type { + result_type body; + format_value(body, "owner", op.owner); + format_chain_properties_v(body, op.props); + return body; + } + +}}} diff --git a/plugins/mongo_db/mongo_db_plugin.cpp b/plugins/mongo_db/mongo_db_plugin.cpp new file mode 100644 index 0000000000..1333474e80 --- /dev/null +++ b/plugins/mongo_db/mongo_db_plugin.cpp @@ -0,0 +1,127 @@ +#include +#include +#include +#include + +#include + +namespace golos { +namespace plugins { +namespace mongo_db { + + using golos::protocol::signed_block; + + class mongo_db_plugin::mongo_db_plugin_impl { + public: + mongo_db_plugin_impl(mongo_db_plugin &plugin) + : pimpl_(plugin), + db_(appbase::app().get_plugin().db()) { + } + + bool initialize(const std::string& uri, const bool write_raw, const std::vector& op) { + return writer.initialize(uri, write_raw, op); + } + + ~mongo_db_plugin_impl() = default; + + void on_block(const signed_block& block) { + writer.on_block(block); + } + + void on_operation(const golos::chain::operation_notification& note) { + if (is_virtual_operation(note.op)) { + writer.on_operation(note); + } + } + + golos::chain::database &database() const { + return db_; + } + + mongo_db_writer writer; + mongo_db_plugin &pimpl_; + + golos::chain::database &db_; + }; + + // Plugin + mongo_db_plugin::mongo_db_plugin() { + } + + mongo_db_plugin::~mongo_db_plugin() { + } + + void mongo_db_plugin::set_program_options( + boost::program_options::options_description &cli, + boost::program_options::options_description &cfg + ) { + cli.add_options() + ("mongodb-uri", + boost::program_options::value()->default_value("mongodb://127.0.0.1:27017/Golos"), + "Mongo DB connection string") + ("mongodb-write-raw-blocks", + boost::program_options::value()->default_value(true), + "Write raw blocks into mongo or not") + ("mongodb-write-operations", + boost::program_options::value>()->multitoken()->zero_tokens()->composing(), + "List of operations to write into mongo"); + cfg.add(cli); + } + + void mongo_db_plugin::plugin_initialize(const boost::program_options::variables_map &options) { + try { + ilog("mongo_db plugin: plugin_initialize() begin"); + + bool raw_blocks = true; + if (options.count("mongodb-write-raw-blocks")) { + raw_blocks = options.at("mongodb-write-raw-blocks").as(); + } + std::vector write_operations; + if (options.count("mongodb-write-operations")) { + write_operations = options.at("mongodb-write-operations").as>(); + } + + // First init mongo db + if (options.count("mongodb-uri")) { + std::string uri_str = options.at("mongodb-uri").as(); + ilog("Connecting MongoDB to ${u}", ("u", uri_str)); + + pimpl_ = std::make_unique(*this); + + if (!pimpl_->initialize(uri_str, raw_blocks, write_operations)) { + ilog("Cannot initialize MongoDB plugin. Plugin disabled."); + pimpl_.reset(); + return; + } + // Set applied block listener + auto &db = pimpl_->database(); + + db.applied_block.connect([&](const signed_block &b) { + pimpl_->on_block(b); + }); + + db.post_apply_operation.connect([&](const operation_notification &o) { + pimpl_->on_operation(o); + }); + + } else { + ilog("Mongo plugin configured, but no mongodb-uri specified. Plugin disabled."); + } + + ilog("mongo_db plugin: plugin_initialize() end"); + } FC_CAPTURE_AND_RETHROW() + } + + void mongo_db_plugin::plugin_startup() { + ilog("mongo_db plugin: plugin_startup() begin"); + + ilog("mongo_db plugin: plugin_startup() end"); + } + + void mongo_db_plugin::plugin_shutdown() { + ilog("mongo_db plugin: plugin_shutdown() begin"); + + ilog("mongo_db plugin: plugin_shutdown() end"); + } + + }}} // namespace golos::plugins::mongo_db diff --git a/plugins/mongo_db/mongo_db_state.cpp b/plugins/mongo_db/mongo_db_state.cpp new file mode 100644 index 0000000000..32e228c244 --- /dev/null +++ b/plugins/mongo_db/mongo_db_state.cpp @@ -0,0 +1,555 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +namespace golos { +namespace plugins { +namespace mongo_db { + + using bsoncxx::builder::stream::array; + using bsoncxx::builder::stream::document; + using bsoncxx::builder::stream::open_document; + using bsoncxx::builder::stream::close_document; + using namespace golos::plugins::follow; + + state_writer::state_writer(db_map& bmi_to_add, const signed_block& block) : + db_(appbase::app().get_plugin().db()), + state_block(block), + all_docs(bmi_to_add) { + } + + named_document state_writer::create_document(const std::string& name, + const std::string& key, const std::string& keyval) { + named_document doc; + doc.collection_name = name; + doc.key = key; + doc.keyval = keyval; + doc.is_removal = false; + return doc; + } + + named_document state_writer::create_removal_document(const std::string& name, + const std::string& key, const std::string& keyval) { + named_document doc; + doc.collection_name = name; + doc.key = key; + doc.keyval = keyval; + doc.is_removal = true; + return doc; + } + + void state_writer::format_comment(const std::string& auth, const std::string& perm) { + try { + auto& comment = db_.get_comment(auth, perm); + auto oid = std::string(auth).append("/").append(perm); + auto oid_hash = fc::sha1::hash(oid).str().substr(0, 24); + + auto doc = create_document("comment_object", "_id", oid_hash); + auto& body = doc.doc; + + body << "$set" << open_document; + + format_oid(body, oid); + + format_value(body, "removed", false); + + format_value(body, "author", auth); + format_value(body, "permlink", perm); + format_value(body, "abs_rshares", comment.abs_rshares); + format_value(body, "active", comment.active); + + format_value(body, "allow_curation_rewards", comment.allow_curation_rewards); + format_value(body, "allow_replies", comment.allow_replies); + format_value(body, "allow_votes", comment.allow_votes); + format_value(body, "author_rewards", comment.author_rewards); + format_value(body, "beneficiary_payout", comment.beneficiary_payout_value); + format_value(body, "cashout_time", comment.cashout_time); + format_value(body, "children", comment.children); + format_value(body, "children_abs_rshares", comment.children_abs_rshares); + format_value(body, "children_rshares2", comment.children_rshares2); + format_value(body, "created", comment.created); + format_value(body, "curator_payout", comment.curator_payout_value); + format_value(body, "depth", comment.depth); + format_value(body, "last_payout", comment.last_payout); + format_value(body, "last_update", comment.last_update); + format_value(body, "max_accepted_payout", comment.max_accepted_payout); + format_value(body, "max_cashout_time", comment.max_cashout_time); + format_value(body, "net_rshares", comment.net_rshares); + format_value(body, "net_votes", comment.net_votes); + format_value(body, "parent_author", comment.parent_author); + format_value(body, "parent_permlink", comment.parent_permlink); + format_value(body, "percent_steem_dollars", comment.percent_steem_dollars); + format_value(body, "reward_weight", comment.reward_weight); + format_value(body, "total_payout", comment.total_payout_value); + format_value(body, "total_vote_weight", comment.total_vote_weight); + format_value(body, "vote_rshares", comment.vote_rshares); + + if (!comment.beneficiaries.empty()) { + array ben_array; + for (auto& b: comment.beneficiaries) { + document tmp; + format_value(tmp, "account", b.account); + format_value(tmp, "weight", b.weight); + ben_array << tmp; + } + body << "beneficiaries" << ben_array; + } + + std::string comment_mode; + switch (comment.mode) { + case first_payout: + comment_mode = "first_payout"; + break; + case second_payout: + comment_mode = "second_payout"; + break; + case archived: + comment_mode = "archived"; + break; + } + + format_value(body, "mode", comment_mode); + + auto& content = db_.get_comment_content(comment_id_type(comment.id)); + + format_value(body, "title", content.title); + format_value(body, "body", content.body); + format_value(body, "json_metadata", content.json_metadata); + + std::string category; + if (comment.parent_author == STEEMIT_ROOT_POST_PARENT) { + category = to_string(comment.parent_permlink); + } else { + category = to_string(db_.get(comment.root_comment).parent_permlink); + } + format_value(body, "category", category); + + body << close_document; + + bmi_insert_or_replace(all_docs, std::move(doc)); + } +// catch (fc::exception& ex) { +// ilog("MongoDB operations fc::exception during formatting comment. ${e}", ("e", ex.what())); +// } + catch (...) { + // ilog("Unknown exception during formatting comment."); + } + } + + auto state_writer::operator()(const vote_operation& op) -> result_type { + format_comment(op.author, op.permlink); + + try { + auto& vote_idx = db_.get_index().indices().get(); + auto& comment = db_.get_comment(op.author, op.permlink); + auto& voter = db_.get_account(op.voter); + auto itr = vote_idx.find(std::make_tuple(comment.id, voter.id)); + if (vote_idx.end() != itr) { + auto comment_oid = std::string(op.author).append("/").append(op.permlink); + auto oid = comment_oid + "/" + op.voter; + auto oid_hash = fc::sha1::hash(oid).str().substr(0, 24); + + auto doc = create_document("comment_vote_object", "_id", oid_hash); + document comment_index; + comment_index << "comment" << 1; + doc.indexes_to_create.push_back(std::move(comment_index)); + auto &body = doc.doc; + + body << "$set" << open_document; + + format_oid(body, oid); + format_oid(body, "comment", comment_oid); + + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "voter", op.voter); + + format_value(body, "weight", itr->weight); + format_value(body, "rshares", itr->rshares); + format_value(body, "vote_percent", itr->vote_percent); + format_value(body, "last_update", itr->last_update); + format_value(body, "num_changes", itr->num_changes); + + body << close_document; + + bmi_insert_or_replace(all_docs, std::move(doc)); + } + } +// catch (fc::exception& ex) { +// ilog("MongoDB operations fc::exception during formatting vote. ${e}", ("e", ex.what())); +// } + catch (...) { + // ilog("Unknown exception during formatting vote."); + } + } + + auto state_writer::operator()(const comment_operation& op) -> result_type { + format_comment(op.author, op.permlink); + } + + auto state_writer::operator()(const comment_options_operation& op) -> result_type { + format_comment(op.author, op.permlink); + } + + auto state_writer::operator()(const delete_comment_operation& op) -> result_type { + + std::string author = op.author; + + auto comment_oid = std::string(op.author).append("/").append(op.permlink); + auto comment_oid_hash = fc::sha1::hash(comment_oid).str().substr(0, 24); + + // Will be updated with the following fields. If no one - created with these fields. + auto comment = create_document("comment_object", "_id", comment_oid_hash); + + auto& body = comment.doc; + + body << "$set" << open_document; + + format_oid(body, comment_oid); + + format_value(body, "removed", true); + + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + + body << close_document; + + bmi_insert_or_replace(all_docs, std::move(comment)); + + // Will be updated with removed = true. If no one - nothing to do. + auto comment_vote = create_removal_document("comment_vote_object", "comment", comment_oid_hash); + + bmi_insert_or_replace(all_docs, std::move(comment_vote)); + } + + auto state_writer::operator()(const transfer_operation& op) -> result_type { + + } + + auto state_writer::operator()(const transfer_to_vesting_operation& op) -> result_type { + + } + + auto state_writer::operator()(const withdraw_vesting_operation& op) -> result_type { + + } + + auto state_writer::operator()(const limit_order_create_operation& op) -> result_type { + + } + + auto state_writer::operator()(const limit_order_cancel_operation& op) -> result_type { + + } + + auto state_writer::operator()(const feed_publish_operation& op) -> result_type { + + } + + auto state_writer::operator()(const convert_operation& op) -> result_type { + + } + + auto state_writer::operator()(const account_create_operation& op) -> result_type { + + } + + auto state_writer::operator()(const account_update_operation& op) -> result_type { + + } + + auto state_writer::operator()(const witness_update_operation& op) -> result_type { + + } + + auto state_writer::operator()(const account_witness_vote_operation& op) -> result_type { + + } + + auto state_writer::operator()(const account_witness_proxy_operation& op) -> result_type { + + } + + auto state_writer::operator()(const pow_operation& op) -> result_type { + + } + + auto state_writer::operator()(const custom_operation& op) -> result_type { + + } + + auto state_writer::operator()(const report_over_production_operation& op) -> result_type { + + } + + auto state_writer::operator()(const custom_json_operation& op) -> result_type { + + } + + auto state_writer::operator()(const set_withdraw_vesting_route_operation& op) -> result_type { + + } + + auto state_writer::operator()(const limit_order_create2_operation& op) -> result_type { + + } + + auto state_writer::operator()(const challenge_authority_operation& op) -> result_type { + + } + + auto state_writer::operator()(const prove_authority_operation& op) -> result_type { + + } + + auto state_writer::operator()(const request_account_recovery_operation& op) -> result_type { + + } + + auto state_writer::operator()(const recover_account_operation& op) -> result_type { + + } + + auto state_writer::operator()(const change_recovery_account_operation& op) -> result_type { + + } + + auto state_writer::operator()(const escrow_transfer_operation& op) -> result_type { + + } + + auto state_writer::operator()(const escrow_dispute_operation& op) -> result_type { + + } + + auto state_writer::operator()(const escrow_release_operation& op) -> result_type { + + } + + auto state_writer::operator()(const pow2_operation& op) -> result_type { + + } + + auto state_writer::operator()(const escrow_approve_operation& op) -> result_type { + + } + + auto state_writer::operator()(const transfer_to_savings_operation& op) -> result_type { + + } + + auto state_writer::operator()(const transfer_from_savings_operation& op) -> result_type { + + } + + auto state_writer::operator()(const cancel_transfer_from_savings_operation& op) -> result_type { + + } + + auto state_writer::operator()(const custom_binary_operation& op) -> result_type { + + } + + auto state_writer::operator()(const decline_voting_rights_operation& op) -> result_type { + + } + + auto state_writer::operator()(const reset_account_operation& op) -> result_type { + + } + + auto state_writer::operator()(const set_reset_account_operation& op) -> result_type { + + } + + auto state_writer::operator()(const delegate_vesting_shares_operation& op) -> result_type { + + } + auto state_writer::operator()(const account_create_with_delegation_operation& op) -> result_type { + + } + auto state_writer::operator()(const account_metadata_operation& op) -> result_type { + + } + auto state_writer::operator()(const proposal_create_operation& op) -> result_type { + + } + auto state_writer::operator()(const proposal_update_operation& op) -> result_type { + + } + auto state_writer::operator()(const proposal_delete_operation& op) -> result_type { + + } + + auto state_writer::operator()(const fill_convert_request_operation& op) -> result_type { + + } + + auto state_writer::operator()(const liquidity_reward_operation& op) -> result_type { + + } + + auto state_writer::operator()(const interest_operation& op) -> result_type { + + } + + auto state_writer::operator()(const fill_vesting_withdraw_operation& op) -> result_type { + + } + + auto state_writer::operator()(const fill_order_operation& op) -> result_type { + + } + + auto state_writer::operator()(const shutdown_witness_operation& op) -> result_type { + + } + + auto state_writer::operator()(const fill_transfer_from_savings_operation& op) -> result_type { + + } + + auto state_writer::operator()(const hardfork_operation& op) -> result_type { + + } + + auto state_writer::operator()(const comment_payout_update_operation& op) -> result_type { + format_comment(op.author, op.permlink); + } + + auto state_writer::operator()(const author_reward_operation& op) -> result_type { + try { + auto comment_oid = std::string(op.author).append("/").append(op.permlink); + auto comment_oid_hash = fc::sha1::hash(comment_oid).str().substr(0, 24); + + auto doc = create_document("author_reward", "_id", comment_oid_hash); + auto &body = doc.doc; + + body << "$set" << open_document; + + format_value(body, "removed", false); + format_oid(body, comment_oid); + format_oid(body, "comment", comment_oid); + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "timestamp", state_block.timestamp); + format_value(body, "sbd_payout", op.sbd_payout); + format_value(body, "steem_payout", op.steem_payout); + format_value(body, "vesting_payout", op.vesting_payout); + + body << close_document; + + bmi_insert_or_replace(all_docs, std::move(doc)); + + } catch (...) { + // + } + } + + auto state_writer::operator()(const curation_reward_operation& op) -> result_type { + try { + auto comment_oid = std::string(op.comment_author).append("/").append(op.comment_permlink); + auto vote_oid = comment_oid + "/" + op.curator; + auto vote_oid_hash = fc::sha1::hash(vote_oid).str().substr(0, 24); + + auto doc = create_document("curation_reward", "_id", vote_oid_hash); + document comment_index; + comment_index << "comment" << 1; + doc.indexes_to_create.push_back(std::move(comment_index)); + auto &body = doc.doc; + + body << "$set" << open_document; + + format_value(body, "removed", false); + format_oid(body, vote_oid); + format_oid(body, "comment", comment_oid); + format_oid(body, "vote", vote_oid); + format_value(body, "author", op.comment_author); + format_value(body, "permlink", op.comment_permlink); + format_value(body, "timestamp", state_block.timestamp); + format_value(body, "reward", op.reward); + format_value(body, "curator", op.curator); + + body << close_document; + + bmi_insert_or_replace(all_docs, std::move(doc)); + } catch (...) { + // + } + } + + auto state_writer::operator()(const comment_reward_operation& op) -> result_type { + try { + auto comment_oid = std::string(op.author).append("/").append(op.permlink); + auto comment_oid_hash = fc::sha1::hash(comment_oid).str().substr(0, 24); + + auto doc = create_document("comment_reward", "_id", comment_oid_hash); + auto &body = doc.doc; + + body << "$set" << open_document; + + format_value(body, "removed", false); + format_oid(body, comment_oid); + format_oid(body, "comment", comment_oid); + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "timestamp", state_block.timestamp); + format_value(body, "payout", op.payout); + + body << close_document; + + bmi_insert_or_replace(all_docs, std::move(doc)); + } catch (...) { + // + } + } + + auto state_writer::operator()(const comment_benefactor_reward_operation& op) -> result_type { + try { + auto comment_oid = std::string(op.author).append("/").append(op.permlink); + auto benefactor_oid = comment_oid + "/" + op.benefactor; + auto benefactor_oid_hash = fc::sha1::hash(benefactor_oid).str().substr(0, 24); + + auto doc = create_document("benefactor_reward", "_id", benefactor_oid_hash); + document comment_index; + comment_index << "comment" << 1; + doc.indexes_to_create.push_back(std::move(comment_index)); + auto &body = doc.doc; + + body << "$set" << open_document; + + format_value(body, "removed", false); + format_oid(body, benefactor_oid); + format_oid(body, "comment", comment_oid); + format_value(body, "author", op.author); + format_value(body, "permlink", op.permlink); + format_value(body, "timestamp", state_block.timestamp); + format_value(body, "reward", op.reward); + format_value(body, "benefactor", op.benefactor); + + body << close_document; + + bmi_insert_or_replace(all_docs, std::move(doc)); + } catch (...) { + // + } + } + + auto state_writer::operator()(const return_vesting_delegation_operation& op) -> result_type { + + } + + auto state_writer::operator()(const chain_properties_update_operation& op) -> result_type { + + } + +}}} \ No newline at end of file diff --git a/plugins/mongo_db/mongo_db_types.cpp b/plugins/mongo_db/mongo_db_types.cpp new file mode 100644 index 0000000000..1d08a1c49f --- /dev/null +++ b/plugins/mongo_db/mongo_db_types.cpp @@ -0,0 +1,17 @@ +#include + +namespace golos { +namespace plugins { +namespace mongo_db { + + void bmi_insert_or_replace(db_map& bmi, named_document doc) { + auto it = bmi.get().find(std::make_tuple( + doc.collection_name, + doc.key, doc.keyval, doc.is_removal)); + if (it != bmi.get().end()) + bmi.get().erase(it); + bmi.push_back(std::move(doc)); + } +} +} +} \ No newline at end of file diff --git a/plugins/mongo_db/mongo_db_writer.cpp b/plugins/mongo_db/mongo_db_writer.cpp new file mode 100644 index 0000000000..faa60207af --- /dev/null +++ b/plugins/mongo_db/mongo_db_writer.cpp @@ -0,0 +1,296 @@ +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace golos { +namespace plugins { +namespace mongo_db { + + using namespace bsoncxx::types; + using namespace bsoncxx::builder; + using bsoncxx::builder::stream::array; + using bsoncxx::builder::basic::kvp; + using bsoncxx::builder::stream::finalize; + using bsoncxx::builder::stream::open_document; + using bsoncxx::builder::stream::close_document; + + mongo_db_writer::mongo_db_writer() : + _db(appbase::app().get_plugin().db()) { + } + + mongo_db_writer::~mongo_db_writer() { + } + + bool mongo_db_writer::initialize(const std::string& uri_str, const bool write_raw, const std::vector& ops) { + try { + uri = mongocxx::uri {uri_str}; + mongo_conn = mongocxx::client {uri}; + db_name = uri.database().empty() ? "Golos" : uri.database(); + mongo_database = mongo_conn[db_name]; + bulk_opts.ordered(false); + write_raw_blocks = write_raw; + + for (auto& op : ops) { + if (!op.empty()) { + write_operations.insert(op); + } + } + + ilog("MongoDB writer initialized."); + + return true; + } + catch (mongocxx::exception & ex) { + wlog("Exception in MongoDB initialize: ${p}", ("p", ex.what())); + return false; + } + catch (...) { + wlog("Unknown exception in MongoDB writer"); + return false; + } + } + + void mongo_db_writer::on_block(const signed_block& block) { + + try { + + blocks[block.block_num()] = block; + + // Update last irreversible block number + last_irreversible_block_num = _db.last_non_undoable_block_num(); + if (last_irreversible_block_num >= blocks.begin()->first) { + + db_map all_docs; + + // Write all the blocks that has num less then last irreversible block + while (!blocks.empty() && blocks.begin()->first <= last_irreversible_block_num) { + auto head_iter = blocks.begin(); + + try { + if (write_raw_blocks) { + write_raw_block(head_iter->second, virtual_ops[head_iter->first]); + } + + state_writer st_writer(all_docs, block); + + // Parsing all transactions. st_writer writes all results to all_docs + + for (const auto& tran : head_iter->second.transactions) { + for (const auto& op : tran.operations) { + op.visit(st_writer); + } + } + + write_block_operations(st_writer, head_iter->second, virtual_ops[head_iter->first]); + } + catch (...) { + // If some block causes any problems lets remove it from buffer and move on + blocks.erase(head_iter); + virtual_ops.erase(head_iter->first); + throw; + } + blocks.erase(head_iter); + virtual_ops.erase(head_iter->first); + } + + // End of blocks series. Writing all docs to bulk + + for (auto& it : all_docs) { + if (!it.is_removal) { + //wlog(it.collection_name); + write_document(it); + } else { + //wlog(it.collection_name + std::string("_REM")); + remove_document(it); + } + } + + // Writing bulk to mongo + + write_data(); + } + + ++processed_blocks; + } + catch (const std::exception& e) { + wlog("Unknown exception in MongoDB ${e}", ("e", e.what())); + } + } + + void mongo_db_writer::on_operation(const golos::chain::operation_notification& note) { + virtual_ops[note.block].push_back(note.op); + // remove ops if there were forks and rollbacks + auto itr = virtual_ops.find(note.block); + ++itr; + virtual_ops.erase(itr, virtual_ops.end()); + } + + void mongo_db_writer::write_raw_block(const signed_block& block, const operations& ops) { + + operation_writer op_writer; + document block_doc; + format_block_info(block, block_doc); + + array transactions_array; + + // Now write every transaction from Block + for (const auto& tran : block.transactions) { + + document tran_doc; + format_transaction_info(tran, tran_doc); + + if (!tran.operations.empty()) { + + array operations_array; + + // Write every operation in transaction + for (const auto& op : tran.operations) { + + try { + operations_array << op.visit(op_writer); + } + catch (std::exception& ex) { + wlog("Mongodb write_raw_block std exception ${e}", ("e", ex.what())); + } catch (...) { + wlog("Mongodb write_raw_block unknown exception "); + } + } + + static const std::string operations = "operations"; + tran_doc << operations << operations_array; + } + + transactions_array << tran_doc; + } + + if (!ops.empty()) { + array operations_array; + for (auto &op: ops) { + try { + operations_array << op.visit(op_writer); + } catch (...) { + // + } + } + static const std::string operations = "virtual_operations"; + block_doc << operations << operations_array; + } + + static const std::string transactions = "transactions"; + block_doc << transactions << transactions_array; + + static const std::string blocks = "blocks"; + if (formatted_blocks.find(blocks) == formatted_blocks.end()) { + formatted_blocks[blocks] = std::make_unique(bulk_opts); + } + + mongocxx::model::insert_one insert_msg{block_doc.view()}; + formatted_blocks[blocks]->append(insert_msg); + } + + void mongo_db_writer::write_document(named_document const& named_doc) { + if (formatted_blocks.find(named_doc.collection_name) == formatted_blocks.end()) { + formatted_blocks[named_doc.collection_name] = std::make_unique(bulk_opts); + } + + //auto view = named_doc.doc.view(); + //auto itr = view.find("_id"); + //if (view.end() == itr) { + // mongocxx::model::insert_one msg{std::move(view)}; + // formatted_blocks[named_doc.collection_name]->append(msg); + //} else { + document filter; + + filter << "_id" << bsoncxx::oid(named_doc.keyval); + + mongocxx::model::update_one msg{filter.view(), + named_doc.doc.view()}; + msg.upsert(true); + + if (indexes.find(named_doc.collection_name) == indexes.end()) { + for (auto& index_to_create : named_doc.indexes_to_create) { + mongo_database[named_doc.collection_name].create_index(index_to_create.view()); + indexes[named_doc.collection_name] = "created"; + } + } + + formatted_blocks[named_doc.collection_name]->append(msg); + //} + } + + void mongo_db_writer::remove_document(named_document const& named_doc) { + if (formatted_blocks.find(named_doc.collection_name) == formatted_blocks.end()) { + formatted_blocks[named_doc.collection_name] = std::make_unique(bulk_opts); + } + + document filter; + filter << named_doc.key << bsoncxx::oid(named_doc.keyval); + auto v1 = filter.view(); + document newval; + newval << "$set" << open_document << "removed" << true << close_document; + auto v2 = newval.view(); + mongocxx::model::update_many msg{v1, v2}; + formatted_blocks[named_doc.collection_name]->append(msg); + } + + void mongo_db_writer::write_block_operations(state_writer& st_writer, const signed_block& block, const operations& ops) { + + for (auto& op: ops) { + op.visit(st_writer); + } + } + + void mongo_db_writer::format_block_info(const signed_block& block, document& doc) { + doc << "block_num" << static_cast(block.block_num()) + << "block_id" << block.id().str() + << "block_prev_block_id" << block.previous.str() + << "block_timestamp" << block.timestamp + << "block_witness" << block.witness + << "block_created_at" << fc::time_point::now(); + } + + void mongo_db_writer::format_transaction_info(const signed_transaction& tran, document& doc) { + doc << "transaction_id" << tran.id().str() + << "transaction_ref_block_num" << static_cast(tran.ref_block_num) + << "transaction_expiration" << tran.expiration; + } + + void mongo_db_writer::write_data() { + auto iter = formatted_blocks.begin(); + for (; iter != formatted_blocks.end(); ++iter) { + + auto& oper = *iter; + try { + const std::string& collection_name = oper.first; + mongocxx::collection _collection = mongo_database[collection_name]; + + auto& bulkp = oper.second; + if (!_collection.bulk_write(*bulkp)) { + wlog("Failed to write blocks to Mongo DB"); + } + } + catch (const std::exception& e) { + wlog("Unknown exception while writing blocks to mongo: ${e}", ("e", e.what())); + // If we got some errors writing block into mongo just skip this block and move on + formatted_blocks.erase(iter); + throw; + } + } + formatted_blocks.clear(); + } +}}} diff --git a/programs/golosd/CMakeLists.txt b/programs/golosd/CMakeLists.txt index 96ada8c81c..eee1ccc126 100644 --- a/programs/golosd/CMakeLists.txt +++ b/programs/golosd/CMakeLists.txt @@ -39,6 +39,7 @@ target_link_libraries( golos::block_info golos::json_rpc golos::follow + ${MONGO_LIB} golos_protocol fc ${CMAKE_DL_LIBS} diff --git a/programs/golosd/main.cpp b/programs/golosd/main.cpp index 5c6981d897..3bd6d3b69a 100644 --- a/programs/golosd/main.cpp +++ b/programs/golosd/main.cpp @@ -23,6 +23,9 @@ #include #include #include +#ifdef MONGODB_PLUGIN_BUILT + #include +#endif #include #include @@ -76,6 +79,9 @@ namespace golos { appbase::app().register_plugin(); appbase::app().register_plugin(); appbase::app().register_plugin(); + #ifdef MONGODB_PLUGIN_BUILT + appbase::app().register_plugin(); + #endif ///plugins }; } diff --git a/share/golosd/config/config_debug_mongo.ini b/share/golosd/config/config_debug_mongo.ini new file mode 100644 index 0000000000..65e629d1f5 --- /dev/null +++ b/share/golosd/config/config_debug_mongo.ini @@ -0,0 +1,149 @@ +# Endpoint for P2P node to listen on +# p2p-endpoint = + +# Maxmimum number of incoming connections on P2P endpoint +# p2p-max-connections = + +# P2P nodes to connect to on startup (may specify multiple times) +# p2p-seed-node = + +# Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints. +# checkpoint = + +# Number of threads for rpc-clients. Optimal value `-1` +webserver-thread-pool-size = 2 + +# IP:PORT for HTTP connections +webserver-http-endpoint = 0.0.0.0:8090 + +# IP:PORT for WebSocket connections +webserver-ws-endpoint = 0.0.0.0:8091 + +# Maximum microseconds for trying to get read lock +read-wait-micro = 500000 + +# Maximum retries to get read lock. Each retry is read-wait-micro microseconds. +# When all retries are made, the rpc-client receives error 'Unable to acquire READ lock'. +max-read-wait-retries = 2 + +# Maximum microseconds for trying to get write lock on broadcast transaction. +write-wait-micro = 500000 + +# Maximum retries to get write lock. Each retry is write-wait-micro microseconds. +# When all retries are made, the rpc-client receives error 'Unable to acquire WRITE lock'. +max-write-wait-retries = 3 + +# Do all write operations (push_block/push_transaction) in the single thread. +# Write lock of database is very heavy. When many threads tries to lock database on writing, rpc-clients +# receive many errors 'Unable to acquire READ lock' ('Unable to acquire WRITE lock'). +# Enabling of this options can increase performance. +single-write-thread = true + +# Enable plugin notifications about operations in a pushed transaction, which should be included to the next generated +# block. Plugins doesn't validate data in operations, they only update its own indexes, so notifications can be +# disabled on push_transaction() without any side-effects. The option doesn't have effect on a pushing signed blocks, +# so it is safe. +# Disabling of this option can increase performance. +enable-plugins-on-push-transaction = true + +# A start size for shared memory file when it doesn't have any data. Possible cases: +# - If shared memory has data and the value is greater then the size of shared_memory.bin, +# the file will be grown to requested size. +# - If shared memory has data and the value is less then the size of shared_memory.bin, nothing happens. +# Changing of this parameter doesn't require the replaying. +shared-file-size = 100M + +# The minimum free space in the shared memory file. When free space reaches the following value, the size of the +# shared_memory.bin increases by the value of inc-shared-file-size. +min-free-shared-file-size = 50M + +# Step of increasing size of shared_memory.bin. When the free memory size reaches min-free-shared-file-size, +# the shared memory size increases by the following value. +inc-shared-file-size = 100M + +# How often do checking the free space in shared_memory.bin. A very frequent checking can decrease performance. +# It's not critical if the free size became very small, because the daemon catches the `bad_alloc` exception +# and resizes. The optimal strategy is do checking of the free space, but not very often. +block-num-check-free-size = 10 # each 30 seconds + +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history operation_history statsd block_info raw_block debug_node witness_api mongo_db + +# For connect to mongodb which is running outside Docker (if golosd running inside) +mongodb-uri = mongodb://172.17.0.1:27017/Golos + +# Remove votes before defined block, should increase performance +clear-votes-before-block = 0 # don't clear votes + +# Virtual operations will not be passed to the plugins, enabling of the option helps to save some memory. +skip-virtual-ops = false + +# Defines a range of accounts to track by the account_history plugin as a json pair ["from","to"] [from,to] +# track-account-range = + +# Defines a list of operations which will be explicitly logged by the account_history plugin. +# history-whitelist-ops = + +# Defines a list of operations which will be explicitly ignored by the account_history plugin. +# history-blacklist-ops = + +# Defines starting block from which recording stats by the account_history plugin. +# history-start-block = + +# Set the maximum size of cached feed for an account +follow-max-feed-size = 500 + +# Track market history by grouping orders into buckets of equal size measured in seconds specified as a JSON array of numbers +bucket-size = [15,60,300,3600,86400] + +# How far back in time to track history for each bucket size, measured in the number of buckets (default: 5760) +history-per-size = 5760 + +# Defines a range of accounts to private messages to/from as a json pair ["from","to"] [from,to) +# pm-account-range = + +# Enable block production, even if the chain is stale. +enable-stale-production = true + +# Percent of witnesses (0-99) that must be participating in order to produce blocks +required-participation = 0 + +# name of witness controlled by this node (e.g. initwitness ) +witness = "cyberfounder" + +# name of miner and its private key (e.g. ["account","WIF PRIVATE KEY"] ) +miner = ["cyberfounder","5JVFFWRLwz6JoP9kguuRFfytToGU6cLgBVTL9t6NB3D3BQLbUBS"] + +# Number of threads to use for proof of work mining +mining-threads = 1 + +# WIF PRIVATE KEY to be used by one or more witnesses or miners +private-key = 5JVFFWRLwz6JoP9kguuRFfytToGU6cLgBVTL9t6NB3D3BQLbUBS + +# Account creation fee to be voted on upon successful POW - Minimum fee is 100.000 STEEM (written as 100000) +# miner-account-creation-fee = + +# Maximum block size (in bytes) to be voted on upon successful POW - Max block size must be between 128 KB and 750 MB +# miner-maximum-block-size = + +# SBD interest rate to be vote on upon successful POW - Default interest rate is 10% (written as 1000) +# miner-sbd-interest-rate = + +# declare an appender named "stderr" that writes messages to the console +[log.console_appender.stderr] +stream=std_error + +# declare an appender named "p2p" that writes messages to p2p.log +[log.file_appender.p2p] +filename=logs/p2p/p2p.log +# filename can be absolute or relative to this config file + +# route any messages logged to the default logger to the "stderr" logger we +# declared above, if they are info level are higher +[logger.default] +level=info +appenders=stderr + +# route messages sent to the "p2p" logger to stderr too +[logger.p2p] +level=info +appenders=stderr diff --git a/share/golosd/config/config_mongo.ini b/share/golosd/config/config_mongo.ini new file mode 100644 index 0000000000..f8ac3fcf25 --- /dev/null +++ b/share/golosd/config/config_mongo.ini @@ -0,0 +1,149 @@ +# Endpoint for P2P node to listen on +p2p-endpoint = 0.0.0.0:4243 + +# Maxmimum number of incoming connections on P2P endpoint +# p2p-max-connections = + +# P2P nodes to connect to on startup (may specify multiple times) +# p2p-seed-node = + +# Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints. +# checkpoint = + +# Number of threads for rpc-clients. The optimal value is `-1` +webserver-thread-pool-size = 2 + +# IP:PORT for HTTP connections +webserver-http-endpoint = 0.0.0.0:8090 + +# IP:PORT for WebSocket connections +webserver-ws-endpoint = 0.0.0.0:8091 + +# Maximum microseconds for trying to get read lock +read-wait-micro = 500000 + +# Maximum retries to get read lock. Each retry is read-wait-micro microseconds. +# When all retries are made, the rpc-client receives error 'Unable to acquire READ lock'. +max-read-wait-retries = 2 + +# Maximum microseconds for trying to get write lock on broadcast transaction. +write-wait-micro = 500000 + +# Maximum retries to get write lock. Each retry is write-wait-micro microseconds. +# When all retries are made, the rpc-client receives error 'Unable to acquire WRITE lock'. +max-write-wait-retries = 3 + +# Do all write operations (push_block/push_transaction) in the single thread. +# Write lock of database is very heavy. When many threads tries to lock database on writing, rpc-clients +# receive many errors 'Unable to acquire READ lock' ('Unable to acquire WRITE lock'). +# Enabling of this options can increase performance. +single-write-thread = true + +# Enable plugin notifications about operations in a pushed transaction, which should be included to the next generated +# block. Plugins doesn't validate data in operations, they only update its own indexes, so notifications can be +# disabled on push_transaction() without any side-effects. The option doesn't have effect on a pushing signed blocks, +# so it is safe. +# Disabling of this option can increase performance. +enable-plugins-on-push-transaction = false + +# A start size for shared memory file when it doesn't have any data. Possible cases: +# - If shared memory has data and the value is greater then the size of shared_memory.bin, +# the file will be grown to requested size. +# - If shared memory has data and the value is less then the size of shared_memory.bin, nothing happens. +# Changing of this parameter doesn't require the replaying. +shared-file-size = 2G + +# The minimum free space in the shared memory file. When free space reaches the following value, the size of the +# shared_memory.bin increases by the value of inc-shared-file-size. +min-free-shared-file-size = 500M + +# Step of increasing size of shared_memory.bin. When the free memory size reaches min-free-shared-file-size, +# the shared memory size increases by the following value. +inc-shared-file-size = 2G + +# How often do checking the free space in shared_memory.bin. A very frequent checking can decrease performance. +# It's not critical if the free size became very small, because the daemon catches the `bad_alloc` exception +# and resizes. The optimal strategy is do checking of the free space, but not very often. +block-num-check-free-size = 1000 # each 3000 seconds + +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_history account_history statsd block_info raw_block witness_api mongo_db + +# For connect to mongodb which is running outside Docker (if golosd running inside) +mongodb-uri = mongodb://172.17.0.1:27017/Golos + +# Remove votes before defined block, should increase performance +clear-votes-before-block = 4294967295 # clear votes after each cashout + +# Virtual operations will not be passed to the plugins, enabling of the option helps to save some memory. +skip-virtual-ops = false + +# Defines a range of accounts to track by the account_history plugin as a json pair ["from","to"] [from,to] +# track-account-range = + +# Defines a list of operations which will be explicitly logged by the account_history plugin. +# history-whitelist-ops = account_create_operation account_update_operation comment_operation delete_comment_operation vote_operation author_reward_operation curation_reward_operation liquidity_reward_operation interest_operation fill_convert_request_operation transfer_operation transfer_to_vesting_operation withdraw_vesting_operation witness_update_operation account_witness_vote_operation account_witness_proxy_operation feed_publish_operation limit_order_create_operation fill_order_operation limit_order_cancel_operation pow_operation fill_vesting_withdraw_operation shutdown_witness_operation custom_operation request_account_recovery_operation recover_account_operation change_recovery_account_operation escrow_transfer_operation escrow_approve_operation escrow_dispute_operation escrow_release_operation transfer_to_savings_operation transfer_from_savings_operation cancel_transfer_from_savings_operation decline_voting_rights_operation comment_benefactor_reward_operation + +# Defines a list of operations which will be explicitly ignored by the account_history plugin. +# history-blacklist-ops = + +# Defines starting block from which recording stats by the account_history plugin. +# history-start-block = 0 + +# Set the maximum size of cached feed for an account +follow-max-feed-size = 500 + +# Track market history by grouping orders into buckets of equal size measured in seconds specified as a JSON array of numbers +bucket-size = [15,60,300,3600,86400] + +# How far back in time to track history for each bucket size, measured in the number of buckets (default: 5760) +history-per-size = 5760 + +# Defines a range of accounts to private messages to/from as a json pair ["from","to"] [from,to) +# pm-account-range = + +# Enable block production, even if the chain is stale. +enable-stale-production = false + +# Percent of witnesses (0-99) that must be participating in order to produce blocks +required-participation = 0 + +# name of witness controlled by this node (e.g. initwitness ) +# witness = + +# name of miner and its private key (e.g. ["account","WIF PRIVATE KEY"] ) +# miner = + +# Number of threads to use for proof of work mining +mining-threads = 0 + +# WIF PRIVATE KEY to be used by one or more witnesses or miners +# private-key = + +# Account creation fee to be voted on upon successful POW - Minimum fee is 100.000 STEEM (written as 100000) +# miner-account-creation-fee = + +# Maximum block size (in bytes) to be voted on upon successful POW - Max block size must be between 128 KB and 750 MB +# miner-maximum-block-size = + +# SBD interest rate to be vote on upon successful POW - Default interest rate is 10% (written as 1000) +# miner-sbd-interest-rate = + +# declare an appender named "stderr" that writes messages to the console +[log.console_appender.stderr] +stream=std_error + +# declare an appender named "p2p" that writes messages to p2p.log +[log.file_appender.p2p] +filename=logs/p2p/p2p.log +# filename can be absolute or relative to this config file + +# route any messages logged to the default logger to the "stderr" logger we +# declared above, if they are info level are higher +[logger.default] +level=debug +appenders=stderr + +# route messages sent to the "p2p" logger to stderr too +[logger.p2p] +level=error +appenders=stderr diff --git a/share/golosd/docker/Dockerfile-lowmem b/share/golosd/docker/Dockerfile-lowmem index 806a9019c4..5cc8039761 100644 --- a/share/golosd/docker/Dockerfile-lowmem +++ b/share/golosd/docker/Dockerfile-lowmem @@ -43,6 +43,7 @@ RUN \ -DBUILD_SHARED_LIBRARIES=FALSE \ -DLOW_MEMORY_NODE=TRUE \ -DCHAINBASE_CHECK_LOCKING=FALSE \ + -DENABLE_MONGO_PLUGIN=FALSE \ .. \ && \ make -j$(nproc) && \ diff --git a/share/golosd/docker/Dockerfile-lowmem-small b/share/golosd/docker/Dockerfile-lowmem-small index 39a770a348..1f92b381bb 100644 --- a/share/golosd/docker/Dockerfile-lowmem-small +++ b/share/golosd/docker/Dockerfile-lowmem-small @@ -40,6 +40,7 @@ RUN \ -DBUILD_SHARED_LIBRARIES=FALSE \ -DLOW_MEMORY_NODE=TRUE \ -DCHAINBASE_CHECK_LOCKING=FALSE \ + -DENABLE_MONGO_PLUGIN=FALSE \ .. \ && \ make -j$(nproc) && \ diff --git a/share/golosd/docker/Dockerfile-mongo b/share/golosd/docker/Dockerfile-mongo new file mode 100644 index 0000000000..e113a7f09e --- /dev/null +++ b/share/golosd/docker/Dockerfile-mongo @@ -0,0 +1,159 @@ +FROM phusion/baseimage:0.9.19 + +ENV LANG=en_US.UTF-8 + +RUN \ + apt-get update && \ + apt-get install -y \ + autoconf \ + automake \ + autotools-dev \ + bsdmainutils \ + build-essential \ + cmake \ + doxygen \ + git \ + ccache\ + libboost-all-dev \ + libreadline-dev \ + libssl-dev \ + libtool \ + ncurses-dev \ + pbzip2 \ + pkg-config \ + python3 \ + python3-dev \ + python3-pip \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + pip3 install gcovr + +# installing mongo drivers +RUN \ + echo "Installing mongo-c-driver" && \ + apt-get -qq update && \ + apt-get install -y \ + pkg-config \ + libssl-dev \ + libsasl2-dev \ + wget \ + && \ + wget https://github.com/mongodb/mongo-c-driver/releases/download/1.9.5/mongo-c-driver-1.9.5.tar.gz && \ + tar xzf mongo-c-driver-1.9.5.tar.gz && \ + cd mongo-c-driver-1.9.5 && \ + ./configure --disable-automatic-init-and-cleanup --enable-static && \ + make && \ + make install && \ + cd .. && \ + rm -rf mongo-c-driver-1.9.5 && \ + echo "Installing mongo-cxx-driver" && \ + git clone https://github.com/mongodb/mongo-cxx-driver.git --branch releases/v3.2 --depth 1 && \ + cd mongo-cxx-driver/build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make EP_mnmlstc_core && \ + make && \ + make install && \ + cd ../.. && \ + rm -rf mongo-cxx-driver +# end + +ADD . /usr/local/src/golos + +RUN \ + cd /usr/local/src/golos && \ + git submodule deinit -f . && \ + git submodule update --init --recursive -f && \ + mkdir build && \ + cd build && \ + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_GOLOS_TESTNET=FALSE \ + -DBUILD_SHARED_LIBRARIES=FALSE \ + -DLOW_MEMORY_NODE=FALSE \ + -DCHAINBASE_CHECK_LOCKING=FALSE \ + -DENABLE_MONGO_PLUGIN=TRUE \ + .. \ + && \ + make -j$(nproc) && \ + make install && \ + rm -rf /usr/local/src/golos + +RUN \ + apt-get remove -y \ + automake \ + autotools-dev \ + bsdmainutils \ + build-essential \ + cmake \ + doxygen \ + dpkg-dev \ + git \ + libboost-all-dev \ + libc6-dev \ + libexpat1-dev \ + libgcc-5-dev \ + libhwloc-dev \ + libibverbs-dev \ + libicu-dev \ + libltdl-dev \ + libncurses5-dev \ + libnuma-dev \ + libopenmpi-dev \ + libpython-dev \ + libpython2.7-dev \ + libreadline-dev \ + libreadline6-dev \ + libssl-dev \ + libstdc++-5-dev \ + libtinfo-dev \ + libtool \ + linux-libc-dev \ + m4 \ + make \ + manpages \ + manpages-dev \ + mpi-default-dev \ + python-dev \ + python2.7-dev \ + python3-dev \ + && \ + apt-get autoremove -y && \ + rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /var/cache/* \ + /usr/include \ + /usr/local/include + +RUN useradd -s /bin/bash -m -d /var/lib/golosd golosd + +RUN mkdir /var/cache/golosd && \ + chown golosd:golosd -R /var/cache/golosd + +# add blockchain cache to image +#ADD $STEEMD_BLOCKCHAIN /var/cache/golosd/blocks.tbz2 + +ENV HOME /var/lib/golosd +RUN chown golosd:golosd -R /var/lib/golosd + +ADD share/golosd/snapshot5392323.json /var/lib/golosd + +# rpc service: +# http +EXPOSE 8090 +# ws +EXPOSE 8091 +# p2p service: +EXPOSE 2001 + +RUN mkdir -p /etc/service/golosd +ADD share/golosd/golosd.sh /etc/service/golosd/run +RUN chmod +x /etc/service/golosd/run + +# add seednodes from documentation to image +ADD share/golosd/seednodes /etc/golosd/seednodes + +# the following adds lots of logging info to stdout +ADD share/golosd/config/config_mongo.ini /etc/golosd/config.ini diff --git a/share/golosd/docker/Dockerfile-small b/share/golosd/docker/Dockerfile-small index ecb7df8fce..433cc3e115 100644 --- a/share/golosd/docker/Dockerfile-small +++ b/share/golosd/docker/Dockerfile-small @@ -40,6 +40,7 @@ RUN \ -DBUILD_SHARED_LIBRARIES=FALSE \ -DLOW_MEMORY_NODE=FALSE \ -DCHAINBASE_CHECK_LOCKING=FALSE \ + -DENABLE_MONGO_PLUGIN=FALSE \ .. \ && \ make -j$(nproc) && \ diff --git a/share/golosd/docker/Dockerfile-test b/share/golosd/docker/Dockerfile-test index aff3143b2c..909c98ecf5 100644 --- a/share/golosd/docker/Dockerfile-test +++ b/share/golosd/docker/Dockerfile-test @@ -42,6 +42,7 @@ RUN \ -DBUILD_GOLOS_TESTNET=TRUE \ -DLOW_MEMORY_NODE=FALSE \ -DMAX_19_VOTED_WITNESSES=TRUE \ + -DENABLE_MONGO_PLUGIN=FALSE \ .. && \ make -j$(nproc) chain_test plugin_test && \ ./tests/chain_test --log_level=message --report_level=detailed && \ @@ -61,6 +62,7 @@ RUN \ # -DBUILD_GOLOS_TESTNET=TRUE \ # -DLOW_MEMORY_NODE=FALSE \ # -DMAX_19_VOTED_WITNESSES=TRUE \ +# -DENABLE_MONGO_PLUGIN=FALSE \ # .. && \ # make -j$(nproc) chain_test plugin_test && \ # ./tests/chain_test && \ diff --git a/share/golosd/docker/Dockerfile-testnet b/share/golosd/docker/Dockerfile-testnet index 9e012724ec..00962773bc 100644 --- a/share/golosd/docker/Dockerfile-testnet +++ b/share/golosd/docker/Dockerfile-testnet @@ -42,6 +42,7 @@ RUN \ -DBUILD_SHARED_LIBRARIES=FALSE \ -DLOW_MEMORY_NODE=FALSE \ -DCHAINBASE_CHECK_LOCKING=FALSE \ + -DENABLE_MONGO_PLUGIN=FALSE \ .. \ && \ make -j$(nproc) && \ diff --git a/share/golosd/docker/Dockerfile-testnet-mongo b/share/golosd/docker/Dockerfile-testnet-mongo new file mode 100644 index 0000000000..0e66673f77 --- /dev/null +++ b/share/golosd/docker/Dockerfile-testnet-mongo @@ -0,0 +1,156 @@ +FROM phusion/baseimage:0.9.22 + +ENV LANG=en_US.UTF-8 + +RUN apt-get update && \ + apt-get install -y \ + autoconf \ + automake \ + autotools-dev \ + bsdmainutils \ + build-essential \ + cmake \ + ccache \ + doxygen \ + git \ + libboost-all-dev \ + libreadline-dev \ + libssl-dev \ + libtool \ + ncurses-dev \ + pbzip2 \ + pkg-config \ + python3 \ + python3-dev \ + python3-pip \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + pip3 install gcovr + +# installing mongo drivers +RUN \ + echo "Installing mongo-c-driver" && \ + apt-get -qq update && \ + apt-get install -y \ + pkg-config \ + libssl-dev \ + libsasl2-dev \ + wget \ + && \ + wget https://github.com/mongodb/mongo-c-driver/releases/download/1.9.5/mongo-c-driver-1.9.5.tar.gz && \ + tar xzf mongo-c-driver-1.9.5.tar.gz && \ + cd mongo-c-driver-1.9.5 && \ + ./configure --disable-automatic-init-and-cleanup --enable-static && \ + make && \ + make install && \ + cd .. && \ + rm -rf mongo-c-driver-1.9.5 && \ + echo "Installing mongo-cxx-driver" && \ + git clone https://github.com/mongodb/mongo-cxx-driver.git --branch releases/v3.2 --depth 1 && \ + cd mongo-cxx-driver/build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ + make EP_mnmlstc_core && \ + make && \ + make install && \ + cd ../.. && \ + rm -rf mongo-cxx-driver +# end + +ADD . /usr/local/src/golos + +RUN \ + cd /usr/local/src/golos && \ + git submodule deinit -f . && \ + git submodule update --init --recursive -f && \ + mkdir build && \ + cd build && \ + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_GOLOS_TESTNET=TRUE \ + -DBUILD_SHARED_LIBRARIES=FALSE \ + -DLOW_MEMORY_NODE=FALSE \ + -DCHAINBASE_CHECK_LOCKING=FALSE \ + -DENABLE_MONGO_PLUGIN=TRUE \ + .. \ + && \ + make -j$(nproc) && \ + make install && \ + rm -rf /usr/local/src/golos + +RUN \ + apt-get remove -y \ + automake \ + autotools-dev \ + bsdmainutils \ + build-essential \ + cmake \ + doxygen \ + dpkg-dev \ + git \ + libboost-all-dev \ + libc6-dev \ + libexpat1-dev \ + libgcc-5-dev \ + libhwloc-dev \ + libibverbs-dev \ + libicu-dev \ + libltdl-dev \ + libncurses5-dev \ + libnuma-dev \ + libopenmpi-dev \ + libpython-dev \ + libpython2.7-dev \ + libreadline-dev \ + libreadline6-dev \ + libssl-dev \ + libstdc++-5-dev \ + libtinfo-dev \ + libtool \ + linux-libc-dev \ + m4 \ + make \ + manpages \ + manpages-dev \ + mpi-default-dev \ + python-dev \ + python2.7-dev \ + python3-dev \ + && \ + apt-get autoremove -y && \ + rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /var/cache/* \ + /usr/include \ + /usr/local/include + +RUN useradd -s /bin/bash -m -d /var/lib/golosd golosd + +RUN mkdir /var/cache/golosd && \ + chown golosd:golosd -R /var/cache/golosd + +# add blockchain cache to image +#ADD $STEEMD_BLOCKCHAIN /var/cache/golosd/blocks.tbz2 + +ENV HOME /var/lib/golosd +RUN chown golosd:golosd -R /var/lib/golosd + +# rpc service: +# http +EXPOSE 8090 +# ws +EXPOSE 8091 +# p2p service: +EXPOSE 2001 + +RUN mkdir -p /etc/service/golosd +ADD share/golosd/golosd.sh /etc/service/golosd/run +RUN chmod +x /etc/service/golosd/run + +# the following adds lots of logging info to stdout +ADD share/golosd/config/config_debug_mongo.ini /etc/golosd/config.ini + +# put blockchain data into separate docker volume +VOLUME /var/lib/golosd \ No newline at end of file From b422dbb5620bff460938ebeffc618b7910c0e149 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Mon, 4 Jun 2018 15:03:27 +0700 Subject: [PATCH 11/22] Fix tests for chain_properties_update. #693 --- tests/tests/operation_tests.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index c5d536ab75..b47cab48cc 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -1560,6 +1560,9 @@ BOOST_FIXTURE_TEST_SUITE(operation_tests, clean_database_fixture) BOOST_TEST_MESSAGE("--- Test upgrading an account to a witness"); + signed_transaction tx; + tx.set_expiration(db->head_block_time() + STEEMIT_MAX_TIME_UNTIL_EXPIRATION); + witness_update_operation op; op.owner = "alice"; op.url = "foo.bar"; @@ -1567,12 +1570,14 @@ BOOST_FIXTURE_TEST_SUITE(operation_tests, clean_database_fixture) op.block_signing_key = signing_key.get_public_key(); op.props.account_creation_fee = asset(STEEMIT_MIN_ACCOUNT_CREATION_FEE + 10, STEEM_SYMBOL); op.props.maximum_block_size = STEEMIT_MIN_BLOCK_SIZE_LIMIT + 100; - - signed_transaction tx; - tx.set_expiration(db->head_block_time() + STEEMIT_MAX_TIME_UNTIL_EXPIRATION); tx.operations.push_back(op); - tx.sign(alice_private_key, db->get_chain_id()); + chain_properties_update_operation op1; + op1.owner = op.owner; + op1.props = op.props; + tx.operations.push_back(op1); + + tx.sign(alice_private_key, db->get_chain_id()); db->push_transaction(tx, 0); const witness_object &alice_witness = db->get_witness("alice"); From 70befc95540e42753db5afb30b57338585303738 Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Mon, 4 Jun 2018 19:36:21 +0300 Subject: [PATCH 12/22] Remove old_account_bandwidth #692 1. Remove unused (old) objects and methods 2. Remove fields from api_account_object 3. Comment unused code from disabled reset_ operations 4. Fix STEEMIT_VESTING_WITHDRAW_INTERVAL_SECONDS in testnet 5. Some code style fixes --- libraries/api/account_api_object.cpp | 26 ++---- .../include/golos/api/account_api_object.hpp | 10 +-- libraries/chain/database.cpp | 87 +++---------------- .../chain/include/golos/chain/database.hpp | 8 +- .../golos/chain/steem_object_types.hpp | 17 +--- libraries/chain/steem_evaluator.cpp | 37 ++++---- .../include/golos/protocol/config.hpp | 2 +- .../include/golos/protocol/operations.hpp | 12 ++- 8 files changed, 48 insertions(+), 151 deletions(-) diff --git a/libraries/api/account_api_object.cpp b/libraries/api/account_api_object.cpp index e462bba0ad..b0c20c1999 100644 --- a/libraries/api/account_api_object.cpp +++ b/libraries/api/account_api_object.cpp @@ -49,38 +49,22 @@ account_api_object::account_api_object(const account_object& a, const golos::cha json_metadata = golos::chain::to_string(meta.json_metadata); #endif - auto old_forum = db.find( - std::make_tuple(name, bandwidth_type::old_forum)); - if (old_forum != nullptr) { - average_bandwidth = old_forum->average_bandwidth; - lifetime_bandwidth = old_forum->lifetime_bandwidth; - last_bandwidth_update = old_forum->last_bandwidth_update; - } - - auto old_market = db.find( - std::make_tuple(name, bandwidth_type::old_market)); - if (old_market != nullptr) { - average_market_bandwidth = old_market->average_bandwidth; - last_market_bandwidth_update = old_market->last_bandwidth_update; - } - - auto post = db.find( - std::make_tuple(name, bandwidth_type::post)); + auto post = db.find(std::make_tuple(name, bandwidth_type::post)); if (post != nullptr) { last_root_post = post->last_bandwidth_update; post_bandwidth = post->average_bandwidth; } - auto forum = db.find( - std::make_tuple(name, bandwidth_type::forum)); + auto forum = db.find(std::make_tuple(name, bandwidth_type::forum)); if (forum != nullptr) { new_average_bandwidth = forum->average_bandwidth; + last_bandwidth_update = forum->last_bandwidth_update; } - auto market = db.find( - std::make_tuple(name, bandwidth_type::market)); + auto market = db.find(std::make_tuple(name, bandwidth_type::market)); if (market != nullptr) { new_average_market_bandwidth = market->average_bandwidth; + last_market_bandwidth_update = market->last_bandwidth_update; } } diff --git a/libraries/api/include/golos/api/account_api_object.hpp b/libraries/api/include/golos/api/account_api_object.hpp index 8dd5ff5dd5..a9914a8e3c 100644 --- a/libraries/api/include/golos/api/account_api_object.hpp +++ b/libraries/api/include/golos/api/account_api_object.hpp @@ -81,11 +81,7 @@ struct account_api_object { uint16_t witnesses_voted_for; - share_type average_bandwidth = 0; - share_type lifetime_bandwidth = 0; time_point_sec last_bandwidth_update; - - share_type average_market_bandwidth = 0; time_point_sec last_market_bandwidth_update; time_point_sec last_post; time_point_sec last_root_post; @@ -94,7 +90,7 @@ struct account_api_object { share_type new_average_bandwidth; share_type new_average_market_bandwidth; set witness_votes; - + fc::optional reputation; }; @@ -110,8 +106,8 @@ FC_REFLECT((golos::api::account_api_object), (savings_sbd_seconds)(savings_sbd_seconds_last_update)(savings_sbd_last_interest_payment) (savings_withdraw_requests)(vesting_shares)(delegated_vesting_shares)(received_vesting_shares) (vesting_withdraw_rate)(next_vesting_withdrawal)(withdrawn)(to_withdraw)(withdraw_routes) - (curation_rewards)(posting_rewards)(proxied_vsf_votes)(witnesses_voted_for)(average_bandwidth) - (lifetime_bandwidth)(last_bandwidth_update)(average_market_bandwidth)(last_market_bandwidth_update) + (curation_rewards)(posting_rewards)(proxied_vsf_votes)(witnesses_voted_for) + (last_bandwidth_update)(last_market_bandwidth_update) (last_post)(last_root_post)(post_bandwidth)(new_average_bandwidth)(new_average_market_bandwidth) (witness_votes)(reputation)) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index f373100394..da451e24f1 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -581,69 +581,12 @@ namespace golos { namespace chain { adjust_supply(-fee); } - void database::old_update_account_bandwidth(const account_object &a, uint32_t trx_size, const bandwidth_type type) { - try { - const auto &props = get_dynamic_global_properties(); - if (props.total_vesting_shares.amount > 0) { - FC_ASSERT(a.vesting_shares.amount > - 0, "Only accounts with a postive vesting balance may transact."); - - auto band = find(boost::make_tuple(a.name, type)); - - if (band == nullptr) { - band = &create([&](account_bandwidth_object &b) { - b.account = a.name; - b.type = type; - }); - } - - modify(*band, [&](account_bandwidth_object &b) { - b.lifetime_bandwidth += - trx_size * STEEMIT_BANDWIDTH_PRECISION; - - auto now = head_block_time(); - auto delta_time = (now - - b.last_bandwidth_update).to_seconds(); - uint64_t N = trx_size * STEEMIT_BANDWIDTH_PRECISION; - if (delta_time >= - STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS) { - b.average_bandwidth = N; - } else { - auto old_weight = b.average_bandwidth * - (STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS - - delta_time); - auto new_weight = delta_time * N; - b.average_bandwidth = (old_weight + new_weight) / - STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS; - } - - b.last_bandwidth_update = now; - }); - - fc::uint128_t account_vshares(a.vesting_shares.amount.value); - fc::uint128_t total_vshares(props.total_vesting_shares.amount.value); - - fc::uint128_t account_average_bandwidth(band->average_bandwidth.value); - fc::uint128_t max_virtual_bandwidth(props.max_virtual_bandwidth); - - FC_ASSERT((account_vshares * max_virtual_bandwidth) > - (account_average_bandwidth * total_vshares), - "Account exceeded maximum allowed bandwidth per vesting share.", - ("account_vshares", account_vshares) - ("account_average_bandwidth", account_average_bandwidth) - ("max_virtual_bandwidth", max_virtual_bandwidth) - ("total_vesting_shares", total_vshares)); - } - } FC_CAPTURE_AND_RETHROW() - } - bool database::update_account_bandwidth(const account_object &a, uint32_t trx_size, const bandwidth_type type) { const auto &props = get_dynamic_global_properties(); bool has_bandwidth = true; if (props.total_vesting_shares.amount > 0) { auto band = find(boost::make_tuple(a.name, type)); - if (band == nullptr) { band = &create([&](account_bandwidth_object &b) { b.account = a.name; @@ -652,11 +595,8 @@ namespace golos { namespace chain { } share_type new_bandwidth; - share_type trx_bandwidth = - trx_size * STEEMIT_BANDWIDTH_PRECISION; - auto delta_time = (head_block_time() - - band->last_bandwidth_update).to_seconds(); - + share_type trx_bandwidth = trx_size * STEEMIT_BANDWIDTH_PRECISION; + auto delta_time = (head_block_time() - band->last_bandwidth_update).to_seconds(); if (delta_time > STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS) { new_bandwidth = 0; } else { @@ -681,16 +621,14 @@ namespace golos { namespace chain { fc::uint128_t account_average_bandwidth(band->average_bandwidth.value); fc::uint128_t max_virtual_bandwidth(props.max_virtual_bandwidth); - has_bandwidth = (account_vshares * max_virtual_bandwidth) > - (account_average_bandwidth * total_vshares); + has_bandwidth = (account_vshares * max_virtual_bandwidth) > (account_average_bandwidth * total_vshares); if (is_producing()) - FC_ASSERT(has_bandwidth, - "Account exceeded maximum allowed bandwidth per vesting share.", - ("account_vshares", account_vshares) - ("account_average_bandwidth", account_average_bandwidth) - ("max_virtual_bandwidth", max_virtual_bandwidth) - ("total_vesting_shares", total_vshares)); + FC_ASSERT(has_bandwidth, "Account exceeded maximum allowed bandwidth per vesting share.", + ("account_vshares", account_vshares) + ("account_average_bandwidth", account_average_bandwidth) + ("max_virtual_bandwidth", max_virtual_bandwidth) + ("total_vesting_shares", total_vshares)); } return has_bandwidth; @@ -3525,14 +3463,11 @@ namespace golos { namespace chain { auto trx_size = fc::raw::pack_size(trx); - for (const auto &auth : required) { - const auto &acnt = get_account(auth); - - old_update_account_bandwidth(acnt, trx_size, bandwidth_type::old_forum); + for (const auto& auth : required) { + const auto& acnt = get_account(auth); update_account_bandwidth(acnt, trx_size, bandwidth_type::forum); - for (const auto &op : trx.operations) { + for (const auto& op : trx.operations) { if (is_market_operation(op)) { - old_update_account_bandwidth(acnt, trx_size, bandwidth_type::old_market); update_account_bandwidth(acnt, trx_size * 10, bandwidth_type::market); break; } diff --git a/libraries/chain/include/golos/chain/database.hpp b/libraries/chain/include/golos/chain/database.hpp index 0b3f1bd26a..c20c96b5eb 100644 --- a/libraries/chain/include/golos/chain/database.hpp +++ b/libraries/chain/include/golos/chain/database.hpp @@ -13,8 +13,7 @@ #include -namespace golos { - namespace chain { +namespace golos { namespace chain { using golos::protocol::signed_transaction; using golos::protocol::operation; @@ -192,8 +191,6 @@ namespace golos { */ void pay_fee(const account_object &a, asset fee); - void old_update_account_bandwidth(const account_object &a, uint32_t trx_size, const bandwidth_type type); - /** * Update an account's bandwidth and returns if the account had the requisite bandwidth for the trx */ @@ -624,5 +621,4 @@ namespace golos { std::string _json_schema; }; - } -} +} } // golos::chain diff --git a/libraries/chain/include/golos/chain/steem_object_types.hpp b/libraries/chain/include/golos/chain/steem_object_types.hpp index b49a923bb7..350625a480 100644 --- a/libraries/chain/include/golos/chain/steem_object_types.hpp +++ b/libraries/chain/include/golos/chain/steem_object_types.hpp @@ -5,15 +5,13 @@ #include #include -//#include #include #include #include -namespace golos { - namespace chain { +namespace golos { namespace chain { using namespace boost::multi_index; @@ -142,13 +140,10 @@ namespace golos { enum bandwidth_type { post, ///< Rate limiting posting reward eligibility over time forum, ///< Rate limiting for all forum related actins - market, ///< Rate limiting for all other actions - old_forum, ///< Rate limiting for all forum related actions (deprecated) - old_market ///< Rate limiting for all other actions (deprecated) + market ///< Rate limiting for all other actions }; - } -} //golos::chain +} } //golos::chain namespace fc { class variant; @@ -211,10 +206,6 @@ namespace fc { } } -namespace fc { - -} - FC_REFLECT_ENUM(golos::chain::object_type, (dynamic_global_property_object_type) (account_object_type) @@ -251,4 +242,4 @@ FC_REFLECT_ENUM(golos::chain::object_type, FC_REFLECT_TYPENAME((golos::chain::shared_string)) FC_REFLECT_TYPENAME((golos::chain::buffer_type)) -FC_REFLECT_ENUM(golos::chain::bandwidth_type, (post)(forum)(market)(old_forum)(old_market)) +FC_REFLECT_ENUM(golos::chain::bandwidth_type, (post)(forum)(market)) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 4923261b65..f8d5faf242 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -506,8 +506,7 @@ namespace golos { namespace chain { } } - auto band = _db.find(boost::make_tuple(o.author, bandwidth_type::post)); - + auto band = _db.find(std::make_tuple(o.author, bandwidth_type::post)); if (band == nullptr) { band = &_db.create([&](account_bandwidth_object &b) { b.account = o.author; @@ -2162,36 +2161,34 @@ namespace golos { namespace chain { } void reset_account_evaluator::do_apply(const reset_account_operation &op) { - database &_db = db(); FC_ASSERT(false, "Reset Account Operation is currently disabled."); - - const auto &acnt = _db.get_account(op.account_to_reset); - auto band = _db.find(boost::make_tuple(op.account_to_reset, bandwidth_type::old_forum)); +/* + database& _db = db(); + const auto& acnt = _db.get_account(op.account_to_reset); + auto band = _db.find(std::make_tuple(op.account_to_reset, bandwidth_type::old_forum)); if (band != nullptr) - FC_ASSERT( - (_db.head_block_time() - band->last_bandwidth_update) > - fc::days(60), "Account must be inactive for 60 days to be eligible for reset"); - FC_ASSERT(acnt.reset_account == - op.reset_account, "Reset account does not match reset account on account."); - + FC_ASSERT((_db.head_block_time() - band->last_bandwidth_update) > fc::days(60), + "Account must be inactive for 60 days to be eligible for reset"); + FC_ASSERT(acnt.reset_account == op.reset_account, "Reset account does not match reset account on account."); _db.update_owner_authority(acnt, op.new_owner_authority); +*/ } void set_reset_account_evaluator::do_apply(const set_reset_account_operation &op) { - database &_db = db(); FC_ASSERT(false, "Set Reset Account Operation is currently disabled."); - - const auto &acnt = _db.get_account(op.account); +/* + database& _db = db(); + const auto& acnt = _db.get_account(op.account); _db.get_account(op.reset_account); - FC_ASSERT(acnt.reset_account == - op.current_reset_account, "Current reset account does not match reset account on account."); - FC_ASSERT(acnt.reset_account != - op.reset_account, "Reset account must change"); + FC_ASSERT(acnt.reset_account == op.current_reset_account, + "Current reset account does not match reset account on account."); + FC_ASSERT(acnt.reset_account != op.reset_account, "Reset account must change"); - _db.modify(acnt, [&](account_object &a) { + _db.modify(acnt, [&](account_object& a) { a.reset_account = op.reset_account; }); +*/ } void delegate_vesting_shares_evaluator::do_apply(const delegate_vesting_shares_operation& op) { diff --git a/libraries/protocol/include/golos/protocol/config.hpp b/libraries/protocol/include/golos/protocol/config.hpp index 9af9597932..6980a3aa21 100644 --- a/libraries/protocol/include/golos/protocol/config.hpp +++ b/libraries/protocol/include/golos/protocol/config.hpp @@ -63,7 +63,7 @@ #define STEEMIT_MAX_PROXY_RECURSION_DEPTH 4 #define STEEMIT_VESTING_WITHDRAW_INTERVALS_PRE_HF_16 104 #define STEEMIT_VESTING_WITHDRAW_INTERVALS 13 -#define STEEMIT_VESTING_WITHDRAW_INTERVAL_SECONDS (60*60*24*7) // 1 week per interval +#define STEEMIT_VESTING_WITHDRAW_INTERVAL_SECONDS (60*60*1) // 1 hour per interval #define STEEMIT_MAX_WITHDRAW_ROUTES 10 #define STEEMIT_SAVINGS_WITHDRAW_TIME (fc::days(3)) #define STEEMIT_SAVINGS_WITHDRAW_REQUEST_LIMIT 100 diff --git a/libraries/protocol/include/golos/protocol/operations.hpp b/libraries/protocol/include/golos/protocol/operations.hpp index 581eef5c7f..59e1511068 100644 --- a/libraries/protocol/include/golos/protocol/operations.hpp +++ b/libraries/protocol/include/golos/protocol/operations.hpp @@ -5,8 +5,7 @@ #include #include -namespace golos { - namespace protocol { +namespace golos { namespace protocol { /** NOTE: do not change the order of any operations prior to the virtual operations * or it will trigger a hardfork. @@ -89,7 +88,7 @@ namespace golos { flat_set& active, flat_set& owner, flat_set& posting, - vector& other ); + vector& other ); void operation_validate( const operation& op );*/ @@ -103,12 +102,11 @@ namespace golos { operation op; }; - } -} // golos::protocol +} } // golos::protocol /*namespace fc { - void to_variant( const golos::protocol::operation& var, fc::variant& vo ); - void from_variant( const fc::variant& var, golos::protocol::operation& vo ); + void to_variant(const golos::protocol::operation& var, fc::variant& vo); + void from_variant(const fc::variant& var, golos::protocol::operation& vo); }*/ DECLARE_OPERATION_TYPE(golos::protocol::operation) From 1c98c1d0c1ebe2a7ba7f3be675662ae1c741f614 Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Mon, 4 Jun 2018 19:41:54 +0300 Subject: [PATCH 13/22] Kill spaces #692 --- libraries/api/discussion_helper.cpp | 10 ++--- .../include/golos/api/discussion_helper.hpp | 6 +-- libraries/chain/database.cpp | 24 +++++----- .../golos/protocol/steem_operations.hpp | 10 ++--- .../wallet/include/golos/wallet/wallet.hpp | 44 +++++++++---------- libraries/wallet/wallet.cpp | 41 +++++++++-------- .../golos/plugins/auth_util/plugin.hpp | 2 +- plugins/debug_node/plugin.cpp | 9 ++-- .../include/golos/plugins/follow/plugin.hpp | 2 +- plugins/operation_history/plugin.cpp | 11 +++-- programs/cli_wallet/main.cpp | 4 +- tests/tests/operation_tests.cpp | 2 +- tests/tests/proposal_tests.cpp | 2 +- 13 files changed, 81 insertions(+), 86 deletions(-) diff --git a/libraries/api/discussion_helper.cpp b/libraries/api/discussion_helper.cpp index ba96ef3439..38753bc0e5 100644 --- a/libraries/api/discussion_helper.cpp +++ b/libraries/api/discussion_helper.cpp @@ -91,7 +91,7 @@ namespace golos { namespace api { std::function fill_promoted_; }; -// get_discussion +// get_discussion discussion discussion_helper::impl::get_discussion(const comment_object& c, uint32_t vote_limit) const { discussion d = create_discussion(c); set_url(d); @@ -100,9 +100,9 @@ namespace golos { namespace api { return d; } - discussion discussion_helper::get_discussion(const comment_object& c, uint32_t vote_limit) const { + discussion discussion_helper::get_discussion(const comment_object& c, uint32_t vote_limit) const { return pimpl->get_discussion(c, vote_limit); - } + } // // select_active_votes @@ -202,7 +202,7 @@ namespace golos { namespace api { void discussion_helper::set_url(discussion& d) const { pimpl->set_url(d); } -// +// // create_discussion discussion discussion_helper::impl::create_discussion(const comment_object& o) const { return discussion(o, database_); @@ -218,7 +218,7 @@ namespace golos { namespace api { std::function fill_promoted ) { pimpl = std::make_unique(db, fill_reputation, fill_promoted); - } + } discussion_helper::~discussion_helper() = default; diff --git a/libraries/api/include/golos/api/discussion_helper.hpp b/libraries/api/include/golos/api/discussion_helper.hpp index c7ec4a72ac..1cdc78f9df 100644 --- a/libraries/api/include/golos/api/discussion_helper.hpp +++ b/libraries/api/include/golos/api/discussion_helper.hpp @@ -18,17 +18,17 @@ namespace golos { namespace api { golos::chain::database& db, std::function&)> fill_reputation, std::function fill_promoted); - ~discussion_helper() ; + ~discussion_helper(); void set_pending_payout(discussion& d) const; - + void set_url(discussion& d) const; void select_active_votes( std::vector& result, uint32_t& total_count, const std::string& author, const std::string& permlink, uint32_t limit - ) const ; + ) const; discussion create_discussion(const comment_object& o) const; diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index da451e24f1..2b7301e576 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -344,7 +344,7 @@ namespace golos { namespace chain { return block_id_type(); } - // Reversible blocks are *usually* in the TAPOS buffer. Since this + // Reversible blocks are *usually* in the TAPOS buffer. Since this // is the fastest check, we do it first. block_summary_id_type bsid = block_num & 0xFFFF; const block_summary_object *bs = find(bsid); @@ -355,7 +355,7 @@ namespace golos { namespace chain { } } - // Next we query the block log. Irreversible blocks are here. + // Next we query the block log. Irreversible blocks are here. auto b = _block_log.read_block_by_num(block_num); if (b.valid()) { @@ -710,7 +710,7 @@ namespace golos { namespace chain { } /** - * Push block "may fail" in which case every partial change is unwound. After + * Push block "may fail" in which case every partial change is unwound. After * push block is successful the block is appended to the chain database on disk. * * @return true if we switched forks as a result of this push. @@ -872,7 +872,7 @@ namespace golos { namespace chain { // Create a temporary undo session as a child of _pending_tx_session. // The temporary session will be discarded by the destructor if - // _apply_transaction fails. If we make it to merge(), we + // _apply_transaction fails. If we make it to merge(), we // apply the changes. auto temp_session = start_undo_session(); @@ -934,7 +934,7 @@ namespace golos { namespace chain { // This rebuild is necessary because pending transactions' validity // and semantics may have changed since they were received, because // time-based semantics are evaluated based on the current block - // time. These changes can only be reflected in the database when + // time. These changes can only be reflected in the database when // the value of the "when" variable is known, which means we need to // re-apply pending transactions in this method. // @@ -1026,7 +1026,7 @@ namespace golos { namespace chain { pending_block.sign(block_signing_private_key); } - // TODO: Move this to _push_block() so session is restored. + // TODO: Move this to _push_block() so session is restored. if (!(skip & skip_block_size_check)) { FC_ASSERT(fc::raw::pack_size(pending_block) <= STEEMIT_MAX_BLOCK_SIZE); } @@ -1205,7 +1205,7 @@ namespace golos { namespace chain { * The ratio of total_vesting_shares / total_vesting_fund_steem should not * change as the result of the user adding funds * - * V / C = (V+Vn) / (C+Cn) + * V / C = (V+Vn) / (C+Cn) * * Simplifies to Vn = (V * Cn ) / C * @@ -1901,7 +1901,7 @@ namespace golos { namespace chain { * Let V = total vesting shares * Let v = total vesting shares being cashed out * - * The user may withdraw vT / V tokens + * The user may withdraw vT / V tokens */ share_type to_withdraw; if (from_account.to_withdraw - from_account.withdrawn < @@ -2207,7 +2207,7 @@ namespace golos { namespace chain { void database::process_comment_cashout() { /// don't allow any content to get paid out until the website is ready to launch - /// and people have had a week to start posting. The first cashout will be the biggest because it + /// and people have had a week to start posting. The first cashout will be the biggest because it /// will represent 2+ months of rewards. // if (!has_hardfork(STEEMIT_FIRST_CASHOUT_TIME)) { // return; @@ -3386,7 +3386,7 @@ namespace golos { namespace chain { } } else { - if ( wit.last_sbd_exchange_update < now + STEEMIT_MAX_FEED_AGE && !wit.sbd_exchange_rate.is_null() ) { + if ( wit.last_sbd_exchange_update < now + STEEMIT_MAX_FEED_AGE && !wit.sbd_exchange_rate.is_null() ) { feeds.push_back(wit.sbd_exchange_rate); } @@ -3598,7 +3598,7 @@ namespace golos { namespace chain { * their capacity. * * When the reserve ratio is at its max (check STEEMIT_MAX_RESERVE_RATIO) a 50% - * reduction will take 3 to 4 days to return back to maximum. When it is at its + * reduction will take 3 to 4 days to return back to maximum. When it is at its * minimum it will return back to its prior level in just a few minutes. * * If the network reserve ratio falls under 100 then it is probably time to @@ -3909,7 +3909,7 @@ namespace golos { namespace chain { }); /** * There are times when the AMOUNT_FOR_SALE * SALE_PRICE == 0 which means that we - * have hit the limit where the seller is asking for nothing in return. When this + * have hit the limit where the seller is asking for nothing in return. When this * happens we must refund any balance back to the seller, it is too small to be * sold at the sale price. */ diff --git a/libraries/protocol/include/golos/protocol/steem_operations.hpp b/libraries/protocol/include/golos/protocol/steem_operations.hpp index 1ff4cda079..4c7370d00f 100644 --- a/libraries/protocol/include/golos/protocol/steem_operations.hpp +++ b/libraries/protocol/include/golos/protocol/steem_operations.hpp @@ -422,7 +422,7 @@ namespace golos { namespace protocol { /** * Witnesses must vote on how to set certain chain properties to ensure a smooth - * and well functioning network. Any time @owner is in the active set of witnesses these + * and well functioning network. Any time @owner is in the active set of witnesses these * properties will be used to control the blockchain configuration. */ struct chain_properties_17 { @@ -456,7 +456,7 @@ namespace golos { namespace protocol { /** * Witnesses must vote on how to set certain chain properties to ensure a smooth - * and well functioning network. Any time @owner is in the active set of witnesses these + * and well functioning network. Any time @owner is in the active set of witnesses these * properties will be used to control the blockchain configuration. */ struct chain_properties_18: public chain_properties_17 { @@ -524,13 +524,13 @@ namespace golos { namespace protocol { * the current witnesses to apply for the position and allow voting * to begin. * - * If the owner isn't a witness they will become a witness. Witnesses + * If the owner isn't a witness they will become a witness. Witnesses * are charged a fee equal to 1 weeks worth of witness pay which in - * turn is derived from the current share supply. The fee is + * turn is derived from the current share supply. The fee is * only applied if the owner is not already a witness. * * If the block_signing_key is null then the witness is removed from - * contention. The network will pick the top 21 witnesses for + * contention. The network will pick the top 21 witnesses for * producing blocks. */ struct witness_update_operation : public base_operation { diff --git a/libraries/wallet/include/golos/wallet/wallet.hpp b/libraries/wallet/include/golos/wallet/wallet.hpp index cdec75f236..cfb19e3c6f 100644 --- a/libraries/wallet/include/golos/wallet/wallet.hpp +++ b/libraries/wallet/include/golos/wallet/wallet.hpp @@ -38,7 +38,7 @@ namespace golos { namespace wallet { try { if( str.size() > sizeof(memo_data) && str[0] == '#') { auto data = fc::from_base58( str.substr(1) ); - auto m = fc::raw::unpack( data ); + auto m = fc::raw::unpack( data ); FC_ASSERT( string(m) == str ); return m; } @@ -183,7 +183,7 @@ namespace golos { namespace wallet { * * @param author The author of the proposal * @param title The proposal to modify. - * @param delta Members contain approvals to create or remove. In JSON you can leave empty members undefined. + * @param delta Members contain approvals to create or remove. In JSON you can leave empty members undefined. * @param broadcast true if you wish to broadcast the transaction * @return the signed version of the transaction */ @@ -248,11 +248,11 @@ namespace golos { namespace wallet { /** Lists all accounts registered in the blockchain. * This returns a list of all account names and their account ids, sorted by account name. * - * Use the \c lowerbound and limit parameters to page through the list. To retrieve all accounts, + * Use the \c lowerbound and limit parameters to page through the list. To retrieve all accounts, * start by setting \c lowerbound to the empty string \c "", and then each iteration, pass * the last account name returned as the \c lowerbound for the next \c list_accounts() call. * - * @param lowerbound the name of the first account to return. If the named account does not exist, + * @param lowerbound the name of the first account to return. If the named account does not exist, * the list will start at the account that comes after \c lowerbound * @param limit the maximum number of accounts to return (max: 1000) * @returns a list of accounts mapping account names to account ids @@ -284,7 +284,7 @@ namespace golos { namespace wallet { string get_wallet_filename() const; /** - * Get the WIF private key corresponding to a public key. The + * Get the WIF private key corresponding to a public key. The * private key must already be in the wallet. */ string get_private_key( public_key_type pubkey )const; @@ -295,7 +295,7 @@ namespace golos { namespace wallet { * @param password - the password to be used at key generation * @return public key corresponding to generated private key, and private key in WIF format. */ - pair get_private_key_from_password( string account, string role, string password )const; + pair get_private_key_from_password(string account, string role, string password) const; /** @@ -343,7 +343,7 @@ namespace golos { namespace wallet { /** Dumps all private keys owned by the wallet. * - * The keys are printed in WIF format. You can import these keys into another wallet + * The keys are printed in WIF format. You can import these keys into another wallet * using \c import_key() * @returns a map containing the private keys, indexed by their public key */ @@ -353,7 +353,7 @@ namespace golos { namespace wallet { * @param method the name of the API command you want help with * @returns a multi-line string suitable for displaying on a terminal */ - string gethelp(const string& method)const; + string gethelp(const string& method)const; /** Loads a specified Graphene wallet. * @@ -374,10 +374,10 @@ namespace golos { namespace wallet { * * @warning This does not change the wallet filename that will be used for future * writes, so think of this function as 'Save a Copy As...' instead of - * 'Save As...'. Use \c set_wallet_filename() to make the filename + * 'Save As...'. Use \c set_wallet_filename() to make the filename * persist. * @param wallet_filename the filename of the new wallet JSON file to create - * or overwrite. If \c wallet_filename is empty, + * or overwrite. If \c wallet_filename is empty, * save to the current filename. */ void save_wallet_file(string wallet_filename = ""); @@ -399,7 +399,7 @@ namespace golos { namespace wallet { /** Suggests a safe brain key to use for creating your account. * \c create_account_with_brain_key() requires you to specify a 'brain key', * a long passphrase that provides enough entropy to generate cyrptographic - * keys. This function will suggest a suitably random string that should + * keys. This function will suggest a suitably random string that should * be easy to write down (and, with effort, memorize). * @returns a suggested brain_key */ @@ -410,7 +410,7 @@ namespace golos { namespace wallet { * TODO: I don't see a broadcast_transaction() function, do we need one? * * @param tx the transaction to serialize - * @returns the binary form of the transaction. It will not be hex encoded, + * @returns the binary form of the transaction. It will not be hex encoded, * this returns a raw string that may have null characters embedded * in it */ @@ -427,7 +427,7 @@ namespace golos { namespace wallet { /** Transforms a brain key to reduce the chance of errors when re-entering the key from memory. * * This takes a user-supplied brain key and normalizes it into the form used - * for generating private keys. In particular, this upper-cases all ASCII characters + * for generating private keys. In particular, this upper-cases all ASCII characters * and collapses multiple spaces into one. * @param s the brain key as supplied by the user * @returns the brain key in its normalized form @@ -624,13 +624,13 @@ namespace golos { namespace wallet { /** Lists all witnesses registered in the blockchain. * This returns a list of all account names that own witnesses, and the associated witness id, - * sorted by name. This lists witnesses whether they are currently voted in or not. + * sorted by name. This lists witnesses whether they are currently voted in or not. * - * Use the \c lowerbound and limit parameters to page through the list. To retrieve all witnesss, + * Use the \c lowerbound and limit parameters to page through the list. To retrieve all witnesss, * start by setting \c lowerbound to the empty string \c "", and then each iteration, pass * the last witness name returned as the \c lowerbound for the next \c list_witnesss() call. * - * @param lowerbound the name of the first witness to return. If the named witness does not exist, + * @param lowerbound the name of the first witness to return. If the named witness does not exist, * the list will start at the witness that comes after \c lowerbound * @param limit the maximum number of witnesss to return (max: 1000) * @returns a list of witnesss mapping witness names to witness ids @@ -656,8 +656,8 @@ namespace golos { namespace wallet { * Update a witness object owned by the given account. * * @param witness_name The name of the witness account. - * @param url A URL containing some information about the witness. The empty string makes it remain the same. - * @param block_signing_key The new block signing public key. The empty string disables block production. + * @param url A URL containing some information about the witness. The empty string makes it remain the same. + * @param block_signing_key The new block signing public key. The empty string disables block production. * @param props The chain properties the witness is voting on. * @param broadcast true if you wish to broadcast the transaction. */ @@ -688,7 +688,7 @@ namespace golos { namespace wallet { * to allow another account to vote their stake. * * Setting a vote proxy does not remove your previous votes from the blockchain, - * they remain there but are ignored. If you later null out your vote proxy, + * they remain there but are ignored. If you later null out your vote proxy, * your previous votes will take effect again. * * This setting can be changed at any time. @@ -920,8 +920,8 @@ namespace golos { namespace wallet { * * Any operation the blockchain supports can be created using the transaction builder's * \c add_operation_to_builder_transaction() , but to do that from the CLI you need to - * know what the JSON form of the operation looks like. This will give you a template - * you can fill in. It's better than nothing. + * know what the JSON form of the operation looks like. This will give you a template + * you can fill in. It's better than nothing. * * @param operation_type the type of operation to return, must be one of the * operations defined in `steem/chain/operations.hpp` @@ -1041,7 +1041,7 @@ namespace golos { namespace wallet { FC_TODO(Supplement API argument description) /** - * Marks one account as following another account. Requires the posting authority of the follower. + * Marks one account as following another account. Requires the posting authority of the follower. * * @param follower * @param following diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index d99ee9645f..81fc07dd8a 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -110,7 +110,7 @@ namespace golos { namespace wallet { while (i < n) { char c = s[i++]; switch (c) { - case ' ': case '\t': case '\r': case '\n': case '\v': case '\f': + case ' ': case '\t': case '\r': case '\n': case '\v': case '\f': preceded_by_whitespace = true; continue; @@ -539,8 +539,7 @@ namespace golos { namespace wallet { } bool load_wallet_file(string wallet_filename = "") { - // TODO: Merge imported wallet with existing wallet, - // instead of replacing it + // TODO: Merge imported wallet with existing wallet, instead of replacing it if( wallet_filename == "" ) wallet_filename = _wallet_filename; @@ -572,7 +571,7 @@ namespace golos { namespace wallet { enable_umask_protection(); // // Parentheses on the following declaration fails to compile, - // due to the Most Vexing Parse. Thanks, C++ + // due to the Most Vexing Parse. Thanks, C++ // // http://en.wikipedia.org/wiki/Most_vexing_parse // @@ -588,7 +587,7 @@ namespace golos { namespace wallet { } // This function generates derived keys starting with index 0 and keeps incrementing - // the index until it finds a key that isn't registered in the block chain. To be + // the index until it finds a key that isn't registered in the block chain. To be // safer, it continues checking for a few more keys to make sure there wasn't a short gap // caused by a failed registration or the like. int find_first_unused_derived_key_index(const fc::ecc::private_key& parent_key) { @@ -677,10 +676,10 @@ namespace golos { namespace wallet { annotated_signed_transaction sign_transaction(signed_transaction tx, bool broadcast = false) { - flat_set< account_name_type > req_active_approvals; - flat_set< account_name_type > req_owner_approvals; - flat_set< account_name_type > req_posting_approvals; - vector< authority > other_auths; + flat_set< account_name_type > req_active_approvals; + flat_set< account_name_type > req_owner_approvals; + flat_set< account_name_type > req_posting_approvals; + vector< authority > other_auths; tx.get_required_authorities( req_active_approvals, req_owner_approvals, req_posting_approvals, other_auths ); @@ -716,7 +715,7 @@ namespace golos { namespace wallet { i++; continue; } - approving_account_lut[ approving_acct->name ] = *approving_acct; + approving_account_lut[ approving_acct->name ] = *approving_acct; i++; } auto get_account_from_lut = [&]( const std::string& name ) -> const golos::api::account_api_object& { @@ -817,7 +816,7 @@ namespace golos { namespace wallet { rtrx.transaction_num = result.trx_num; return rtrx; } catch (const fc::exception& e) { - elog("Caught exception while broadcasting tx ${id}: ${e}", ("id", tx.id().str())("e", e.to_detail_string()) ); + elog("Caught exception while broadcasting tx ${id}: ${e}", ("id", tx.id().str())("e", e.to_detail_string()) ); throw; } } @@ -843,8 +842,8 @@ namespace golos { namespace wallet { asset total_sbd(0, SBD_SYMBOL ); for( const auto& a : accounts ) { total_steem += a.balance; - total_vest += a.vesting_shares; - total_sbd += a.sbd_balance; + total_vest += a.vesting_shares; + total_sbd += a.sbd_balance; out << std::left << std::setw( 17 ) << std::string(a.name) << std::right << std::setw(18) << fc::variant(a.balance).as_string() <<" " << std::right << std::setw(26) << fc::variant(a.vesting_shares).as_string() <<" " @@ -1543,7 +1542,7 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st account_update_operation op; op.account = account_name; - op.owner = authority( 1, owner, 1 ); + op.owner = authority( 1, owner, 1 ); op.active = authority( 1, active, 1); op.posting = authority( 1, posting, 1); op.memo_key = memo; @@ -2149,7 +2148,7 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st /** * Transfers into savings happen immediately, transfers from savings take 72 hours */ - annotated_signed_transaction wallet_api::transfer_to_savings( string from, string to, asset amount, string memo, bool broadcast ) + annotated_signed_transaction wallet_api::transfer_to_savings( string from, string to, asset amount, string memo, bool broadcast) { FC_ASSERT( !is_locked() ); check_memo( memo, get_account( from ) ); @@ -2169,7 +2168,7 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st /** * @param request_id - an unique ID assigned by from account, the id is used to cancel the operation and can be reused after the transfer completes */ - annotated_signed_transaction wallet_api::transfer_from_savings( string from, uint32_t request_id, string to, asset amount, string memo, bool broadcast ) + annotated_signed_transaction wallet_api::transfer_from_savings( string from, uint32_t request_id, string to, asset amount, string memo, bool broadcast) { FC_ASSERT( !is_locked() ); check_memo( memo, get_account( from ) ); @@ -2191,7 +2190,7 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st * @param request_id the id used in transfer_from_savings * @param from the account that initiated the transfer */ - annotated_signed_transaction wallet_api::cancel_transfer_from_savings( string from, uint32_t request_id, bool broadcast ) + annotated_signed_transaction wallet_api::cancel_transfer_from_savings( string from, uint32_t request_id, bool broadcast) { FC_ASSERT( !is_locked() ); cancel_transfer_from_savings_operation op; @@ -2291,7 +2290,7 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st fc::sha512 shared_secret; auto from_key = my->try_get_private_key( m->from ); if( !from_key ) { - auto to_key = my->try_get_private_key( m->to ); + auto to_key = my->try_get_private_key( m->to ); if( !to_key ) return encrypted_memo; shared_secret = to_key->get_shared_secret( m->from ); } else { @@ -2362,7 +2361,7 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st return my->_remote_database_api->get_open_orders( owner ); } - annotated_signed_transaction wallet_api::create_order( string owner, uint32_t order_id, asset amount_to_sell, asset min_to_receive, bool fill_or_kill, uint32_t expiration_sec, bool broadcast ) { + annotated_signed_transaction wallet_api::create_order(string owner, uint32_t order_id, asset amount_to_sell, asset min_to_receive, bool fill_or_kill, uint32_t expiration_sec, bool broadcast) { FC_ASSERT( !is_locked() ); limit_order_create_operation op; op.owner = owner; @@ -2395,7 +2394,7 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st annotated_signed_transaction wallet_api::post_comment( string author, string permlink, string parent_author, string parent_permlink, string title, string body, string json, bool broadcast ) { FC_ASSERT( !is_locked() ); comment_operation op; - op.parent_author = parent_author; + op.parent_author = parent_author; op.parent_permlink = parent_permlink; op.author = author; op.permlink = permlink; @@ -2508,7 +2507,7 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st const bool broadcast) { string _following = following; - auto follwer_account = get_account( follower ); + auto follwer_account = get_account( follower ); FC_ASSERT( _following.size() ); if( _following[0] != '@' || _following[0] != '#' ) { _following = '@' + _following; diff --git a/plugins/auth_util/include/golos/plugins/auth_util/plugin.hpp b/plugins/auth_util/include/golos/plugins/auth_util/plugin.hpp index 39a5f4e68b..021e8c64bd 100644 --- a/plugins/auth_util/include/golos/plugins/auth_util/plugin.hpp +++ b/plugins/auth_util/include/golos/plugins/auth_util/plugin.hpp @@ -13,7 +13,7 @@ namespace golos { namespace plugins { namespace auth_util { -using golos::plugins::json_rpc::msg_pack; +using golos::plugins::json_rpc::msg_pack; DEFINE_API_ARGS ( check_authority_signature, msg_pack, std::vector) diff --git a/plugins/debug_node/plugin.cpp b/plugins/debug_node/plugin.cpp index 187aa07a84..c78312a333 100644 --- a/plugins/debug_node/plugin.cpp +++ b/plugins/debug_node/plugin.cpp @@ -470,13 +470,12 @@ bool plugin::plugin_impl::debug_has_hardfork( uint32_t hardfork_id ) { // ``` // You can pass from 1 to 5 params as you wish. // Just don't forget that it work for testnet only and you have to write this in ur config.ini: -// +// // witness = "cyberfounder" // private-key = 5JVFFWRLwz6JoP9kguuRFfytToGU6cLgBVTL9t6NB3D3BQLbUBS // enable-stale-production = true // required-participation = 0 -// -// +// #include @@ -491,7 +490,6 @@ DEFINE_PLUGIN_API ( debug_generate_blocks ) { (uint32_t, miss_blocks, 0) (bool, edit_if_needed, true) ) - return my->debug_generate_blocks( debug_key, count, skip, miss_blocks, edit_if_needed ); } @@ -501,7 +499,6 @@ DEFINE_PLUGIN_API ( debug_push_blocks ) { (uint32_t, count) (bool, skip_validate_invariants, false) ) - return my->debug_push_blocks( src_filename, count, skip_validate_invariants ); } @@ -513,7 +510,7 @@ DEFINE_PLUGIN_API ( debug_push_json_blocks ) { ) // `skip_flags` can be set to 577 if loading mainnet blocks // 577 = skip_witness_signature | skip_authority_check | skip_witness_schedule_check - + auto &db = my->database(); return db.with_weak_read_lock([&]() { return my->debug_push_json_blocks(json_filename, count, skip_flags); diff --git a/plugins/follow/include/golos/plugins/follow/plugin.hpp b/plugins/follow/include/golos/plugins/follow/plugin.hpp index 9be91cf9f6..44a0bbb813 100644 --- a/plugins/follow/include/golos/plugins/follow/plugin.hpp +++ b/plugins/follow/include/golos/plugins/follow/plugin.hpp @@ -13,7 +13,7 @@ namespace golos { namespace plugins { namespace follow { const account_name_type& account, fc::optional& reputation ); - + /// API, args, return DEFINE_API_ARGS(get_followers, msg_pack, std::vector) DEFINE_API_ARGS(get_following, msg_pack, std::vector) diff --git a/plugins/operation_history/plugin.cpp b/plugins/operation_history/plugin.cpp index 6c9167d726..5138b9bbfc 100644 --- a/plugins/operation_history/plugin.cpp +++ b/plugins/operation_history/plugin.cpp @@ -75,7 +75,6 @@ namespace golos { namespace plugins { namespace operation_history { if (database.head_block_num() < start_block) { return; } - if (filter.find(fc::get_typename::name()) != filter.end()) { if (!blacklist) { operation_visitor::operator()(op); @@ -175,15 +174,15 @@ namespace golos { namespace plugins { namespace operation_history { boost::program_options::value()->composing(), "Defines starting block from which recording stats." ); - + cfg.add(cli); } void plugin::plugin_initialize(const boost::program_options::variables_map& options) { ilog("operation_history plugin: plugin_initialize() begin"); - + pimpl = std::make_unique(); - + pimpl->database.pre_apply_operation.connect([&](golos::chain::operation_notification& note){ pimpl->on_operation(note); }); @@ -207,7 +206,7 @@ namespace golos { namespace plugins { namespace operation_history { FC_ASSERT( !options.count("history-blacklist-ops"), "history-blacklist-ops and history-whitelist-ops can't be specified together"); - + pimpl->filter_content = true; pimpl->blacklist = false; split_list(options.at("history-whitelist-ops").as>()); @@ -215,7 +214,7 @@ namespace golos { namespace plugins { namespace operation_history { } else if (options.count("history-blacklist-ops")) { pimpl->filter_content = true; pimpl->blacklist = true; - split_list(options.at("history-blacklist-ops").as>()); + split_list(options.at("history-blacklist-ops").as>()); ilog("operation_history: blacklisting ops ${o}", ("o", pimpl->ops_list)); } diff --git a/programs/cli_wallet/main.cpp b/programs/cli_wallet/main.cpp index 2151e0588b..0c49e16fa0 100644 --- a/programs/cli_wallet/main.cpp +++ b/programs/cli_wallet/main.cpp @@ -122,7 +122,7 @@ int unsafe_main(int argc, char** argv) { // Note: each logging option have default value, no need to check options.count() auto ll_default = fc::variant(options["logger.default.level"].as()).as(); auto ll_rpc = fc::variant(options["logger.rpc.level"].as()).as(); - + fc::logging_config cfg; fc::file_appender::config ac; if (ll_rpc < fc::log_level::off) { @@ -146,7 +146,7 @@ int unsafe_main(int argc, char** argv) { fc::configure_logging(cfg); // - // TODO: We read wallet_data twice, once in main() to grab the + // TODO: We read wallet_data twice, once in main() to grab the // socket info, again in wallet_api when we do // load_wallet_file(). Seems like this could be better // designed. diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index b47cab48cc..3dc9b988ea 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -1977,7 +1977,7 @@ BOOST_FIXTURE_TEST_SUITE(operation_tests, clean_database_fixture) BOOST_TEST_MESSAGE("--- Test adding a grandchild proxy"); // alice - // bob-> sam-> dave + // bob-> sam-> dave tx.operations.clear(); tx.signatures.clear(); diff --git a/tests/tests/proposal_tests.cpp b/tests/tests/proposal_tests.cpp index 92ff88a6dd..6b2f50b8a2 100644 --- a/tests/tests/proposal_tests.cpp +++ b/tests/tests/proposal_tests.cpp @@ -396,7 +396,7 @@ BOOST_AUTO_TEST_CASE(update_proposal3) { try { * * Mega Corp. Well 30, Yaya 30 T=40 * Nova Ltd. Alice 10, Well 10 T=20 - * Odle Intl. Dan 10, Yaya 10, Zyzz 10, Dave 5 T=20 + * Odle Intl. Dan 10, Yaya 10, Zyzz 10, Dave 5 T=20 * Poxx LLC Well 10, Xylo 10, Yaya 20, Zyzz 20 T=40 */ BOOST_AUTO_TEST_CASE(nested_signatures) { try { From 8cb9690f1a7ef1eec8651da554d40bd7770d7f3b Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Tue, 5 Jun 2018 09:29:34 +0300 Subject: [PATCH 14/22] Rename bandwidth fields of account object and add lifetime bandwidth #692 --- libraries/api/account_api_object.cpp | 8 +++++--- libraries/api/include/golos/api/account_api_object.hpp | 9 ++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/libraries/api/account_api_object.cpp b/libraries/api/account_api_object.cpp index b0c20c1999..8558ae8490 100644 --- a/libraries/api/account_api_object.cpp +++ b/libraries/api/account_api_object.cpp @@ -51,19 +51,21 @@ account_api_object::account_api_object(const account_object& a, const golos::cha auto post = db.find(std::make_tuple(name, bandwidth_type::post)); if (post != nullptr) { - last_root_post = post->last_bandwidth_update; post_bandwidth = post->average_bandwidth; + last_root_post = post->last_bandwidth_update; } auto forum = db.find(std::make_tuple(name, bandwidth_type::forum)); if (forum != nullptr) { - new_average_bandwidth = forum->average_bandwidth; + average_bandwidth = forum->average_bandwidth; + lifetime_bandwidth = forum->lifetime_bandwidth; last_bandwidth_update = forum->last_bandwidth_update; } auto market = db.find(std::make_tuple(name, bandwidth_type::market)); if (market != nullptr) { - new_average_market_bandwidth = market->average_bandwidth; + average_market_bandwidth = market->average_bandwidth; + lifetime_market_bandwidth = market->lifetime_bandwidth; last_market_bandwidth_update = market->last_bandwidth_update; } } diff --git a/libraries/api/include/golos/api/account_api_object.hpp b/libraries/api/include/golos/api/account_api_object.hpp index a9914a8e3c..2dd7b26714 100644 --- a/libraries/api/include/golos/api/account_api_object.hpp +++ b/libraries/api/include/golos/api/account_api_object.hpp @@ -81,14 +81,16 @@ struct account_api_object { uint16_t witnesses_voted_for; + share_type average_bandwidth; + share_type average_market_bandwidth; + share_type lifetime_bandwidth; + share_type lifetime_market_bandwidth; time_point_sec last_bandwidth_update; time_point_sec last_market_bandwidth_update; time_point_sec last_post; time_point_sec last_root_post; share_type post_bandwidth = STEEMIT_100_PERCENT; - share_type new_average_bandwidth; - share_type new_average_market_bandwidth; set witness_votes; fc::optional reputation; @@ -107,8 +109,9 @@ FC_REFLECT((golos::api::account_api_object), (savings_withdraw_requests)(vesting_shares)(delegated_vesting_shares)(received_vesting_shares) (vesting_withdraw_rate)(next_vesting_withdrawal)(withdrawn)(to_withdraw)(withdraw_routes) (curation_rewards)(posting_rewards)(proxied_vsf_votes)(witnesses_voted_for) + (average_bandwidth)(average_market_bandwidth)(lifetime_bandwidth)(lifetime_market_bandwidth) (last_bandwidth_update)(last_market_bandwidth_update) - (last_post)(last_root_post)(post_bandwidth)(new_average_bandwidth)(new_average_market_bandwidth) + (last_post)(last_root_post)(post_bandwidth) (witness_votes)(reputation)) #endif //GOLOS_ACCOUNT_API_OBJ_HPP From bff56fb90c044dac056ffd6703b2010260dd06a0 Mon Sep 17 00:00:00 2001 From: AKorpusenko Date: Tue, 5 Jun 2018 11:09:39 +0300 Subject: [PATCH 15/22] Fixed unitialized fied promoted in get_discussions_by_promoted. #688 --- plugins/tags/plugin.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/tags/plugin.cpp b/plugins/tags/plugin.cpp index 409a21f879..bda112c2ff 100644 --- a/plugins/tags/plugin.cpp +++ b/plugins/tags/plugin.cpp @@ -10,6 +10,8 @@ // These visitors creates additional tables, we don't really need them in LOW_MEM mode #include #include +#include +#include #define CHECK_ARG_SIZE(_S) \ FC_ASSERT( \ @@ -404,15 +406,16 @@ namespace golos { namespace plugins { namespace tags { } discussion d = create_discussion(*comment); - if (!select(d) || !query.is_good_tags(d)) { - continue; - } fill_discussion(d, query); - d.promoted = itr->promoted_balance; + d.promoted = asset(itr->promoted_balance, SBD_SYMBOL); d.hot = itr->hot; d.trending = itr->trending; + if (!select(d) || !query.is_good_tags(d)) { + continue; + } + if (query.has_start_comment() && !query.is_good_start(d.id) && !order(query.start_comment, d)) { continue; } From bb0f850863e756e4b7d1bb2da0aefd78336bc663 Mon Sep 17 00:00:00 2001 From: AKorpusenko Date: Tue, 5 Jun 2018 11:13:36 +0300 Subject: [PATCH 16/22] Removed extra includes needed for debub. #688 --- plugins/tags/plugin.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/tags/plugin.cpp b/plugins/tags/plugin.cpp index bda112c2ff..cc87ff5d26 100644 --- a/plugins/tags/plugin.cpp +++ b/plugins/tags/plugin.cpp @@ -10,8 +10,6 @@ // These visitors creates additional tables, we don't really need them in LOW_MEM mode #include #include -#include -#include #define CHECK_ARG_SIZE(_S) \ FC_ASSERT( \ From cff576b9a81579c6088a4217794fb10d2f03c8bf Mon Sep 17 00:00:00 2001 From: maslenitsa93 Date: Mon, 4 Jun 2018 17:48:38 +0300 Subject: [PATCH 17/22] Mongo_db add root_comment field into comment_object #702 --- plugins/mongo_db/mongo_db_state.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/mongo_db/mongo_db_state.cpp b/plugins/mongo_db/mongo_db_state.cpp index 32e228c244..519c1160d9 100644 --- a/plugins/mongo_db/mongo_db_state.cpp +++ b/plugins/mongo_db/mongo_db_state.cpp @@ -125,13 +125,20 @@ namespace mongo_db { format_value(body, "body", content.body); format_value(body, "json_metadata", content.json_metadata); - std::string category; + std::string category, root_oid; if (comment.parent_author == STEEMIT_ROOT_POST_PARENT) { category = to_string(comment.parent_permlink); + root_oid = oid; } else { - category = to_string(db_.get(comment.root_comment).parent_permlink); + auto& root_comment = db_.get(comment.root_comment); + category = to_string(root_comment.parent_permlink); + root_oid = std::string(root_comment.author).append("/").append(root_comment.permlink.c_str()); } format_value(body, "category", category); + format_oid(body, "root_comment", root_oid); + document root_comment_index; + root_comment_index << "root_comment" << 1; + doc.indexes_to_create.push_back(std::move(root_comment_index)); body << close_document; From 7590e665fdfffa7106fb02fea51ebf4ec7e2f423 Mon Sep 17 00:00:00 2001 From: AKorpusenko Date: Tue, 5 Jun 2018 13:06:51 +0300 Subject: [PATCH 18/22] Bug fix. #688 --- plugins/tags/plugin.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/tags/plugin.cpp b/plugins/tags/plugin.cpp index cc87ff5d26..1ae5b421de 100644 --- a/plugins/tags/plugin.cpp +++ b/plugins/tags/plugin.cpp @@ -404,16 +404,16 @@ namespace golos { namespace plugins { namespace tags { } discussion d = create_discussion(*comment); - - fill_discussion(d, query); d.promoted = asset(itr->promoted_balance, SBD_SYMBOL); - d.hot = itr->hot; - d.trending = itr->trending; if (!select(d) || !query.is_good_tags(d)) { continue; } + fill_discussion(d, query); + d.hot = itr->hot; + d.trending = itr->trending; + if (query.has_start_comment() && !query.is_good_start(d.id) && !order(query.start_comment, d)) { continue; } From 203ffecd8d244f5a26cb6c44f85d7e2423ceb463 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Tue, 5 Jun 2018 18:31:10 +0700 Subject: [PATCH 19/22] Fix updating of tags after cashout window. #708 --- .../golos/plugins/tags/tag_visitor.hpp | 5 +- plugins/tags/tag_visitor.cpp | 193 ++++++++++-------- 2 files changed, 116 insertions(+), 82 deletions(-) diff --git a/plugins/tags/include/golos/plugins/tags/tag_visitor.hpp b/plugins/tags/include/golos/plugins/tags/tag_visitor.hpp index 4ac8074aba..0a889f9fcc 100644 --- a/plugins/tags/include/golos/plugins/tags/tag_visitor.hpp +++ b/plugins/tags/include/golos/plugins/tags/tag_visitor.hpp @@ -50,9 +50,10 @@ namespace golos { namespace plugins { namespace tags { inline double calculate_trending(const share_type& score, const time_point_sec& created) const; - /** finds tags that have been added or removed or updated */ - void update_tags(const account_name_type& author, const std::string& permlink, bool parse_tags = false) const; + void create_update_tags(const account_name_type& author, const std::string& permlink) const; + void update_tags(const account_name_type& author, const std::string& permlink) const; + void remove_tags(const account_name_type& author, const std::string& permlink) const; void operator()(const comment_operation& op) const; diff --git a/plugins/tags/tag_visitor.cpp b/plugins/tags/tag_visitor.cpp index e2657b7411..d0099ba567 100644 --- a/plugins/tags/tag_visitor.cpp +++ b/plugins/tags/tag_visitor.cpp @@ -85,25 +85,21 @@ namespace golos { namespace plugins { namespace tags { const tag_object& current, const comment_object& comment, double hot, double trending ) const { auto cashout_time = db_.calculate_discussion_payout_time(comment); - if (cashout_time != fc::time_point_sec::maximum()) { - remove_stats(current); - db_.modify(current, [&](tag_object& obj) { - obj.active = comment.active; - obj.cashout = cashout_time; - obj.children = comment.children; - obj.net_rshares = comment.net_rshares.value; - obj.net_votes = comment.net_votes; - obj.children_rshares2 = comment.children_rshares2; - obj.hot = hot; - obj.trending = trending; - if (obj.cashout == fc::time_point_sec()) { - obj.promoted_balance = 0; - } - }); - add_stats(current); - } else { - remove_tag(current); - } + remove_stats(current); + db_.modify(current, [&](tag_object& obj) { + obj.active = comment.active; + obj.cashout = cashout_time; + obj.children = comment.children; + obj.net_rshares = comment.net_rshares.value; + obj.net_votes = comment.net_votes; + obj.children_rshares2 = comment.children_rshares2; + obj.hot = hot; + obj.trending = trending; + if (obj.cashout == fc::time_point_sec()) { + obj.promoted_balance = 0; + } + }); + add_stats(current); } void operation_visitor::create_tag( @@ -166,79 +162,101 @@ namespace golos { namespace plugins { namespace tags { return calculate_score<10000000, 10000>(score, created); } - double - operation_visitor::calculate_trending(const share_type& score, const time_point_sec& created) const { + double operation_visitor::calculate_trending(const share_type& score, const time_point_sec& created) const { return calculate_score<10000000, 480000>(score, created); } - void operation_visitor::update_tags( - const account_name_type& author, const std::string& permlink, bool parse_tags - ) const { + void operation_visitor::create_update_tags( + const account_name_type& author, const std::string& permlink + ) const { try { const auto& comment = db_.get_comment(author, permlink); - try { - auto hot = calculate_hot(comment.net_rshares, comment.created); - auto trending = calculate_trending(comment.net_rshares, comment.created); - const auto& comment_idx = db_.get_index().indices().get(); - - if (parse_tags) { - auto meta = get_metadata(comment_api_object(comment, db_)); - auto citr = comment_idx.lower_bound(comment.id); - const tag_object* language_tag = nullptr; - - if (meta.tags.empty()) { - meta.tags.insert(std::string()); - } + auto hot = calculate_hot(comment.net_rshares, comment.created); + auto trending = calculate_trending(comment.net_rshares, comment.created); + const auto& comment_idx = db_.get_index().indices().get(); - std::map existing_tags; - std::vector remove_queue; - for (; citr != comment_idx.end() && citr->comment == comment.id; ++citr) { - const tag_object* tag = &*citr; - switch (tag->type) { - case tag_type::tag: - if (meta.tags.find(tag->name) == meta.tags.end()) { - remove_queue.push_back(tag); - } else { - existing_tags[tag->name] = tag; - } - break; + auto meta = get_metadata(comment_api_object(comment, db_)); + auto citr = comment_idx.lower_bound(comment.id); + const tag_object* language_tag = nullptr; - case tag_type::language: - language_tag = tag; - if (meta.language != language_tag->name) { - remove_queue.push_back(tag); - } - break; - } - } + if (meta.tags.empty()) { + meta.tags.insert(std::string()); + } - for (const auto& name : meta.tags) { - auto existing = existing_tags.find(name); - if (existing == existing_tags.end()) { - create_tag(name, tag_type::tag, comment, hot, trending); + std::map existing_tags; + std::vector remove_queue; + for (; citr != comment_idx.end() && citr->comment == comment.id; ++citr) { + const tag_object* tag = &*citr; + switch (tag->type) { + case tag_type::tag: + if (meta.tags.find(tag->name) == meta.tags.end()) { + remove_queue.push_back(tag); } else { - update_tag(*existing->second, comment, hot, trending); + existing_tags[tag->name] = tag; } - } + break; - if (!meta.language.empty() && (!language_tag || meta.language != language_tag->name)) { - create_tag(meta.language, tag_type::language, comment, hot, trending); - } + case tag_type::language: + language_tag = tag; + if (meta.language != language_tag->name) { + remove_queue.push_back(tag); + } + break; + } + } - for (const auto& item : remove_queue) { - remove_tag(*item); - } + for (const auto& name : meta.tags) { + auto existing = existing_tags.find(name); + if (existing == existing_tags.end()) { + create_tag(name, tag_type::tag, comment, hot, trending); } else { - auto citr = comment_idx.lower_bound(comment.id); - while (citr != comment_idx.end() && citr->comment == comment.id) { - update_tag(*citr, comment, hot, trending); - ++citr; - } + update_tag(*existing->second, comment, hot, trending); } + } - if (comment.parent_author.size()) { - update_tags(comment.parent_author, to_string(comment.parent_permlink)); - } - } FC_CAPTURE_LOG_AND_RETHROW(()) + if (!meta.language.empty() && (!language_tag || meta.language != language_tag->name)) { + create_tag(meta.language, tag_type::language, comment, hot, trending); + } + + for (const auto& item : remove_queue) { + remove_tag(*item); + } + } FC_CAPTURE_LOG_AND_RETHROW(()) } + + void operation_visitor::update_tags(const account_name_type& author, const std::string& permlink) const { + const auto& comment = db_.get_comment(author, permlink); + auto hot = calculate_hot(comment.net_rshares, comment.created); + auto trending = calculate_trending(comment.net_rshares, comment.created); + const auto& comment_idx = db_.get_index().indices().get(); + + auto citr = comment_idx.lower_bound(comment.id); + for (; citr != comment_idx.end() && citr->comment == comment.id; ++citr) { + update_tag(*citr, comment, hot, trending); + } + + if (comment.parent_author.size()) { + update_tags(comment.parent_author, to_string(comment.parent_permlink)); + } + } + + void operation_visitor::remove_tags(const account_name_type& author, const std::string& permlink) const { + const auto& comment = db_.get_comment(author, permlink); + const auto& comment_idx = db_.get_index().indices().get(); + std::vector remove_queue; + + remove_queue.reserve(10); + auto citr = comment_idx.lower_bound(comment.id); + for (; citr != comment_idx.end() && citr->comment == comment.id; ++citr) { + const tag_object* tag = &*citr; + remove_queue.push_back(tag); + } + + for (const auto& tag : remove_queue) { + remove_tag(*tag); + } + + if (comment.parent_author.size()) { + update_tags(comment.parent_author, to_string(comment.parent_permlink)); + } } void operation_visitor::operator()(const transfer_operation& op) const { @@ -285,6 +303,7 @@ namespace golos { namespace plugins { namespace tags { } void operation_visitor::operator()(const comment_reward_operation& op) const { + // only update existing tags update_tags(op.author, op.permlink); const auto& comment = db_.get_comment(op.author, op.permlink); @@ -320,15 +339,29 @@ namespace golos { namespace plugins { namespace tags { } void operation_visitor::operator()(const comment_operation& op) const { - update_tags(op.author, op.permlink, true); + const auto& comment = db_.get_comment(op.author, op.permlink); + + if (db_.calculate_discussion_payout_time(comment) != fc::time_point_sec::maximum()) { + // in a cashout window + create_update_tags(op.author, op.permlink); + } } void operation_visitor::operator()(const vote_operation& op) const { + // only update existing tags update_tags(op.author, op.permlink); } void operation_visitor::operator()(const comment_payout_update_operation& op) const { - update_tags(op.author, op.permlink); + const auto& comment = db_.get_comment(op.author, op.permlink); + const auto cashout_time = db_.calculate_discussion_payout_time(comment); + + if (cashout_time != fc::time_point_sec::maximum()) { + update_tags(op.author, op.permlink); + } else { + // it can be the end of a cashout window + remove_tags(op.author, op.permlink); + } } } } } // golos::plugins::tags \ No newline at end of file From cf3de51ab0bbb2870e6fdcf703a2c025212adb15 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Tue, 5 Jun 2018 19:00:44 +0700 Subject: [PATCH 20/22] Change HF18 time to 20 Jun 2018. #711 --- libraries/chain/hardfork.d/0_18.hf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/hardfork.d/0_18.hf b/libraries/chain/hardfork.d/0_18.hf index 285e296434..df8fe4b5b7 100644 --- a/libraries/chain/hardfork.d/0_18.hf +++ b/libraries/chain/hardfork.d/0_18.hf @@ -10,7 +10,7 @@ #ifdef STEEMIT_BUILD_TESTNET #define STEEMIT_HARDFORK_0_18_TIME 1523869200 // 16 apr 2018 12:00:00 MSK #else -#define STEEMIT_HARDFORK_0_18_TIME 1908003600 // 18 jun 2030 12:00:00 MSK - fake date, will be corrected later +#define STEEMIT_HARDFORK_0_18_TIME 1529485200 // 20 jun 2018 12:00:00 MSK #endif #define STEEMIT_HARDFORK_0_18_VERSION hardfork_version( 0, 18 ) From 4795c90723a0a28f8ced1ebde9032ec19b990eca Mon Sep 17 00:00:00 2001 From: maslenitsa93 Date: Tue, 5 Jun 2018 21:29:47 +0300 Subject: [PATCH 21/22] Mongo_db Add transfer_operation implementation #703 --- .../golos/plugins/mongo_db/mongo_db_state.hpp | 2 +- .../golos/plugins/mongo_db/mongo_db_types.hpp | 6 +- plugins/mongo_db/mongo_db_state.cpp | 62 ++++++++++++++----- plugins/mongo_db/mongo_db_writer.cpp | 31 +++++----- 4 files changed, 68 insertions(+), 33 deletions(-) diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp index cfb5eb7e2f..3013de7de9 100644 --- a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp @@ -90,7 +90,7 @@ namespace mongo_db { db_map &all_docs; - void format_comment(const std::string& auth, const std::string& perm); + bool format_comment(const std::string& auth, const std::string& perm); named_document create_document(const std::string& name, const std::string& key, const std::string& keyval); diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_types.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_types.hpp index 6d8c93c0f9..877477f0a2 100644 --- a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_types.hpp +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_types.hpp @@ -71,8 +71,12 @@ namespace mongo_db { using named_document_ptr = std::unique_ptr; + inline std::string hash_oid(const std::string& value) { + return fc::sha1::hash(value).str().substr(0, 24); + } + inline void format_oid(document& doc, const std::string& name, const std::string& value) { - auto oid = fc::sha1::hash(value).str().substr(0, 24); + auto oid = hash_oid(value); doc << name << bsoncxx::oid(oid); } diff --git a/plugins/mongo_db/mongo_db_state.cpp b/plugins/mongo_db/mongo_db_state.cpp index 519c1160d9..1b3ade6855 100644 --- a/plugins/mongo_db/mongo_db_state.cpp +++ b/plugins/mongo_db/mongo_db_state.cpp @@ -10,6 +10,7 @@ #include #include +#include namespace golos { namespace plugins { @@ -47,11 +48,11 @@ namespace mongo_db { return doc; } - void state_writer::format_comment(const std::string& auth, const std::string& perm) { + bool state_writer::format_comment(const std::string& auth, const std::string& perm) { try { auto& comment = db_.get_comment(auth, perm); auto oid = std::string(auth).append("/").append(perm); - auto oid_hash = fc::sha1::hash(oid).str().substr(0, 24); + auto oid_hash = hash_oid(oid); auto doc = create_document("comment_object", "_id", oid_hash); auto& body = doc.doc; @@ -143,12 +144,15 @@ namespace mongo_db { body << close_document; bmi_insert_or_replace(all_docs, std::move(doc)); + + return true; } // catch (fc::exception& ex) { // ilog("MongoDB operations fc::exception during formatting comment. ${e}", ("e", ex.what())); // } catch (...) { // ilog("Unknown exception during formatting comment."); + return false; } } @@ -163,7 +167,7 @@ namespace mongo_db { if (vote_idx.end() != itr) { auto comment_oid = std::string(op.author).append("/").append(op.permlink); auto oid = comment_oid + "/" + op.voter; - auto oid_hash = fc::sha1::hash(oid).str().substr(0, 24); + auto oid_hash = hash_oid(oid); auto doc = create_document("comment_vote_object", "_id", oid_hash); document comment_index; @@ -212,7 +216,7 @@ namespace mongo_db { std::string author = op.author; auto comment_oid = std::string(op.author).append("/").append(op.permlink); - auto comment_oid_hash = fc::sha1::hash(comment_oid).str().substr(0, 24); + auto comment_oid_hash = hash_oid(comment_oid); // Will be updated with the following fields. If no one - created with these fields. auto comment = create_document("comment_object", "_id", comment_oid_hash); @@ -239,7 +243,30 @@ namespace mongo_db { } auto state_writer::operator()(const transfer_operation& op) -> result_type { - + auto doc = create_document("transfer", "", ""); + auto& body = doc.doc; + + format_value(body, "from", op.from); + format_value(body, "to", op.to); + format_value(body, "amount", op.amount); + format_value(body, "memo", op.memo); + + std::vector part; + auto path = op.memo; + boost::split(part, path, boost::is_any_of("/")); + if (part.size() >= 2 && part[0][0] == '@') { + auto acnt = part[0].substr(1); + auto perm = part[1]; + + if (format_comment(acnt, perm)) { + auto comment_oid = acnt.append("/").append(perm); + format_oid(body, "comment", comment_oid); + } else { + ilog("unable to find body"); + } + } + + all_docs.push_back(std::move(doc)); } auto state_writer::operator()(const transfer_to_vesting_operation& op) -> result_type { @@ -274,6 +301,14 @@ namespace mongo_db { } + auto state_writer::operator()(const account_create_with_delegation_operation& op) -> result_type { + + } + + auto state_writer::operator()(const account_metadata_operation& op) -> result_type { + + } + auto state_writer::operator()(const witness_update_operation& op) -> result_type { } @@ -381,18 +416,15 @@ namespace mongo_db { auto state_writer::operator()(const delegate_vesting_shares_operation& op) -> result_type { } - auto state_writer::operator()(const account_create_with_delegation_operation& op) -> result_type { - - } - auto state_writer::operator()(const account_metadata_operation& op) -> result_type { - - } + auto state_writer::operator()(const proposal_create_operation& op) -> result_type { } + auto state_writer::operator()(const proposal_update_operation& op) -> result_type { } + auto state_writer::operator()(const proposal_delete_operation& op) -> result_type { } @@ -436,7 +468,7 @@ namespace mongo_db { auto state_writer::operator()(const author_reward_operation& op) -> result_type { try { auto comment_oid = std::string(op.author).append("/").append(op.permlink); - auto comment_oid_hash = fc::sha1::hash(comment_oid).str().substr(0, 24); + auto comment_oid_hash = hash_oid(comment_oid); auto doc = create_document("author_reward", "_id", comment_oid_hash); auto &body = doc.doc; @@ -466,7 +498,7 @@ namespace mongo_db { try { auto comment_oid = std::string(op.comment_author).append("/").append(op.comment_permlink); auto vote_oid = comment_oid + "/" + op.curator; - auto vote_oid_hash = fc::sha1::hash(vote_oid).str().substr(0, 24); + auto vote_oid_hash = hash_oid(vote_oid); auto doc = create_document("curation_reward", "_id", vote_oid_hash); document comment_index; @@ -497,7 +529,7 @@ namespace mongo_db { auto state_writer::operator()(const comment_reward_operation& op) -> result_type { try { auto comment_oid = std::string(op.author).append("/").append(op.permlink); - auto comment_oid_hash = fc::sha1::hash(comment_oid).str().substr(0, 24); + auto comment_oid_hash = hash_oid(comment_oid); auto doc = create_document("comment_reward", "_id", comment_oid_hash); auto &body = doc.doc; @@ -524,7 +556,7 @@ namespace mongo_db { try { auto comment_oid = std::string(op.author).append("/").append(op.permlink); auto benefactor_oid = comment_oid + "/" + op.benefactor; - auto benefactor_oid_hash = fc::sha1::hash(benefactor_oid).str().substr(0, 24); + auto benefactor_oid_hash = hash_oid(benefactor_oid); auto doc = create_document("benefactor_reward", "_id", benefactor_oid_hash); document comment_index; diff --git a/plugins/mongo_db/mongo_db_writer.cpp b/plugins/mongo_db/mongo_db_writer.cpp index faa60207af..a092bf52b9 100644 --- a/plugins/mongo_db/mongo_db_writer.cpp +++ b/plugins/mongo_db/mongo_db_writer.cpp @@ -208,19 +208,21 @@ namespace mongo_db { formatted_blocks[named_doc.collection_name] = std::make_unique(bulk_opts); } - //auto view = named_doc.doc.view(); - //auto itr = view.find("_id"); - //if (view.end() == itr) { - // mongocxx::model::insert_one msg{std::move(view)}; - // formatted_blocks[named_doc.collection_name]->append(msg); - //} else { - document filter; - - filter << "_id" << bsoncxx::oid(named_doc.keyval); - - mongocxx::model::update_one msg{filter.view(), - named_doc.doc.view()}; - msg.upsert(true); + auto view = named_doc.doc.view(); + auto itr = view.find("$set"); + if (view.end() == itr) { + mongocxx::model::insert_one msg{std::move(view)}; + formatted_blocks[named_doc.collection_name]->append(msg); + } else { + document filter; + + filter << "_id" << bsoncxx::oid(named_doc.keyval); + + mongocxx::model::update_one msg{filter.view(), + view}; + msg.upsert(true); + formatted_blocks[named_doc.collection_name]->append(msg); + } if (indexes.find(named_doc.collection_name) == indexes.end()) { for (auto& index_to_create : named_doc.indexes_to_create) { @@ -228,9 +230,6 @@ namespace mongo_db { indexes[named_doc.collection_name] = "created"; } } - - formatted_blocks[named_doc.collection_name]->append(msg); - //} } void mongo_db_writer::remove_document(named_document const& named_doc) { From 6b7e61d778cbb4a43f05de28e256d1709fce16e9 Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Wed, 6 Jun 2018 09:15:22 +0300 Subject: [PATCH 22/22] Fetch witness.url only if needed (cli_wallet update_witness) #713 --- libraries/wallet/wallet.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 81fc07dd8a..50c8d46113 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -1864,18 +1864,14 @@ fc::ecc::private_key wallet_api::derive_private_key(const std::string& prefix_st signed_transaction tx; witness_update_operation op; - auto wit = my->_remote_witness_api->get_witness_by_account(witness_account_name); - if (!wit.valid()) { - op.url = url; - } else { - FC_ASSERT(wit->owner == witness_account_name); - if (!url.empty()) { - op.url = url; - } else { - op.url = wit->url; + if (url.empty()) { + auto wit = my->_remote_witness_api->get_witness_by_account(witness_account_name); + if (wit.valid()) { + FC_ASSERT(wit->owner == witness_account_name); + url = wit->url; } } - + op.url = url; op.owner = witness_account_name; op.block_signing_key = block_signing_key;