From dbb3f6736c91c84bafb8dc72fa1d6716e255fd41 Mon Sep 17 00:00:00 2001 From: Keke Li Date: Thu, 17 Dec 2020 12:02:36 -0800 Subject: [PATCH 1/3] Merge back pull https://github.com/EOSIO/eos/pull/9582 to 2.0.x, Fix stdout console logging --- libraries/fc | 2 +- programs/nodeos/logging.json | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libraries/fc b/libraries/fc index b53dbdcbfe6..a3694752d2d 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit b53dbdcbfe65c40dbc108b86961e130510af852d +Subproject commit a3694752d2dd2021185955eafc1a504efaabfa79 diff --git a/programs/nodeos/logging.json b/programs/nodeos/logging.json index 1bcf5aa4304..694e97ad01e 100644 --- a/programs/nodeos/logging.json +++ b/programs/nodeos/logging.json @@ -15,7 +15,8 @@ "level": "error", "color": "red" } - ] + ], + "flush": true }, "enabled": true },{ @@ -33,7 +34,8 @@ "level": "error", "color": "red" } - ] + ], + "flush": true }, "enabled": true },{ From c2ad122addfffbb3ddc0d1faf773a430589a1ce7 Mon Sep 17 00:00:00 2001 From: johndebord Date: Tue, 29 Dec 2020 12:45:15 -0500 Subject: [PATCH 2/3] Consolidated Security Fixes for 2.0.9 - Fixes to packed_transaction cache. - Transaction account fail limit refactor. Co-Authored-By: Kevin Heifner --- libraries/chain/controller.cpp | 1 + .../chain/include/eosio/chain/transaction.hpp | 6 ++ plugins/producer_plugin/producer_plugin.cpp | 88 ++++++++++++++++++- 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index df67940eadb..0b742fd2369 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1875,6 +1875,7 @@ struct controller_impl { if( receipt.trx.contains()) { const auto& pt = receipt.trx.get(); transaction_metadata_ptr trx_meta_ptr = trx_lookup ? trx_lookup( pt.id() ) : transaction_metadata_ptr{}; + if( trx_meta_ptr && *trx_meta_ptr->packed_trx() != pt ) trx_meta_ptr = nullptr; if( trx_meta_ptr && ( skip_auth_checks || !trx_meta_ptr->recovered_keys().empty() ) ) { trx_metas.emplace_back( std::move( trx_meta_ptr ), recover_keys_future{} ); } else if( skip_auth_checks ) { diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index a9d523a9e0a..39c7c965324 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -164,6 +164,12 @@ namespace eosio { namespace chain { packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); packed_transaction( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + friend bool operator==(const packed_transaction& lhs, const packed_transaction& rhs) { + return std::tie(lhs.signatures, lhs.compression, lhs.packed_context_free_data, lhs.packed_trx) == + std::tie(rhs.signatures, rhs.compression, rhs.packed_context_free_data, rhs.packed_trx); + } + friend bool operator!=(const packed_transaction& lhs, const packed_transaction& rhs) { return !(lhs == rhs); } + uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index dc650e5cf5b..5a738957cc8 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -215,6 +215,7 @@ class producer_plugin_impl : public std::enable_shared_from_this()->default_value( 1024 ), "Maximum size (in MiB) of the incoming transaction queue. Exceeding this value will subjectively drop transaction with resource exhaustion.") + ("disable-api-persisted-trx", bpo::bool_switch()->default_value(false), + "Disable the re-apply of API transactions.") ("producer-threads", bpo::value()->default_value(config::default_controller_thread_pool_size), "Number of worker threads in producer thread pool") ("snapshots-dir", bpo::value()->default_value("snapshots"), @@ -910,6 +913,8 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ my->_incoming_defer_ratio = options.at("incoming-defer-ratio").as(); + my->_disable_persist_until_expired = options.at("disable-api-persisted-trx").as(); + auto thread_pool_size = options.at( "producer-threads" ).as(); EOS_ASSERT( thread_pool_size > 0, plugin_config_exception, "producer-threads ${num} must be greater than 0", ("num", thread_pool_size)); @@ -1739,10 +1744,79 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point return !exhausted; } +namespace { +// track multiple deadline / transaction cpu exceeded exceptions on unapplied transactions +class account_failures { +public: + constexpr static uint32_t max_failures_per_account = 3; + + void add( const account_name& n, int64_t exception_code ) { + if( exception_code == deadline_exception::code_value || exception_code == tx_cpu_usage_exceeded::code_value ) { + auto& fa = failed_accounts[n]; + ++fa.num_failures; + fa.add( exception_code ); + } + } + + // return true if exceeds max_failures_per_account and should be dropped + bool failure_limit( const account_name& n ) { + auto fitr = failed_accounts.find( n ); + if( fitr != failed_accounts.end() && fitr->second.num_failures >= max_failures_per_account ) { + ++fitr->second.num_failures; + return true; + } + return false; + } + + void report() const { + if( _log.is_enabled( fc::log_level::debug ) ) { + for( const auto& e : failed_accounts ) { + std::string reason; + if( e.second.num_failures > max_failures_per_account ) { + reason.clear(); + if( e.second.is_deadline() ) reason += "deadline"; + if( e.second.is_tx_cpu_usage() ) { + if( !reason.empty() ) reason += ", "; + reason += "tx_cpu_usage"; + } + fc_dlog( _log, "Dropped ${n} trxs, account: ${a}, reason: ${r} exceeded", + ("n", e.second.num_failures - max_failures_per_account)("a", e.first)("r", reason) ); + } + } + } + } + +private: + struct account_failure { + enum class ex_fields : uint8_t { + ex_deadline_exception = 1, + ex_tx_cpu_usage_exceeded = 2 + }; + + void add(int64_t exception_code = 0) { + if( exception_code == tx_cpu_usage_exceeded::code_value ) { + ex_flags = set_field( ex_flags, ex_fields::ex_tx_cpu_usage_exceeded ); + } else if( exception_code == deadline_exception::code_value ) { + ex_flags = set_field( ex_flags, ex_fields::ex_deadline_exception ); + } + } + bool is_deadline() const { return has_field( ex_flags, ex_fields::ex_deadline_exception ); } + bool is_tx_cpu_usage() const { return has_field( ex_flags, ex_fields::ex_tx_cpu_usage_exceeded ); } + + uint32_t num_failures = 0; + uint8_t ex_flags = 0; + }; + + std::map failed_accounts; +}; + +} // anonymous namespace + bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadline ) { bool exhausted = false; if( !_unapplied_transactions.empty() ) { + account_failures account_fails; chain::controller& chain = chain_plug->chain(); int num_applied = 0, num_failed = 0, num_processed = 0; auto unapplied_trxs_size = _unapplied_transactions.size(); @@ -1761,11 +1835,16 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin try { auto start = fc::time_point::now(); auto trx_deadline = start + fc::milliseconds( _max_transaction_time_ms ); + if( account_fails.failure_limit( trx->packed_trx()->get_transaction().first_authorizer() ) ) { + ++num_failed; + itr = _unapplied_transactions.erase( itr ); + continue; + } auto prev_billed_cpu_time_us = trx->billed_cpu_time_us; if( prev_billed_cpu_time_us > 0 ) { - auto prev_billed_plus50 = prev_billed_cpu_time_us + EOS_PERCENT( prev_billed_cpu_time_us, 50 * config::percent_1 ); - auto trx_dl = start + fc::microseconds( prev_billed_plus50 ); + auto prev_billed_plus100 = prev_billed_cpu_time_us + EOS_PERCENT( prev_billed_cpu_time_us, 100 * config::percent_1 ); + auto trx_dl = start + fc::microseconds( prev_billed_plus100 ); if( trx_dl < trx_deadline ) trx_deadline = trx_dl; } bool deadline_is_subjective = false; @@ -1784,10 +1863,12 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin break; } } else { + auto failure_code = trace->except->code(); // this failed our configured maximum transaction time, we don't want to replay it fc_dlog( _log, "Failed ${c} trx, prev billed: ${p}us, ran: ${r}us, id: ${id}", ("c", trace->except->code())("p", prev_billed_cpu_time_us) ("r", fc::time_point::now() - start)("id", trx->id()) ); + account_fails.add( trx->packed_trx()->get_transaction().first_authorizer(), failure_code ); ++num_failed; itr = _unapplied_transactions.erase( itr ); continue; @@ -1803,6 +1884,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin fc_dlog( _log, "Processed ${m} of ${n} previously applied transactions, Applied ${applied}, Failed/Dropped ${failed}", ("m", num_processed)( "n", unapplied_trxs_size )("applied", num_applied)("failed", num_failed) ); + account_fails.report(); } return !exhausted; } From ec5f76e12d1e00c9b8ec6ca6c3e37a597da7179e Mon Sep 17 00:00:00 2001 From: johndebord Date: Tue, 29 Dec 2020 15:14:21 -0500 Subject: [PATCH 3/3] Bump version to 2.0.9 --- CMakeLists.txt | 2 +- README.md | 12 ++++++------ docs/00_install/00_install-prebuilt-binaries.md | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e387eac3ac3..de23e74c7d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 2) set(VERSION_MINOR 0) -set(VERSION_PATCH 8) +set(VERSION_PATCH 9) #set(VERSION_SUFFIX rc3) if(VERSION_SUFFIX) diff --git a/README.md b/README.md index 3b857fa48a7..3a34934f191 100644 --- a/README.md +++ b/README.md @@ -74,13 +74,13 @@ brew remove eosio #### Ubuntu 18.04 Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.8/eosio_2.0.8-1-ubuntu-18.04_amd64.deb -sudo apt install ./eosio_2.0.8-1-ubuntu-18.04_amd64.deb +wget https://github.com/eosio/eos/releases/download/v2.0.9/eosio_2.0.9-1-ubuntu-18.04_amd64.deb +sudo apt install ./eosio_2.0.9-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.8/eosio_2.0.8-1-ubuntu-16.04_amd64.deb -sudo apt install ./eosio_2.0.8-1-ubuntu-16.04_amd64.deb +wget https://github.com/eosio/eos/releases/download/v2.0.9/eosio_2.0.9-1-ubuntu-16.04_amd64.deb +sudo apt install ./eosio_2.0.9-1-ubuntu-16.04_amd64.deb ``` #### Ubuntu Package Uninstall ```sh @@ -91,8 +91,8 @@ sudo apt remove eosio #### RPM Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.8/eosio-2.0.8-1.el7.x86_64.rpm -sudo yum install ./eosio-2.0.8-1.el7.x86_64.rpm +wget https://github.com/eosio/eos/releases/download/v2.0.9/eosio-2.0.9-1.el7.x86_64.rpm +sudo yum install ./eosio-2.0.9-1.el7.x86_64.rpm ``` #### RPM Package Uninstall ```sh diff --git a/docs/00_install/00_install-prebuilt-binaries.md b/docs/00_install/00_install-prebuilt-binaries.md index ab0d814213c..f014b88f1ee 100644 --- a/docs/00_install/00_install-prebuilt-binaries.md +++ b/docs/00_install/00_install-prebuilt-binaries.md @@ -25,13 +25,13 @@ brew remove eosio #### Ubuntu 18.04 Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.8/eosio_2.0.8-1-ubuntu-18.04_amd64.deb -sudo apt install ./eosio_2.0.8-1-ubuntu-18.04_amd64.deb +wget https://github.com/eosio/eos/releases/download/v2.0.9/eosio_2.0.9-1-ubuntu-18.04_amd64.deb +sudo apt install ./eosio_2.0.9-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.8/eosio_2.0.8-1-ubuntu-16.04_amd64.deb -sudo apt install ./eosio_2.0.8-1-ubuntu-16.04_amd64.deb +wget https://github.com/eosio/eos/releases/download/v2.0.9/eosio_2.0.9-1-ubuntu-16.04_amd64.deb +sudo apt install ./eosio_2.0.9-1-ubuntu-16.04_amd64.deb ``` #### Ubuntu Package Uninstall ```sh @@ -42,8 +42,8 @@ sudo apt remove eosio #### RPM Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.8/eosio-2.0.8-1.el7.x86_64.rpm -sudo yum install ./eosio-2.0.8-1.el7.x86_64.rpm +wget https://github.com/eosio/eos/releases/download/v2.0.9/eosio-2.0.9-1.el7.x86_64.rpm +sudo yum install ./eosio-2.0.9-1.el7.x86_64.rpm ``` #### RPM Package Uninstall ```sh