From 5c049780c8b310428cf72fb304bf0c1071742785 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sat, 27 Jun 2020 13:13:51 +0300 Subject: [PATCH 0001/1090] test: Add test for erase orphan tx from peer --- test/functional/p2p_invalid_tx.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 8fef2d173f786..037a4249cc768 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -60,7 +60,6 @@ def run_test(self): block.solve() # Save the coinbase for later block1 = block - tip = block.sha256 node.p2ps[0].send_blocks_and_test([block], node, success=True) self.log.info("Mature the block.") @@ -157,6 +156,7 @@ def run_test(self): with node.assert_debug_log(['orphanage overflow, removed 1 tx']): node.p2ps[0].send_txs_and_test(orphan_tx_pool, node, success=False) + self.log.info('Test orphan with rejected parents') rejected_parent = CTransaction() rejected_parent.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_2_invalid.sha256, 0))) rejected_parent.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) @@ -164,6 +164,10 @@ def run_test(self): with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(rejected_parent.hash)]): node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False) + self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool') + with node.assert_debug_log(['Erased 100 orphan tx from peer=25']): + self.reconnect_p2p(num_connections=1) + if __name__ == '__main__': InvalidTxRequestTest().main() From fa45bb21193ae0c220cfc224d5e3ea0e7f3ec988 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sat, 27 Jun 2020 13:15:55 +0300 Subject: [PATCH 0002/1090] test: Add test for erase orphan tx included by block --- test/functional/p2p_invalid_tx.py | 33 +++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 037a4249cc768..0eb099b1d5ab2 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -92,24 +92,24 @@ def run_test(self): SCRIPT_PUB_KEY_OP_TRUE = b'\x51\x75' * 15 + b'\x51' tx_withhold = CTransaction() tx_withhold.vin.append(CTxIn(outpoint=COutPoint(block1.vtx[0].sha256, 0))) - tx_withhold.vout.append(CTxOut(nValue=50 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_withhold.vout = [CTxOut(nValue=25 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 2 tx_withhold.calc_sha256() # Our first orphan tx with some outputs to create further orphan txs tx_orphan_1 = CTransaction() tx_orphan_1.vin.append(CTxIn(outpoint=COutPoint(tx_withhold.sha256, 0))) - tx_orphan_1.vout = [CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 3 + tx_orphan_1.vout = [CTxOut(nValue=8 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 3 tx_orphan_1.calc_sha256() # A valid transaction with low fee tx_orphan_2_no_fee = CTransaction() tx_orphan_2_no_fee.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 0))) - tx_orphan_2_no_fee.vout.append(CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_2_no_fee.vout.append(CTxOut(nValue=8 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) # A valid transaction with sufficient fee tx_orphan_2_valid = CTransaction() tx_orphan_2_valid.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 1))) - tx_orphan_2_valid.vout.append(CTxOut(nValue=10 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_2_valid.vout.append(CTxOut(nValue=8 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) tx_orphan_2_valid.calc_sha256() # An invalid transaction with negative fee @@ -168,6 +168,31 @@ def run_test(self): with node.assert_debug_log(['Erased 100 orphan tx from peer=25']): self.reconnect_p2p(num_connections=1) + self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool') + tx_withhold_until_block_A = CTransaction() + tx_withhold_until_block_A.vin.append(CTxIn(outpoint=COutPoint(tx_withhold.sha256, 1))) + tx_withhold_until_block_A.vout = [CTxOut(nValue=12 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 2 + tx_withhold_until_block_A.calc_sha256() + + tx_orphan_include_by_block_A = CTransaction() + tx_orphan_include_by_block_A.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_A.sha256, 0))) + tx_orphan_include_by_block_A.vout.append(CTxOut(nValue=12 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_include_by_block_A.calc_sha256() + + self.log.info('Send the orphan ... ') + node.p2ps[0].send_txs_and_test([tx_orphan_include_by_block_A], node, success=False) + + tip = int(node.getbestblockhash(), 16) + height = node.getblockcount() + 1 + block_A = create_block(tip, create_coinbase(height)) + block_A.vtx.extend([tx_withhold, tx_withhold_until_block_A, tx_orphan_include_by_block_A]) + block_A.hashMerkleRoot = block_A.calc_merkle_root() + block_A.solve() + + self.log.info('Send the block that includes the previous orphan ... ') + with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): + node.p2ps[0].send_blocks_and_test([block_A], node, success=True) + if __name__ == '__main__': InvalidTxRequestTest().main() From c0a5fceee9858afd24fe0bf655b7b30728e96e78 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sat, 27 Jun 2020 13:21:12 +0300 Subject: [PATCH 0003/1090] test: Add test for erase orphan tx conflicted by block --- test/functional/p2p_invalid_tx.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 0eb099b1d5ab2..54d4f966d34c0 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -193,6 +193,35 @@ def run_test(self): with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): node.p2ps[0].send_blocks_and_test([block_A], node, success=True) + self.log.info('Test that a transaction in the orphan pool conflicts with a new tip block causes erase this transaction from the orphan pool') + tx_withhold_until_block_B = CTransaction() + tx_withhold_until_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_A.sha256, 1))) + tx_withhold_until_block_B.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_withhold_until_block_B.calc_sha256() + + tx_orphan_include_by_block_B = CTransaction() + tx_orphan_include_by_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_B.sha256, 0))) + tx_orphan_include_by_block_B.vout.append(CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_include_by_block_B.calc_sha256() + + tx_orphan_conflict_by_block_B = CTransaction() + tx_orphan_conflict_by_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_B.sha256, 0))) + tx_orphan_conflict_by_block_B.vout.append(CTxOut(nValue=9 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_conflict_by_block_B.calc_sha256() + self.log.info('Send the orphan ... ') + node.p2ps[0].send_txs_and_test([tx_orphan_conflict_by_block_B], node, success=False) + + tip = int(node.getbestblockhash(), 16) + height = node.getblockcount() + 1 + block_B = create_block(tip, create_coinbase(height)) + block_B.vtx.extend([tx_withhold_until_block_B, tx_orphan_include_by_block_B]) + block_B.hashMerkleRoot = block_B.calc_merkle_root() + block_B.solve() + + self.log.info('Send the block that includes a transaction which conflicts with the previous orphan ... ') + with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): + node.p2ps[0].send_blocks_and_test([block_B], node, success=True) + if __name__ == '__main__': InvalidTxRequestTest().main() From 98868633d1d5cd2591b9613545bd2ce2e81af212 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Wed, 12 Jan 2022 22:02:21 +0000 Subject: [PATCH 0004/1090] Bugfix: configure: bitcoin-{cli,tx,util} don't need UPnP, NAT-PMP, or ZMQ --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 9b080eb0c8037..b9567c9780560 100644 --- a/configure.ac +++ b/configure.ac @@ -1349,7 +1349,7 @@ if test "$use_usdt" != "no"; then ) fi -if test "$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nonononononono"; then +if test "$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononono"; then use_upnp=no use_natpmp=no use_zmq=no From 0811cbfc2868ee80c522fd426f188f10b06cd421 Mon Sep 17 00:00:00 2001 From: brunoerg Date: Sat, 15 Jan 2022 15:37:59 -0300 Subject: [PATCH 0005/1090] doc: add info about status code 404 for some rest endpoints --- doc/REST-interface.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 51a73b89fcecf..1e0e701df552c 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -31,6 +31,7 @@ Supported API `GET /rest/tx/.` Given a transaction hash: returns a transaction in binary, hex-encoded binary, or JSON formats. +Responds with 404 if the transaction doesn't exist. By default, this endpoint will only search the mempool. To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. @@ -70,6 +71,7 @@ Responds with 404 if the block doesn't exist. `GET /rest/blockhashbyheight/.` Given a height: returns hash of block in best-block-chain at height provided. +Responds with 404 if block not found. #### Chaininfos `GET /rest/chaininfo.json` From 817326a828d6148dc63d9ef08f641b9c0c522411 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Fri, 29 Mar 2019 17:10:11 -0400 Subject: [PATCH 0006/1090] wallet: avoid rescans if under the snapshot Refuse to load a wallet if it requires a rescan lower than the height of an unvalidated snapshot we're running -- in more general terms, if we don't have data for the blocks. --- src/interfaces/chain.h | 3 +++ src/node/interfaces.cpp | 5 +++++ src/wallet/wallet.cpp | 17 ++++++++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 4f5105a5c123e..07acf880c6428 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -286,6 +286,9 @@ class Chain //! to be prepared to handle this by ignoring notifications about unknown //! removed transactions and already added new transactions. virtual void requestMempoolTransactions(Notifications& notifications) = 0; + + //! Return true if an assumed-valid chain is in use. + virtual bool hasAssumedValidChain() = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 1a48957f0fdfd..855db7b0ec2b3 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -723,6 +723,11 @@ class ChainImpl : public Chain notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } + bool hasAssumedValidChain() override + { + return Assert(m_node.chainman)->IsSnapshotActive(); + } + NodeContext& m_node; }; } // namespace diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 3fcb086d2de46..6726ea8e4d4f7 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2903,20 +2903,31 @@ bool CWallet::AttachChain(const std::shared_ptr& walletInstance, interf if (tip_height && *tip_height != rescan_height) { - if (chain.havePruned()) { + // Technically we could execute the code below in any case, but performing the + // `while` loop below can make startup very slow, so only check blocks on disk + // if necessary. + if (chain.havePruned() || chain.hasAssumedValidChain()) { int block_height = *tip_height; while (block_height > 0 && chain.haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - // We can't rescan beyond non-pruned blocks, stop and throw an error. + // We can't rescan beyond blocks we don't have data for, stop and throw an error. // This might happen if a user uses an old wallet within a pruned node // or if they ran -disablewallet for a longer time, then decided to re-enable // Exit early and print an error. + // It also may happen if an assumed-valid chain is in use and therefore not + // all block data is available. // If a block is pruned after this check, we will load the wallet, // but fail the rescan with a generic error. - error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); + + error = chain.hasAssumedValidChain() ? + _( + "Assumed-valid: last wallet synchronisation goes beyond " + "available block data. You need to wait for the background " + "validation chain to download more blocks.") : + _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); return false; } } From 0f40d653218789aa176ca2f844e3222d2ad890a3 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Wed, 16 Feb 2022 21:17:21 -0500 Subject: [PATCH 0007/1090] refactor: remove duplicate code from BlockAssembler --- src/node/miner.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/node/miner.cpp b/src/node/miner.cpp index 6e9bde84d8eba..303738db4bc3f 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -266,13 +266,9 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already modtxiter mit = mapModifiedTx.find(desc); if (mit == mapModifiedTx.end()) { CTxMemPoolModifiedEntry modEntry(desc); - modEntry.nSizeWithAncestors -= it->GetTxSize(); - modEntry.nModFeesWithAncestors -= it->GetModifiedFee(); - modEntry.nSigOpCostWithAncestors -= it->GetSigOpCost(); - mapModifiedTx.insert(modEntry); - } else { - mapModifiedTx.modify(mit, update_for_parent_inclusion(it)); + mit = mapModifiedTx.insert(modEntry).first; } + mapModifiedTx.modify(mit, update_for_parent_inclusion(it)); } } return nDescendantsUpdated; From fa8671018766b2f0e18c94cff3ab2a67c6b3a41d Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 14 Mar 2022 16:52:55 +0100 Subject: [PATCH 0008/1090] Clarify that CheckSequenceLocksAtTip is a validation function --- src/validation.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validation.h b/src/validation.h index 965ed4225eb3b..2199de3197568 100644 --- a/src/validation.h +++ b/src/validation.h @@ -273,7 +273,7 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx const Package& txns, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/* Transaction policy functions */ +/* Mempool validation helper functions */ /** * Check if transaction will be final in the next block to be created. From f59959e3818692c5b3c2dfa51c14e515085e940f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Barbosa?= Date: Fri, 25 Mar 2022 22:41:21 +0000 Subject: [PATCH 0009/1090] wallet: Prevent wallet unload on GetWalletForJSONRPCRequest Don't extend shared ownership of all wallets to GetWalletForJSONRPCRequest scope. --- src/wallet/rpc/util.cpp | 9 ++++----- src/wallet/wallet.cpp | 7 +++++++ src/wallet/wallet.h | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 59683c5fd8c4e..f788d0d2d33f2 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -64,12 +64,11 @@ std::shared_ptr GetWalletForJSONRPCRequest(const JSONRPCRequest& reques return pwallet; } - std::vector> wallets = GetWallets(context); - if (wallets.size() == 1) { - return wallets[0]; - } + size_t count{0}; + auto wallet = GetDefaultWallet(context, count); + if (wallet) return wallet; - if (wallets.empty()) { + if (count == 0) { throw JSONRPCError( RPC_WALLET_NOT_FOUND, "No wallet is loaded. Load a wallet using loadwallet or create a new one with createwallet. (Note: A default wallet is no longer automatically created)"); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index be64b4cdbbc06..96b9a69d45565 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -151,6 +151,13 @@ std::vector> GetWallets(WalletContext& context) return context.wallets; } +std::shared_ptr GetDefaultWallet(WalletContext& context, size_t& count) +{ + LOCK(context.wallets_mutex); + count = context.wallets.size(); + return count == 1 ? context.wallets[0] : nullptr; +} + std::shared_ptr GetWallet(WalletContext& context, const std::string& name) { LOCK(context.wallets_mutex); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 0490d321abe64..686e10229407a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -62,6 +62,7 @@ bool AddWallet(WalletContext& context, const std::shared_ptr& wallet); bool RemoveWallet(WalletContext& context, const std::shared_ptr& wallet, std::optional load_on_start, std::vector& warnings); bool RemoveWallet(WalletContext& context, const std::shared_ptr& wallet, std::optional load_on_start); std::vector> GetWallets(WalletContext& context); +std::shared_ptr GetDefaultWallet(WalletContext& context, size_t& count); std::shared_ptr GetWallet(WalletContext& context, const std::string& name); std::shared_ptr LoadWallet(WalletContext& context, const std::string& name, std::optional load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings); std::shared_ptr CreateWallet(WalletContext& context, const std::string& name, std::optional load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings); From c456302d4258e3abc4b8afde20fba808632771b2 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Thu, 3 Mar 2022 11:56:20 +0100 Subject: [PATCH 0010/1090] doc: minor improvements in getutxos REST endpoint synopsis Describing an optional sub-path as in the synopsis could be misleading as the angle brackets normally indicate that the field has to be replaced a custom value. Clarify that by showing two variants instead, similar to the block endpoint with the notxdetails option. Further improvements: - uppercase and , to match the description of the other endpoints - s/getutxo command/getutxos endpoint/ - describe what the checkmempool option does - s/serialisation/serialization/ (the US spelling is more dominant than the UK spelling in the project, and there is indeed no other instance of the string "serialis*" in the source tree, except once in a release note) - link to BIP64 within the text instead of only showing bare URL - mention that BIP64 is only relevant for bin and hex output formats - show two endpoint formats of the block section as list --- doc/REST-interface.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 1f0a07a2840d9..02315c359c531 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -36,8 +36,8 @@ By default, this endpoint will only search the mempool. To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. #### Blocks -`GET /rest/block/.` -`GET /rest/block/notxdetails/.` +- `GET /rest/block/.` +- `GET /rest/block/notxdetails/.` Given a block hash: returns a block, in binary, hex-encoded binary or JSON formats. Responds with 404 if the block doesn't exist. @@ -89,11 +89,13 @@ Only supports JSON as output format. * softforks : (array) status of softforks in progress #### Query UTXO set -`GET /rest/getutxos//-/-/.../-.` +- `GET /rest/getutxos/-/-/.../-.` +- `GET /rest/getutxos/checkmempool/-/-/.../-.` -The getutxo command allows querying of the UTXO set given a set of outpoints. -See BIP64 for input and output serialisation: -https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki +The getutxos endpoint allows querying the UTXO set, given a set of outpoints. +With the `/checkmempool/` option, the mempool is also taken into account. +See [BIP64](https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki) for +input and output serialization (relevant for `bin` and `hex` output formats). Example: ``` From 395767e9f15b7a1b5203da68f1fbe3df281ae906 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 6 Apr 2022 18:11:27 -0400 Subject: [PATCH 0011/1090] Add test case mimicking issue 24765 --- test/functional/feature_taproot.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index c3925dbb00865..daa7ac1221ce8 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -1143,6 +1143,12 @@ def predict_sigops_ratio(n, dummy_size): tap = taproot_construct(pubs[0], scripts) add_spender(spenders, "alwaysvalid/notsuccessx", tap=tap, leaf="op_success", inputs=[], standard=False, failure={"leaf": "normal"}) # err_msg differs based on opcode + # == Test case for https://github.com/bitcoin/bitcoin/issues/24765 == + + zero_fn = lambda h: bytes([0 for _ in range(32)]) + tap = taproot_construct(pubs[0], [("leaf", CScript([pubs[1], OP_CHECKSIG, pubs[1], OP_CHECKSIGADD, OP_2, OP_EQUAL])), zero_fn]) + add_spender(spenders, "case24765", tap=tap, leaf="leaf", inputs=[getter("sign"), getter("sign")], key=secs[1], no_fail=True) + # == Legacy tests == # Also add a few legacy spends into the mix, so that transactions which combine taproot and pre-taproot spends get tested too. From 511eb7fdeac36da9d6576c878a1c8d390b38f1bd Mon Sep 17 00:00:00 2001 From: Chris Geihsler Date: Wed, 13 Apr 2022 22:50:10 -0400 Subject: [PATCH 0012/1090] Ignore problematic blocks in DisconnectBlock When using checklevel=4, block verification fails because of duplicate coinbase transactions involving blocks 91812 and 91722. There was already a check in place for ConnectBlock to ignore the problematic blocks, but DisconnectBlock did not contain a similar check. This change ignores the blocks where these inconsistencies surface so that block verification will succeed at checklevel=4. --- src/validation.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/validation.cpp b/src/validation.cpp index f4b316f67aa94..6d87ab88b3243 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1837,6 +1837,15 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI return DISCONNECT_FAILED; } + // Ignore blocks that contain transactions which are 'overwritten' by later transactions, + // unless those are already completely spent. + // See https://github.com/bitcoin/bitcoin/issues/22596 for additional information. + // Note: the blocks specified here are different than the ones used in ConnectBlock because DisconnectBlock + // unwinds the blocks in reverse. As a result, the inconsistency is not discovered until the earlier + // blocks with the duplicate coinbase transactions are disconnected. + bool fEnforceBIP30 = !((pindex->nHeight==91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (pindex->nHeight==91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))); + // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = *(block.vtx[i]); @@ -1850,7 +1859,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI COutPoint out(hash, o); Coin coin; bool is_spent = view.SpendCoin(out, &coin); - if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase) { + if (fEnforceBIP30 && (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase)) { fClean = false; // transaction output mismatch } } From e899d4ca6f44785b6e5481e200a6a9a8d2448612 Mon Sep 17 00:00:00 2001 From: Chris Geihsler Date: Thu, 14 Apr 2022 11:48:19 -0400 Subject: [PATCH 0013/1090] init: limit bip30 exceptions to coinbase txs Co-authored-by: James O'Beirne --- src/validation.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/validation.cpp b/src/validation.cpp index 6d87ab88b3243..ffad7e8402081 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1851,6 +1851,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI const CTransaction &tx = *(block.vtx[i]); uint256 hash = tx.GetHash(); bool is_coinbase = tx.IsCoinBase(); + bool is_bip30_exception = (is_coinbase && !fEnforceBIP30); // Check that all outputs are available and match the outputs in the block itself // exactly. @@ -1859,8 +1860,10 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI COutPoint out(hash, o); Coin coin; bool is_spent = view.SpendCoin(out, &coin); - if (fEnforceBIP30 && (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase)) { - fClean = false; // transaction output mismatch + if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase) { + if (!is_bip30_exception) { + fClean = false; // transaction output mismatch + } } } } From e7a5bf6be79e341e037305a4c2d8a1a510a8d709 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 21 Feb 2022 17:03:27 +0100 Subject: [PATCH 0014/1090] fees: make the class FeeFilterRounder thread-safe So that its methods can be called concurrently by different threads on the same object. Currently it has just one method (`round()`). Co-authored-by: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> --- src/policy/fees.cpp | 6 ++++-- src/policy/fees.h | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 6499dbd97f07a..9f576e738a558 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1010,8 +1010,10 @@ FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) CAmount FeeFilterRounder::round(CAmount currentMinFee) { std::set::iterator it = feeset.lower_bound(currentMinFee); - if ((it != feeset.begin() && insecure_rand.rand32() % 3 != 0) || it == feeset.end()) { - it--; + if (it == feeset.end() || + (it != feeset.begin() && + WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) { + --it; } return static_cast(*it); } diff --git a/src/policy/fees.h b/src/policy/fees.h index 6e25bb42b8af1..c7fa1a0b77793 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -299,12 +299,13 @@ class FeeFilterRounder /** Create new FeeFilterRounder */ explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); - /** Quantize a minimum fee for privacy purpose before broadcast. Not thread-safe due to use of FastRandomContext */ + /** Quantize a minimum fee for privacy purpose before broadcast. */ CAmount round(CAmount currentMinFee); private: std::set feeset; - FastRandomContext insecure_rand; + Mutex m_insecure_rand_mutex; + FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; #endif // BITCOIN_POLICY_FEES_H From 8b4ad203d06c5ded6ecebbd7277b29a442d88bcf Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 21 Feb 2022 17:11:59 +0100 Subject: [PATCH 0015/1090] fees: make FeeFilterRounder::feeset const It is only set in the constructor, thus improve readability by marking it as `const` and setting it from the initializer list using a helper function to derive its value. The idea was suggested by Anthony Towns in https://github.com/bitcoin/bitcoin/pull/19268#discussion_r439929792 --- src/policy/fees.cpp | 17 ++++++++++++++--- src/policy/fees.h | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 9f576e738a558..d64360e82d123 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -998,13 +998,24 @@ void CBlockPolicyEstimator::FlushUnconfirmed() { LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, (endclear - startclear)*0.000001); } -FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) +static std::set MakeFeeSet(const CFeeRate& minIncrementalFee, + double max_filter_fee_rate, + double fee_filter_spacing) { - CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); + std::set feeset; + + const CAmount minFeeLimit{std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2)}; feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { + for (double bucketBoundary = minFeeLimit; bucketBoundary <= max_filter_fee_rate; bucketBoundary *= fee_filter_spacing) { feeset.insert(bucketBoundary); } + + return feeset; +} + +FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) + : feeset{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} +{ } CAmount FeeFilterRounder::round(CAmount currentMinFee) diff --git a/src/policy/fees.h b/src/policy/fees.h index c7fa1a0b77793..e7f45c3151e6c 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -303,7 +303,7 @@ class FeeFilterRounder CAmount round(CAmount currentMinFee); private: - std::set feeset; + const std::set feeset; Mutex m_insecure_rand_mutex; FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; From 8173f160e085186c9bcc7f3506205c309ee66af6 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 15 Mar 2022 16:57:05 +0100 Subject: [PATCH 0016/1090] style: rename variables to match coding style Rename the variables that were touched by the previous commit (split logical from style changes). minIncrementalFee -> min_incremental_fee minFeeLimit -> min_fee_limit bucketBoundary -> bucket_boundary feeset -> fee_set FeeFilterRounder::feeset -> FeeFilterRounder::m_fee_set --- src/policy/fees.cpp | 25 ++++++++++++++----------- src/policy/fees.h | 4 ++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index d64360e82d123..44024ce7acb1b 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -998,31 +998,34 @@ void CBlockPolicyEstimator::FlushUnconfirmed() { LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, (endclear - startclear)*0.000001); } -static std::set MakeFeeSet(const CFeeRate& minIncrementalFee, +static std::set MakeFeeSet(const CFeeRate& min_incremental_fee, double max_filter_fee_rate, double fee_filter_spacing) { - std::set feeset; + std::set fee_set; - const CAmount minFeeLimit{std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2)}; - feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= max_filter_fee_rate; bucketBoundary *= fee_filter_spacing) { - feeset.insert(bucketBoundary); + const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)}; + fee_set.insert(0); + for (double bucket_boundary = min_fee_limit; + bucket_boundary <= max_filter_fee_rate; + bucket_boundary *= fee_filter_spacing) { + + fee_set.insert(bucket_boundary); } - return feeset; + return fee_set; } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) - : feeset{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} + : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} { } CAmount FeeFilterRounder::round(CAmount currentMinFee) { - std::set::iterator it = feeset.lower_bound(currentMinFee); - if (it == feeset.end() || - (it != feeset.begin() && + std::set::iterator it = m_fee_set.lower_bound(currentMinFee); + if (it == m_fee_set.end() || + (it != m_fee_set.begin() && WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) { --it; } diff --git a/src/policy/fees.h b/src/policy/fees.h index e7f45c3151e6c..b44ba4dc7abe7 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -297,13 +297,13 @@ class FeeFilterRounder public: /** Create new FeeFilterRounder */ - explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); + explicit FeeFilterRounder(const CFeeRate& min_incremental_fee); /** Quantize a minimum fee for privacy purpose before broadcast. */ CAmount round(CAmount currentMinFee); private: - const std::set feeset; + const std::set m_fee_set; Mutex m_insecure_rand_mutex; FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; From 1276090705060fcc97072481c2383bbaaa556194 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 25 Mar 2022 21:36:22 +0100 Subject: [PATCH 0017/1090] util, refactor: Use GetPathArg to read "-conf" value Also "includeconf" values been normalized. --- src/bitcoin-cli.cpp | 2 +- src/init/common.cpp | 2 +- src/qt/guiutil.cpp | 2 +- src/util/system.cpp | 14 +++++++------- src/util/system.h | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index dea46693bc4d7..8f7b836a7c2f8 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -801,7 +801,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( "Could not locate RPC credentials. No authentication cookie could be found, and RPC password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)", - fs::PathToString(GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME))))); + fs::PathToString(GetConfigFile(gArgs.GetPathArg("-conf", BITCOIN_CONF_FILENAME))))); } else { throw std::runtime_error("Authorization failed: Incorrect rpcuser or rpcpassword"); } diff --git a/src/init/common.cpp b/src/init/common.cpp index 688471b35da75..3d46857f3cad3 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -137,7 +137,7 @@ bool StartLogging(const ArgsManager& args) LogPrintf("Using data directory %s\n", fs::PathToString(gArgs.GetDataDirNet())); // Only log conf file usage message if conf file actually exists. - fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); + fs::path config_file_path = GetConfigFile(args.GetPathArg("-conf", BITCOIN_CONF_FILENAME)); if (fs::exists(config_file_path)) { LogPrintf("Config file: %s\n", fs::PathToString(config_file_path)); } else if (args.IsArgSet("-conf")) { diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 6fb5fce5b309a..e70ca3053d4f3 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -431,7 +431,7 @@ void openDebugLogfile() bool openBitcoinConf() { - fs::path pathConfig = GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)); + fs::path pathConfig = GetConfigFile(gArgs.GetPathArg("-conf", BITCOIN_CONF_FILENAME)); /* Create the file */ std::ofstream configFile{pathConfig, std::ios_base::app}; diff --git a/src/util/system.cpp b/src/util/system.cpp index a7e66defcdf6d..5acc9a8e5894f 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -826,9 +826,9 @@ bool CheckDataDirOption() return datadir.empty() || fs::is_directory(fs::absolute(datadir)); } -fs::path GetConfigFile(const std::string& confPath) +fs::path GetConfigFile(const fs::path& configuration_file_path) { - return AbsPathForConfigVal(fs::PathFromString(confPath), false); + return AbsPathForConfigVal(configuration_file_path, /*net_specific=*/false); } static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector>& options, std::list& sections) @@ -912,17 +912,17 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) m_config_sections.clear(); } - const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME); - std::ifstream stream{GetConfigFile(confPath)}; + const fs::path conf_path = GetPathArg("-conf", BITCOIN_CONF_FILENAME); + std::ifstream stream{GetConfigFile(conf_path)}; // not ok to have a config file specified that cannot be opened if (IsArgSet("-conf") && !stream.good()) { - error = strprintf("specified config file \"%s\" could not be opened.", confPath); + error = strprintf("specified config file \"%s\" could not be opened.", fs::PathToString(conf_path)); return false; } // ok to not have a config file if (stream.good()) { - if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { + if (!ReadConfigStream(stream, fs::PathToString(conf_path), error, ignore_invalid_keys)) { return false; } // `-includeconf` cannot be included in the command line arguments except @@ -960,7 +960,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) const size_t default_includes = add_includes({}); for (const std::string& conf_file_name : conf_file_names) { - std::ifstream conf_file_stream{GetConfigFile(conf_file_name)}; + std::ifstream conf_file_stream{GetConfigFile(fs::PathFromString(conf_file_name))}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; diff --git a/src/util/system.h b/src/util/system.h index a66b597d41231..796205fbfc2c3 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -98,7 +98,7 @@ bool TryCreateDirectories(const fs::path& p); fs::path GetDefaultDataDir(); // Return true if -datadir option points to a valid directory or is not specified. bool CheckDataDirOption(); -fs::path GetConfigFile(const std::string& confPath); +fs::path GetConfigFile(const fs::path& configuration_file_path); #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif From 138c668e2b4d64279ddefbe07c1d9b7c3d3c537c Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 25 Mar 2022 21:37:28 +0100 Subject: [PATCH 0018/1090] util, refactor: Use GetPathArg to read "-rpccookiefile" value --- src/rpc/request.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index 95a7c25b9393e..c2f1b15d9cd01 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -66,16 +66,16 @@ UniValue JSONRPCError(int code, const std::string& message) */ static const std::string COOKIEAUTH_USER = "__cookie__"; /** Default name for auth cookie file */ -static const std::string COOKIEAUTH_FILE = ".cookie"; +static const char* const COOKIEAUTH_FILE = ".cookie"; /** Get name of RPC authentication cookie file */ static fs::path GetAuthCookieFile(bool temp=false) { - std::string arg = gArgs.GetArg("-rpccookiefile", COOKIEAUTH_FILE); + fs::path arg = gArgs.GetPathArg("-rpccookiefile", COOKIEAUTH_FILE); if (temp) { arg += ".tmp"; } - return AbsPathForConfigVal(fs::PathFromString(arg)); + return AbsPathForConfigVal(arg); } bool GenerateAuthCookie(std::string *cookie_out) From b01f336708019f8c8274ea701d3446e4123e7af2 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 25 Mar 2022 21:42:58 +0100 Subject: [PATCH 0019/1090] util, refactor: Drop explicit conversion to fs::path Removes unhelpful noise/verbosity. See: https://github.com/bitcoin/bitcoin/pull/24306#discussion_r809363741 --- src/util/system.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/system.cpp b/src/util/system.cpp index 5acc9a8e5894f..bfb1a79b400d7 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -528,7 +528,7 @@ bool ArgsManager::InitSettings(std::string& error) bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const { - fs::path settings = GetPathArg("-settings", fs::path{BITCOIN_SETTINGS_FILENAME}); + fs::path settings = GetPathArg("-settings", BITCOIN_SETTINGS_FILENAME); if (settings.empty()) { return false; } From 734b9669ff7b2f5e2820993443a6f868f6b0b20a Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Fri, 22 Apr 2022 11:09:45 +0200 Subject: [PATCH 0020/1090] test: add getblockfrompeer coverage of invalid inputs --- test/functional/rpc_getblockfrompeer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index b65322d920b06..cc9ccf306628d 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -49,14 +49,17 @@ def run_test(self): assert_equal(len(peers), 1) peer_0_peer_1_id = peers[0]["id"] - self.log.info("Arguments must be sensible") - assert_raises_rpc_error(-8, "hash must be of length 64 (not 4, for '1234')", self.nodes[0].getblockfrompeer, "1234", 0) + self.log.info("Arguments must be valid") + assert_raises_rpc_error(-8, "hash must be of length 64 (not 4, for '1234')", self.nodes[0].getblockfrompeer, "1234", peer_0_peer_1_id) + assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].getblockfrompeer, 1234, peer_0_peer_1_id) + assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].getblockfrompeer, short_tip, "0") self.log.info("We must already have the header") assert_raises_rpc_error(-1, "Block header missing", self.nodes[0].getblockfrompeer, "00" * 32, 0) self.log.info("Non-existent peer generates error") - assert_raises_rpc_error(-1, "Peer does not exist", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id + 1) + for peer_id in [-1, peer_0_peer_1_id + 1]: + assert_raises_rpc_error(-1, "Peer does not exist", self.nodes[0].getblockfrompeer, short_tip, peer_id) self.log.info("Successful fetch") result = self.nodes[0].getblockfrompeer(short_tip, peer_0_peer_1_id) From 304ece994504220c355577170409b9200941f2af Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 31 May 2021 16:55:56 +0200 Subject: [PATCH 0021/1090] rpc: document bools in FillPSBT() calls --- src/wallet/rpc/spend.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 07119133b7ad6..13df213065273 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -82,10 +82,10 @@ static UniValue FinishTransaction(const std::shared_ptr pwallet, const PartiallySignedTransaction psbtx(rawTx); // First fill transaction with our data without signing, - // so external signers are not asked sign more than once. + // so external signers are not asked to sign more than once. bool complete; - pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false, true); - const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, true, false)}; + pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); + const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/true, /*bip32derivs=*/false)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } @@ -1100,7 +1100,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) } else { PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false /* sign */, true /* bip32derivs */); + const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); CHECK_NONFATAL(err == TransactionError::OK); CHECK_NONFATAL(!complete); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); @@ -1675,7 +1675,7 @@ RPCHelpMan walletcreatefundedpsbt() // Fill transaction with out data but don't sign bool bip32derivs = request.params[4].isNull() ? true : request.params[4].get_bool(); bool complete = true; - const TransactionError err{wallet.FillPSBT(psbtx, complete, 1, false, bip32derivs)}; + const TransactionError err{wallet.FillPSBT(psbtx, complete, 1, /*sign=*/false, /*bip32derivs=*/bip32derivs)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } From 7e02a3329797211ed5d35e5f5e7b919c099b78ba Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 25 Apr 2022 18:13:23 +0200 Subject: [PATCH 0022/1090] rpc: bumpfee signer support --- src/wallet/feebumper.cpp | 17 ++++++++++++++++- src/wallet/rpc/spend.cpp | 5 ++++- test/functional/wallet_signer.py | 23 ++++++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 73042424ad498..c66754bfcb43b 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -239,7 +239,22 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx) { LOCK(wallet.cs_wallet); - return wallet.SignTransaction(mtx); + + if (wallet.IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { + // Make a blank psbt + PartiallySignedTransaction psbtx(mtx); + + // First fill transaction with our data without signing, + // so external signers are not asked to sign more than once. + bool complete; + wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); + const TransactionError err = wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true /* sign */, false /* bip32derivs */); + if (err != TransactionError::OK) return false; + complete = FinalizeAndExtractPSBT(psbtx, mtx); + return complete; + } else { + return wallet.SignTransaction(mtx); + } } Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector& errors, uint256& bumped_txid) diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 13df213065273..602d7aec96e31 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -1010,7 +1010,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) std::shared_ptr const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !want_psbt) { + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER) && !want_psbt) { throw JSONRPCError(RPC_WALLET_ERROR, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead."); } @@ -1088,6 +1088,9 @@ static RPCHelpMan bumpfee_helper(std::string method_name) // For psbtbumpfee, return the base64-encoded unsigned PSBT of the new transaction. if (!want_psbt) { if (!feebumper::SignTransaction(*pwallet, mtx)) { + if (pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Transaction incomplete. Try psbtbumpfee instead."); + } throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 8e4e1f5d36b67..0839dbf9f4925 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -13,6 +13,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_greater_than, assert_raises_rpc_error, ) @@ -150,7 +151,7 @@ def test_valid_signer(self): assert_equal(result[1], {'success': True}) assert_equal(mock_wallet.getwalletinfo()["txcount"], 1) dest = self.nodes[0].getnewaddress(address_type='bech32') - mock_psbt = mock_wallet.walletcreatefundedpsbt([], {dest:0.5}, 0, {}, True)['psbt'] + mock_psbt = mock_wallet.walletcreatefundedpsbt([], {dest:0.5}, 0, {'replaceable': True}, True)['psbt'] mock_psbt_signed = mock_wallet.walletprocesspsbt(psbt=mock_psbt, sign=True, sighashtype="ALL", bip32derivs=True) mock_psbt_final = mock_wallet.finalizepsbt(mock_psbt_signed["psbt"]) mock_tx = mock_psbt_final["hex"] @@ -190,6 +191,7 @@ def test_valid_signer(self): self.log.info('Test send using hww1') + # Don't broadcast transaction yet so the RPC returns the raw hex res = hww.send(outputs={dest:0.5},options={"add_to_wallet": False}) assert(res["complete"]) assert_equal(res["hex"], mock_tx) @@ -199,6 +201,25 @@ def test_valid_signer(self): res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()],options={"add_to_wallet": False}) assert(res["complete"]) assert_equal(res["hex"], mock_tx) + # Broadcast transaction so we can bump the fee + hww.sendrawtransaction(res["hex"]) + + self.log.info('Prepare fee bumped mock PSBT') + + # Now that the transaction is broadcast, bump fee in mock wallet: + orig_tx_id = res["txid"] + mock_psbt_bumped = mock_wallet.psbtbumpfee(orig_tx_id)["psbt"] + mock_psbt_bumped_signed = mock_wallet.walletprocesspsbt(psbt=mock_psbt_bumped, sign=True, sighashtype="ALL", bip32derivs=True) + + with open(os.path.join(self.nodes[1].cwd, "mock_psbt"), "w", encoding="utf8") as f: + f.write(mock_psbt_bumped_signed["psbt"]) + + self.log.info('Test bumpfee using hww1') + + # Bump fee + res = hww.bumpfee(orig_tx_id) + assert_greater_than(res["fee"], res["origfee"]) + assert_equal(res["errors"], []) # # Handle error thrown by script # self.set_mock_result(self.nodes[4], "2") From 2c07cfacd1745844a1d3c57f2e8617549b9815d7 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 25 Apr 2022 18:14:28 +0200 Subject: [PATCH 0023/1090] gui: bumpfee signer support Specifically this enables the Send button in the fee bump dialog for wallets with external signer support. Similar to 2efdfb88aab6496dcf2b98e0de30635bc6bade85. --- src/qt/walletmodel.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 5ee32e79d5d4e..8c2f8e568d7df 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -508,7 +508,9 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); } - auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString, "", "", SEND_CONFIRM_DELAY, !m_wallet->privateKeysDisabled(), getOptionsModel()->getEnablePSBTControls(), nullptr); + const bool enable_send{!wallet().privateKeysDisabled() || wallet().hasExternalSigner()}; + const bool always_show_unsigned{getOptionsModel()->getEnablePSBTControls()}; + auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString, "", "", SEND_CONFIRM_DELAY, enable_send, always_show_unsigned, nullptr); confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); // TODO: Replace QDialog::exec() with safer QDialog::show(). const auto retval = static_cast(confirmationDialog->exec()); @@ -526,6 +528,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) // Short-circuit if we are returning a bumped transaction PSBT to clipboard if (retval == QMessageBox::Save) { + // "Create Unsigned" clicked PartiallySignedTransaction psbtx(mtx); bool complete = false; const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, nullptr, psbtx, complete); @@ -541,7 +544,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return true; } - assert(!m_wallet->privateKeysDisabled()); + assert(!m_wallet->privateKeysDisabled() || wallet().hasExternalSigner()); // sign bumped transaction if (!m_wallet->signBumpTransaction(mtx)) { From 2c3ee4c347838ecadb17a011932dffc077e46630 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Fri, 12 Nov 2021 16:09:54 -0500 Subject: [PATCH 0024/1090] gui: Load Base64 PSBT string from file Some .psbt files may have the PSBT as a base64 string instead of in binary. We should be able to load those files. --- src/qt/walletframe.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index dc4e25a02b230..abeb6a827baca 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -215,6 +215,14 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) } std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary}; data.assign(std::istream_iterator{in}, {}); + + // Some psbt files may be base64 strings in the file rather than binary data + std::string b64_str{data.begin(), data.end()}; + b64_str.erase(b64_str.find_last_not_of(" \t\n\r\f\v") + 1); // Trim trailing whitespace + auto b64_dec = DecodeBase64(b64_str); + if (b64_dec.has_value()) { + data = b64_dec.value(); + } } std::string error; From b0a53d50d9142bed51a8372eeb848816bfa94da8 Mon Sep 17 00:00:00 2001 From: Patrick Strateman Date: Sun, 14 Jun 2020 20:14:34 -0400 Subject: [PATCH 0025/1090] Make sanity check in GCSFilter constructor optional BlockFilterIndex will perform the cheaper check of verifying the filter hash when reading the filter from disk. --- src/blockfilter.cpp | 8 +++++--- src/blockfilter.h | 6 +++--- src/index/blockfilterindex.cpp | 13 +++++++++---- src/index/blockfilterindex.h | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 63a9ba498f522..1ad687214358a 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -47,7 +47,7 @@ GCSFilter::GCSFilter(const Params& params) : m_params(params), m_N(0), m_F(0), m_encoded{0} {} -GCSFilter::GCSFilter(const Params& params, std::vector encoded_filter) +GCSFilter::GCSFilter(const Params& params, std::vector encoded_filter, bool skip_decode_check) : m_params(params), m_encoded(std::move(encoded_filter)) { SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded}; @@ -59,6 +59,8 @@ GCSFilter::GCSFilter(const Params& params, std::vector encoded_fi } m_F = static_cast(m_N) * static_cast(m_params.m_M); + if (skip_decode_check) return; + // Verify that the encoded filter contains exactly N elements. If it has too much or too little // data, a std::ios_base::failure exception will be raised. BitStreamReader bitreader{stream}; @@ -219,14 +221,14 @@ static GCSFilter::ElementSet BasicFilterElements(const CBlock& block, } BlockFilter::BlockFilter(BlockFilterType filter_type, const uint256& block_hash, - std::vector filter) + std::vector filter, bool skip_decode_check) : m_filter_type(filter_type), m_block_hash(block_hash) { GCSFilter::Params params; if (!BuildParams(params)) { throw std::invalid_argument("unknown filter_type"); } - m_filter = GCSFilter(params, std::move(filter)); + m_filter = GCSFilter(params, std::move(filter), skip_decode_check); } BlockFilter::BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo) diff --git a/src/blockfilter.h b/src/blockfilter.h index 96cefbf3b2f9f..d6a51e95c24e2 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -59,7 +59,7 @@ class GCSFilter explicit GCSFilter(const Params& params = Params()); /** Reconstructs an already-created filter from an encoding. */ - GCSFilter(const Params& params, std::vector encoded_filter); + GCSFilter(const Params& params, std::vector encoded_filter, bool skip_decode_check); /** Builds a new filter from the params and set of elements. */ GCSFilter(const Params& params, const ElementSet& elements); @@ -122,7 +122,7 @@ class BlockFilter //! Reconstruct a BlockFilter from parts. BlockFilter(BlockFilterType filter_type, const uint256& block_hash, - std::vector filter); + std::vector filter, bool skip_decode_check); //! Construct a new BlockFilter of the specified type from a block. BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo); @@ -164,7 +164,7 @@ class BlockFilter if (!BuildParams(params)) { throw std::ios_base::failure("unknown filter_type"); } - m_filter = GCSFilter(params, std::move(encoded_filter)); + m_filter = GCSFilter(params, std::move(encoded_filter), /*skip_decode_check=*/false); } }; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 4f99eddfd77af..a8e1860481249 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -143,18 +144,22 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) return BaseIndex::CommitInternal(batch); } -bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const +bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const { CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { return false; } + // Check that the hash of the encoded_filter matches the one stored in the db. uint256 block_hash; std::vector encoded_filter; try { filein >> block_hash >> encoded_filter; - filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter)); + uint256 result; + CHash256().Write(encoded_filter).Finalize(result); + if (result != hash) return error("Checksum mismatch in filter decode."); + filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter), /*skip_decode_check=*/true); } catch (const std::exception& e) { return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what()); @@ -381,7 +386,7 @@ bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& return false; } - return ReadFilterFromDisk(entry.pos, filter_out); + return ReadFilterFromDisk(entry.pos, entry.hash, filter_out); } bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) @@ -425,7 +430,7 @@ bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* st filters_out.resize(entries.size()); auto filter_pos_it = filters_out.begin(); for (const auto& entry : entries) { - if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) { + if (!ReadFilterFromDisk(entry.pos, entry.hash, *filter_pos_it)) { return false; } ++filter_pos_it; diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index a049019c020fe..09c3c78c6310c 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -31,7 +31,7 @@ class BlockFilterIndex final : public BaseIndex FlatFilePos m_next_filter_pos; std::unique_ptr m_filter_fileseq; - bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const; + bool ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const; size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); Mutex m_cs_headers_cache; From bcb0cacac28e98a39dc856c574a0872fe17059e9 Mon Sep 17 00:00:00 2001 From: mruddy <6440430+mruddy@users.noreply.github.com> Date: Thu, 14 Apr 2022 18:44:34 -0400 Subject: [PATCH 0026/1090] reindex, log, test: fixes #21379 This fixes a blk file size calculation made during reindex that results in increased blk file malformity. The fix is to avoid double counting the size of the serialization header during reindex. This adds a unit test to reproduce the bug before the fix and to ensure that it does not recur. These changes include a log message change also so as to not be as alarming. This is a common and recoverable data corruption. These messages can now be filtered by the debug log reindex category. --- src/Makefile.test.include | 1 + src/node/blockstorage.cpp | 13 ++++++---- src/node/blockstorage.h | 4 ++++ src/test/blockmanager_tests.cpp | 42 +++++++++++++++++++++++++++++++++ src/validation.cpp | 13 +++++++++- 5 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 src/test/blockmanager_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 02a3f9ae7db9a..09148f9d36a6f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -77,6 +77,7 @@ BITCOIN_TESTS =\ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ test/blockfilter_tests.cpp \ + test/blockmanager_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 17ab226a30d38..826e4150fe126 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -788,19 +788,24 @@ bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, c return true; } -/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; - if (dbp != nullptr) { + const auto position_known {dbp != nullptr}; + if (position_known) { blockPos = *dbp; + } else { + // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for + // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). + // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. + nBlockSize += static_cast(BLOCK_SERIALIZATION_HEADER_SIZE); } - if (!FindBlockPos(blockPos, nBlockSize + 8, nHeight, active_chain, block.GetBlockTime(), dbp != nullptr)) { + if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) { error("%s: FindBlockPos failed", __func__); return FlatFilePos(); } - if (dbp == nullptr) { + if (!position_known) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 488713dbd8c1c..8e49254e76cec 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -44,6 +44,9 @@ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB +/** Size of header written by WriteBlockToDisk before a serialized CBlock */ +static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); + extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; /** Pruning-related variables and constants */ @@ -171,6 +174,7 @@ class BlockManager bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); /** Calculate the amount of disk space the block & undo files currently use */ diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp new file mode 100644 index 0000000000000..dd7c32cc46c0e --- /dev/null +++ b/src/test/blockmanager_tests.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include + +#include +#include + +using node::BlockManager; +using node::BLOCK_SERIALIZATION_HEADER_SIZE; + +// use BasicTestingSetup here for the data directory configuration, setup, and cleanup +BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) +{ + const auto params {CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)}; + BlockManager blockman {}; + CChain chain {}; + // simulate adding a genesis block normally + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // simulate what happens during reindex + // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file + // the block is found at offset 8 because there is an 8 byte serialization header + // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. + FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // now simulate what happens after reindex for the first new block processed + // the actual block contents don't matter, just that it's a block. + // verify that the write position is at offset 0x12d. + // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur + // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 + // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, *params, nullptr)}; + BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/validation.cpp b/src/validation.cpp index b5d6a660886fe..e8cb92a1f1a7b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4378,7 +4378,18 @@ void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) } } } catch (const std::exception& e) { - LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what()); + // historical bugs added extra data to the block files that does not deserialize cleanly. + // commonly this data is between readable blocks, but it does not really matter. such data is not fatal to the import process. + // the code that reads the block files deals with invalid data by simply ignoring it. + // it continues to search for the next {4 byte magic message start bytes + 4 byte length + block} that does deserialize cleanly + // and passes all of the other block validation checks dealing with POW and the merkle root, etc... + // we merely note with this informational log message when unexpected data is encountered. + // we could also be experiencing a storage system read error, or a read of a previous bad write. these are possible, but + // less likely scenarios. we don't have enough information to tell a difference here. + // the reindex process is not the place to attempt to clean and/or compact the block files. if so desired, a studious node operator + // may use knowledge of the fact that the block files are not entirely pristine in order to prepare a set of pristine, and + // perhaps ordered, block files for later reindexing. + LogPrint(BCLog::REINDEX, "%s: unexpected data at file offset 0x%x - %s. continuing\n", __func__, (nRewind - 1), e.what()); } } } catch (const std::runtime_error& e) { From 299023c1d9962628d158fac0306f8531506a0123 Mon Sep 17 00:00:00 2001 From: Patrick Strateman Date: Mon, 15 Jun 2020 12:30:22 -0400 Subject: [PATCH 0027/1090] Add GCSFilterDecode and GCSBlockFilterGetHash benchmarks. All of the benchmarks are standardized on the BASIC filter parameters so we can compare between all the benchmarks. All the GCS benchmarks are renamed to have "GCS" as the prefix. --- src/bench/gcs_filter.cpp | 57 +++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 607e4392b7a1a..c96e9b7f31162 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -5,7 +5,7 @@ #include #include -static void ConstructGCSFilter(benchmark::Bench& bench) +static const GCSFilter::ElementSet GenerateGCSTestElements() { GCSFilter::ElementSet elements; for (int i = 0; i < 10000; ++i) { @@ -15,29 +15,56 @@ static void ConstructGCSFilter(benchmark::Bench& bench) elements.insert(std::move(element)); } + return elements; +} + +static void GCSBlockFilterGetHash(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + BlockFilter block_filter(BlockFilterType::BASIC, {}, filter.GetEncoded(), /*skip_decode_check=*/false); + + bench.run([&] { + block_filter.GetHash(); + }); +} + +static void GCSFilterConstruct(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + uint64_t siphash_k0 = 0; bench.batch(elements.size()).unit("elem").run([&] { - GCSFilter filter({siphash_k0, 0, 20, 1 << 20}, elements); + GCSFilter filter({siphash_k0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); siphash_k0++; }); } -static void MatchGCSFilter(benchmark::Bench& bench) +static void GCSFilterDecode(benchmark::Bench& bench) { - GCSFilter::ElementSet elements; - for (int i = 0; i < 10000; ++i) { - GCSFilter::Element element(32); - element[0] = static_cast(i); - element[1] = static_cast(i >> 8); - elements.insert(std::move(element)); - } - GCSFilter filter({0, 0, 20, 1 << 20}, elements); + auto elements = GenerateGCSTestElements(); - bench.unit("elem").run([&] { - filter.Match(GCSFilter::Element()); + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + auto encoded = filter.GetEncoded(); + + bench.run([&] { + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/false); }); } -BENCHMARK(ConstructGCSFilter); -BENCHMARK(MatchGCSFilter); +static void GCSFilterMatch(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + + bench.run([&] { + filter.Match(GCSFilter::Element()); + }); +} +BENCHMARK(GCSBlockFilterGetHash); +BENCHMARK(GCSFilterConstruct); +BENCHMARK(GCSFilterDecode); +BENCHMARK(GCSFilterMatch); From aee9a8140b3a58b744766f9e89572f1d953a808b Mon Sep 17 00:00:00 2001 From: Patrick Strateman Date: Mon, 15 Jun 2020 19:41:18 -0400 Subject: [PATCH 0028/1090] Add GCSFilterDecodeSkipCheck benchmark This benchmark allows us to compare the differences between doing the sanity check for corruption via GolombRiceDecode() vs checking the hash of the encoded block filter. --- src/bench/gcs_filter.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index c96e9b7f31162..0b43652453035 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -54,6 +54,18 @@ static void GCSFilterDecode(benchmark::Bench& bench) }); } +static void GCSFilterDecodeSkipCheck(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + auto encoded = filter.GetEncoded(); + + bench.run([&] { + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/true); + }); +} + static void GCSFilterMatch(benchmark::Bench& bench) { auto elements = GenerateGCSTestElements(); @@ -67,4 +79,5 @@ static void GCSFilterMatch(benchmark::Bench& bench) BENCHMARK(GCSBlockFilterGetHash); BENCHMARK(GCSFilterConstruct); BENCHMARK(GCSFilterDecode); +BENCHMARK(GCSFilterDecodeSkipCheck); BENCHMARK(GCSFilterMatch); From e734228d8585c0870c71ce8ba8c037f8cf8b249a Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Sun, 22 May 2022 14:17:15 +0900 Subject: [PATCH 0029/1090] Update GCSFilter benchmarks Element count used in the GCSFilter benchmarks are increased to 100,000 from 10,000. Testing the benchmarks with different element counts showed that a filter with 100,000 elements resulted in the same ns/op. This this a desirable thing to have as it allows us to reason about how long a single filter element takes to process, letting us easily calculate how long a filter with N elements (where N > 100,000) would take to process. GCSFilterConstruct benchmark is now called without batch. This makes intra-bench results more intuitive as all benchmarks are in ns/op instead of a custom unit. There are no downsides to this change as testing showed that there is no observable difference in error rates in the benchmarks when calling without batch. --- src/bench/gcs_filter.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 0b43652453035..80babb213b3e0 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -8,7 +8,12 @@ static const GCSFilter::ElementSet GenerateGCSTestElements() { GCSFilter::ElementSet elements; - for (int i = 0; i < 10000; ++i) { + + // Testing the benchmarks with different number of elements show that a filter + // with at least 100,000 elements results in benchmarks that have the same + // ns/op. This makes it easy to reason about how long (in nanoseconds) a single + // filter element takes to process. + for (int i = 0; i < 100000; ++i) { GCSFilter::Element element(32); element[0] = static_cast(i); element[1] = static_cast(i >> 8); @@ -35,7 +40,7 @@ static void GCSFilterConstruct(benchmark::Bench& bench) auto elements = GenerateGCSTestElements(); uint64_t siphash_k0 = 0; - bench.batch(elements.size()).unit("elem").run([&] { + bench.run([&]{ GCSFilter filter({siphash_k0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); siphash_k0++; From 7fa851fba8570ef256317f7d5759aa3de9088bf1 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Fri, 31 Dec 2021 16:25:35 +0100 Subject: [PATCH 0030/1090] rpc: Pruned nodes can not fetch unsynced blocks While a node is still catching up to the tip that it is aware of via the headers, the user can currently use to fetch blocks close to the tip. These blocks are stored in the current block/rev file which otherwise contains blocks the node is receiving as part of the syncing process. This creates a problem for pruned nodes: The files containing a fetched block are not pruned during syncing because they contain a block close to the tip. This means the entire file will not be pruned until the tip have moved on far enough from the fetched block. In extreme cases with heavy pruning (550) and multiple blocks being fetched this could mean that the disc usage far exceeds what the user expects, potentially running out of space. --- src/rpc/blockchain.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f2186c131fd36..4fbed3da60bc2 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -453,6 +453,12 @@ static RPCHelpMan getblockfrompeer() throw JSONRPCError(RPC_MISC_ERROR, "Block header missing"); } + // Fetching blocks before the node has syncing past their height can prevent block files from + // being pruned, so we avoid it if the node is in prune mode. + if (index->nHeight > chainman.ActiveChain().Tip()->nHeight && node::fPruneMode) { + throw JSONRPCError(RPC_MISC_ERROR, "In prune mode, only blocks that the node has already synced previously can be fetched from a peer"); + } + const bool block_has_data = WITH_LOCK(::cs_main, return index->nStatus & BLOCK_HAVE_DATA); if (block_has_data) { throw JSONRPCError(RPC_MISC_ERROR, "Block already downloaded"); From 5826bf546e83478947edbdf49978414f0b69eb1a Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 2 Jan 2022 17:04:58 +0100 Subject: [PATCH 0031/1090] test: Add test for getblockfrompeer on syncing pruned nodes --- test/functional/rpc_getblockfrompeer.py | 32 ++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index a7628b55915a4..8986b65eacecf 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -5,7 +5,12 @@ """Test the getblockfrompeer RPC.""" from test_framework.authproxy import JSONRPCException -from test_framework.messages import NODE_WITNESS +from test_framework.messages import ( + CBlock, + from_hex, + msg_headers, + NODE_WITNESS, +) from test_framework.p2p import ( P2P_SERVICES, P2PInterface, @@ -16,6 +21,7 @@ assert_raises_rpc_error, ) + class GetBlockFromPeerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -78,6 +84,30 @@ def run_test(self): self.log.info("Don't fetch blocks we already have") assert_raises_rpc_error(-1, "Block already downloaded", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id) + self.log.info("Don't fetch blocks while the node has not synced past it yet") + # For this test we need node 1 in prune mode and as a side effect this also disconnects + # the nodes which is also necessary for the rest of the test. + self.restart_node(1, ["-prune=550"]) + + # Generate a block on the disconnected node that the pruning node is not connected to + blockhash = self.generate(self.nodes[0], 1, sync_fun=self.no_op)[0] + block_hex = self.nodes[0].getblock(blockhash=blockhash, verbosity=0) + block = from_hex(CBlock(), block_hex) + + # Connect a P2PInterface to the pruning node and have it submit only the header of the + # block that the pruning node has not seen + node1_interface = self.nodes[1].add_p2p_connection(P2PInterface()) + node1_interface.send_message(msg_headers([block])) + + # Get the peer id of the P2PInterface from the pruning node + node1_peers = self.nodes[1].getpeerinfo() + assert_equal(len(node1_peers), 1) + node1_interface_id = node1_peers[0]["id"] + + # Trying to fetch this block from the P2PInterface should not be possible + error_msg = "In prune mode, only blocks that the node has already synced previously can be fetched from a peer" + assert_raises_rpc_error(-1, error_msg, self.nodes[1].getblockfrompeer, blockhash, node1_interface_id) + if __name__ == '__main__': GetBlockFromPeerTest().main() From a413595c37f51557f9506e0a279cd80fc9a6fb36 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Thu, 9 Jun 2022 14:38:26 +0200 Subject: [PATCH 0032/1090] build: Fix `capnp` package build for Android --- depends/packages/capnp.mk | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index 8a3a14810d6ed..cd9b91cf7cf27 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -6,8 +6,13 @@ $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) +define $(package)_set_vars := +$(package)_config_opts := --with-external-capnp +$(package)_config_opts_android := --disable-shared +endef + define $(package)_config_cmds - $($(package)_autoconf) --with-external-capnp + $($(package)_autoconf) endef define $(package)_build_cmds From 8b8edc25c13a3e613770bf38b21a2556192e6315 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Thu, 9 Jun 2022 15:49:54 +0200 Subject: [PATCH 0033/1090] build: Specify native binaries explicitly when building `capnp` package From `configure --help`: --with-external-capnp use the system capnp binary (or the one specified with $CAPNP) instead of compiling a new one (useful for cross-compiling) --- depends/packages/capnp.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index cd9b91cf7cf27..f4778c1ecdc32 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -8,6 +8,8 @@ $(package)_dependencies=native_$(package) define $(package)_set_vars := $(package)_config_opts := --with-external-capnp +$(package)_config_opts += CAPNP="$$(native_capnp_prefixbin)/capnp" +$(package)_config_opts += CAPNP_CXX="$$(native_capnp_prefixbin)/capnp-c++" $(package)_config_opts_android := --disable-shared endef From 292b1a3e9c98b9ba74b28d149df8554d4ad8e5c0 Mon Sep 17 00:00:00 2001 From: amadeuszpawlik Date: Sat, 28 May 2022 20:40:51 +0200 Subject: [PATCH 0034/1090] GetExternalSigner(): fail if multiple signers are found If there are multiple external signers, `GetExternalSigner()` will just pick the first one in the list. If the user has two or more hardware wallets connected at the same time, he might not notice this. This PR adds a check and fails with suitable message. --- src/qt/walletcontroller.cpp | 4 +++ .../external_signer_scriptpubkeyman.cpp | 3 +- test/functional/mocks/invalid_signer.py | 2 +- test/functional/mocks/multi_signers.py | 30 +++++++++++++++++++ test/functional/mocks/signer.py | 2 +- test/functional/rpc_signer.py | 5 +--- test/functional/wallet_signer.py | 15 ++++++++++ 7 files changed, 54 insertions(+), 7 deletions(-) create mode 100755 test/functional/mocks/multi_signers.py diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index d27ddf1aba52b..fae4c7cdf1311 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -293,6 +293,10 @@ void CreateWalletActivity::create() } catch (const std::runtime_error& e) { QMessageBox::critical(nullptr, tr("Can't list signers"), e.what()); } + if (signers.size() > 1) { + QMessageBox::critical(nullptr, tr("Too many external signers found"), QString::fromStdString("More than one external signer found. Please connect only one at a time.")); + signers.clear(); + } m_create_wallet_dialog->setSigners(signers); m_create_wallet_dialog->setWindowModality(Qt::ApplicationModal); diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index 9d5f58b784c6a..76de51ac3e0d2 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -45,7 +45,8 @@ ExternalSigner ExternalSignerScriptPubKeyMan::GetExternalSigner() { std::vector signers; ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); if (signers.empty()) throw std::runtime_error(std::string(__func__) + ": No external signers found"); - // TODO: add fingerprint argument in case of multiple signers + // TODO: add fingerprint argument instead of failing in case of multiple signers. + if (signers.size() > 1) throw std::runtime_error(std::string(__func__) + ": More than one external signer found. Please connect only one at a time."); return signers[0]; } diff --git a/test/functional/mocks/invalid_signer.py b/test/functional/mocks/invalid_signer.py index e30cc9e20b053..14f9fed72e888 100755 --- a/test/functional/mocks/invalid_signer.py +++ b/test/functional/mocks/invalid_signer.py @@ -18,7 +18,7 @@ def perform_pre_checks(): sys.exit(int(mock_result[0])) def enumerate(args): - sys.stdout.write(json.dumps([{"fingerprint": "b3c19bfc", "type": "trezor", "model": "trezor_t"}, {"fingerprint": "00000002"}])) + sys.stdout.write(json.dumps([{"fingerprint": "b3c19bfc", "type": "trezor", "model": "trezor_t"}])) def getdescriptors(args): xpub_pkh = "xpub6CRhJvXV8x2AKWvqi1ZSMFU6cbxzQiYrv3dxSUXCawjMJ1JzpqVsveH4way1yCmJm29KzH1zrVZmVwes4Qo6oXVE1HFn4fdiKrYJngqFFc6" diff --git a/test/functional/mocks/multi_signers.py b/test/functional/mocks/multi_signers.py new file mode 100755 index 0000000000000..88f93e23dea0a --- /dev/null +++ b/test/functional/mocks/multi_signers.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import json +import sys + +def enumerate(args): + sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}, + {"fingerprint": "00000002", "type": "trezor", "model": "trezor_one"}])) + +parser = argparse.ArgumentParser(prog='./multi_signers.py', description='External multi-signer mock') + +subparsers = parser.add_subparsers(description='Commands', dest='command') +subparsers.required = True + +parser_enumerate = subparsers.add_parser('enumerate', help='list available signers') +parser_enumerate.set_defaults(func=enumerate) + + +if not sys.stdin.isatty(): + buffer = sys.stdin.read() + if buffer and buffer.rstrip() != "": + sys.argv.extend(buffer.rstrip().split(" ")) + +args = parser.parse_args() + +args.func(args) diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py index b732b26a53d07..c5a8f7b1e9234 100755 --- a/test/functional/mocks/signer.py +++ b/test/functional/mocks/signer.py @@ -18,7 +18,7 @@ def perform_pre_checks(): sys.exit(int(mock_result[0])) def enumerate(args): - sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}, {"fingerprint": "00000002"}])) + sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}])) def getdescriptors(args): xpub = "tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B" diff --git a/test/functional/rpc_signer.py b/test/functional/rpc_signer.py index f1107197c53a4..de17b2b929228 100755 --- a/test/functional/rpc_signer.py +++ b/test/functional/rpc_signer.py @@ -77,10 +77,7 @@ def run_test(self): ) self.clear_mock_result(self.nodes[1]) - result = self.nodes[1].enumeratesigners() - assert_equal(len(result['signers']), 2) - assert_equal(result['signers'][0]["fingerprint"], "00000001") - assert_equal(result['signers'][0]["name"], "trezor_t") + assert_equal({'fingerprint': '00000001', 'name': 'trezor_t'} in self.nodes[1].enumeratesigners()['signers'], True) if __name__ == '__main__': RPCSignerTest().main() diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 8e4e1f5d36b67..5609ac9bf5845 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -32,6 +32,13 @@ def mock_invalid_signer_path(self): else: return path + def mock_multi_signers_path(self): + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mocks', 'multi_signers.py') + if platform.system() == "Windows": + return "py " + path + else: + return path + def set_test_params(self): self.num_nodes = 2 # The experimental syscall sandbox feature (-sandbox) is not compatible with -signer (which @@ -58,6 +65,8 @@ def run_test(self): self.test_valid_signer() self.restart_node(1, [f"-signer={self.mock_invalid_signer_path()}", "-keypool=10"]) self.test_invalid_signer() + self.restart_node(1, [f"-signer={self.mock_multi_signers_path()}", "-keypool=10"]) + self.test_multiple_signers() def test_valid_signer(self): self.log.debug(f"-signer={self.mock_signer_path()}") @@ -212,5 +221,11 @@ def test_invalid_signer(self): self.log.info('Test invalid external signer') assert_raises_rpc_error(-1, "Invalid descriptor", self.nodes[1].createwallet, wallet_name='hww_invalid', disable_private_keys=True, descriptors=True, external_signer=True) + def test_multiple_signers(self): + self.log.debug(f"-signer={self.mock_multi_signers_path()}") + self.log.info('Test multiple external signers') + + assert_raises_rpc_error(-1, "GetExternalSigner: More than one external signer found", self.nodes[1].createwallet, wallet_name='multi_hww', disable_private_keys=True, descriptors=True, external_signer=True) + if __name__ == '__main__': WalletSignerTest().main() From 192eb1e61c3c43baec7f32c498ab0ce0656a58f7 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 10:45:08 -0300 Subject: [PATCH 0035/1090] refactor: getAddress don't access m_address_book, use FindAddressEntry function --- src/wallet/interfaces.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index e1203817e0de9..5cfcf16e16424 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -191,18 +191,16 @@ class WalletImpl : public Wallet std::string* purpose) override { LOCK(m_wallet->cs_wallet); - auto it = m_wallet->m_address_book.find(dest); - if (it == m_wallet->m_address_book.end() || it->second.IsChange()) { - return false; - } + const auto& entry = m_wallet->FindAddressBookEntry(dest, /*allow_change=*/false); + if (!entry) return false; // addr not found if (name) { - *name = it->second.GetLabel(); + *name = entry->GetLabel(); } if (is_mine) { *is_mine = m_wallet->IsMine(dest); } if (purpose) { - *purpose = it->second.purpose; + *purpose = entry->purpose; } return true; } From 09649bc95d5f2855a54a8cf02e65215a3b333c92 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:06:22 -0300 Subject: [PATCH 0036/1090] refactor: implement general 'ListAddrBookAddresses' for addressbook destinations lookup --- src/wallet/rpc/coins.cpp | 6 +++--- src/wallet/wallet.cpp | 15 +++++++-------- src/wallet/wallet.h | 13 ++++++++++++- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 2649fa586c102..6050ad7b4ba21 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -18,10 +18,10 @@ namespace wallet { static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { - std::set addresses; + std::vector addresses; if (by_label) { // Get the set of addresses assigned to label - addresses = wallet.GetLabelAddresses(LabelFromValue(params[0])); + addresses = wallet.ListAddrBookAddresses(CWallet::AddrBookFilter{LabelFromValue(params[0])}); if (addresses.empty()) throw JSONRPCError(RPC_WALLET_ERROR, "Label not found in wallet"); } else { // Get the address @@ -29,7 +29,7 @@ static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool b if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } - addresses.insert(dest); + addresses.emplace_back(dest); } // Filter by own scripts only diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 910562e66901a..8ca8ef0a198c7 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2348,17 +2348,16 @@ void CWallet::MarkDestinationsDirty(const std::set& destinations } } -std::set CWallet::GetLabelAddresses(const std::string& label) const +std::vector CWallet::ListAddrBookAddresses(const std::optional& _filter) const { AssertLockHeld(cs_wallet); - std::set result; - for (const std::pair& item : m_address_book) - { - if (item.second.IsChange()) continue; - const CTxDestination& address = item.first; + std::vector result; + AddrBookFilter filter = _filter ? *_filter : AddrBookFilter(); + for (const std::pair& item : m_address_book) { + if (filter.ignore_change && item.second.IsChange()) continue; const std::string& strName = item.second.GetLabel(); - if (strName == label) - result.insert(address); + if (filter.m_op_label && *filter.m_op_label != strName) continue; + result.emplace_back(item.first); } return result; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 7da601c3b7fd3..970d3e2e757d9 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -635,7 +635,18 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati std::optional GetOldestKeyPoolTime() const; - std::set GetLabelAddresses(const std::string& label) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + // Filter struct for 'ListAddrBookAddresses' + struct AddrBookFilter { + // Fetch addresses with the provided label + std::optional m_op_label{std::nullopt}; + // Don't include change addresses by default + bool ignore_change{true}; + }; + + /** + * Filter and retrieve destinations stored in the addressbook + */ + std::vector ListAddrBookAddresses(const std::optional& filter) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Marks all outputs in each one of the destinations dirty, so their cache is From 032842ae4196aaed5ea3567ea01a61ed75ab2edd Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:35:14 -0300 Subject: [PATCH 0037/1090] wallet: implement ForEachAddrBookEntry method --- src/wallet/wallet.cpp | 23 +++++++++++++++++------ src/wallet/wallet.h | 7 +++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 8ca8ef0a198c7..f7eb0bbc03f7e 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2348,17 +2348,28 @@ void CWallet::MarkDestinationsDirty(const std::set& destinations } } +void CWallet::ForEachAddrBookEntry(const ListAddrBookFunc& func) const +{ + AssertLockHeld(cs_wallet); + for (const std::pair& item : m_address_book) { + const auto& entry = item.second; + func(item.first, entry.GetLabel(), entry.purpose, entry.IsChange()); + } +} + std::vector CWallet::ListAddrBookAddresses(const std::optional& _filter) const { AssertLockHeld(cs_wallet); std::vector result; AddrBookFilter filter = _filter ? *_filter : AddrBookFilter(); - for (const std::pair& item : m_address_book) { - if (filter.ignore_change && item.second.IsChange()) continue; - const std::string& strName = item.second.GetLabel(); - if (filter.m_op_label && *filter.m_op_label != strName) continue; - result.emplace_back(item.first); - } + ForEachAddrBookEntry([&result, &filter](const CTxDestination& dest, const std::string& label, const std::string& purpose, bool is_change) { + // Filter by change + if (filter.ignore_change && is_change) return; + // Filter by label + if (filter.m_op_label && *filter.m_op_label != label) return; + // All good + result.emplace_back(dest); + }); return result; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 970d3e2e757d9..3775f325ba40c 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -648,6 +648,13 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati */ std::vector ListAddrBookAddresses(const std::optional& filter) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** + * Walk-through the address book entries. + * Stops when the provided 'ListAddrBookFunc' returns false. + */ + using ListAddrBookFunc = std::function; + void ForEachAddrBookEntry(const ListAddrBookFunc& func) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** * Marks all outputs in each one of the destinations dirty, so their cache is * reset and does not return outdated information. From 2b48642499016cb357e4bcec32481cd50361194e Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:38:19 -0300 Subject: [PATCH 0038/1090] refactor: use ForEachAddrBookEntry in interfaces::getAddresses --- src/interfaces/wallet.h | 2 +- src/wallet/interfaces.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index f26ac866dcf95..b3cb0ae3875d8 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -114,7 +114,7 @@ class Wallet std::string* purpose) = 0; //! Get wallet address list. - virtual std::vector getAddresses() = 0; + virtual std::vector getAddresses() const = 0; //! Get receive requests. virtual std::vector getAddressReceiveRequests() = 0; diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 5cfcf16e16424..823deed71ba91 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -204,14 +204,14 @@ class WalletImpl : public Wallet } return true; } - std::vector getAddresses() override + std::vector getAddresses() const override { LOCK(m_wallet->cs_wallet); std::vector result; - for (const auto& item : m_wallet->m_address_book) { - if (item.second.IsChange()) continue; - result.emplace_back(item.first, m_wallet->IsMine(item.first), item.second.GetLabel(), item.second.purpose); - } + m_wallet->ForEachAddrBookEntry([&](const CTxDestination& dest, const std::string& label, const std::string& purpose, bool is_change) EXCLUSIVE_LOCKS_REQUIRED(m_wallet->cs_wallet) { + if (is_change) return; + result.emplace_back(dest, m_wallet->IsMine(dest), label, purpose); + }); return result; } std::vector getAddressReceiveRequests() override { From 83e42c4b94e376a19d3eb0a2379769b8b8ac5fc8 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:39:25 -0300 Subject: [PATCH 0039/1090] refactor: use 'ForEachAddrBookEntry' in RPC 'getaddressesbylabel' --- src/wallet/rpc/addresses.cpp | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index f25ad595282b0..55dd2f935c477 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -637,17 +637,6 @@ RPCHelpMan getaddressinfo() }; } -/** Convert CAddressBookData to JSON record. */ -static UniValue AddressBookDataToJSON(const CAddressBookData& data, const bool verbose) -{ - UniValue ret(UniValue::VOBJ); - if (verbose) { - ret.pushKV("name", data.GetLabel()); - } - ret.pushKV("purpose", data.purpose); - return ret; -} - RPCHelpMan getaddressesbylabel() { return RPCHelpMan{"getaddressesbylabel", @@ -680,10 +669,10 @@ RPCHelpMan getaddressesbylabel() // Find all addresses that have the given label UniValue ret(UniValue::VOBJ); std::set addresses; - for (const std::pair& item : pwallet->m_address_book) { - if (item.second.IsChange()) continue; - if (item.second.GetLabel() == label) { - std::string address = EncodeDestination(item.first); + pwallet->ForEachAddrBookEntry([&](const CTxDestination& _dest, const std::string& _label, const std::string& _purpose, bool _is_change) { + if (_is_change) return; + if (_label == label) { + std::string address = EncodeDestination(_dest); // CWallet::m_address_book is not expected to contain duplicate // address strings, but build a separate set as a precaution just in // case it does. @@ -693,9 +682,11 @@ RPCHelpMan getaddressesbylabel() // and since duplicate addresses are unexpected (checked with // std::set in O(log(N))), UniValue::__pushKV is used instead, // which currently is O(1). - ret.__pushKV(address, AddressBookDataToJSON(item.second, false)); + UniValue value(UniValue::VOBJ); + value.pushKV("purpose", _purpose); + ret.__pushKV(address, value); } - } + }); if (ret.empty()) { throw JSONRPCError(RPC_WALLET_INVALID_LABEL_NAME, std::string("No addresses with label " + label)); From fa9f2ab8fd53075d2a3ec93ddac4908e73525c46 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:46:14 -0300 Subject: [PATCH 0040/1090] refactor: RPC 'listlabels', encapsulate 'CWallet::ListAddrBookLabels' functionality Mainly to not access 'm_address_book' externally. --- src/wallet/rpc/addresses.cpp | 8 +------- src/wallet/wallet.cpp | 14 ++++++++++++++ src/wallet/wallet.h | 5 +++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 55dd2f935c477..da4cc44ee68c7 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -733,13 +733,7 @@ RPCHelpMan listlabels() } // Add to a set to sort by label name, then insert into Univalue array - std::set label_set; - for (const std::pair& entry : pwallet->m_address_book) { - if (entry.second.IsChange()) continue; - if (purpose.empty() || entry.second.purpose == purpose) { - label_set.insert(entry.second.GetLabel()); - } - } + std::set label_set = pwallet->ListAddrBookLabels(purpose); UniValue ret(UniValue::VARR); for (const std::string& name : label_set) { diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f7eb0bbc03f7e..3c211fb3482cc 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2373,6 +2373,20 @@ std::vector CWallet::ListAddrBookAddresses(const std::optional CWallet::ListAddrBookLabels(const std::string& purpose) const +{ + AssertLockHeld(cs_wallet); + std::set label_set; + ForEachAddrBookEntry([&](const CTxDestination& _dest, const std::string& _label, + const std::string& _purpose, bool _is_change) { + if (_is_change) return; + if (purpose.empty() || _purpose == purpose) { + label_set.insert(_label); + } + }); + return label_set; +} + bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool internal, bilingual_str& error) { m_spk_man = pwallet->GetScriptPubKeyMan(type, internal); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 3775f325ba40c..8bc1189bec15b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -648,6 +648,11 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati */ std::vector ListAddrBookAddresses(const std::optional& filter) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** + * Retrieve all the known labels in the address book + */ + std::set ListAddrBookLabels(const std::string& purpose) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** * Walk-through the address book entries. * Stops when the provided 'ListAddrBookFunc' returns false. From b459fc122feace9e9a738c48aab21961cf15dddc Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 12:00:33 -0300 Subject: [PATCH 0041/1090] refactor: RPC 'ListReceived', encapsulate m_address_book access --- src/wallet/rpc/transactions.cpp | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index fae9bf3ea5a0e..5d9b0965bcaac 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -138,26 +138,12 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons UniValue ret(UniValue::VARR); std::map label_tally; - // Create m_address_book iterator - // If we aren't filtering, go from begin() to end() - auto start = wallet.m_address_book.begin(); - auto end = wallet.m_address_book.end(); - // If we are filtering, find() the applicable entry - if (has_filtered_address) { - start = wallet.m_address_book.find(filtered_address); - if (start != end) { - end = std::next(start); - } - } + const auto& func = [&](const CTxDestination& address, const std::string& label, const std::string& purpose, bool is_change) { + if (is_change) return; // no change addresses - for (auto item_it = start; item_it != end; ++item_it) - { - if (item_it->second.IsChange()) continue; - const CTxDestination& address = item_it->first; - const std::string& label = item_it->second.GetLabel(); auto it = mapTally.find(address); if (it == mapTally.end() && !fIncludeEmpty) - continue; + return; CAmount nAmount = 0; int nConf = std::numeric_limits::max(); @@ -196,6 +182,14 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons obj.pushKV("txids", transactions); ret.push_back(obj); } + }; + + if (has_filtered_address) { + const auto& entry = wallet.FindAddressBookEntry(filtered_address, /*allow_change=*/false); + if (entry) func(filtered_address, entry->GetLabel(), entry->purpose, /*is_change=*/false); + } else { + // No filtered addr, walk-through the addressbook entry + wallet.ForEachAddrBookEntry(func); } if (by_label) From 324f00a6420bbd64c67c264e50632e6fa36ae732 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 12:05:54 -0300 Subject: [PATCH 0042/1090] refactor: 'ListReceived' use optional for filtered address Plus remove open bracket jump line --- src/wallet/rpc/transactions.cpp | 45 ++++++++++++--------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index 5d9b0965bcaac..6e42c0736d1f3 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -85,14 +85,12 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons filter |= ISMINE_WATCH_ONLY; } - bool has_filtered_address = false; - CTxDestination filtered_address = CNoDestination(); + std::optional filtered_address{std::nullopt}; if (!by_label && !params[3].isNull() && !params[3].get_str().empty()) { if (!IsValidDestinationString(params[3].get_str())) { throw JSONRPCError(RPC_WALLET_ERROR, "address_filter parameter was invalid"); } filtered_address = DecodeDestination(params[3].get_str()); - has_filtered_address = true; } // Tally @@ -106,23 +104,21 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons // Coinbase with less than 1 confirmation is no longer in the main chain if ((wtx.IsCoinBase() && (nDepth < 1)) - || (wallet.IsTxImmatureCoinBase(wtx) && !include_immature_coinbase)) - { + || (wallet.IsTxImmatureCoinBase(wtx) && !include_immature_coinbase)) { continue; } - for (const CTxOut& txout : wtx.tx->vout) - { + for (const CTxOut& txout : wtx.tx->vout) { CTxDestination address; if (!ExtractDestination(txout.scriptPubKey, address)) continue; - if (has_filtered_address && !(filtered_address == address)) { + if (filtered_address && !(filtered_address == address)) { continue; } isminefilter mine = wallet.IsMine(address); - if(!(mine & filter)) + if (!(mine & filter)) continue; tallyitem& item = mapTally[address]; @@ -148,34 +144,27 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons CAmount nAmount = 0; int nConf = std::numeric_limits::max(); bool fIsWatchonly = false; - if (it != mapTally.end()) - { + if (it != mapTally.end()) { nAmount = (*it).second.nAmount; nConf = (*it).second.nConf; fIsWatchonly = (*it).second.fIsWatchonly; } - if (by_label) - { + if (by_label) { tallyitem& _item = label_tally[label]; _item.nAmount += nAmount; _item.nConf = std::min(_item.nConf, nConf); _item.fIsWatchonly = fIsWatchonly; - } - else - { + } else { UniValue obj(UniValue::VOBJ); - if(fIsWatchonly) - obj.pushKV("involvesWatchonly", true); + if (fIsWatchonly) obj.pushKV("involvesWatchonly", true); obj.pushKV("address", EncodeDestination(address)); obj.pushKV("amount", ValueFromAmount(nAmount)); obj.pushKV("confirmations", (nConf == std::numeric_limits::max() ? 0 : nConf)); obj.pushKV("label", label); UniValue transactions(UniValue::VARR); - if (it != mapTally.end()) - { - for (const uint256& _item : (*it).second.txids) - { + if (it != mapTally.end()) { + for (const uint256& _item : (*it).second.txids) { transactions.push_back(_item.GetHex()); } } @@ -184,18 +173,16 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons } }; - if (has_filtered_address) { - const auto& entry = wallet.FindAddressBookEntry(filtered_address, /*allow_change=*/false); - if (entry) func(filtered_address, entry->GetLabel(), entry->purpose, /*is_change=*/false); + if (filtered_address) { + const auto& entry = wallet.FindAddressBookEntry(*filtered_address, /*allow_change=*/false); + if (entry) func(*filtered_address, entry->GetLabel(), entry->purpose, /*is_change=*/false); } else { // No filtered addr, walk-through the addressbook entry wallet.ForEachAddrBookEntry(func); } - if (by_label) - { - for (const auto& entry : label_tally) - { + if (by_label) { + for (const auto& entry : label_tally) { CAmount nAmount = entry.second.nAmount; int nConf = entry.second.nConf; UniValue obj(UniValue::VOBJ); From d69045e291e32e02d105d1b5ff1c8b86db0ae69e Mon Sep 17 00:00:00 2001 From: furszy Date: Wed, 22 Jun 2022 12:46:43 -0300 Subject: [PATCH 0043/1090] test: add coverage for 'listreceivedbyaddress' no change addrs return --- test/functional/wallet_listreceivedby.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index db1d8eb54aec9..7ae3a40eaf39c 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -57,6 +57,11 @@ def run_test(self): {"address": empty_addr}, {"address": empty_addr, "label": "", "amount": 0, "confirmations": 0, "txids": []}) + # No returned addy should be a change addr + for node in self.nodes: + for addr_obj in node.listreceivedbyaddress(): + assert_equal(node.getaddressinfo(addr_obj["address"])["ischange"], False) + # Test Address filtering # Only on addr expected = {"address": addr, "label": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]} From a89ddfbe22b6db5beda678c9493e08fec6144122 Mon Sep 17 00:00:00 2001 From: w0xlt <94266259+w0xlt@users.noreply.github.com> Date: Fri, 29 Apr 2022 19:10:57 -0300 Subject: [PATCH 0044/1090] wallet: Save wallet scan progress Currently, the wallet scan progress is not saved. If it is interrupted, it will be necessary to start from scratch on the next load. With this change, progress is saved every 60 seconds. Co-authored-by: furszy Co-authored-by: Jon Atack Co-authored-by: Ryan Ofsky --- src/interfaces/chain.h | 4 ++++ src/node/interfaces.cpp | 17 ++++++++++++----- src/qt/test/wallettests.cpp | 2 +- src/wallet/rpc/transactions.cpp | 2 +- src/wallet/test/util.cpp | 2 +- src/wallet/test/wallet_tests.cpp | 8 ++++---- src/wallet/wallet.cpp | 22 +++++++++++++++++----- src/wallet/wallet.h | 2 +- 8 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index ddfb4bda95b11..df9e55874f38f 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -111,6 +111,10 @@ class Chain //! Get locator for the current chain tip. virtual CBlockLocator getTipLocator() = 0; + //! Return a locator that refers to a block in the active chain. + //! If specified block is not in the active chain, return locator for the latest ancestor that is in the chain. + virtual CBlockLocator getActiveChainLocator(const uint256& block_hash) = 0; + //! Return height of the highest block on chain in common with the locator, //! which will either be the original block used to create the locator, //! or one of its ancestors. diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 4810ae1f68cbd..1930d0a0dbfda 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -516,20 +516,27 @@ class ChainImpl : public Chain } bool haveBlockOnDisk(int height) override { - LOCK(cs_main); + LOCK(::cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); CBlockIndex* block = active[height]; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } CBlockLocator getTipLocator() override { - LOCK(cs_main); + LOCK(::cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); return active.GetLocator(); } + CBlockLocator getActiveChainLocator(const uint256& block_hash) override + { + LOCK(::cs_main); + const CBlockIndex* index = chainman().m_blockman.LookupBlockIndex(block_hash); + if (!index) return {}; + return chainman().ActiveChain().GetLocator(index); + } std::optional findLocatorFork(const CBlockLocator& locator) override { - LOCK(cs_main); + LOCK(::cs_main); const CChainState& active = Assert(m_node.chainman)->ActiveChainstate(); if (const CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { return fork->nHeight; @@ -585,7 +592,7 @@ class ChainImpl : public Chain void findCoins(std::map& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override { - LOCK(cs_main); + LOCK(::cs_main); return GuessVerificationProgress(chainman().GetParams().TxData(), chainman().m_blockman.LookupBlockIndex(block_hash)); } bool hasBlocks(const uint256& block_hash, int min_height, std::optional max_height) override @@ -684,7 +691,7 @@ class ChainImpl : public Chain CFeeRate relayDustFee() override { return ::dustRelayFee; } bool havePruned() override { - LOCK(cs_main); + LOCK(::cs_main); return m_node.chainman->m_blockman.m_have_pruned; } bool isReadyToBroadcast() override { return !node::fImporting && !node::fReindex && !isInitialBlockDownload(); } diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index bc06f0f23be44..3740aa9801930 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -173,7 +173,7 @@ void TestGUI(interfaces::Node& node) { WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, 0 /* block height */, {} /* max height */, reserver, true /* fUpdate */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/true, /*save_progress=*/false); QCOMPARE(result.status, CWallet::ScanResult::SUCCESS); QCOMPARE(result.last_scanned_block, node.context()->chainman->ActiveChain().Tip()->GetBlockHash()); QVERIFY(result.last_failed_block.IsNull()); diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index fae9bf3ea5a0e..f4e46a5982eb6 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -908,7 +908,7 @@ RPCHelpMan rescanblockchain() } CWallet::ScanResult result = - pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, true /* fUpdate */); + pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, /*fUpdate=*/true, /*save_progress=*/false); switch (result.status) { case CWallet::ScanResult::SUCCESS: break; diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index aa3121511d85e..ab72721f9d305 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -38,7 +38,7 @@ std::unique_ptr CreateSyncedWallet(interfaces::Chain& chain, CChain& cc } WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(cchain.Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(cchain.Genesis()->GetBlockHash(), /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK_EQUAL(result.last_scanned_block, cchain.Tip()->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, cchain.Height()); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 70863f5464ab8..4fbce7fc47f7c 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -112,7 +112,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/{}, /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); @@ -132,7 +132,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -161,7 +161,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -188,7 +188,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6c333c709b7d3..56fa3eb1b0021 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1674,7 +1674,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r if (start) { // TODO: this should take into account failure by ScanResult::USER_ABORT - ScanResult result = ScanForWalletTransactions(start_block, start_height, {} /* max_height */, reserver, update); + ScanResult result = ScanForWalletTransactions(start_block, start_height, /*max_height=*/{}, reserver, /*fUpdate=*/update, /*save_progress=*/false); if (result.status == ScanResult::FAILURE) { int64_t time_max; CHECK_NONFATAL(chain().findBlock(result.last_failed_block, FoundBlock().maxTime(time_max))); @@ -1705,10 +1705,10 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * the main chain after to the addition of any new keys you want to detect * transactions for. */ -CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate) +CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress) { using Clock = std::chrono::steady_clock; - constexpr auto LOG_INTERVAL{60s}; + constexpr auto INTERVAL_TIME{60s}; auto current_time{Clock::now()}; auto start_time{Clock::now()}; @@ -1737,7 +1737,9 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc if (block_height % 100 == 0 && progress_end - progress_begin > 0.0) { ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } - if (Clock::now() >= current_time + LOG_INTERVAL) { + + bool next_interval = Clock::now() >= current_time + INTERVAL_TIME; + if (next_interval) { current_time = Clock::now(); WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } @@ -1768,6 +1770,16 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; result.last_scanned_height = block_height; + + if (save_progress && next_interval) { + CBlockLocator loc = m_chain->getActiveChainLocator(block_hash); + + if (!loc.IsNull()) { + WalletLogPrintf("Saving scan progress %d.\n", block_height); + WalletBatch batch(GetDatabase()); + batch.WriteBestBlock(loc); + } + } } else { // could not scan block, keep scanning but record this block as the most recent failure result.last_failed_block = block_hash; @@ -3026,7 +3038,7 @@ bool CWallet::AttachChain(const std::shared_ptr& walletInstance, interf { WalletRescanReserver reserver(*walletInstance); - if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(chain.getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { + if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(chain.getBlockHash(rescan_height), rescan_height, /*max_height=*/{}, reserver, /*fUpdate=*/true, /*save_progress=*/true).status)) { error = _("Failed to rescan the wallet during initialization"); return false; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 7da601c3b7fd3..206a4417a7e3a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -528,7 +528,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati //! USER_ABORT. uint256 last_failed_block; }; - ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate); + ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress); void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); From 230a2f4cc3fab9f66b6c24ba809ddbea77755cb7 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 8 Jun 2022 23:05:16 -0300 Subject: [PATCH 0045/1090] wallet test: Add unit test for wallet scan save_progress option --- src/wallet/test/wallet_tests.cpp | 19 +++++++++++++++++-- src/wallet/wallet.cpp | 11 +++++------ src/wallet/wallet.h | 7 +++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 4fbce7fc47f7c..3483f05292b04 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -123,7 +123,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -131,13 +131,28 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); + std::chrono::steady_clock::time_point fake_time; + reserver.setNow([&] { fake_time += 60s; return fake_time; }); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); + + { + CBlockLocator locator; + BOOST_CHECK(!WalletBatch{wallet.GetDatabase()}.ReadBestBlock(locator)); + BOOST_CHECK(locator.IsNull()); + } + + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/true); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); BOOST_CHECK_EQUAL(GetBalance(wallet).m_mine_immature, 100 * COIN); + + { + CBlockLocator locator; + BOOST_CHECK(WalletBatch{wallet.GetDatabase()}.ReadBestBlock(locator)); + BOOST_CHECK(!locator.IsNull()); + } } // Prune the older block file. diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 56fa3eb1b0021..91106cde83229 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1707,10 +1707,9 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r */ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress) { - using Clock = std::chrono::steady_clock; constexpr auto INTERVAL_TIME{60s}; - auto current_time{Clock::now()}; - auto start_time{Clock::now()}; + auto current_time{reserver.now()}; + auto start_time{reserver.now()}; assert(reserver.isReserved()); @@ -1738,9 +1737,9 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } - bool next_interval = Clock::now() >= current_time + INTERVAL_TIME; + bool next_interval = reserver.now() >= current_time + INTERVAL_TIME; if (next_interval) { - current_time = Clock::now(); + current_time = reserver.now(); WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } @@ -1817,7 +1816,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else { - auto duration_milliseconds = std::chrono::duration_cast(Clock::now() - start_time); + auto duration_milliseconds = std::chrono::duration_cast(reserver.now() - start_time); WalletLogPrintf("Rescan completed in %15dms\n", duration_milliseconds.count()); } return result; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 206a4417a7e3a..5be2ad166a187 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -896,8 +896,11 @@ void MaybeResendWalletTxs(WalletContext& context); class WalletRescanReserver { private: + using Clock = std::chrono::steady_clock; + using NowFn = std::function; CWallet& m_wallet; bool m_could_reserve; + NowFn m_now; public: explicit WalletRescanReserver(CWallet& w) : m_wallet(w), m_could_reserve(false) {} @@ -918,6 +921,10 @@ class WalletRescanReserver return (m_could_reserve && m_wallet.fScanningWallet); } + Clock::time_point now() const { return m_now ? m_now() : Clock::now(); }; + + void setNow(NowFn now) { m_now = std::move(now); } + ~WalletRescanReserver() { if (m_could_reserve) { From fa8a1c06961f4b1826696e0db8dce81dce627721 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Thu, 23 Jun 2022 21:48:43 +0200 Subject: [PATCH 0046/1090] rpc: Fix Univalue push_backV OOM in listtransactions --- src/univalue/include/univalue.h | 10 ++++++++++ src/wallet/rpc/transactions.cpp | 12 ++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 22be0311e835e..7f9a6aaffd989 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -84,6 +84,8 @@ class UniValue { bool push_back(const UniValue& val); bool push_backV(const std::vector& vec); + template + bool push_backV(It first, It last); void __pushKV(const std::string& key, const UniValue& val); bool pushKV(const std::string& key, const UniValue& val); @@ -137,6 +139,14 @@ class UniValue { friend const UniValue& find_value( const UniValue& obj, const std::string& name); }; +template +bool UniValue::push_backV(It first, It last) +{ + if (typ != VARR) return false; + values.insert(values.end(), first, last); + return true; +} + enum jtokentype { JTOK_ERR = -1, JTOK_NONE = 0, // eof diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index fae9bf3ea5a0e..cb303d1cf6a76 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -329,11 +329,12 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) * @param wtx The wallet transaction. * @param nMinDepth The minimum confirmation depth. * @param fLong Whether to include the JSON version of the transaction. - * @param ret The UniValue into which the result is stored. + * @param ret The vector into which the result is stored. * @param filter_ismine The "is mine" filter flags. * @param filter_label Optional label string to filter incoming transactions. */ -static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +template +static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nMinDepth, bool fLong, Vec& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { CAmount nFee; std::list listReceived; @@ -519,8 +520,7 @@ RPCHelpMan listtransactions() if (nFrom < 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative from"); - UniValue ret(UniValue::VARR); - + std::vector ret; { LOCK(pwallet->cs_wallet); @@ -542,9 +542,9 @@ RPCHelpMan listtransactions() if ((nFrom + nCount) > (int)ret.size()) nCount = ret.size() - nFrom; - const std::vector& txs = ret.getValues(); + auto txs_rev_it{std::make_move_iterator(ret.rend())}; UniValue result{UniValue::VARR}; - result.push_backV({ txs.rend() - nFrom - nCount, txs.rend() - nFrom }); // Return oldest to newest + result.push_backV(txs_rev_it - nFrom - nCount, txs_rev_it - nFrom); // Return oldest to newest return result; }, }; From 2ef5294a5bb68ceb3797d2638567a172cc21699f Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Fri, 22 Apr 2022 11:11:39 +0200 Subject: [PATCH 0047/1090] rpc: add RPCTypeCheck for getblockfrompeer inputs --- src/rpc/blockchain.cpp | 5 +++++ test/functional/rpc_getblockfrompeer.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f46e5e9feff3b..659e9dcd6a14f 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -439,6 +439,11 @@ static RPCHelpMan getblockfrompeer() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + RPCTypeCheck(request.params, { + UniValue::VSTR, // blockhash + UniValue::VNUM, // peer_id + }); + const NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); PeerManager& peerman = EnsurePeerman(node); diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index cc9ccf306628d..a379132db4e42 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -51,8 +51,8 @@ def run_test(self): self.log.info("Arguments must be valid") assert_raises_rpc_error(-8, "hash must be of length 64 (not 4, for '1234')", self.nodes[0].getblockfrompeer, "1234", peer_0_peer_1_id) - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].getblockfrompeer, 1234, peer_0_peer_1_id) - assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].getblockfrompeer, short_tip, "0") + assert_raises_rpc_error(-3, "Expected type string, got number", self.nodes[0].getblockfrompeer, 1234, peer_0_peer_1_id) + assert_raises_rpc_error(-3, "Expected type number, got string", self.nodes[0].getblockfrompeer, short_tip, "0") self.log.info("We must already have the header") assert_raises_rpc_error(-1, "Block header missing", self.nodes[0].getblockfrompeer, "00" * 32, 0) From e7a3f698b5f3f7dde8632c4911abd4e5bc1f06cb Mon Sep 17 00:00:00 2001 From: w0xlt <94266259+w0xlt@users.noreply.github.com> Date: Thu, 16 Dec 2021 18:22:09 -0300 Subject: [PATCH 0048/1090] gui: Add Wallet Restore in the GUI Co-authored-by: Shashwat Vangani <85434418+shaavan@users.noreply.github.com> Co-authored-by: furszy --- src/qt/bitcoingui.cpp | 33 +++++++++++++++++++++++++++- src/qt/bitcoingui.h | 1 + src/qt/walletcontroller.cpp | 43 +++++++++++++++++++++++++++++++++++++ src/qt/walletcontroller.h | 20 +++++++++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index bfcdf6f3164e9..0f05ef2a3bb56 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -348,6 +349,12 @@ void BitcoinGUI::createActions() m_create_wallet_action->setEnabled(false); m_create_wallet_action->setStatusTip(tr("Create a new wallet")); + //: Name of the menu item that restores wallet from a backup file. + m_restore_wallet_action = new QAction(tr("Restore Wallet…"), this); + m_restore_wallet_action->setEnabled(false); + //: Status tip for Restore Wallet menu item + m_restore_wallet_action->setStatusTip(tr("Restore a wallet from a backup file")); + m_close_all_wallets_action = new QAction(tr("Close All Wallets…"), this); m_close_all_wallets_action->setStatusTip(tr("Close all wallets")); @@ -412,6 +419,27 @@ void BitcoinGUI::createActions() action->setEnabled(false); } }); + connect(m_restore_wallet_action, &QAction::triggered, [this] { + //: Name of the wallet data file format. + QString name_data_file = tr("Wallet Data"); + + //: The title for Restore Wallet File Windows + QString title_windows = tr("Load Wallet Backup"); + + QString backup_file = GUIUtil::getOpenFileName(this, title_windows, QString(), name_data_file + QLatin1String(" (*.dat)"), nullptr); + if (backup_file.isEmpty()) return; + + bool wallet_name_ok; + //: Title of the Restore Wallet input dialog (where the wallet name is entered) + QString wallet_name = QInputDialog::getText(this, tr("Restore Name"), tr("Wallet Name:"), QLineEdit::Normal, "", &wallet_name_ok); + if (!wallet_name_ok || wallet_name.isEmpty()) return; + + auto activity = new RestoreWalletActivity(m_wallet_controller, this); + connect(activity, &RestoreWalletActivity::restored, this, &BitcoinGUI::setCurrentWallet, Qt::QueuedConnection); + + auto backup_file_path = fs::PathFromString(backup_file.toStdString()); + activity->restore(backup_file_path, wallet_name.toStdString()); + }); connect(m_close_wallet_action, &QAction::triggered, [this] { m_wallet_controller->closeWallet(walletFrame->currentWalletModel(), this); }); @@ -450,8 +478,10 @@ void BitcoinGUI::createMenuBar() file->addAction(m_close_wallet_action); file->addAction(m_close_all_wallets_action); file->addSeparator(); - file->addAction(openAction); file->addAction(backupWalletAction); + file->addAction(m_restore_wallet_action); + file->addSeparator(); + file->addAction(openAction); file->addAction(signMessageAction); file->addAction(verifyMessageAction); file->addAction(m_load_psbt_action); @@ -642,6 +672,7 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) m_create_wallet_action->setEnabled(true); m_open_wallet_action->setEnabled(true); m_open_wallet_action->setMenu(m_open_wallet_menu); + m_restore_wallet_action->setEnabled(true); GUIUtil::ExceptionSafeConnect(wallet_controller, &WalletController::walletAdded, this, &BitcoinGUI::addWallet); connect(wallet_controller, &WalletController::walletRemoved, this, &BitcoinGUI::removeWallet); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 6272d5d947fc8..b2e13245e1d15 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -157,6 +157,7 @@ class BitcoinGUI : public QMainWindow QAction* m_create_wallet_action{nullptr}; QAction* m_open_wallet_action{nullptr}; QMenu* m_open_wallet_menu{nullptr}; + QAction* m_restore_wallet_action{nullptr}; QAction* m_close_wallet_action{nullptr}; QAction* m_close_all_wallets_action{nullptr}; QAction* m_wallet_selector_label_action = nullptr; diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index d27ddf1aba52b..11140c5da973f 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -373,3 +373,46 @@ void LoadWalletsActivity::load() QTimer::singleShot(0, this, [this] { Q_EMIT finished(); }); }); } + +RestoreWalletActivity::RestoreWalletActivity(WalletController* wallet_controller, QWidget* parent_widget) + : WalletControllerActivity(wallet_controller, parent_widget) +{ +} + +void RestoreWalletActivity::restore(const fs::path& backup_file, const std::string& wallet_name) +{ + QString name = QString::fromStdString(wallet_name); + + showProgressDialog( + //: Title of progress window which is displayed when wallets are being restored. + tr("Restore Wallet"), + /*: Descriptive text of the restore wallets progress window which indicates to + the user that wallets are currently being restored.*/ + tr("Restoring Wallet %1…").arg(name.toHtmlEscaped())); + + QTimer::singleShot(0, worker(), [this, backup_file, wallet_name] { + std::unique_ptr wallet = node().walletLoader().restoreWallet(backup_file, wallet_name, m_error_message, m_warning_message); + + if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(wallet)); + + QTimer::singleShot(0, this, &RestoreWalletActivity::finish); + }); +} + +void RestoreWalletActivity::finish() +{ + if (!m_error_message.empty()) { + //: Title of message box which is displayed when the wallet could not be restored. + QMessageBox::critical(m_parent_widget, tr("Restore wallet failed"), QString::fromStdString(m_error_message.translated)); + } else if (!m_warning_message.empty()) { + //: Title of message box which is displayed when the wallet is restored with some warning. + QMessageBox::warning(m_parent_widget, tr("Restore wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); + } else { + //: Title of message box which is displayed when the wallet is successfully restored. + QMessageBox::information(m_parent_widget, tr("Restore wallet message"), QString::fromStdString(Untranslated("Wallet restored successfully \n").translated)); + } + + if (m_wallet_model) Q_EMIT restored(m_wallet_model); + + Q_EMIT finished(); +} diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index 24f85c258c805..fcd65756c67c7 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -33,6 +33,10 @@ class Node; class Wallet; } // namespace interfaces +namespace fs { +class path; +} + class AskPassphraseDialog; class CreateWalletActivity; class CreateWalletDialog; @@ -155,4 +159,20 @@ class LoadWalletsActivity : public WalletControllerActivity void load(); }; +class RestoreWalletActivity : public WalletControllerActivity +{ + Q_OBJECT + +public: + RestoreWalletActivity(WalletController* wallet_controller, QWidget* parent_widget); + + void restore(const fs::path& backup_file, const std::string& wallet_name); + +Q_SIGNALS: + void restored(WalletModel* wallet_model); + +private: + void finish(); +}; + #endif // BITCOIN_QT_WALLETCONTROLLER_H From a94659c84ee10ac5915eb5a6b654435183d88521 Mon Sep 17 00:00:00 2001 From: S3RK <1466284+S3RK@users.noreply.github.com> Date: Mon, 27 Jun 2022 09:09:09 +0200 Subject: [PATCH 0049/1090] wallet: replace GetTxSpendSize with CalculateMaximumSignedInputSize --- src/bench/coin_selection.cpp | 3 ++- src/wallet/spend.cpp | 12 ++++-------- src/wallet/spend.h | 3 --- src/wallet/test/coinselector_tests.cpp | 3 ++- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index b2958bcc9f0f9..f8914e762ce93 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -58,7 +58,8 @@ static void CoinSelection(benchmark::Bench& bench) // Create coins std::vector coins; for (const auto& wtx : wtxs) { - coins.emplace_back(COutPoint(wtx->GetHash(), 0), wtx->tx->vout.at(0), /*depth=*/6 * 24, GetTxSpendSize(wallet, *wtx, 0), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); + const auto txout = wtx->tx->vout.at(0); + coins.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, false), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); } const CoinEligibilityFilter filter_standard(1, 6, 0); diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 5799a9ff2ab1e..5d9cecc3984e1 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -27,11 +27,6 @@ using interfaces::FoundBlock; namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; -int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig) -{ - return CalculateMaximumSignedInputSize(wtx.tx->vout[out], &wallet, use_max_sig); -} - int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) { CMutableTransaction txn; @@ -198,7 +193,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, // Filter by spendable outputs only if (!spendable && only_spendable) continue; - int input_bytes = GetTxSpendSize(wallet, wtx, i, (coinControl && coinControl->fAllowWatchOnly)); + int input_bytes = CalculateMaximumSignedInputSize(output, provider.get(), coinControl && coinControl->fAllowWatchOnly); result.coins.emplace_back(outpoint, output, nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me, feerate); result.total_amount += output.nValue; @@ -289,8 +284,9 @@ std::map> ListCoins(const CWallet& wallet) ) { CTxDestination address; if (ExtractDestination(FindNonChangeParentOutput(wallet, *wtx.tx, output.n).scriptPubKey, address)) { + const auto out = wtx.tx->vout.at(output.n); result[address].emplace_back( - COutPoint(wtx.GetHash(), output.n), wtx.tx->vout.at(output.n), depth, GetTxSpendSize(wallet, wtx, output.n), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); + COutPoint(wtx.GetHash(), output.n), out, depth, CalculateMaximumSignedInputSize(out, &wallet, false), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); } } } @@ -447,7 +443,7 @@ std::optional SelectCoins(const CWallet& wallet, const std::vec if (ptr_wtx->tx->vout.size() <= outpoint.n) { return std::nullopt; } - input_bytes = GetTxSpendSize(wallet, *ptr_wtx, outpoint.n, false); + input_bytes = CalculateMaximumSignedInputSize(ptr_wtx->tx->vout[outpoint.n], &wallet, false); txout = ptr_wtx->tx->vout.at(outpoint.n); } else { // The input is external. We did not find the tx in mapWallet. diff --git a/src/wallet/spend.h b/src/wallet/spend.h index cba42d6fae5c3..a7d91eb9c417a 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -16,9 +16,6 @@ namespace wallet { /** Get the marginal bytes if spending the specified output from this transaction. * use_max_sig indicates whether to use the maximum sized, 72 byte signature when calculating the * size of the input spend. This should only be set when watch-only outputs are allowed */ -int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig = false); - -//Get the marginal bytes of spending the specified output int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index d6f47e99548af..0664b64244bbd 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -86,7 +86,8 @@ static void add_coin(std::vector& coins, CWallet& wallet, const CAmount auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); CWalletTx& wtx = (*ret.first).second; - coins.emplace_back(COutPoint(wtx.GetHash(), nInput), wtx.tx->vout.at(nInput), nAge, GetTxSpendSize(wallet, wtx, nInput), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); + const auto txout = wtx.tx->vout.at(nInput); + coins.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, false), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); } /** Check if SelectionResult a is equivalent to SelectionResult b. From d54c5c8b1b1a38b5b38e6878aea0fa8d6c1ad7e9 Mon Sep 17 00:00:00 2001 From: S3RK <1466284+S3RK@users.noreply.github.com> Date: Mon, 27 Jun 2022 09:11:09 +0200 Subject: [PATCH 0050/1090] wallet: use CCoinControl to estimate signature size --- src/bench/coin_selection.cpp | 2 +- src/wallet/spend.cpp | 18 +++++++++--------- src/wallet/spend.h | 14 +++++--------- src/wallet/test/coinselector_tests.cpp | 4 ++-- src/wallet/wallet.cpp | 14 +++++++------- src/wallet/wallet.h | 2 +- 6 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index f8914e762ce93..eaefb9b63a884 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -59,7 +59,7 @@ static void CoinSelection(benchmark::Bench& bench) std::vector coins; for (const auto& wtx : wtxs) { const auto txout = wtx->tx->vout.at(0); - coins.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, false), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); + coins.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); } const CoinEligibilityFilter filter_standard(1, 6, 0); diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 5d9cecc3984e1..583c021b8f9fe 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -27,20 +27,20 @@ using interfaces::FoundBlock; namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; -int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const COutPoint outpoint, const SigningProvider* provider, const CCoinControl* coin_control) { CMutableTransaction txn; - txn.vin.push_back(CTxIn(COutPoint())); - if (!provider || !DummySignInput(*provider, txn.vin[0], txout, use_max_sig)) { + txn.vin.push_back(CTxIn(outpoint)); + if (!provider || !DummySignInput(*provider, txn.vin[0], txout, coin_control)) { return -1; } return GetVirtualTransactionInputSize(txn.vin[0]); } -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, const CCoinControl* coin_control) { const std::unique_ptr provider = wallet->GetSolvingProvider(txout.scriptPubKey); - return CalculateMaximumSignedInputSize(txout, provider.get(), use_max_sig); + return CalculateMaximumSignedInputSize(txout, COutPoint(), provider.get(), coin_control); } // txouts needs to be in the order of tx.vin @@ -193,7 +193,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, // Filter by spendable outputs only if (!spendable && only_spendable) continue; - int input_bytes = CalculateMaximumSignedInputSize(output, provider.get(), coinControl && coinControl->fAllowWatchOnly); + int input_bytes = CalculateMaximumSignedInputSize(output, COutPoint(), provider.get(), coinControl); result.coins.emplace_back(outpoint, output, nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me, feerate); result.total_amount += output.nValue; @@ -286,7 +286,7 @@ std::map> ListCoins(const CWallet& wallet) if (ExtractDestination(FindNonChangeParentOutput(wallet, *wtx.tx, output.n).scriptPubKey, address)) { const auto out = wtx.tx->vout.at(output.n); result[address].emplace_back( - COutPoint(wtx.GetHash(), output.n), out, depth, CalculateMaximumSignedInputSize(out, &wallet, false), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); + COutPoint(wtx.GetHash(), output.n), out, depth, CalculateMaximumSignedInputSize(out, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); } } } @@ -443,14 +443,14 @@ std::optional SelectCoins(const CWallet& wallet, const std::vec if (ptr_wtx->tx->vout.size() <= outpoint.n) { return std::nullopt; } - input_bytes = CalculateMaximumSignedInputSize(ptr_wtx->tx->vout[outpoint.n], &wallet, false); txout = ptr_wtx->tx->vout.at(outpoint.n); + input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); } else { // The input is external. We did not find the tx in mapWallet. if (!coin_control.GetExternalOutput(outpoint, txout)) { return std::nullopt; } - input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /*use_max_sig=*/true); + input_bytes = CalculateMaximumSignedInputSize(txout, outpoint, &coin_control.m_external_provider, &coin_control); } // If available, override calculated size with coin control specified size if (coin_control.HasInputWeight(outpoint)) { diff --git a/src/wallet/spend.h b/src/wallet/spend.h index a7d91eb9c417a..21f0095e77588 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -14,20 +14,16 @@ namespace wallet { /** Get the marginal bytes if spending the specified output from this transaction. - * use_max_sig indicates whether to use the maximum sized, 72 byte signature when calculating the - * size of the input spend. This should only be set when watch-only outputs are allowed */ -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); -int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); - + * Use CoinControl to determine whether to expect signature grinding when calculating the size of the input spend. */ +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, const CCoinControl* coin_control = nullptr); +int CalculateMaximumSignedInputSize(const CTxOut& txout, const COutPoint outpoint, const SigningProvider* pwallet, const CCoinControl* coin_control = nullptr); struct TxSize { int64_t vsize{-1}; int64_t weight{-1}; }; -/** Calculate the size of the transaction assuming all signatures are max size -* Use DummySignatureCreator, which inserts 71 byte signatures everywhere. -* NOTE: this requires that all inputs must be in mapWallet (eg the tx should -* be AllInputsMine). */ +/** Calculate the size of the transaction using CoinControl to determine + * whether to expect signature grinding when calculating the size of the input spend. */ TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector& txouts, const CCoinControl* coin_control = nullptr); TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const CCoinControl* coin_control = nullptr) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 0664b64244bbd..e85bf58d4e3a2 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -86,8 +86,8 @@ static void add_coin(std::vector& coins, CWallet& wallet, const CAmount auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); CWalletTx& wtx = (*ret.first).second; - const auto txout = wtx.tx->vout.at(nInput); - coins.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, false), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); + const auto& txout = wtx.tx->vout.at(nInput); + coins.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); } /** Check if SelectionResult a is equivalent to SelectionResult b. diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index d0b093bbb7c91..b90ef98850e8d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1504,13 +1504,16 @@ bool CWallet::AddWalletFlags(uint64_t flags) } // Helper for producing a max-sized low-S low-R signature (eg 71 bytes) -// or a max-sized low-S signature (e.g. 72 bytes) if use_max_sig is true -bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) +// or a max-sized low-S signature (e.g. 72 bytes) depending on coin_control +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, const CCoinControl* coin_control) { // Fill in dummy signatures for fee calculation. const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; + // Use max sig if watch only inputs were used or if this particular input is an external input + // to ensure a sufficient fee is attained for the requested feerate. + const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(tx_in.prevout)); if (!ProduceSignature(provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { return false; } @@ -1577,12 +1580,9 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector nIn++; continue; } - // Use max sig if watch only inputs were used or if this particular input is an external input - // to ensure a sufficient fee is attained for the requested feerate. - const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(txin.prevout)); const std::unique_ptr provider = GetSolvingProvider(txout.scriptPubKey); - if (!provider || !DummySignInput(*provider, txin, txout, use_max_sig)) { - if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, use_max_sig)) { + if (!provider || !DummySignInput(*provider, txin, txout, coin_control)) { + if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, coin_control)) { return false; } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index bf42b3da9ec51..1cf91d6ef665b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -932,7 +932,7 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); //! Remove wallet name from persistent configuration so it will not be loaded on startup. bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); -bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig); +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, const CCoinControl* coin_control = nullptr); bool FillInputToWeight(CTxIn& txin, int64_t target_weight); } // namespace wallet From 796b020c37c793674f9d614d5d70fd1ed65f0938 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Fri, 20 May 2022 09:54:41 +0200 Subject: [PATCH 0051/1090] wallet: add taproot support to external signer --- src/external_signer.cpp | 3 +++ test/functional/mocks/signer.py | 10 +++++++--- test/functional/wallet_signer.py | 15 +++++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/external_signer.cpp b/src/external_signer.cpp index d125fe479b3a5..537e3a0b13e4d 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -81,6 +81,9 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str for (const auto& entry : input.hd_keypaths) { if (parsed_m_fingerprint == MakeUCharSpan(entry.second.fingerprint)) return true; } + for (const auto& entry : input.m_tap_bip32_paths) { + if (parsed_m_fingerprint == MakeUCharSpan(entry.second.second.fingerprint)) return true; + } return false; }; diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py index b732b26a53d07..0b4f964c47955 100755 --- a/test/functional/mocks/signer.py +++ b/test/functional/mocks/signer.py @@ -27,12 +27,15 @@ def getdescriptors(args): "receive": [ "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/0/*)#vt6w3l3j", "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/0/*))#r0grqw5x", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs" + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs", + "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/0/*)#sng9rd4t" ], "internal": [ "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/1/*)#all0v2p2", "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/1/*))#kwx4c3pe", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg" + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg", + "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/1/*)#p8dy7c9n" + ] })) @@ -44,7 +47,8 @@ def displayaddress(args): return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) expected_desc = [ - "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r" + "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r", + "tr([00000001/86'/1'/0'/0/0]c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#4vdj9jqk", ] if args.desc not in expected_desc: return sys.stdout.write(json.dumps({"error": "Unexpected descriptor", "desc": args.desc})) diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 8e4e1f5d36b67..4bb60a9f58eb5 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -89,7 +89,7 @@ def test_valid_signer(self): # ) # self.clear_mock_result(self.nodes[1]) - assert_equal(hww.getwalletinfo()["keypoolsize"], 30) + assert_equal(hww.getwalletinfo()["keypoolsize"], 40) address1 = hww.getnewaddress(address_type="bech32") assert_equal(address1, "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g") @@ -112,6 +112,13 @@ def test_valid_signer(self): assert_equal(address_info['ismine'], True) assert_equal(address_info['hdkeypath'], "m/44'/1'/0'/0/0") + address4 = hww.getnewaddress(address_type="bech32m") + assert_equal(address4, "bcrt1phw4cgpt6cd30kz9k4wkpwm872cdvhss29jga2xpmftelhqll62ms4e9sqj") + address_info = hww.getaddressinfo(address4) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/86'/1'/0'/0/0") + self.log.info('Test walletdisplayaddress') result = hww.walletdisplayaddress(address1) assert_equal(result, {"address": address1}) @@ -124,7 +131,7 @@ def test_valid_signer(self): self.clear_mock_result(self.nodes[1]) self.log.info('Prepare mock PSBT') - self.nodes[0].sendtoaddress(address1, 1) + self.nodes[0].sendtoaddress(address4, 1) self.generate(self.nodes[0], 1) # Load private key into wallet to generate a signed PSBT for the mock @@ -133,14 +140,14 @@ def test_valid_signer(self): assert mock_wallet.getwalletinfo()['private_keys_enabled'] result = mock_wallet.importdescriptors([{ - "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0", + "desc": "tr([00000001/86'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#0jtt2jc9", "timestamp": 0, "range": [0,1], "internal": False, "active": True }, { - "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#j6uzqvuh", + "desc": "tr([00000001/86'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#7xw2h8ga", "timestamp": 0, "range": [0, 0], "internal": True, From 7f2450871b3ea0b4d02d56bd2ca365fcc25cf90e Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:02:55 -0500 Subject: [PATCH 0052/1090] Move handling of unconnecting headers into own function --- src/net_processing.cpp | 91 +++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 28 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index a55618072124a..6965695ad92a8 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -560,6 +560,11 @@ class PeerManagerImpl final : public PeerManager const std::vector& headers, bool via_compact_block) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + /** Various helpers for headers processing, invoked by ProcessHeadersMessage() */ + /** Deal with state tracking and headers sync for peers that send the + * occasional non-connecting header (this can happen due to BIP 130 headers + * announcements for blocks interacting with the 2hr (MAX_FUTURE_BLOCK_TIME) rule). */ + void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector& headers); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2194,6 +2199,48 @@ void PeerManagerImpl::SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlo m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); } +/** + * Special handling for unconnecting headers that might be part of a block + * announcement. + * + * We'll send a getheaders message in response to try to connect the chain. + * + * The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + * don't connect before given DoS points. + * + * Once a headers message is received that is valid and does connect, + * nUnconnectingHeaders gets reset back to 0. + */ +void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, + const std::vector& headers) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + + nodestate->nUnconnectingHeaders++; + + // Try to fill in the missing headers. + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + m_chainman.m_best_header->nHeight, + pfrom.GetId(), nodestate->nUnconnectingHeaders); + + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); + + // The peer may just be broken, so periodically assign DoS points if this + // condition persists. + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(peer, 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); + } +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2208,36 +2255,24 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, bool received_new_header = false; const CBlockIndex *pindexLast = nullptr; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom.GetId()); - // If this looks like it could be a block announcement (nCount <= - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (!m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) && nCount <= MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - m_chainman.m_best_header->nHeight, - pfrom.GetId(), nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(peer, 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); - } - return; + // Do these headers connect to something in our block index? + bool headers_connect_blockindex{WITH_LOCK(::cs_main, return m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) != nullptr)}; + + if (!headers_connect_blockindex) { + if (nCount <= MAX_BLOCKS_TO_ANNOUNCE) { + // If this looks like it could be a BIP 130 block announcement, use + // special logic for handling headers that don't connect, as this + // could be benign. + HandleFewUnconnectingHeaders(pfrom, peer, headers); + } else { + Misbehaving(peer, 10, "invalid header received"); } + return; + } + + { + LOCK(cs_main); uint256 hashLastBlock; for (const CBlockHeader& header : headers) { From 9492e93bf9f4a841bf43ca4b593871c0863d5b63 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:11:30 -0500 Subject: [PATCH 0053/1090] Add helper function for checking header continuity --- src/net_processing.cpp | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6965695ad92a8..75e76d28cc13a 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -565,6 +565,8 @@ class PeerManagerImpl final : public PeerManager * occasional non-connecting header (this can happen due to BIP 130 headers * announcements for blocks interacting with the 2hr (MAX_FUTURE_BLOCK_TIME) rule). */ void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector& headers); + /** Return true if the headers connect to each other, false otherwise */ + bool CheckHeadersAreContinuous(const std::vector& headers) const; void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2241,6 +2243,18 @@ void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, } } +bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector& headers) const +{ + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + return false; + } + hashLastBlock = header.GetHash(); + } + return true; +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2271,21 +2285,18 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } + // At this point, the headers connect to something in our block index. + if (!CheckHeadersAreContinuous(headers)) { + Misbehaving(peer, 20, "non-continuous headers sequence"); + return; + } + { LOCK(cs_main); - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(peer, 20, "non-continuous headers sequence"); - return; - } - hashLastBlock = header.GetHash(); - } - // If we don't have the last header, then they'll have given us // something new (if these headers are valid). - if (!m_chainman.m_blockman.LookupBlockIndex(hashLastBlock)) { + if (!m_chainman.m_blockman.LookupBlockIndex(headers.back().GetHash())) { received_new_header = true; } } From bf8ea6df75749c27f753b562c4724b3f8d263ad4 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:15:20 -0500 Subject: [PATCH 0054/1090] Move additional headers fetching to own function Also moves the call to happen directly after validation of a headers message (rather than mixed in with other state updates for the peer), and removes an incorrect comment in favor of one that explains why headers sync must continue from the last header a peer has sent. --- src/net_processing.cpp | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 75e76d28cc13a..795851c4910e6 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -567,6 +567,8 @@ class PeerManagerImpl final : public PeerManager void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector& headers); /** Return true if the headers connect to each other, false otherwise */ bool CheckHeadersAreContinuous(const std::vector& headers) const; + /** Request further headers from this peer from a given block header */ + void FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2255,6 +2257,25 @@ bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector& return true; } +/* + * Continue fetching headers from a given point. + * pindexLast should be the last header we learned from a peer in their prior + * headers message. + * + * This is used for headers sync with a peer; even if pindexLast is an ancestor + * of a known chain (such as our tip) we don't yet know where the peer's chain + * might fork from what we know, so we continue exactly from where the peer + * left off. + */ +void PeerManagerImpl::FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", + pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2309,6 +2330,12 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, } } + // Consider fetching more headers. + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + FetchMoreHeaders(pfrom, pindexLast, peer); + } + { LOCK(cs_main); CNodeState *nodestate = State(pfrom.GetId()); @@ -2328,15 +2355,6 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, nodestate->m_last_block_announcement = GetTime(); } - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of m_chainman.ActiveChain().Tip or m_chainman.m_best_header, continue - // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", - pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); - } - // If this set of headers is valid and ends in a block with at least as // much work as our tip, download as much as possible. if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { From 29c45185223441943ab610e62937a118c7c3a5b2 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:33:32 -0500 Subject: [PATCH 0055/1090] Move headers-direct-fetch logic into own function --- src/net_processing.cpp | 126 +++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 795851c4910e6..189556e03e609 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -569,6 +569,8 @@ class PeerManagerImpl final : public PeerManager bool CheckHeadersAreContinuous(const std::vector& headers) const; /** Request further headers from this peer from a given block header */ void FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer); + /** Potentially fetch blocks from this peer upon receipt of new headers tip */ + void HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2276,6 +2278,73 @@ void PeerManagerImpl::FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLa m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); } +/* + * Given a new headers tip ending in pindexLast, potentially request blocks towards that tip. + * We require that the given tip have at least as much work as our tip, and for + * our current tip to be "close to synced" (see CanDirectFetch()). + */ +void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + + if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { + + std::vector vToFetch; + const CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !IsBlockRequested(pindexWalk->GetBlockHash()) && + (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!m_chainman.ActiveChain().Contains(pindexWalk)) { + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + std::vector vGetData; + // Download as much as possible, from earliest to latest. + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + uint32_t nFetchFlags = GetFetchFlags(pfrom); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + BlockRequested(pfrom.GetId(), *pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom.GetId()); + } + if (vGetData.size() > 1) { + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (!m_ignore_incoming_txs && + nodestate->m_provides_cmpctblocks && + vGetData.size() == 1 && + mapBlocksInFlight.size() == 1 && + pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2355,60 +2424,9 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, nodestate->m_last_block_announcement = GetTime(); } - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { - std::vector vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !IsBlockRequested(pindexWalk->GetBlockHash()) && - (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!m_chainman.ActiveChain().Contains(pindexWalk)) { - LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - std::vector vGetData; - // Download as much as possible, from earliest to latest. - for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - uint32_t nFetchFlags = GetFetchFlags(pfrom); - vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - BlockRequested(pfrom.GetId(), *pindex); - LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom.GetId()); - } - if (vGetData.size() > 1) { - LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (!m_ignore_incoming_txs && - nodestate->m_provides_cmpctblocks && - vGetData.size() == 1 && - mapBlocksInFlight.size() == 1 && - pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } + // Consider immediately downloading blocks. + HeadersDirectFetchBlocks(pfrom, pindexLast); + // If we're in IBD, we want outbound peers that will serve us a useful // chain. Disconnect peers that are on chains with insufficient work. if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { From 2b341db731793844f12944363186edea23eabdeb Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:36:33 -0500 Subject: [PATCH 0056/1090] Move headers direct fetch to end of ProcessHeadersMessage --- src/net_processing.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 189556e03e609..2be5d5f50d286 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2424,9 +2424,6 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, nodestate->m_last_block_announcement = GetTime(); } - // Consider immediately downloading blocks. - HeadersDirectFetchBlocks(pfrom, pindexLast); - // If we're in IBD, we want outbound peers that will serve us a useful // chain. Disconnect peers that are on chains with insufficient work. if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { @@ -2462,6 +2459,9 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, } } + // Consider immediately downloading blocks. + HeadersDirectFetchBlocks(pfrom, pindexLast); + return; } From 6d95cd3e7444ebaaabb64a76783ea3551530f1d7 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:40:33 -0500 Subject: [PATCH 0057/1090] Move peer state updates from headers message into separate function --- src/net_processing.cpp | 117 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 53 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2be5d5f50d286..5f19f8b8ec599 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -571,6 +571,8 @@ class PeerManagerImpl final : public PeerManager void FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer); /** Potentially fetch blocks from this peer upon receipt of new headers tip */ void HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast); + /** Update peer state based on received headers message */ + void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2345,6 +2347,67 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* } } +/** + * Given receipt of headers from a peer ending in pindexLast, along with + * whether that header was new and whether the headers message was full, + * update the state we keep for the peer. + */ +void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, + const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers) +{ + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); + + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + + if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) { + // If the peer has no more headers to give us, then we know we have + // their tip. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if it is an outbound disconnection + // candidate. + // Note: We compare their tip to nMinimumChainWork (rather than + // m_chainman.ActiveChain().Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (pfrom.IsOutboundOrBlockRelayConn()) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); + pfrom.fDisconnect = true; + } + } + } + + // If this is an outbound full-relay peer, check to see if we should protect + // it from the bad/lagging chain logic. + // Note that outbound block-relay peers are excluded from this protection, and + // thus always subject to eviction under the bad/lagging chain logic. + // See ChainSyncTimeoutState. + if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { + if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); + nodestate->m_chain_sync.m_protect = true; + ++m_outbound_peers_with_protect_from_disconnect; + } + } +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2405,59 +2468,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, FetchMoreHeaders(pfrom, pindexLast, peer); } - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom.GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); - - // From here, pindexBestKnownBlock should be guaranteed to be non-null, - // because it is set in UpdateBlockAvailability. Some nullptr checks - // are still present, however, as belt-and-suspenders. - - if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { - nodestate->m_last_block_announcement = GetTime(); - } - - // If we're in IBD, we want outbound peers that will serve us a useful - // chain. Disconnect peers that are on chains with insufficient work. - if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { - // When nCount < MAX_HEADERS_RESULTS, we know we have no more - // headers to fetch from this peer. - if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { - // This peer has too little work on their headers chain to help - // us sync -- disconnect if it is an outbound disconnection - // candidate. - // Note: We compare their tip to nMinimumChainWork (rather than - // m_chainman.ActiveChain().Tip()) because we won't start block download - // until we have a headers chain that has at least - // nMinimumChainWork, even if a peer has a chain past our tip, - // as an anti-DoS measure. - if (pfrom.IsOutboundOrBlockRelayConn()) { - LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); - pfrom.fDisconnect = true; - } - } - } - - // If this is an outbound full-relay peer, check to see if we should protect - // it from the bad/lagging chain logic. - // Note that outbound block-relay peers are excluded from this protection, and - // thus always subject to eviction under the bad/lagging chain logic. - // See ChainSyncTimeoutState. - if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { - if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { - LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); - nodestate->m_chain_sync.m_protect = true; - ++m_outbound_peers_with_protect_from_disconnect; - } - } - } + UpdatePeerStateForReceivedHeaders(pfrom, pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); // Consider immediately downloading blocks. HeadersDirectFetchBlocks(pfrom, pindexLast); From ffe87db247b19ffb8bfba329c5dd0be39ef5a53f Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:43:38 -0500 Subject: [PATCH 0058/1090] Cleanup received_new_header calculation to use WITH_LOCK --- src/net_processing.cpp | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 5f19f8b8ec599..975ee16615d9e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2420,7 +2420,6 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } - bool received_new_header = false; const CBlockIndex *pindexLast = nullptr; // Do these headers connect to something in our block index? @@ -2444,15 +2443,9 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } - { - LOCK(cs_main); - - // If we don't have the last header, then they'll have given us - // something new (if these headers are valid). - if (!m_chainman.m_blockman.LookupBlockIndex(headers.back().GetHash())) { - received_new_header = true; - } - } + // If we don't have the last header, then this peer will have given us + // something new (if these headers are valid). + bool received_new_header{WITH_LOCK(::cs_main, return m_chainman.m_blockman.LookupBlockIndex(headers.back().GetHash()) == nullptr)}; BlockValidationState state; if (!m_chainman.ProcessNewBlockHeaders(headers, state, &pindexLast)) { From abf5d16c24cb08b0451bdbd4d1de63a12930e8f5 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Thu, 3 Mar 2022 14:48:38 -0500 Subject: [PATCH 0059/1090] Don't send getheaders message when another request is outstanding Change getheaders messages so that we wait up to 2 minutes for a response to a prior getheaders message before issuing a new one. Also change the handling of the getheaders message sent in response to a block INV, so that we no longer use the hashstop variable (including the hash stop will just mean that if our peer's headers chain is longer, then we won't learn it, so there's no benefit to using hashstop). Also, now respond to a getheaders during IBD with an empty headers message (rather than nothing) -- this better conforms to the intent of the new logic that it's better to not ignore a peer's getheaders message, even if you have nothing to give. This also avoids a lot of functional tests breaking. p2p_segwit.py is modified to use this same strategy, as the test logic (of expecting a getheaders after a block inv) would otherwise be broken. --- src/net_processing.cpp | 110 +++++++++++++++--------- test/functional/feature_minchainwork.py | 2 +- test/functional/p2p_segwit.py | 4 + 3 files changed, 72 insertions(+), 44 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 975ee16615d9e..fe1b06b18cdff 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -61,6 +61,8 @@ static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; * Timeout = base + per_header * (expected number of headers) */ static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_BASE = 15min; static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1ms; +/** How long to wait for a peer to respond to a getheaders request */ +static constexpr auto HEADERS_RESPONSE_TIME{2min}; /** Protect at least this many outbound peers from disconnection due to slow/ * behind headers chain. */ @@ -355,6 +357,9 @@ struct Peer { /** Work queue of items requested by this peer **/ std::deque m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); + /** Time of the last getheaders message to this peer */ + std::atomic m_last_getheaders_timestamp{0s}; + Peer(NodeId id) : m_id{id} {} @@ -501,7 +506,7 @@ class PeerManagerImpl final : public PeerManager private: /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ - void ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ void EvictExtraOutboundPeers(std::chrono::seconds now) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -567,9 +572,12 @@ class PeerManagerImpl final : public PeerManager void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector& headers); /** Return true if the headers connect to each other, false otherwise */ bool CheckHeadersAreContinuous(const std::vector& headers) const; - /** Request further headers from this peer from a given block header */ - void FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer); - /** Potentially fetch blocks from this peer upon receipt of new headers tip */ + /** Request further headers from this peer with a given locator. + * We don't issue a getheaders message if we have a recent one outstanding. + * This returns true if a getheaders is actually sent, and false otherwise. + */ + bool MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer); + /** Potentially fetch blocks from this peer upon receipt of a new headers tip */ void HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast); /** Update peer state based on received headers message */ void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers); @@ -2228,15 +2236,14 @@ void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, CNodeState *nodestate = State(pfrom.GetId()); nodestate->nUnconnectingHeaders++; - // Try to fill in the missing headers. - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), peer)) { + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), m_chainman.m_best_header->nHeight, pfrom.GetId(), nodestate->nUnconnectingHeaders); - + } // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. @@ -2261,23 +2268,19 @@ bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector& return true; } -/* - * Continue fetching headers from a given point. - * pindexLast should be the last header we learned from a peer in their prior - * headers message. - * - * This is used for headers sync with a peer; even if pindexLast is an ancestor - * of a known chain (such as our tip) we don't yet know where the peer's chain - * might fork from what we know, so we continue exactly from where the peer - * left off. - */ -void PeerManagerImpl::FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer) +bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer) { const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", - pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); + const auto current_time = GetTime(); + // Only allow a new getheaders message to go out if we don't have a recent + // one already in-flight + if (peer.m_last_getheaders_timestamp.load() < current_time - HEADERS_RESPONSE_TIME) { + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); + peer.m_last_getheaders_timestamp = current_time; + return true; + } + return false; } /* @@ -2458,7 +2461,10 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, // Consider fetching more headers. if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. - FetchMoreHeaders(pfrom, pindexLast, peer); + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(pindexLast), peer)) { + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", + pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); + } } UpdatePeerStateForReceivedHeaders(pfrom, pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); @@ -3228,8 +3234,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (best_block != nullptr) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *best_block)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", m_chainman.m_best_header->nHeight, best_block->ToString(), pfrom.GetId()); + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer)) { + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", + m_chainman.m_best_header->nHeight, best_block->ToString(), + pfrom.GetId()); + } } return; @@ -3402,7 +3411,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // others. if (m_chainman.ActiveTip() == nullptr || (m_chainman.ActiveTip()->nChainWork < nMinimumChainWork && !pfrom.HasPermission(NetPermissionFlags::Download))) { - LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work\n", pfrom.GetId()); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); + // Just respond with an empty headers message, to tell the peer to + // go away but not treat us as unresponsive. + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector())); return; } @@ -3683,8 +3695,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!m_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers - if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); + if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer); + } return; } @@ -3958,6 +3971,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Assume that this is in response to any outstanding getheaders + // request we may have sent, and clear out the time of our last request + peer->m_last_getheaders_timestamp = 0s; + std::vector headers; // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. @@ -4386,7 +4403,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic& interrupt return fMoreWork; } -void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) +void PeerManagerImpl::ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seconds time_in_seconds) { AssertLockHeld(cs_main); @@ -4424,10 +4441,15 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_ pto.fDisconnect = true; } else { assert(state.m_chain_sync.m_work_header); + // Here, we assume that the getheaders message goes out, + // because it'll either go out or be skipped because of a + // getheaders in-flight already, in which case the peer should + // still respond to us with a sufficiently high work chain tip. + MaybeSendGetHeaders(pto, + m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), + peer); LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); - m_connman.PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; - constexpr auto HEADERS_RESPONSE_TIME{2min}; // Bump the timeout to allow a response, which could clear the timeout // (if the response shows the peer has synced), reset the timeout (if // the peer syncs to the required work but not to our tip), or result @@ -4835,15 +4857,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && sync_blocks_and_headers_from_peer) || m_chainman.m_best_header->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { - state.fSyncStarted = true; - state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + - ( - // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling - // to maintain precision - std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * - (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing - ); - nSyncStarted++; const CBlockIndex* pindexStart = m_chainman.m_best_header; /* If possible, start at the block preceding the currently best known header. This ensures that we always get a @@ -4854,8 +4867,19 @@ bool PeerManagerImpl::SendMessages(CNode* pto) got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexStart), uint256())); + if (MaybeSendGetHeaders(*pto, m_chainman.ActiveChain().GetLocator(pindexStart), *peer)) { + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); + + state.fSyncStarted = true; + state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + + ( + // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling + // to maintain precision + std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * + (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing + ); + nSyncStarted++; + } } } @@ -5201,7 +5225,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Check that outbound peers have reasonable chains // GetTime() is used by this anti-DoS logic so we can test this using mocktime - ConsiderEviction(*pto, GetTime()); + ConsiderEviction(*pto, *peer, GetTime()); // // Message: getdata (blocks) diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index fa10855a987b4..9d0ad5ef9d01a 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -82,7 +82,7 @@ def run_test(self): msg.hashstop = 0 peer.send_and_ping(msg) time.sleep(5) - assert "headers" not in peer.last_message + assert ("headers" not in peer.last_message or len(peer.last_message["headers"].headers) == 0) self.log.info("Generating one more block") self.generate(self.nodes[0], 1) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 952f1e5cc5a2a..eedb7e6fa1f9d 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -371,6 +371,10 @@ def test_block_relay(self): block1 = self.build_next_block() block1.solve() + # Send an empty headers message, to clear out any prior getheaders + # messages that our peer may be waiting for us on. + self.test_node.send_message(msg_headers()) + self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block1, True) From 6666803c897e4ad27b45cb74e3a9aa74a335f1bf Mon Sep 17 00:00:00 2001 From: MacroFake Date: Mon, 6 Jun 2022 17:11:03 +0200 Subject: [PATCH 0060/1090] streams: Add AutoFile without ser-type and ser-version The moved parts can be reviewed with "--color-moved=dimmed-zebra". The one-char changes can be reviewed with "--word-diff-regex=.". --- src/streams.h | 69 ++++++++++++++++++------------ test/functional/feature_addrman.py | 2 +- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/streams.h b/src/streams.h index 96b7696f72c56..f14d347380ac6 100644 --- a/src/streams.h +++ b/src/streams.h @@ -465,35 +465,28 @@ class BitStreamWriter }; - /** Non-refcounted RAII wrapper for FILE* * * Will automatically close the file when it goes out of scope if not null. * If you're returning the file pointer, return file.release(). * If you need to close the file early, use file.fclose() instead of fclose(file). */ -class CAutoFile +class AutoFile { -private: - const int nType; - const int nVersion; - +protected: FILE* file; public: - CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) - { - file = filenew; - } + explicit AutoFile(FILE* filenew) : file{filenew} {} - ~CAutoFile() + ~AutoFile() { fclose(); } // Disallow copies - CAutoFile(const CAutoFile&) = delete; - CAutoFile& operator=(const CAutoFile&) = delete; + AutoFile(const AutoFile&) = delete; + AutoFile& operator=(const AutoFile&) = delete; void fclose() { @@ -504,14 +497,14 @@ class CAutoFile } /** Get wrapped FILE* with transfer of ownership. - * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller + * @note This will invalidate the AutoFile object, and makes it the responsibility of the caller * of this function to clean up the returned FILE*. */ FILE* release() { FILE* ret = file; file = nullptr; return ret; } /** Get wrapped FILE* without transfer of ownership. * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the - * CAutoFile outlives use of the passed pointer. + * AutoFile outlives use of the passed pointer. */ FILE* Get() const { return file; } @@ -522,40 +515,62 @@ class CAutoFile // // Stream subset // - int GetType() const { return nType; } - int GetVersion() const { return nVersion; } - void read(Span dst) { - if (!file) - throw std::ios_base::failure("CAutoFile::read: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); if (fread(dst.data(), 1, dst.size(), file) != dst.size()) { - throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed"); + throw std::ios_base::failure(feof(file) ? "AutoFile::read: end of file" : "AutoFile::read: fread failed"); } } void ignore(size_t nSize) { - if (!file) - throw std::ios_base::failure("CAutoFile::ignore: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::ignore: file handle is nullptr"); unsigned char data[4096]; while (nSize > 0) { size_t nNow = std::min(nSize, sizeof(data)); if (fread(data, 1, nNow, file) != nNow) - throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed"); + throw std::ios_base::failure(feof(file) ? "AutoFile::ignore: end of file" : "AutoFile::read: fread failed"); nSize -= nNow; } } void write(Span src) { - if (!file) - throw std::ios_base::failure("CAutoFile::write: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); if (fwrite(src.data(), 1, src.size(), file) != src.size()) { - throw std::ios_base::failure("CAutoFile::write: write failed"); + throw std::ios_base::failure("AutoFile::write: write failed"); } } + template + AutoFile& operator<<(const T& obj) + { + if (!file) throw std::ios_base::failure("AutoFile::operator<<: file handle is nullptr"); + ::Serialize(*this, obj); + return *this; + } + + template + AutoFile& operator>>(T&& obj) + { + if (!file) throw std::ios_base::failure("AutoFile::operator>>: file handle is nullptr"); + ::Unserialize(*this, obj); + return *this; + } +}; + +class CAutoFile : public AutoFile +{ +private: + const int nType; + const int nVersion; + +public: + CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : AutoFile{filenew}, nType(nTypeIn), nVersion(nVersionIn) {} + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } + template CAutoFile& operator<<(const T& obj) { diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 5e49d0214a2f8..63abf0d9f8f9d 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -95,7 +95,7 @@ def run_test(self): with open(peers_dat, "wb") as f: f.write(serialize_addrman()[:-1]) self.nodes[0].assert_start_raises_init_error( - expected_msg=init_error("CAutoFile::read: end of file.*"), + expected_msg=init_error("AutoFile::read: end of file.*"), match=ErrorMatch.FULL_REGEX, ) From facc2fa7b8a218a0df6a19772e1641ea68dda2e3 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Mon, 6 Jun 2022 17:22:59 +0200 Subject: [PATCH 0061/1090] Use AutoFile where possible --- src/index/blockfilterindex.cpp | 8 ++++---- src/net.cpp | 2 +- src/policy/fees.cpp | 16 ++++++++-------- src/policy/fees.h | 6 +++--- src/rpc/blockchain.cpp | 4 ++-- src/rpc/blockchain.h | 2 +- src/test/flatfile_tests.cpp | 12 ++++++------ src/test/fuzz/autofile.cpp | 4 +--- src/test/fuzz/policy_estimator.cpp | 2 +- src/test/fuzz/policy_estimator_io.cpp | 2 +- src/test/fuzz/util.h | 7 +++---- src/test/fuzz/utxo_snapshot.cpp | 4 ++-- src/test/util/chainstate.h | 6 +++--- src/test/validation_chainstatemanager_tests.cpp | 10 +++++----- src/util/asmap.cpp | 2 +- src/validation.cpp | 4 ++-- src/validation.h | 4 ++-- 17 files changed, 46 insertions(+), 49 deletions(-) diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index c92b8c7e195f3..bc44a90c0ea52 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -131,7 +131,7 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) const FlatFilePos& pos = m_next_filter_pos; // Flush current filter file to disk. - CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile file{m_filter_fileseq->Open(pos)}; if (file.IsNull()) { return error("%s: Failed to open filter file %d", __func__, pos.nFile); } @@ -145,7 +145,7 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const { - CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION); + AutoFile filein{m_filter_fileseq->Open(pos, true)}; if (filein.IsNull()) { return false; } @@ -173,7 +173,7 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& // If writing the filter would overflow the file, flush and move to the next one. if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) { - CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile last_file{m_filter_fileseq->Open(pos)}; if (last_file.IsNull()) { LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); return 0; @@ -199,7 +199,7 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& return 0; } - CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile fileout{m_filter_fileseq->Open(pos)}; if (fileout.IsNull()) { LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); return 0; diff --git a/src/net.cpp b/src/net.cpp index 7f4e571c8dc58..a91abae944f15 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -3054,7 +3054,7 @@ void CaptureMessageToFile(const CAddress& addr, fs::create_directories(base_path); fs::path path = base_path / (is_incoming ? "msgs_recv.dat" : "msgs_sent.dat"); - CAutoFile f(fsbridge::fopen(path, "ab"), SER_DISK, CLIENT_VERSION); + AutoFile f{fsbridge::fopen(path, "ab")}; ser_writedata64(f, now.count()); f.write(MakeByteSpan(msg_type)); diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 27a6ab221fec0..2b940be07ed07 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -161,13 +161,13 @@ class TxConfirmStats unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } /** Write state of estimation data to a file*/ - void Write(CAutoFile& fileout) const; + void Write(AutoFile& fileout) const; /** * Read saved state of estimation data from a file and replace all internal data structures and * variables with this state. */ - void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); + void Read(AutoFile& filein, int nFileVersion, size_t numBuckets); }; @@ -390,7 +390,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, return median; } -void TxConfirmStats::Write(CAutoFile& fileout) const +void TxConfirmStats::Write(AutoFile& fileout) const { fileout << Using(decay); fileout << scale; @@ -400,7 +400,7 @@ void TxConfirmStats::Write(CAutoFile& fileout) const fileout << Using>>(failAvg); } -void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) +void TxConfirmStats::Read(AutoFile& filein, int nFileVersion, size_t numBuckets) { // Read data file and do some very basic sanity checking // buckets and bucketMap are not updated yet, so don't access them @@ -546,7 +546,7 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath longStats = std::unique_ptr(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); // If the fee estimation file is present, read recorded estimations - CAutoFile est_file(fsbridge::fopen(m_estimation_filepath, "rb"), SER_DISK, CLIENT_VERSION); + AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")}; if (est_file.IsNull() || !Read(est_file)) { LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); } @@ -904,13 +904,13 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation void CBlockPolicyEstimator::Flush() { FlushUnconfirmed(); - CAutoFile est_file(fsbridge::fopen(m_estimation_filepath, "wb"), SER_DISK, CLIENT_VERSION); + AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "wb")}; if (est_file.IsNull() || !Write(est_file)) { LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); } } -bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const +bool CBlockPolicyEstimator::Write(AutoFile& fileout) const { try { LOCK(m_cs_fee_estimator); @@ -935,7 +935,7 @@ bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const return true; } -bool CBlockPolicyEstimator::Read(CAutoFile& filein) +bool CBlockPolicyEstimator::Read(AutoFile& filein) { try { LOCK(m_cs_fee_estimator); diff --git a/src/policy/fees.h b/src/policy/fees.h index 9ee5c2938a212..e4628bf85334c 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -20,7 +20,7 @@ #include #include -class CAutoFile; +class AutoFile; class CTxMemPoolEntry; class TxConfirmStats; @@ -220,11 +220,11 @@ class CBlockPolicyEstimator EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Write estimation data to a file */ - bool Write(CAutoFile& fileout) const + bool Write(AutoFile& fileout) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Read estimation data from a file */ - bool Read(CAutoFile& filein) + bool Read(AutoFile& filein) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index af9458206edc6..c1b4e863b5adf 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2293,7 +2293,7 @@ static RPCHelpMan dumptxoutset() } FILE* file{fsbridge::fopen(temppath, "wb")}; - CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + AutoFile afile{file}; if (afile.IsNull()) { throw JSONRPCError( RPC_INVALID_PARAMETER, @@ -2314,7 +2314,7 @@ static RPCHelpMan dumptxoutset() UniValue CreateUTXOSnapshot( NodeContext& node, CChainState& chainstate, - CAutoFile& afile, + AutoFile& afile, const fs::path& path, const fs::path& temppath) { diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 5fbd9d5fd373d..a332fd4892773 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -55,7 +55,7 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], UniValue CreateUTXOSnapshot( node::NodeContext& node, CChainState& chainstate, - CAutoFile& afile, + AutoFile& afile, const fs::path& path, const fs::path& tmppath); diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index d54d6b6471f88..605faa08e411f 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -41,26 +41,26 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Write first line to file. { - CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos1))}; file << LIMITED_STRING(line1, 256); } // Attempt to append to file opened in read-only mode. { - CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2), true)}; BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure); } // Append second line to file. { - CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2))}; file << LIMITED_STRING(line2, 256); } // Read text from file in read-only mode. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos1), true)}; file >> LIMITED_STRING(text, 256); BOOST_CHECK_EQUAL(text, line1); @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Read text from file with position offset. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2))}; file >> LIMITED_STRING(text, 256); BOOST_CHECK_EQUAL(text, line2); @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Ensure another file in the sequence has no data. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(1, pos2))}; BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure); } } diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index 3b410930ed876..1a8957d0905f3 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -18,7 +18,7 @@ FUZZ_TARGET(autofile) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile auto_file = fuzzed_auto_file_provider.open(); + AutoFile auto_file{fuzzed_auto_file_provider.open()}; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, @@ -53,8 +53,6 @@ FUZZ_TARGET(autofile) }); } (void)auto_file.Get(); - (void)auto_file.GetType(); - (void)auto_file.GetVersion(); (void)auto_file.IsNull(); if (fuzzed_data_provider.ConsumeBool()) { FILE* f = auto_file.release(); diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 58c19a91cb7f0..637ba503c6851 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -76,7 +76,7 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) } { FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile fuzzed_auto_file = fuzzed_auto_file_provider.open(); + AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; block_policy_estimator.Write(fuzzed_auto_file); block_policy_estimator.Read(fuzzed_auto_file); } diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp index 77402c260a8b5..436873c95544c 100644 --- a/src/test/fuzz/policy_estimator_io.cpp +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -26,7 +26,7 @@ FUZZ_TARGET_INIT(policy_estimator_io, initialize_policy_estimator_io) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile fuzzed_auto_file = fuzzed_auto_file_provider.open(); + AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; // Re-using block_policy_estimator across runs to avoid costly creation of CBlockPolicyEstimator object. static CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args)}; if (block_policy_estimator.Read(fuzzed_auto_file)) { diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 4b89ad9bdc4b5..f6a4e5570f7a8 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -361,17 +361,16 @@ class FuzzedFileProvider class FuzzedAutoFileProvider { - FuzzedDataProvider& m_fuzzed_data_provider; FuzzedFileProvider m_fuzzed_file_provider; public: - FuzzedAutoFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_data_provider{fuzzed_data_provider}, m_fuzzed_file_provider{fuzzed_data_provider} + FuzzedAutoFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_file_provider{fuzzed_data_provider} { } - CAutoFile open() + AutoFile open() { - return {m_fuzzed_file_provider.open(), m_fuzzed_data_provider.ConsumeIntegral(), m_fuzzed_data_provider.ConsumeIntegral()}; + return AutoFile{m_fuzzed_file_provider.open()}; } }; diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index 33496a457e12d..0b596492bef5b 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -39,13 +39,13 @@ FUZZ_TARGET_INIT(utxo_snapshot, initialize_chain) Assert(!chainman.SnapshotBlockhash()); { - CAutoFile outfile{fsbridge::fopen(snapshot_path, "wb"), SER_DISK, CLIENT_VERSION}; + AutoFile outfile{fsbridge::fopen(snapshot_path, "wb")}; const auto file_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; outfile << Span{file_data}; } const auto ActivateFuzzedSnapshot{[&] { - CAutoFile infile{fsbridge::fopen(snapshot_path, "rb"), SER_DISK, CLIENT_VERSION}; + AutoFile infile{fsbridge::fopen(snapshot_path, "rb")}; SnapshotMetadata metadata; try { infile >> metadata; diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 5ac504c24faa4..13e0e684b87a7 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -16,7 +16,7 @@ #include -const auto NoMalleation = [](CAutoFile& file, node::SnapshotMetadata& meta){}; +const auto NoMalleation = [](AutoFile& file, node::SnapshotMetadata& meta){}; /** * Create and activate a UTXO snapshot, optionally providing a function to @@ -32,7 +32,7 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); fs::path snapshot_path = root / fs::u8path(tfm::format("test_snapshot.%d.dat", height)); FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; - CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + AutoFile auto_outfile{outfile}; UniValue result = CreateUTXOSnapshot( node, node.chainman->ActiveChainstate(), auto_outfile, snapshot_path, snapshot_path); @@ -42,7 +42,7 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma // Read the written snapshot in and then activate it. // FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; - CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + AutoFile auto_infile{infile}; node::SnapshotMetadata metadata; auto_infile >> metadata; diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 6dc522b42145c..14de96ff41b32 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -193,7 +193,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) // Should not load malleated snapshots BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // A UTXO is missing but count is correct metadata.m_coins_count -= 1; @@ -204,22 +204,22 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) auto_infile >> coin; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Coins count is larger than coins in file metadata.m_coins_count += 1; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Coins count is smaller than coins in file metadata.m_coins_count -= 1; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Wrong hash metadata.m_base_blockhash = uint256::ZERO; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Wrong hash metadata.m_base_blockhash = uint256::ONE; })); diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index ceb8379c1c102..b1aa598453071 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -195,7 +195,7 @@ std::vector DecodeAsmap(fs::path path) { std::vector bits; FILE *filestr = fsbridge::fopen(path, "rb"); - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + AutoFile file{filestr}; if (file.IsNull()) { LogPrintf("Failed to open asmap file from disk\n"); return bits; diff --git a/src/validation.cpp b/src/validation.cpp index 6b21d33871360..36a9083117a5f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4864,7 +4864,7 @@ const AssumeutxoData* ExpectedAssumeutxo( } bool ChainstateManager::ActivateSnapshot( - CAutoFile& coins_file, + AutoFile& coins_file, const SnapshotMetadata& metadata, bool in_memory) { @@ -4959,7 +4959,7 @@ static void FlushSnapshotToDisk(CCoinsViewCache& coins_cache, bool snapshot_load bool ChainstateManager::PopulateAndValidateSnapshot( CChainState& snapshot_chainstate, - CAutoFile& coins_file, + AutoFile& coins_file, const SnapshotMetadata& metadata) { // It's okay to release cs_main before we're done using `coins_cache` because we know diff --git a/src/validation.h b/src/validation.h index 0e27e117fa5a5..cce268ee8ea9a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -820,7 +820,7 @@ class ChainstateManager //! Internal helper for ActivateSnapshot(). [[nodiscard]] bool PopulateAndValidateSnapshot( CChainState& snapshot_chainstate, - CAutoFile& coins_file, + AutoFile& coins_file, const node::SnapshotMetadata& metadata); /** @@ -909,7 +909,7 @@ class ChainstateManager //! - Move the new chainstate to `m_snapshot_chainstate` and make it our //! ChainstateActive(). [[nodiscard]] bool ActivateSnapshot( - CAutoFile& coins_file, const node::SnapshotMetadata& metadata, bool in_memory); + AutoFile& coins_file, const node::SnapshotMetadata& metadata, bool in_memory); //! The most-work chain. CChainState& ActiveChainstate() const; From d5c141f221d67aaeee989da0db0ac5383d7562d3 Mon Sep 17 00:00:00 2001 From: Jarol Rodriguez Date: Wed, 29 Jun 2022 20:02:02 -0400 Subject: [PATCH 0062/1090] qt: apply translator comments to reset options confirmation dialog Follow-up to #617. This applies translator strings to the reset options confirmation dialog and also refactors the way we pass the strings to the dialog in order to allow the comments to be applied. Because the strings were being concatenated, we can not apply translator comments to all of the relevant strings. What we want to do instead is have a variable in which the translatable strings are appended to using the QString append function. This satisfies the Qt translator engine and the comments are then properly applied within the `extracomment` field in the translation file. --- src/qt/optionsdialog.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 462b923d61d9d..42c8b0776326c 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -286,11 +286,19 @@ void OptionsDialog::on_resetButton_clicked() { if (model) { // confirmation dialog + /*: Text explaining that the settings changed will not come into effect + until the client is restarted. */ + QString reset_dialog_text = tr("Client restart required to activate changes.") + "

"; + /*: Text explaining to the user that the client's current settings + will be backed up at a specific location. %1 is a stand-in + argument for the backup location's path. */ + reset_dialog_text.append(tr("Current settings will be backed up at \"%1\".").arg(m_client_model->dataDir()) + "

"); + /*: Text asking the user to confirm if they would like to proceed + with a client shutdown. */ + reset_dialog_text.append(tr("Client will be shut down. Do you want to proceed?")); + //: Window title text of pop-up window shown when the user has chosen to reset options. QMessageBox::StandardButton btnRetVal = QMessageBox::question(this, tr("Confirm options reset"), - tr("Client restart required to activate changes.") + "

" + - tr("Current settings will be backed up at \"%1\".").arg(m_client_model->dataDir()) + "

" + - tr("Client will be shut down. Do you want to proceed?"), - QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel); + reset_dialog_text, QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel); if (btnRetVal == QMessageBox::Cancel) return; From 140d942634f9f1bba191aafa948df57812c0f3fe Mon Sep 17 00:00:00 2001 From: S3RK <1466284+S3RK@users.noreply.github.com> Date: Tue, 5 Apr 2022 08:23:49 +0200 Subject: [PATCH 0063/1090] wallet: don't add change fee to target if subtracting fees from output --- src/wallet/spend.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 81055a5a1a87d..1d22d0993e7e7 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -397,10 +397,13 @@ std::optional AttemptSelection(const CWallet& wallet, const CAm // The knapsack solver has some legacy behavior where it will spend dust outputs. We retain this behavior, so don't filter for positive only here. std::vector all_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, false /* positive_only */); + CAmount target_with_change = nTargetValue; // While nTargetValue includes the transaction fees for non-input things, it does not include the fee for creating a change output. - // So we need to include that for KnapsackSolver as well, as we are expecting to create a change output. - if (auto knapsack_result{KnapsackSolver(all_groups, nTargetValue + coin_selection_params.m_change_fee, - coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { + // So we need to include that for KnapsackSolver and SRD as well, as we are expecting to create a change output. + if (!coin_selection_params.m_subtract_fee_outputs) { + target_with_change += coin_selection_params.m_change_fee; + } + if (auto knapsack_result{KnapsackSolver(all_groups, target_with_change, coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { knapsack_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*knapsack_result); } @@ -409,7 +412,7 @@ std::optional AttemptSelection(const CWallet& wallet, const CAm // barely meets the target. Just use the lower bound change target instead of the randomly // generated one, since SRD will result in a random change amount anyway; avoid making the // target needlessly large. - const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + CHANGE_LOWER; + const CAmount srd_target = target_with_change + CHANGE_LOWER; if (auto srd_result{SelectCoinsSRD(positive_groups, srd_target, coin_selection_params.rng_fast)}) { srd_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*srd_result); From 0ee43d13e993a59fe1ba509f617dd0e01e1068e2 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Thu, 30 Jun 2022 19:12:01 +0530 Subject: [PATCH 0064/1090] test: refactor rpc_signrawtransaction.py rpc_signrawtransaction.py is split into rpc_signrawtransactionwithkey.py and wallet_signrawtransactionwithwallet.py. rpc_signrawtransactionwithkey.py can be run with the wallet disabled. --- .../rpc_signrawtransactionwithkey.py | 140 ++++++++++++++++++ test/functional/test_runner.py | 5 +- ...=> wallet_signrawtransactionwithwallet.py} | 100 +------------ 3 files changed, 146 insertions(+), 99 deletions(-) create mode 100755 test/functional/rpc_signrawtransactionwithkey.py rename test/functional/{rpc_signrawtransaction.py => wallet_signrawtransactionwithwallet.py} (71%) diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py new file mode 100755 index 0000000000000..0da5a99fdb609 --- /dev/null +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test transaction signing using the signrawtransactionwithkey RPC.""" + +from test_framework.blocktools import ( + COINBASE_MATURITY, +) +from test_framework.address import ( + script_to_p2sh, +) +from test_framework.key import ECKey +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + find_vout_for_address, +) +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2pkh_script, + script_to_p2sh_p2wsh_script, + script_to_p2wsh_script, +) +from test_framework.wallet_util import ( + bytes_to_wif, +) + +from decimal import ( + Decimal, +) +from test_framework.wallet import ( + getnewdestination, +) + + +class SignRawTransactionWithKeyTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + + def send_to_address(self, addr, amount): + input = {"txid": self.nodes[0].getblock(self.block_hash[self.blk_idx])["tx"][0], "vout": 0} + output = {addr: amount} + self.blk_idx += 1 + rawtx = self.nodes[0].createrawtransaction([input], output) + txid = self.nodes[0].sendrawtransaction(self.nodes[0].signrawtransactionwithkey(rawtx, [self.nodes[0].get_deterministic_priv_key().key])["hex"], 0) + return txid + + def successful_signing_test(self): + """Create and sign a valid raw transaction with one input. + + Expected results: + + 1) The transaction has a complete set of signatures + 2) No script verification error occurred""" + self.log.info("Test valid raw transaction with one input") + privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N', 'cVKpPfVKSJxKqVpE9awvXNWuLHCa5j5tiE7K6zbUSptFpTEtiFrA'] + + inputs = [ + # Valid pay-to-pubkey scripts + {'txid': '9b907ef1e3c26fc71fe4a4b3580bc75264112f95050014157059c736f0202e71', 'vout': 0, + 'scriptPubKey': '76a91460baa0f494b38ce3c940dea67f3804dc52d1fb9488ac'}, + {'txid': '83a4f6a6b73660e13ee6cb3c6063fa3759c50c9b7521d0536022961898f4fb02', 'vout': 0, + 'scriptPubKey': '76a914669b857c03a5ed269d5d85a1ffac9ed5d663072788ac'}, + ] + + outputs = {'mpLQjfK79b7CCV4VMJWEWAj5Mpx8Up5zxB': 0.1} + + rawTx = self.nodes[0].createrawtransaction(inputs, outputs) + rawTxSigned = self.nodes[0].signrawtransactionwithkey(rawTx, privKeys, inputs) + + # 1) The transaction has a complete set of signatures + assert rawTxSigned['complete'] + + # 2) No script verification error occurred + assert 'errors' not in rawTxSigned + + def witness_script_test(self): + self.log.info("Test signing transaction to P2SH-P2WSH addresses without wallet") + # Create a new P2SH-P2WSH 1-of-1 multisig address: + eckey = ECKey() + eckey.generate() + embedded_privkey = bytes_to_wif(eckey.get_bytes()) + embedded_pubkey = eckey.get_pubkey().get_bytes().hex() + p2sh_p2wsh_address = self.nodes[1].createmultisig(1, [embedded_pubkey], "p2sh-segwit") + # send transaction to P2SH-P2WSH 1-of-1 multisig address + self.block_hash = self.generate(self.nodes[0], COINBASE_MATURITY + 1) + self.blk_idx = 0 + self.send_to_address(p2sh_p2wsh_address["address"], 49.999) + self.generate(self.nodes[0], 1) + # Get the UTXO info from scantxoutset + unspent_output = self.nodes[1].scantxoutset('start', [p2sh_p2wsh_address['descriptor']])['unspents'][0] + spk = script_to_p2sh_p2wsh_script(p2sh_p2wsh_address['redeemScript']).hex() + unspent_output['witnessScript'] = p2sh_p2wsh_address['redeemScript'] + unspent_output['redeemScript'] = script_to_p2wsh_script(unspent_output['witnessScript']).hex() + assert_equal(spk, unspent_output['scriptPubKey']) + # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys + spending_tx = self.nodes[0].createrawtransaction([unspent_output], {getnewdestination()[2]: Decimal("49.998")}) + spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) + # Check the signing completed successfully + assert 'complete' in spending_tx_signed + assert_equal(spending_tx_signed['complete'], True) + + # Now test with P2PKH and P2PK scripts as the witnessScript + for tx_type in ['P2PKH', 'P2PK']: # these tests are order-independent + self.verify_txn_with_witness_script(tx_type) + + def verify_txn_with_witness_script(self, tx_type): + self.log.info("Test with a {} script as the witnessScript".format(tx_type)) + eckey = ECKey() + eckey.generate() + embedded_privkey = bytes_to_wif(eckey.get_bytes()) + embedded_pubkey = eckey.get_pubkey().get_bytes().hex() + witness_script = { + 'P2PKH': key_to_p2pkh_script(embedded_pubkey).hex(), + 'P2PK': key_to_p2pk_script(embedded_pubkey).hex() + }.get(tx_type, "Invalid tx_type") + redeem_script = script_to_p2wsh_script(witness_script).hex() + addr = script_to_p2sh(redeem_script) + script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] + # Fund that address + txid = self.send_to_address(addr, 10) + vout = find_vout_for_address(self.nodes[0], txid, addr) + self.generate(self.nodes[0], 1) + # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys + spending_tx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], {getnewdestination()[2]: Decimal("9.999")}) + spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [{'txid': txid, 'vout': vout, 'scriptPubKey': script_pub_key, 'redeemScript': redeem_script, 'witnessScript': witness_script, 'amount': 10}]) + # Check the signing completed successfully + assert 'complete' in spending_tx_signed + assert_equal(spending_tx_signed['complete'], True) + self.nodes[0].sendrawtransaction(spending_tx_signed['hex']) + + def run_test(self): + self.successful_signing_test() + self.witness_script_test() + + +if __name__ == '__main__': + SignRawTransactionWithKeyTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6a44f9d21d413..07d0016277215 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -181,8 +181,9 @@ 'rpc_whitelist.py', 'feature_proxy.py', 'feature_syscall_sandbox.py', - 'rpc_signrawtransaction.py --legacy-wallet', - 'rpc_signrawtransaction.py --descriptors', + 'wallet_signrawtransactionwithwallet.py --legacy-wallet', + 'wallet_signrawtransactionwithwallet.py --descriptors', + 'rpc_signrawtransactionwithkey.py', 'rpc_rawtransaction.py --legacy-wallet', 'wallet_groups.py --legacy-wallet', 'wallet_transactiontime_rescan.py --descriptors', diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/wallet_signrawtransactionwithwallet.py similarity index 71% rename from test/functional/rpc_signrawtransaction.py rename to test/functional/wallet_signrawtransactionwithwallet.py index 8da2cfa72bf50..6b30386b7e604 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/wallet_signrawtransactionwithwallet.py @@ -2,16 +2,14 @@ # Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test transaction signing using the signrawtransaction* RPCs.""" +"""Test transaction signing using the signrawtransactionwithwallet RPC.""" from test_framework.blocktools import ( COINBASE_MATURITY, ) from test_framework.address import ( - script_to_p2sh, script_to_p2wsh, ) -from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -29,20 +27,13 @@ OP_DROP, OP_TRUE, ) -from test_framework.script_util import ( - key_to_p2pk_script, - key_to_p2pkh_script, - script_to_p2sh_p2wsh_script, - script_to_p2wsh_script, -) -from test_framework.wallet_util import bytes_to_wif from decimal import ( Decimal, getcontext, ) -class SignRawTransactionsTest(BitcoinTestFramework): +class SignRawTransactionWithWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 @@ -50,35 +41,6 @@ def set_test_params(self): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def successful_signing_test(self): - """Create and sign a valid raw transaction with one input. - - Expected results: - - 1) The transaction has a complete set of signatures - 2) No script verification error occurred""" - self.log.info("Test valid raw transaction with one input") - privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N', 'cVKpPfVKSJxKqVpE9awvXNWuLHCa5j5tiE7K6zbUSptFpTEtiFrA'] - - inputs = [ - # Valid pay-to-pubkey scripts - {'txid': '9b907ef1e3c26fc71fe4a4b3580bc75264112f95050014157059c736f0202e71', 'vout': 0, - 'scriptPubKey': '76a91460baa0f494b38ce3c940dea67f3804dc52d1fb9488ac'}, - {'txid': '83a4f6a6b73660e13ee6cb3c6063fa3759c50c9b7521d0536022961898f4fb02', 'vout': 0, - 'scriptPubKey': '76a914669b857c03a5ed269d5d85a1ffac9ed5d663072788ac'}, - ] - - outputs = {'mpLQjfK79b7CCV4VMJWEWAj5Mpx8Up5zxB': 0.1} - - rawTx = self.nodes[0].createrawtransaction(inputs, outputs) - rawTxSigned = self.nodes[0].signrawtransactionwithkey(rawTx, privKeys, inputs) - - # 1) The transaction has a complete set of signatures - assert rawTxSigned['complete'] - - # 2) No script verification error occurred - assert 'errors' not in rawTxSigned - def test_with_lock_outputs(self): self.log.info("Test correct error reporting when trying to sign a locked output") self.nodes[0].encryptwallet("password") @@ -191,60 +153,6 @@ def test_fully_signed_tx(self): assert_equal(signedtx["hex"], signedtx2["hex"]) self.nodes[0].walletlock() - def witness_script_test(self): - self.log.info("Test signing transaction to P2SH-P2WSH addresses without wallet") - # Create a new P2SH-P2WSH 1-of-1 multisig address: - eckey = ECKey() - eckey.generate() - embedded_privkey = bytes_to_wif(eckey.get_bytes()) - embedded_pubkey = eckey.get_pubkey().get_bytes().hex() - p2sh_p2wsh_address = self.nodes[1].createmultisig(1, [embedded_pubkey], "p2sh-segwit") - # send transaction to P2SH-P2WSH 1-of-1 multisig address - self.generate(self.nodes[0], COINBASE_MATURITY + 1) - self.nodes[0].sendtoaddress(p2sh_p2wsh_address["address"], 49.999) - self.generate(self.nodes[0], 1) - # Get the UTXO info from scantxoutset - unspent_output = self.nodes[1].scantxoutset('start', [p2sh_p2wsh_address['descriptor']])['unspents'][0] - spk = script_to_p2sh_p2wsh_script(p2sh_p2wsh_address['redeemScript']).hex() - unspent_output['witnessScript'] = p2sh_p2wsh_address['redeemScript'] - unspent_output['redeemScript'] = script_to_p2wsh_script(unspent_output['witnessScript']).hex() - assert_equal(spk, unspent_output['scriptPubKey']) - # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys - spending_tx = self.nodes[0].createrawtransaction([unspent_output], {self.nodes[1].get_wallet_rpc(self.default_wallet_name).getnewaddress(): Decimal("49.998")}) - spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) - # Check the signing completed successfully - assert 'complete' in spending_tx_signed - assert_equal(spending_tx_signed['complete'], True) - - # Now test with P2PKH and P2PK scripts as the witnessScript - for tx_type in ['P2PKH', 'P2PK']: # these tests are order-independent - self.verify_txn_with_witness_script(tx_type) - - def verify_txn_with_witness_script(self, tx_type): - self.log.info("Test with a {} script as the witnessScript".format(tx_type)) - eckey = ECKey() - eckey.generate() - embedded_privkey = bytes_to_wif(eckey.get_bytes()) - embedded_pubkey = eckey.get_pubkey().get_bytes().hex() - witness_script = { - 'P2PKH': key_to_p2pkh_script(embedded_pubkey).hex(), - 'P2PK': key_to_p2pk_script(embedded_pubkey).hex() - }.get(tx_type, "Invalid tx_type") - redeem_script = script_to_p2wsh_script(witness_script).hex() - addr = script_to_p2sh(redeem_script) - script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] - # Fund that address - txid = self.nodes[0].sendtoaddress(addr, 10) - vout = find_vout_for_address(self.nodes[0], txid, addr) - self.generate(self.nodes[0], 1) - # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys - spending_tx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): Decimal("9.999")}) - spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [{'txid': txid, 'vout': vout, 'scriptPubKey': script_pub_key, 'redeemScript': redeem_script, 'witnessScript': witness_script, 'amount': 10}]) - # Check the signing completed successfully - assert 'complete' in spending_tx_signed - assert_equal(spending_tx_signed['complete'], True) - self.nodes[0].sendrawtransaction(spending_tx_signed['hex']) - def OP_1NEGATE_test(self): self.log.info("Test OP_1NEGATE (0x4f) satisfies BIP62 minimal push standardness rule") hex_str = ( @@ -385,9 +293,7 @@ def test_signing_with_missing_prevtx_info(self): ]) def run_test(self): - self.successful_signing_test() self.script_verification_error_test() - self.witness_script_test() self.OP_1NEGATE_test() self.test_with_lock_outputs() self.test_fully_signed_tx() @@ -397,4 +303,4 @@ def run_test(self): if __name__ == '__main__': - SignRawTransactionsTest().main() + SignRawTransactionWithWalletTest().main() From 76fb300b63c5c0d01d768510ec69d894820432fa Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 30 Jun 2022 11:08:44 -0400 Subject: [PATCH 0065/1090] psbt: Check Taproot tree depth and leaf versions Since TaprootBuilder has assertions for the depth and leaf versions, the PSBT decoder should check these values before calling TaprootBuilder::Add so that the assertions are not triggered on malformed taproot trees. --- src/psbt.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/psbt.h b/src/psbt.h index a143a99988e05..c390bb67d3813 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -866,6 +866,12 @@ struct PSBTOutput s_tree >> depth; s_tree >> leaf_ver; s_tree >> script; + if (depth > TAPROOT_CONTROL_MAX_NODE_COUNT) { + throw std::ios_base::failure("Output Taproot tree has as leaf greater than Taproot maximum depth"); + } + if ((leaf_ver & ~TAPROOT_LEAF_MASK) != 0) { + throw std::ios_base::failure("Output Taproot tree has a leaf with an invalid leaf version"); + } m_tap_tree->Add((int)depth, script, (int)leaf_ver, true /* track */); } if (!m_tap_tree->IsComplete()) { From 2222842ae73f85494797b14753bc18446e4817a2 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:07:55 +0200 Subject: [PATCH 0066/1090] test: Allow absolute fee in MiniWallet create_self_transfer --- test/functional/test_framework/wallet.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 68d5dfa880081..8b5689c09e064 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -197,7 +197,7 @@ def get_utxos(self, *, mark_as_spent=True): return utxos def send_self_transfer(self, *, from_node, **kwargs): - """Create and send a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + """Call create_self_transfer and send the transaction.""" tx = self.create_self_transfer(**kwargs) self.sendrawtransaction(from_node=from_node, tx_hex=tx['hex']) return tx @@ -272,16 +272,18 @@ def create_self_transfer_multi( "tx": tx, } - def create_self_transfer(self, *, fee_rate=Decimal("0.003"), utxo_to_spend=None, locktime=0, sequence=0): - """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), fee=Decimal("0"), utxo_to_spend=None, locktime=0, sequence=0): + """Create and return a tx with the specified fee. If fee is 0, use fee_rate, where the resulting fee may be exact or at most one satoshi higher than needed.""" utxo_to_spend = utxo_to_spend or self.get_utxo() + assert fee_rate >= 0 + assert fee >= 0 if self._mode in (MiniWalletMode.RAW_OP_TRUE, MiniWalletMode.ADDRESS_OP_TRUE): vsize = Decimal(104) # anyone-can-spend elif self._mode == MiniWalletMode.RAW_P2PK: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False - send_value = utxo_to_spend["value"] - (fee_rate * vsize / 1000) + send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) assert send_value > 0 tx = CTransaction() From fac3800d2c962420303ab5c61b2f02f35b9f693a Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:09:56 +0200 Subject: [PATCH 0067/1090] test: Allow amount_per_output in MiniWallet create_self_transfer_multi --- test/functional/test_framework/wallet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 8b5689c09e064..6c83a40347989 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -232,12 +232,14 @@ def create_self_transfer_multi( *, utxos_to_spend: Optional[List[dict]] = None, num_outputs=1, + amount_per_output=0, sequence=0, fee_per_output=1000, ): """ Create and return a transaction that spends the given UTXOs and creates a - certain number of outputs with equal amounts. + certain number of outputs with equal amounts. The output amounts can be + set by amount_per_output or automatically calculated with a fee_per_output. """ utxos_to_spend = utxos_to_spend or [self.get_utxo()] # create simple tx template (1 input, 1 output) @@ -258,7 +260,7 @@ def create_self_transfer_multi( inputs_value_total = sum([int(COIN * utxo['value']) for utxo in utxos_to_spend]) outputs_value_total = inputs_value_total - fee_per_output * num_outputs for o in tx.vout: - o.nValue = outputs_value_total // num_outputs + o.nValue = amount_per_output or (outputs_value_total // num_outputs) txid = tx.rehash() return { "new_utxos": [self._create_utxo( From fa2924582726ee00b392bd0b5591e7d9770e1b90 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:27:59 +0200 Subject: [PATCH 0068/1090] test: Allow setting sequence per input in MiniWallet create_self_transfer_multi Previously it was only possible to set the same sequence in all inputs --- test/functional/test_framework/wallet.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 6c83a40347989..7ab54618c8b41 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -242,13 +242,17 @@ def create_self_transfer_multi( set by amount_per_output or automatically calculated with a fee_per_output. """ utxos_to_spend = utxos_to_spend or [self.get_utxo()] + sequence = [sequence] * len(utxos_to_spend) if type(sequence) is int else sequence + assert_equal(len(utxos_to_spend), len(sequence)) # create simple tx template (1 input, 1 output) tx = self.create_self_transfer( fee_rate=0, - utxo_to_spend=utxos_to_spend[0], sequence=sequence)["tx"] + utxo_to_spend=utxos_to_spend[0])["tx"] # duplicate inputs, witnesses and outputs tx.vin = [deepcopy(tx.vin[0]) for _ in range(len(utxos_to_spend))] + for txin, seq in zip(tx.vin, sequence): + txin.nSequence = seq tx.wit.vtxinwit = [deepcopy(tx.wit.vtxinwit[0]) for _ in range(len(utxos_to_spend))] tx.vout = [deepcopy(tx.vout[0]) for _ in range(num_outputs)] From fa5059b7dfba6d95c14a12f21d02b63fe29b3f2b Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:28:33 +0200 Subject: [PATCH 0069/1090] test: Make the scriptPubKey of MiniWallet created txs mutable This makes individual bytes of the scriptPubKey mutable, previously it could only be re-assigned as a whole. --- test/functional/test_framework/wallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 7ab54618c8b41..216462778148b 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -294,7 +294,7 @@ def create_self_transfer(self, *, fee_rate=Decimal("0.003"), fee=Decimal("0"), u tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=sequence)] - tx.vout = [CTxOut(int(COIN * send_value), self._scriptPubKey)] + tx.vout = [CTxOut(int(COIN * send_value), bytearray(self._scriptPubKey))] tx.nLockTime = locktime if self._mode == MiniWalletMode.RAW_P2PK: self.sign_tx(tx) From 99f4785cad94657dcf349d00fdd6f1d44cac9bb0 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 28 Jun 2022 10:53:02 -0400 Subject: [PATCH 0070/1090] Replace GetTime() with NodeClock in MaybeSendGetHeaders() --- src/net_processing.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index fe1b06b18cdff..8cdc08c865a14 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -358,7 +358,7 @@ struct Peer { std::deque m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); /** Time of the last getheaders message to this peer */ - std::atomic m_last_getheaders_timestamp{0s}; + std::atomic m_last_getheaders_timestamp{NodeSeconds{}}; Peer(NodeId id) : m_id{id} @@ -2272,10 +2272,11 @@ bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& loc { const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - const auto current_time = GetTime(); + const auto current_time = NodeClock::now(); + // Only allow a new getheaders message to go out if we don't have a recent // one already in-flight - if (peer.m_last_getheaders_timestamp.load() < current_time - HEADERS_RESPONSE_TIME) { + if (current_time - peer.m_last_getheaders_timestamp.load() > HEADERS_RESPONSE_TIME) { m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); peer.m_last_getheaders_timestamp = current_time; return true; @@ -3973,7 +3974,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Assume that this is in response to any outstanding getheaders // request we may have sent, and clear out the time of our last request - peer->m_last_getheaders_timestamp = 0s; + peer->m_last_getheaders_timestamp.store(NodeSeconds{}); std::vector headers; From ebe106a754d2da567702e60185142d9e381bf1cd Mon Sep 17 00:00:00 2001 From: glozow Date: Fri, 1 Jul 2022 14:09:14 +0100 Subject: [PATCH 0071/1090] add glozow to trusted-keys --- contrib/verify-commits/trusted-keys | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index 2f8a21009aebc..046589a583d28 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -4,3 +4,4 @@ B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B E777299FC265DD04793070EB944D35F9AC3DB76A D1DBF2C4B96F2DEBF4C16654410108112E7EA81F 152812300785C96444D3334D17565732E08E5E41 +6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C From bc13ec888cdc2791f79eeb6eb76b9134d670043e Mon Sep 17 00:00:00 2001 From: w0xlt <94266259+w0xlt@users.noreply.github.com> Date: Fri, 1 Jul 2022 10:16:42 -0300 Subject: [PATCH 0072/1090] doc: Add a release note about the "restore wallet" menu item --- doc/release-notes/release-notes-471.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release-notes/release-notes-471.md diff --git a/doc/release-notes/release-notes-471.md b/doc/release-notes/release-notes-471.md new file mode 100644 index 0000000000000..7cebedd8b30be --- /dev/null +++ b/doc/release-notes/release-notes-471.md @@ -0,0 +1,4 @@ +GUI changes +-------- + +- A new menu item to restore a wallet from a backup file has been added (#471). \ No newline at end of file From eac1099e0071bfe11fe664a8a490eee6bbc80c47 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Fri, 1 Jul 2022 19:27:34 +0530 Subject: [PATCH 0073/1090] test: remove wallet dependency from mempool_updatefromblock.py This functional test can now be run with the wallet disabled. --- test/functional/mempool_updatefromblock.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 51de582ce0f11..f97c2223a6877 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -12,6 +12,9 @@ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +from test_framework.address import key_to_p2pkh +from test_framework.wallet_util import bytes_to_wif +from test_framework.key import ECKey class MempoolUpdateFromBlockTest(BitcoinTestFramework): @@ -19,8 +22,13 @@ def set_test_params(self): self.num_nodes = 1 self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000', '-limitancestorcount=100']] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def get_new_address(self): + key = ECKey() + key.generate() + pubkey = key.get_pubkey().get_bytes() + address = key_to_p2pkh(pubkey) + self.priv_keys.append(bytes_to_wif(key.get_bytes())) + return address def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', end_address='', fee=Decimal(0.00100000)): """Create an acyclic tournament (a type of directed graph) of transactions and use it for testing. @@ -38,11 +46,12 @@ def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', e More details: https://en.wikipedia.org/wiki/Tournament_(graph_theory) """ + self.priv_keys = [self.nodes[0].get_deterministic_priv_key().key] if not start_input_txid: start_input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0] if not end_address: - end_address = self.nodes[0].getnewaddress() + end_address = self.get_new_address() first_block_hash = '' tx_id = [] @@ -74,7 +83,7 @@ def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', e output_value = ((inputs_value - fee) / Decimal(n_outputs)).quantize(Decimal('0.00000001')) outputs = {} for _ in range(n_outputs): - outputs[self.nodes[0].getnewaddress()] = output_value + outputs[self.get_new_address()] = output_value else: output_value = (inputs_value - fee).quantize(Decimal('0.00000001')) outputs = {end_address: output_value} @@ -84,7 +93,7 @@ def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', e # Create a new transaction. unsigned_raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) - signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(unsigned_raw_tx) + signed_raw_tx = self.nodes[0].signrawtransactionwithkey(unsigned_raw_tx, self.priv_keys) tx_id.append(self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])) tx_size.append(self.nodes[0].getmempoolentry(tx_id[-1])['vsize']) From 220a5a2841172a07d6d7849596316f0e0933e272 Mon Sep 17 00:00:00 2001 From: 0xb10c <0xb10c@gmail.com> Date: Fri, 1 Jul 2022 13:08:49 +0200 Subject: [PATCH 0074/1090] test: hook into PID in tracing tests This makes sure to NOT hook into other bitcoind binaries run in paralell in the test framework. We only want to trace the intended binary. In interface_usdt_utxocache.py: While testing the utxocache flush with pruning, bitcoind is restarted and we need to hook into the new PID again. --- test/functional/interface_usdt_net.py | 2 +- test/functional/interface_usdt_utxocache.py | 15 ++++++++++++--- test/functional/interface_usdt_validation.py | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py index 9522cd8c59588..2235da702b2db 100755 --- a/test/functional/interface_usdt_net.py +++ b/test/functional/interface_usdt_net.py @@ -109,7 +109,7 @@ def __repr__(self): self.log.info( "hook into the net:inbound_message and net:outbound_message tracepoints") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="net:inbound_message", fn_name="trace_inbound_message") ctx.enable_probe(probe="net:outbound_message", diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index f48ff9699d8c1..61587197efba3 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -173,7 +173,7 @@ def test_uncache(self): invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16) self.log.info("hooking into the utxocache:uncache tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:uncache", fn_name="trace_utxocache_uncache") bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0) @@ -238,7 +238,7 @@ def test_add_spent(self): self.log.info( "hook into the utxocache:add and utxocache:spent tracepoints") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add") ctx.enable_probe(probe="utxocache:spent", fn_name="trace_utxocache_spent") @@ -334,7 +334,7 @@ def test_flush(self): self.log.info("test the utxocache:flush tracepoint API") self.log.info("hook into the utxocache:flush tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:flush", fn_name="trace_utxocache_flush") bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) @@ -373,6 +373,7 @@ def handle_utxocache_flush(_, data, __): self.stop_node(0) bpf.perf_buffer_poll(timeout=200) + bpf.cleanup() self.log.info("check that we don't expect additional flushes") assert_equal(0, len(expected_flushes)) @@ -381,6 +382,14 @@ def handle_utxocache_flush(_, data, __): self.log.info("restart the node with -prune") self.start_node(0, ["-fastprune=1", "-prune=1"]) + self.log.info("test the utxocache:flush tracepoint API with pruning") + self.log.info("hook into the utxocache:flush tracepoint") + ctx = USDT(pid=self.nodes[0].process.pid) + ctx.enable_probe(probe="utxocache:flush", + fn_name="trace_utxocache_flush") + bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) + bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) + BLOCKS_TO_MINE = 350 self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") self.generate(self.wallet, BLOCKS_TO_MINE) diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py index d11809273bbd5..fb1a91b91410d 100755 --- a/test/functional/interface_usdt_validation.py +++ b/test/functional/interface_usdt_validation.py @@ -94,7 +94,7 @@ def __repr__(self): expected_blocks = list() self.log.info("hook into the validation:block_connected tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="validation:block_connected", fn_name="trace_block_connected") bpf = BPF(text=validation_blockconnected_program, From dba6f8234217565957e37516a0ea655f1180d99c Mon Sep 17 00:00:00 2001 From: 0xb10c <0xb10c@gmail.com> Date: Fri, 1 Jul 2022 13:17:30 +0200 Subject: [PATCH 0075/1090] test: adopt USDT utxocache interface tests The USDT interface exposes process internals via the tracepoints. This means, the USDT interface tests somewhat awardly depend on these internals. If internals change, the tests have to adopt to that change. Previously, the USDT interface tests weren't run in the CI so changes could break the USDT interface tests without being noticed (e.g. https://github.com/bitcoin/bitcoin/pull/25486). In fa13375aa3fcb4fd5b9e0d4c69ac31cf66c3209a a 'self.rescan_utxos()' call was added in the 'generate()' function of the test framework. 'rescan_utxos()' causes the UTXO cache to be flushed. In the USDT interface tests for the 'utxocache:flush' trancepoint, 'generate()' is used. As the utxo cache is now flushed more often, the number of flushes the tests expectes need to be adopted. Also, the utxo cache has now a different size when being flushed. The utxocache tracepoint is tested by shutting the node down and pruning blocks, to test the 'for_prune' argument. Changes: - A list 'expected_flushes' is now used which contains 'mode', 'for_prune', and 'size' for each expected flush. - When a flush happens, the expected-flush is removed from the list. This list is checked to be empty (unchanged). - Previously, shutting down caused these two flushes: UTXOCacheFlush(duration=*, mode=ALWAYS, size=104, memory=*, for_prune=False) UTXOCacheFlush(duration=*, mode=ALWAYS, size=0, memory=*, for_prune=False) now it causes these flushes: UTXOCacheFlush(duration=*, mode=ALWAYS, size=2, memory=*, for_prune=False) UTXOCacheFlush(duration=*, mode=ALWAYS, size=0, memory=*, for_prune=False) The 104 UTXOs flushed previously were mainly coinbase UTXOs generated in previous tests and the test setup. These are now already flushed. - In the 'for_prune' test we previously hooked into the tracepoint before mining blocks. This changed to only get notified about the tracepoint being triggered for the prune. Here, the utxo cache is empty already as it has just been flushed in 'generate()'. old: UTXOCacheFlush(duration=*, mode=NONE, size=350, memory=*, for_prune=True) new: UTXOCacheFlush(duration=*, mode=NONE, size=0, memory=*, for_prune=True) --- test/functional/interface_usdt_utxocache.py | 33 +++++++++------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index 61587197efba3..2280de14790c1 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -345,16 +345,17 @@ def test_flush(self): # that the handle_* functions succeeded. EXPECTED_HANDLE_FLUSH_SUCCESS = 3 handle_flush_succeeds = 0 - possible_cache_sizes = set() - expected_flushes = [] + expected_flushes = list() def handle_utxocache_flush(_, data, __): nonlocal handle_flush_succeeds event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents self.log.info(f"handle_utxocache_flush(): {event}") - expected = expected_flushes.pop(0) - assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode]) - possible_cache_sizes.remove(event.size) # fails if size not in set + expected_flushes.remove({ + "mode": FLUSHMODE_NAME[event.mode], + "for_prune": event.for_prune, + "size": event.size + }) # sanity checks only assert(event.memory > 0) assert(event.duration > 0) @@ -363,13 +364,12 @@ def handle_utxocache_flush(_, data, __): bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) self.log.info("stop the node to flush the UTXO cache") - UTXOS_IN_CACHE = 104 # might need to be changed if the eariler tests are modified + UTXOS_IN_CACHE = 2 # might need to be changed if the eariler tests are modified # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the # second flush, however it can happen that the order changes. - possible_cache_sizes = {UTXOS_IN_CACHE, 0} - flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False} - expected_flushes.extend([flush_for_shutdown, flush_for_shutdown]) + expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": UTXOS_IN_CACHE}) + expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": 0}) self.stop_node(0) bpf.perf_buffer_poll(timeout=200) @@ -377,11 +377,14 @@ def handle_utxocache_flush(_, data, __): self.log.info("check that we don't expect additional flushes") assert_equal(0, len(expected_flushes)) - assert_equal(0, len(possible_cache_sizes)) self.log.info("restart the node with -prune") self.start_node(0, ["-fastprune=1", "-prune=1"]) + BLOCKS_TO_MINE = 350 + self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") + self.generate(self.wallet, BLOCKS_TO_MINE) + self.log.info("test the utxocache:flush tracepoint API with pruning") self.log.info("hook into the utxocache:flush tracepoint") ctx = USDT(pid=self.nodes[0].process.pid) @@ -390,15 +393,8 @@ def handle_utxocache_flush(_, data, __): bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) - BLOCKS_TO_MINE = 350 - self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") - self.generate(self.wallet, BLOCKS_TO_MINE) - # we added BLOCKS_TO_MINE coinbase UTXOs to the cache - possible_cache_sizes = {BLOCKS_TO_MINE} - expected_flushes.append( - {"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE}) - self.log.info(f"prune blockchain to trigger a flush for pruning") + expected_flushes.append({"mode": "NONE", "for_prune": True, "size": 0}) self.nodes[0].pruneblockchain(315) bpf.perf_buffer_poll(timeout=500) @@ -407,7 +403,6 @@ def handle_utxocache_flush(_, data, __): self.log.info( f"check that we don't expect additional flushes and that the handle_* function succeeded") assert_equal(0, len(expected_flushes)) - assert_equal(0, len(possible_cache_sizes)) assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds) From 1770be72d5eb47cae565d4ac86de5bc16f94fdd6 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Sun, 3 Jul 2022 16:06:56 +0200 Subject: [PATCH 0076/1090] test: pass `dustrelayfee=0` option for tests using dust (instead of `acceptnonstdtxn=1`) By specifying the `dustrelayfee=0` option instead of the more generic `acceptnonstdtxn=1`, we can be more specific about what part of the transaction is non-standard and can be sure that all other aspects follow the standard policy. --- test/functional/feature_dbcrash.py | 8 +++++--- test/functional/wallet_basic.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 62e9bec66321e..f606f26e70619 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -62,8 +62,8 @@ def set_test_params(self): self.node2_args = ["-dbcrashratio=24", "-dbcache=16"] + self.base_args # Node3 is a normal node with default args, except will mine full blocks - # and non-standard txs (e.g. txs with "dust" outputs) - self.node3_args = ["-blockmaxweight=4000000", "-acceptnonstdtxn"] + # and txs with "dust" outputs + self.node3_args = ["-blockmaxweight=4000000", "-dustrelayfee=0"] self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args] def setup_network(self): @@ -211,7 +211,9 @@ def run_test(self): self.crashed_on_restart = 0 # Track count of crashes during recovery # Start by creating a lot of utxos on node3 - utxo_list = self.wallet.send_self_transfer_multi(from_node=self.nodes[3], num_outputs=5000)['new_utxos'] + utxo_list = [] + for _ in range(5): + utxo_list.extend(self.wallet.send_self_transfer_multi(from_node=self.nodes[3], num_outputs=1000)['new_utxos']) self.generate(self.nodes[3], 1, sync_fun=self.no_op) assert_equal(len(self.nodes[3].getrawmempool()), 0) self.log.info(f"Prepped {len(utxo_list)} utxo entries") diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index a6c93ba5f9d72..f66fab19ac372 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -25,7 +25,7 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.extra_args = [[ - "-acceptnonstdtxn=1", "-walletrejectlongchains=0" + "-dustrelayfee=0", "-walletrejectlongchains=0" ]] * self.num_nodes self.setup_clean_chain = True self.supports_cli = False From cccf691c24f9cbc4aedd1b36c1d9ba173910ceca Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Sun, 3 Jul 2022 20:07:44 +0200 Subject: [PATCH 0077/1090] contrib: dedup `get_witness_script` helper in signet miner --- contrib/signet/miner | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/contrib/signet/miner b/contrib/signet/miner index b366b98e2d8d9..61415cb2dd021 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -21,8 +21,8 @@ PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__fi PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) -from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402 -from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, tx_from_hex, uint256_from_str # noqa: E402 +from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, ser_compact_size, ser_string, ser_uint256, tx_from_hex # noqa: E402 from test_framework.script import CScriptOp # noqa: E402 logging.basicConfig( @@ -123,10 +123,6 @@ def create_coinbase(height, value, spk): cb.vout = [CTxOut(value, spk)] return cb -def get_witness_script(witness_root, witness_nonce): - commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce))) - return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment)) - def signet_txs(block, challenge): # assumes signet solution has not been added yet so does not need # to be removed @@ -222,7 +218,7 @@ def generate_psbt(tmpl, reward_spk, *, blocktime=None): cbwit = CTxInWitness() cbwit.scriptWitness.stack = [ser_uint256(witnonce)] block.vtx[0].wit.vtxinwit = [cbwit] - block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce))) + block.vtx[0].vout.append(CTxOut(0, bytes(get_witness_script(witroot, witnonce)))) signme, spendme = signet_txs(block, signet_spk_bin) @@ -627,5 +623,3 @@ def main(): if __name__ == "__main__": main() - - From 236239bd40ae1175537fc932df5af27902326329 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 1 May 2022 14:28:14 +0200 Subject: [PATCH 0078/1090] wallet: Rescan mempool for transactions as well --- src/wallet/test/wallet_tests.cpp | 8 ++++---- src/wallet/wallet.cpp | 7 ++++++- test/functional/wallet_importdescriptors.py | 4 +++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 70863f5464ab8..0ba39b52d53c8 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -55,9 +55,6 @@ static const std::shared_ptr TestLoadWallet(WalletContext& context) auto database = MakeWalletDatabase("", options, status, error); auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); NotifyWalletLoaded(context, wallet); - if (context.chain) { - wallet->postInitProcess(); - } return wallet; } @@ -765,6 +762,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // being blocked wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); + // AddToWallet events for block_tx and mempool_tx BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); @@ -777,6 +775,8 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // transactionAddedToMempool events are processed promise.set_value(); SyncWithValidationInterfaceQueue(); + // AddToWallet events for block_tx and mempool_tx events are counted a + // second time as the notificaiton queue is processed BOOST_CHECK_EQUAL(addtx_count, 4); @@ -800,7 +800,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); }); wallet = TestLoadWallet(context); - BOOST_CHECK_EQUAL(addtx_count, 4); + BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 910562e66901a..e3aebd4f81746 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1687,7 +1687,8 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r /** * Scan the block chain (starting in start_block) for transactions * from or to us. If fUpdate is true, found transactions that already - * exist in the wallet will be updated. + * exist in the wallet will be updated. If max_height is not set, the + * mempool will be scanned as well. * * @param[in] start_block Scan starting block. If block is not on the active * chain, the scan will return SUCCESS immediately. @@ -1797,6 +1798,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc } } } + if (!max_height) { + WalletLogPrintf("Scanning current mempool transactions.\n"); + WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); + } ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index ac74ff248440c..b74a041539e8b 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -480,7 +480,9 @@ def run_test(self): addr = wmulti_pub.getnewaddress('', 'bech32') assert_equal(addr, 'bcrt1qp8s25ckjl7gr6x2q3dx3tn2pytwp05upkjztk6ey857tt50r5aeqn6mvr9') # Derived at m/84'/0'/0'/1 change_addr = wmulti_pub.getrawchangeaddress('bech32') - assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') + assert_equal(change_addr, 'bcrt1qzxl0qz2t88kljdnkzg4n4gapr6kte26390gttrg79x66nt4p04fssj53nl') + assert(send_txid in self.nodes[0].getrawmempool(True)) + assert(send_txid in (x['txid'] for x in wmulti_pub.listunspent(0))) assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 999) # generate some utxos for next tests From 3abdbbb90a4a8f2041fec37506268e66a0b3eb31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Barbosa?= Date: Tue, 12 May 2020 23:33:43 +0100 Subject: [PATCH 0079/1090] rpc, wallet: Document and test mempool scan after importaddress co-authored-by: Fabian Jahr --- src/wallet/rpc/backup.cpp | 4 +++- test/functional/wallet_balance.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index aa9f23c886635..0b73d444f85d6 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -201,6 +201,8 @@ RPCHelpMan importaddress() "\nAdds an address or script (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported address exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "If you have the full public key, you should call importpubkey instead of this.\n" "Hint: use importmulti to import more than one address.\n" "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" @@ -209,7 +211,7 @@ RPCHelpMan importaddress() { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The Bitcoin address (or hex-encoded script)"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, {"p2sh", RPCArg::Type::BOOL, RPCArg::Default{false}, "Add the P2SH version of the script as well"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 0c93821e7f472..94b74c0696d44 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -273,6 +273,22 @@ def test_balances(*, fee_node_1=0): self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY) assert_equal(self.nodes[0].getbalance(minconf=0), total_amount + 1) # The reorg recovered our fee of 1 coin + if not self.options.descriptors: + self.log.info('Check if mempool is taken into account after import*') + address = self.nodes[0].getnewaddress() + privkey = self.nodes[0].dumpprivkey(address) + self.nodes[0].sendtoaddress(address, 0.1) + self.nodes[0].unloadwallet('') + # check importaddress on fresh wallet + self.nodes[0].createwallet('w1', False, True) + self.nodes[0].importaddress(address) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], 0) + assert_equal(self.nodes[0].getbalances()['watchonly']['untrusted_pending'], Decimal('0.1')) + self.nodes[0].importprivkey(privkey) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('0.1')) + assert_equal(self.nodes[0].getbalances()['watchonly']['untrusted_pending'], 0) + self.nodes[0].unloadwallet('w1') + if __name__ == '__main__': WalletTest().main() From 6d3db52e667474b6c0c2e4eeb9fb5b3ba4063205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Barbosa?= Date: Tue, 12 May 2020 23:34:04 +0100 Subject: [PATCH 0080/1090] rpc, wallet: Document and test mempool scan after importprivkey co-authored-by: Fabian Jahr --- src/wallet/rpc/backup.cpp | 4 +++- test/functional/wallet_balance.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 0b73d444f85d6..386c0e90508ad 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -100,11 +100,13 @@ RPCHelpMan importprivkey() "Hint: use importmulti to import more than one private key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key (see dumpprivkey)"}, {"label", RPCArg::Type::STR, RPCArg::DefaultHint{"current label if address exists, otherwise \"\""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 94b74c0696d44..d49bca6855781 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -288,6 +288,10 @@ def test_balances(*, fee_node_1=0): assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('0.1')) assert_equal(self.nodes[0].getbalances()['watchonly']['untrusted_pending'], 0) self.nodes[0].unloadwallet('w1') + # check importprivkey on fresh wallet + self.nodes[0].createwallet('w2', False, True) + self.nodes[0].importprivkey(privkey) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('0.1')) if __name__ == '__main__': From e6d3ef85867545a5a66a211e35e818e8a1b166fa Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 1 May 2022 19:45:32 +0200 Subject: [PATCH 0081/1090] rpc, wallet: Document mempool scan after importpubkey --- src/wallet/rpc/backup.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 386c0e90508ad..0f058e45edf05 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -404,11 +404,13 @@ RPCHelpMan importpubkey() "Hint: use importmulti to import more than one public key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"pubkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The hex-encoded public key"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ From 0e396d1ba701c9ac6280a98bf37f53352167e724 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 1 May 2022 19:53:09 +0200 Subject: [PATCH 0082/1090] rpc, wallet: Document mempool scan after importmulti --- src/wallet/rpc/backup.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 0f058e45edf05..e8a39239eb763 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -1255,6 +1255,8 @@ RPCHelpMan importmulti() "Conversely, if all the private keys are provided and the address/script is spendable, the watchonly option must be set to false, or a warning will be returned.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", @@ -1296,7 +1298,7 @@ RPCHelpMan importmulti() "\"requests\""}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Stating if should rescan the blockchain after all imports"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, "\"options\""}, }, From 833ce76df712932c19e99737e87b5569e2bca34b Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 1 May 2022 20:00:10 +0200 Subject: [PATCH 0083/1090] rpc, wallet: Document mempool rescan after importdescriptor, importwallet --- src/wallet/rpc/backup.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index e8a39239eb763..26c2cccb2f6de 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -489,7 +489,7 @@ RPCHelpMan importwallet() { return RPCHelpMan{"importwallet", "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" - "Note: Use \"getwalletinfo\" to query the scanning progress.\n", + "Note: Blockchain and Mempool will be rescanned after a successful import. Use \"getwalletinfo\" to query the scanning progress.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet file"}, }, @@ -1600,7 +1600,7 @@ RPCHelpMan importdescriptors() " Use the string \"now\" to substitute the current synced blockchain time.\n" " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" - " of all descriptors being imported will be scanned.", + "of all descriptors being imported will be scanned as well as the mempool.", /*oneline_description=*/"", {"timestamp | \"now\"", "integer / string"} }, {"internal", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, From 1be796418934ae7370cb0ed501877db59e738106 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Wed, 1 Jun 2022 01:11:50 +0200 Subject: [PATCH 0084/1090] test, wallet: Add mempool rescan test for import RPCs --- test/functional/wallet_import_rescan.py | 57 ++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index d9acc8cea5b4b..085ad51c79916 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -87,6 +87,7 @@ def check(self, txid=None, amount=None, confirmation_height=None): assert_equal(len(txs), self.expected_txs) addresses = self.node.listreceivedbyaddress(minconf=0, include_watchonly=True, address_filter=self.address['address']) + if self.expected_txs: assert_equal(len(addresses[0]["txids"]), self.expected_txs) @@ -98,13 +99,18 @@ def check(self, txid=None, amount=None, confirmation_height=None): assert_equal(tx["category"], "receive") assert_equal(tx["label"], self.label) assert_equal(tx["txid"], txid) - assert_equal(tx["confirmations"], 1 + current_height - confirmation_height) - assert "trusted" not in tx + + # If no confirmation height is given, the tx is still in the + # mempool. + confirmations = (1 + current_height - confirmation_height) if confirmation_height else 0 + assert_equal(tx["confirmations"], confirmations) + if confirmations: + assert "trusted" not in tx address, = [ad for ad in addresses if txid in ad["txids"]] assert_equal(address["address"], self.address["address"]) assert_equal(address["amount"], self.expected_balance) - assert_equal(address["confirmations"], 1 + current_height - confirmation_height) + assert_equal(address["confirmations"], confirmations) # Verify the transaction is correctly marked watchonly depending on # whether the transaction pays to an imported public key or # imported private key. The test setup ensures that transaction @@ -162,11 +168,12 @@ def setup_network(self): self.import_deterministic_coinbase_privkeys() self.stop_nodes() - self.start_nodes() + self.start_nodes(extra_args=[["-whitelist=noban@127.0.0.1"]] * self.num_nodes) for i in range(1, self.num_nodes): self.connect_nodes(i, 0) def run_test(self): + # Create one transaction on node 0 with a unique amount for # each possible type of wallet import RPC. for i, variant in enumerate(IMPORT_VARIANTS): @@ -207,7 +214,7 @@ def run_test(self): variant.check() # Create new transactions sending to each address. - for i, variant in enumerate(IMPORT_VARIANTS): + for variant in IMPORT_VARIANTS: variant.sent_amount = get_rand_amount() variant.sent_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.sent_amount) self.generate(self.nodes[0], 1) # Generate one block for each send @@ -223,6 +230,46 @@ def run_test(self): variant.expected_txs += 1 variant.check(variant.sent_txid, variant.sent_amount, variant.confirmation_height) + self.log.info('Test that the mempool is rescanned as well if the rescan parameter is set to true') + + # The late timestamp and pruned variants are not necessary when testing mempool rescan + mempool_variants = [variant for variant in IMPORT_VARIANTS if variant.rescan != Rescan.late_timestamp and not variant.prune] + # No further blocks are mined so the timestamp will stay the same + timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] + + # Create one transaction on node 0 with a unique amount for + # each possible type of wallet import RPC. + for i, variant in enumerate(mempool_variants): + variant.label = "mempool label {} {}".format(i, variant) + variant.address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress( + label=variant.label, + address_type=variant.address_type.value, + )) + variant.key = self.nodes[1].dumpprivkey(variant.address["address"]) + variant.initial_amount = get_rand_amount() + variant.initial_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.initial_amount) + variant.confirmation_height = 0 + variant.timestamp = timestamp + + assert_equal(len(self.nodes[0].getrawmempool()), len(mempool_variants)) + self.sync_mempools() + + # For each variation of wallet key import, invoke the import RPC and + # check the results from getbalance and listtransactions. + for variant in mempool_variants: + self.log.info('Run import for mempool variant {}'.format(variant)) + expect_rescan = variant.rescan == Rescan.yes + variant.node = self.nodes[2 + IMPORT_NODES.index(ImportNode(variant.prune, expect_rescan))] + variant.do_import(variant.timestamp) + if expect_rescan: + variant.expected_balance = variant.initial_amount + variant.expected_txs = 1 + variant.check(variant.initial_txid, variant.initial_amount) + else: + variant.expected_balance = 0 + variant.expected_txs = 0 + variant.check() + if __name__ == "__main__": ImportRescanTest().main() From 42aa5d5b6269d27af525d5001907558442e96023 Mon Sep 17 00:00:00 2001 From: dergoegge Date: Thu, 26 May 2022 15:40:21 +0200 Subject: [PATCH 0085/1090] [net] Add NoBan status to NodeEvictionCandidate --- src/net.cpp | 14 ++++++++++++-- src/net.h | 1 + src/test/fuzz/node_eviction.cpp | 1 + src/test/util/net.cpp | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 7f4e571c8dc58..91a1b05b81685 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -948,6 +948,15 @@ static void EraseLastKElements( elements.erase(std::remove_if(elements.end() - eraseSize, elements.end(), predicate), elements.end()); } +void ProtectNoBanConnections(std::vector& eviction_candidates) +{ + eviction_candidates.erase(std::remove_if(eviction_candidates.begin(), eviction_candidates.end(), + [](NodeEvictionCandidate const& n) { + return n.m_noban; + }), + eviction_candidates.end()); +} + void ProtectEvictionCandidatesByRatio(std::vector& eviction_candidates) { // Protect the half of the remaining nodes which have been connected the longest. @@ -1025,6 +1034,8 @@ void ProtectEvictionCandidatesByRatio(std::vector& evicti { // Protect connections with certain characteristics + ProtectNoBanConnections(vEvictionCandidates); + // Deterministically select 4 peers to protect by netgroup. // An attacker cannot predict which netgroups will be protected EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); @@ -1096,8 +1107,6 @@ bool CConnman::AttemptToEvictConnection() LOCK(m_nodes_mutex); for (const CNode* node : m_nodes) { - if (node->HasPermission(NetPermissionFlags::NoBan)) - continue; if (!node->IsInboundConn()) continue; if (node->fDisconnect) @@ -1115,6 +1124,7 @@ bool CConnman::AttemptToEvictConnection() Desig(prefer_evict) node->m_prefer_evict, Desig(m_is_local) node->addr.IsLocal(), Desig(m_network) node->ConnectedThroughNetwork(), + Desig(m_noban) node->HasPermission(NetPermissionFlags::NoBan), }; vEvictionCandidates.push_back(candidate); } diff --git a/src/net.h b/src/net.h index bf169649c0b8c..e46933d972ea3 100644 --- a/src/net.h +++ b/src/net.h @@ -1261,6 +1261,7 @@ struct NodeEvictionCandidate bool prefer_evict; bool m_is_local; Network m_network; + bool m_noban; }; /** diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index 6a363f00f702b..d7721f1bf6739 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -32,6 +32,7 @@ FUZZ_TARGET(node_eviction) /*prefer_evict=*/fuzzed_data_provider.ConsumeBool(), /*m_is_local=*/fuzzed_data_provider.ConsumeBool(), /*m_network=*/fuzzed_data_provider.PickValueInArray(ALL_NETWORKS), + /*m_noban=*/fuzzed_data_provider.ConsumeBool(), }); } // Make a copy since eviction_candidates may be in some valid but otherwise diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 62b770753a037..26b3acc677eff 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -58,6 +58,7 @@ std::vector GetRandomNodeEvictionCandidates(int n_candida /*prefer_evict=*/random_context.randbool(), /*m_is_local=*/random_context.randbool(), /*m_network=*/ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], + /*m_noban=*/false, }); } return candidates; From a3c27070396ab8c2941c437e8099547e8fc9c110 Mon Sep 17 00:00:00 2001 From: dergoegge Date: Thu, 26 May 2022 15:49:10 +0200 Subject: [PATCH 0086/1090] [net] Add connection type to NodeEvictionCandidate --- src/net.cpp | 14 ++++++++++++-- src/net.h | 1 + src/test/fuzz/node_eviction.cpp | 1 + src/test/util/net.cpp | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 91a1b05b81685..46b8975eba974 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -957,6 +957,15 @@ void ProtectNoBanConnections(std::vector& eviction_candid eviction_candidates.end()); } +void ProtectOutboundConnections(std::vector& eviction_candidates) +{ + eviction_candidates.erase(std::remove_if(eviction_candidates.begin(), eviction_candidates.end(), + [](NodeEvictionCandidate const& n) { + return n.m_conn_type != ConnectionType::INBOUND; + }), + eviction_candidates.end()); +} + void ProtectEvictionCandidatesByRatio(std::vector& eviction_candidates) { // Protect the half of the remaining nodes which have been connected the longest. @@ -1036,6 +1045,8 @@ void ProtectEvictionCandidatesByRatio(std::vector& evicti ProtectNoBanConnections(vEvictionCandidates); + ProtectOutboundConnections(vEvictionCandidates); + // Deterministically select 4 peers to protect by netgroup. // An attacker cannot predict which netgroups will be protected EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); @@ -1107,8 +1118,6 @@ bool CConnman::AttemptToEvictConnection() LOCK(m_nodes_mutex); for (const CNode* node : m_nodes) { - if (!node->IsInboundConn()) - continue; if (node->fDisconnect) continue; NodeEvictionCandidate candidate{ @@ -1125,6 +1134,7 @@ bool CConnman::AttemptToEvictConnection() Desig(m_is_local) node->addr.IsLocal(), Desig(m_network) node->ConnectedThroughNetwork(), Desig(m_noban) node->HasPermission(NetPermissionFlags::NoBan), + Desig(m_conn_type) node->m_conn_type, }; vEvictionCandidates.push_back(candidate); } diff --git a/src/net.h b/src/net.h index e46933d972ea3..c76c446dbac94 100644 --- a/src/net.h +++ b/src/net.h @@ -1262,6 +1262,7 @@ struct NodeEvictionCandidate bool m_is_local; Network m_network; bool m_noban; + ConnectionType m_conn_type; }; /** diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index d7721f1bf6739..e27b2545807e9 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -33,6 +33,7 @@ FUZZ_TARGET(node_eviction) /*m_is_local=*/fuzzed_data_provider.ConsumeBool(), /*m_network=*/fuzzed_data_provider.PickValueInArray(ALL_NETWORKS), /*m_noban=*/fuzzed_data_provider.ConsumeBool(), + /*m_conn_type=*/fuzzed_data_provider.PickValueInArray(ALL_CONNECTION_TYPES), }); } // Make a copy since eviction_candidates may be in some valid but otherwise diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 26b3acc677eff..bbcee6a5c82d5 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -59,6 +59,7 @@ std::vector GetRandomNodeEvictionCandidates(int n_candida /*m_is_local=*/random_context.randbool(), /*m_network=*/ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], /*m_noban=*/false, + /*m_conn_type=*/ConnectionType::INBOUND, }); } return candidates; From 31346a31962ab06d49cb45aee8acd92d8806538b Mon Sep 17 00:00:00 2001 From: sogoagain Date: Sun, 3 Jul 2022 00:33:03 +0900 Subject: [PATCH 0087/1090] [ci] apply cache size limit and print ccache statistics in "ARM64 Android APK" --- ci/test/06_script_a.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 218f5eeb63216..13693a2ecfb6b 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -11,16 +11,19 @@ if [ -z "$NO_WERROR" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" fi +CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + if [ -n "$ANDROID_TOOLS_URL" ]; then CI_EXEC make distclean || true CI_EXEC ./autogen.sh CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" + CI_EXEC "${PRINT_CCACHE_STATISTICS}" exit 0 fi BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" -CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" if [ -n "$CONFIG_SHELL" ]; then CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" @@ -57,6 +60,6 @@ fi CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) -CI_EXEC "ccache --version | head -n 1 && ccache --show-stats" +CI_EXEC "${PRINT_CCACHE_STATISTICS}" CI_EXEC du -sh "${DEPENDS_DIR}"/*/ CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" From da8f62de2c5561e091ef8073d6950c033f41aabf Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:39:20 -0300 Subject: [PATCH 0088/1090] wallet: remove always true 'fUseCache' from CachedTxGetImmatureCredit --- src/wallet/receive.cpp | 4 ++-- src/wallet/receive.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8de4017371a34..8512ee9b2c40c 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -164,12 +164,12 @@ CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx) return wtx.nChangeCached; } -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) { AssertLockHeld(wallet.cs_wallet); if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_SPENDABLE); } return 0; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index 1caef293f2729..0283bdcfca024 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -29,7 +29,7 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism //! filter decides which addresses will count towards the debit CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter); CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx); -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache = true) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); From 47b1012677821ce2939e10ba462fbe53ffff17df Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:39:53 -0300 Subject: [PATCH 0089/1090] wallet: remove always true 'fUseCache' from CachedTxGetImmatureWatchOnlyCredit --- src/wallet/receive.cpp | 4 ++-- src/wallet/receive.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8512ee9b2c40c..e9d59d8f0fe02 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -175,12 +175,12 @@ CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) return 0; } -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache) +CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) { AssertLockHeld(wallet.cs_wallet); if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_WATCH_ONLY); } return 0; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index 0283bdcfca024..e365d49edcd4b 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -31,7 +31,7 @@ CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const ismi CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx); CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache = true) +CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); From 4f0ca9bff6299353f595fe168dce720a96a91c41 Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:40:32 -0300 Subject: [PATCH 0090/1090] wallet: remove always false 'recalculate' arg from GetCachableAmount --- src/wallet/receive.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index e9d59d8f0fe02..5bee00baa6979 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -111,10 +111,10 @@ CAmount TxGetChange(const CWallet& wallet, const CTransaction& tx) return nChange; } -static CAmount GetCachableAmount(const CWallet& wallet, const CWalletTx& wtx, CWalletTx::AmountType type, const isminefilter& filter, bool recalculate = false) +static CAmount GetCachableAmount(const CWallet& wallet, const CWalletTx& wtx, CWalletTx::AmountType type, const isminefilter& filter) { auto& amount = wtx.m_amounts[type]; - if (recalculate || !amount.m_cached[filter]) { + if (!amount.m_cached[filter]) { amount.Set(filter, type == CWalletTx::DEBIT ? wallet.GetDebit(*wtx.tx, filter) : TxGetCredit(wallet, *wtx.tx, filter)); wtx.m_is_cache_empty = false; } From 04c6423f7b250ae1e51bb5cd159913e97494fb0e Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:42:35 -0300 Subject: [PATCH 0091/1090] wallet: remove always true 'fUseCache' arg from CachedTxGetAvailableCredit --- src/wallet/receive.cpp | 8 ++++---- src/wallet/receive.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 5bee00baa6979..243c5908ea42d 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -186,7 +186,7 @@ CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletT return 0; } -CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache, const isminefilter& filter) +CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) { AssertLockHeld(wallet.cs_wallet); @@ -197,7 +197,7 @@ CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, if (wallet.IsTxImmatureCoinBase(wtx)) return 0; - if (fUseCache && allow_cache && wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_cached[filter]) { + if (allow_cache && wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_cached[filter]) { return wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_value[filter]; } @@ -332,8 +332,8 @@ Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) const CWalletTx& wtx = entry.second; const bool is_trusted{CachedTxIsTrusted(wallet, wtx, trusted_parents)}; const int tx_depth{wallet.GetTxDepthInMainChain(wtx)}; - const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_WATCH_ONLY | reuse_filter)}; + const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index e365d49edcd4b..ec8d0d1baadce 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -33,7 +33,7 @@ CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); -CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) +CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter = ISMINE_SPENDABLE) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); struct COutputEntry { From 0cb177263c36118094b7cd3b8f94741c0471ff62 Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:47:05 -0300 Subject: [PATCH 0092/1090] wallet: unify CachedTxGetImmatureCredit and CachedTxGetImmatureWatchOnlyCredit --- src/wallet/receive.cpp | 19 ++++--------------- src/wallet/receive.h | 4 +--- src/wallet/test/wallet_tests.cpp | 4 ++-- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 243c5908ea42d..ec990339a3844 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -164,23 +164,12 @@ CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx) return wtx.nChangeCached; } -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) { AssertLockHeld(wallet.cs_wallet); if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_SPENDABLE); - } - - return 0; -} - -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) -{ - AssertLockHeld(wallet.cs_wallet); - - if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_WATCH_ONLY); + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, filter); } return 0; @@ -342,8 +331,8 @@ Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) ret.m_mine_untrusted_pending += tx_credit_mine; ret.m_watchonly_untrusted_pending += tx_credit_watchonly; } - ret.m_mine_immature += CachedTxGetImmatureCredit(wallet, wtx); - ret.m_watchonly_immature += CachedTxGetImmatureWatchOnlyCredit(wallet, wtx); + ret.m_mine_immature += CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE); + ret.m_watchonly_immature += CachedTxGetImmatureCredit(wallet, wtx, ISMINE_WATCH_ONLY); } } return ret; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index ec8d0d1baadce..41a70b089adf6 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -29,9 +29,7 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism //! filter decides which addresses will count towards the debit CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter); CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx); -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) - EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter = ISMINE_SPENDABLE) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 0c03b2f4a2a6a..27aa3f02e9f1e 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -350,13 +350,13 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // Call GetImmatureCredit() once before adding the key to the wallet to // cache the current immature credit amount, which is 0. - BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 0); + BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE), 0); // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 50*COIN); + BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE), 50*COIN); } static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) From bf310b0e8ce82d52bacceeb47c9f5dbb26885f7e Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:53:04 -0300 Subject: [PATCH 0093/1090] wallet: clean InputIsMine code, use GetWalletTx --- src/wallet/receive.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index ec990339a3844..1f76704b93eaf 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -9,15 +9,12 @@ #include namespace wallet { -isminetype InputIsMine(const CWallet& wallet, const CTxIn &txin) +isminetype InputIsMine(const CWallet& wallet, const CTxIn& txin) { AssertLockHeld(wallet.cs_wallet); - std::map::const_iterator mi = wallet.mapWallet.find(txin.prevout.hash); - if (mi != wallet.mapWallet.end()) - { - const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.tx->vout.size()) - return wallet.IsMine(prev.tx->vout[txin.prevout.n]); + const CWalletTx* prev = wallet.GetWalletTx(txin.prevout.hash); + if (prev && txin.prevout.n < prev->tx->vout.size()) { + return wallet.IsMine(prev->tx->vout[txin.prevout.n]); } return ISMINE_NO; } From 757216e31cac7dcd45e11b2a2c6148420b3b99da Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Tue, 5 Jul 2022 15:40:52 +0200 Subject: [PATCH 0094/1090] wallet: don't iter twice when getting the cached debit/credit amount Instead of calling GetCachableAmount twice, which will result in iterating through all the transaction txins/txouts and calling GetDebit/GetCredit (which lock cs_wallet), just merge the filters and do it once. --- src/wallet/receive.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8de4017371a34..d3303c0b1ffa4 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -130,12 +130,10 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism return 0; CAmount credit = 0; - if (filter & ISMINE_SPENDABLE) { + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { // GetBalance can assume transactions in mapWallet won't change - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_WATCH_ONLY); + credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, get_amount_filter); } return credit; } @@ -146,11 +144,9 @@ CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const ismi return 0; CAmount debit = 0; - if (filter & ISMINE_SPENDABLE) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_WATCH_ONLY); + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { + debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, get_amount_filter); } return debit; } From f1c16ed733f416a3bea878f1c21621dbd93b267c Mon Sep 17 00:00:00 2001 From: Jarol Rodriguez Date: Tue, 5 Jul 2022 14:09:36 -0400 Subject: [PATCH 0095/1090] doc: remove note on arm cross-compilation from build-unix.md No reason to have this here with outdated information. We already point users to the depends readme, the doc cross builders should be pointed to , within this doc. --- doc/build-unix.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/doc/build-unix.md b/doc/build-unix.md index bcfa25dc765b4..874015707a25d 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -288,26 +288,3 @@ This example lists the steps necessary to setup and build a command line only di ./src/bitcoind If you intend to work with legacy Berkeley DB wallets, see [Berkeley DB](#berkeley-db) section. - -ARM Cross-compilation -------------------- -These steps can be performed on, for example, an Ubuntu VM. The depends system -will also work on other Linux distributions, however the commands for -installing the toolchain will be different. - -Make sure you install the build requirements mentioned above. -Then, install the toolchain and curl: - - sudo apt-get install g++-arm-linux-gnueabihf curl - -To build executables for ARM: - - cd depends - make HOST=arm-linux-gnueabihf NO_QT=1 - cd .. - ./autogen.sh - CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-reduce-exports LDFLAGS=-static-libstdc++ - make - - -For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory. From e049fd76f0d57c1e6400fbfbaf4cc6ebe540f16f Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Tue, 5 Jul 2022 23:41:38 +0000 Subject: [PATCH 0096/1090] Bugfix: Check for readlink buffer overflow and handle gracefully If readlink returns the size of the buffer, an overflow may have (safely) occurred. Pass a buffer size of MAX_PATH+1 (the size of the actual buffer) to detect this scenario. --- src/qt/guiutil.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 558d4f108c011..cb8be5b9bad56 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -743,9 +743,10 @@ bool SetStartOnSystemStartup(bool fAutoStart) else { char pszExePath[MAX_PATH+1]; - ssize_t r = readlink("/proc/self/exe", pszExePath, sizeof(pszExePath) - 1); - if (r == -1) + ssize_t r = readlink("/proc/self/exe", pszExePath, sizeof(pszExePath)); + if (r == -1 || r > MAX_PATH) { return false; + } pszExePath[r] = '\0'; fs::create_directories(GetAutostartDir()); From fad690ba0a62dfd97ef22711b2344909fcd1fb73 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:44:24 +0200 Subject: [PATCH 0097/1090] test: Remove -acceptnonstdtxn=1 from feature_rbf.py --- test/functional/feature_rbf.py | 388 ++++++++++++++++----------------- 1 file changed, 186 insertions(+), 202 deletions(-) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 91dc222babb7d..40ad2137d4716 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -4,28 +4,18 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" -from copy import deepcopy from decimal import Decimal from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, SEQUENCE_FINAL, ) -from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) -from test_framework.script_util import ( - DUMMY_P2WPKH_SCRIPT, - DUMMY_2_P2WPKH_SCRIPT, -) from test_framework.wallet import MiniWallet from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE @@ -35,7 +25,6 @@ def set_test_params(self): self.num_nodes = 2 self.extra_args = [ [ - "-acceptnonstdtxn=1", "-maxorphantx=1000", "-limitancestorcount=50", "-limitancestorsize=101", @@ -96,15 +85,14 @@ def run_test(self): self.log.info("Passed") - def make_utxo(self, node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRIPT): + def make_utxo(self, node, amount, *, confirmed=True, scriptPubKey=None): """Create a txout with a given amount and scriptPubKey - confirmed - txouts created will be confirmed in the blockchain; + confirmed - txout created will be confirmed in the blockchain; unconfirmed otherwise. """ - txid, n = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey, amount=amount) + txid, n = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey or self.wallet.get_scriptPubKey(), amount=amount) - # If requested, ensure txouts are confirmed. if confirmed: mempool_size = len(node.getrawmempool()) while mempool_size > 0: @@ -115,30 +103,24 @@ def make_utxo(self, node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRI assert new_size < mempool_size mempool_size = new_size - return COutPoint(int(txid, 16), n) + return self.wallet.get_utxo(txid=txid, vout=n) def test_simple_doublespend(self): """Simple doublespend""" # we use MiniWallet to create a transaction template with inputs correctly set, # and modify the output (amount, scriptPubKey) according to our needs - tx_template = self.wallet.create_self_transfer()['tx'] - - tx1a = deepcopy(tx_template) - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx = self.wallet.create_self_transfer()["tx"] + tx1a_txid = self.nodes[0].sendrawtransaction(tx.serialize().hex()) # Should fail because we haven't changed the fee - tx1b = deepcopy(tx_template) - tx1b.vout = [CTxOut(1 * COIN, DUMMY_2_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() + tx.vout[0].scriptPubKey[-1] ^= 1 # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx.serialize().hex(), 0) # Extra 0.1 BTC fee - tx1b.vout[0].nValue -= int(0.1 * COIN) - tx1b_hex = tx1b.serialize().hex() + tx.vout[0].nValue -= int(0.1 * COIN) + tx1b_hex = tx.serialize().hex() # Works when enabled tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) @@ -160,28 +142,28 @@ def test_doublespend_chain(self): chain_txids = [] while remaining_value > 1 * COIN: remaining_value -= int(0.1 * COIN) - tx = CTransaction() - tx.vin = [CTxIn(prevout, nSequence=0)] - tx.vout = [CTxOut(remaining_value, CScript([1, OP_DROP] * 15 + [1]))] - tx_hex = tx.serialize().hex() - txid = self.nodes[0].sendrawtransaction(tx_hex, 0) - chain_txids.append(txid) - prevout = COutPoint(int(txid, 16), 0) + prevout = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=prevout, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] + chain_txids.append(prevout["txid"]) # Whether the double-spend is allowed is evaluated by including all # child fees - 4 BTC - so this attempt is rejected. - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - 3 * COIN, DUMMY_P2WPKH_SCRIPT)] + dbl_tx = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("3"), + )["tx"] dbl_tx_hex = dbl_tx.serialize().hex() # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # Accepted with sufficient fee - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(int(0.1 * COIN), DUMMY_P2WPKH_SCRIPT)] + dbl_tx.vout[0].nValue = int(0.1 * COIN) dbl_tx_hex = dbl_tx.serialize().hex() self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) @@ -205,22 +187,19 @@ def branch(prevout, initial_value, max_txs, tree_width=5, fee=0.00001 * COIN, _t if txout_value < fee: return - vout = [CTxOut(txout_value, CScript([i+1])) - for i in range(tree_width)] - tx = CTransaction() - tx.vin = [CTxIn(prevout, nSequence=0)] - tx.vout = vout - tx_hex = tx.serialize().hex() + tx = self.wallet.send_self_transfer_multi( + utxos_to_spend=[prevout], + from_node=self.nodes[0], + sequence=0, + num_outputs=tree_width, + amount_per_output=txout_value, + ) - assert len(tx.serialize()) < 100000 - txid = self.nodes[0].sendrawtransaction(tx_hex, 0) - yield tx + yield tx["txid"] _total_txs[0] += 1 - txid = int(txid, 16) - - for i, txout in enumerate(tx.vout): - for x in branch(COutPoint(txid, i), txout_value, + for utxo in tx["new_utxos"]: + for x in branch(utxo, txout_value, max_txs, tree_width=tree_width, fee=fee, _total_txs=_total_txs): @@ -232,25 +211,26 @@ def branch(prevout, initial_value, max_txs, tree_width=5, fee=0.00001 * COIN, _t assert_equal(len(tree_txs), n) # Attempt double-spend, will fail because too little fee paid - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - fee * n, DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=(Decimal(fee) / COIN) * n, + )["hex"] # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # 0.1 BTC fee is enough - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - fee * n - int(0.1 * COIN), DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=(Decimal(fee) / COIN) * n + Decimal("0.1"), + )["hex"] self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() - for tx in tree_txs: - tx.rehash() - assert tx.hash not in mempool + for txid in tree_txs: + assert txid not in mempool # Try again, but with more total transactions than the "max txs # double-spent at once" anti-DoS limit. @@ -260,33 +240,36 @@ def branch(prevout, initial_value, max_txs, tree_width=5, fee=0.00001 * COIN, _t tree_txs = list(branch(tx0_outpoint, initial_nValue, n, fee=fee)) assert_equal(len(tree_txs), n) - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - 2 * fee * n, DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=2 * (Decimal(fee) / COIN) * n, + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) - for tx in tree_txs: - tx.rehash() - self.nodes[0].getrawtransaction(tx.hash) + for txid in tree_txs: + self.nodes[0].getrawtransaction(txid) def test_replacement_feeperkb(self): """Replacement requires fee-per-KB to be higher""" tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - self.nodes[0].sendrawtransaction(tx1a_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.1"), + ) # Higher fee, but the fee per KB is much lower, so the replacement is # rejected. - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.001 * COIN), CScript([b'a' * 999000]))] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[tx0_outpoint], + sequence=0, + num_outputs=100, + amount_per_output=1000, + )["hex"] # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -296,37 +279,36 @@ def test_spends_of_conflicting_outputs(self): utxo1 = self.make_utxo(self.nodes[0], int(1.2 * COIN)) utxo2 = self.make_utxo(self.nodes[0], 3 * COIN) - tx1a = CTransaction() - tx1a.vin = [CTxIn(utxo1, nSequence=0)] - tx1a.vout = [CTxOut(int(1.1 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) - - tx1a_txid = int(tx1a_txid, 16) + tx1a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=utxo1, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] # Direct spend an output of the transaction we're replacing. - tx2 = CTransaction() - tx2.vin = [CTxIn(utxo1, nSequence=0), CTxIn(utxo2, nSequence=0)] - tx2.vin.append(CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)) - tx2.vout = tx1a.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[utxo1, utxo2, tx1a_utxo], + sequence=0, + amount_per_output=int(COIN * tx1a_utxo["value"]), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) # Spend tx1a's output to test the indirect case. - tx1b = CTransaction() - tx1b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] - tx1b.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) - tx1b_txid = int(tx1b_txid, 16) + tx1b_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1a_utxo, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] - tx2 = CTransaction() - tx2.vin = [CTxIn(utxo1, nSequence=0), CTxIn(utxo2, nSequence=0), - CTxIn(COutPoint(tx1b_txid, 0))] - tx2.vout = tx1a.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[utxo1, utxo2, tx1b_utxo], + sequence=0, + amount_per_output=int(COIN * tx1a_utxo["value"]), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) @@ -334,18 +316,20 @@ def test_spends_of_conflicting_outputs(self): def test_new_unconfirmed_inputs(self): """Replacements that add new unconfirmed inputs are rejected""" confirmed_utxo = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - unconfirmed_utxo = self.make_utxo(self.nodes[0], int(0.1 * COIN), False) + unconfirmed_utxo = self.make_utxo(self.nodes[0], int(0.1 * COIN), confirmed=False) - tx1 = CTransaction() - tx1.vin = [CTxIn(confirmed_utxo)] - tx1.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1_hex = tx1.serialize().hex() - self.nodes[0].sendrawtransaction(tx1_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=confirmed_utxo, + sequence=0, + fee=Decimal("0.1"), + ) - tx2 = CTransaction() - tx2.vin = [CTxIn(confirmed_utxo), CTxIn(unconfirmed_utxo)] - tx2.vout = tx1.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[confirmed_utxo, unconfirmed_utxo], + sequence=0, + amount_per_output=1 * COIN, + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, 0) @@ -361,45 +345,39 @@ def test_too_many_replacements(self): fee = int(0.0001 * COIN) split_value = int((initial_nValue - fee) / (MAX_REPLACEMENT_LIMIT + 1)) - outputs = [] - for _ in range(MAX_REPLACEMENT_LIMIT + 1): - outputs.append(CTxOut(split_value, CScript([1]))) - - splitting_tx = CTransaction() - splitting_tx.vin = [CTxIn(utxo, nSequence=0)] - splitting_tx.vout = outputs - splitting_tx_hex = splitting_tx.serialize().hex() - - txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, 0) - txid = int(txid, 16) + splitting_tx_utxos = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[utxo], + sequence=0, + num_outputs=MAX_REPLACEMENT_LIMIT + 1, + amount_per_output=split_value, + )["new_utxos"] # Now spend each of those outputs individually - for i in range(MAX_REPLACEMENT_LIMIT + 1): - tx_i = CTransaction() - tx_i.vin = [CTxIn(COutPoint(txid, i), nSequence=0)] - tx_i.vout = [CTxOut(split_value - fee, DUMMY_P2WPKH_SCRIPT)] - tx_i_hex = tx_i.serialize().hex() - self.nodes[0].sendrawtransaction(tx_i_hex, 0) + for utxo in splitting_tx_utxos: + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=utxo, + sequence=0, + fee=Decimal(fee) / COIN, + ) # Now create doublespend of the whole lot; should fail. # Need a big enough fee to cover all spending transactions and have # a higher fee rate double_spend_value = (split_value - 100 * fee) * (MAX_REPLACEMENT_LIMIT + 1) - inputs = [] - for i in range(MAX_REPLACEMENT_LIMIT + 1): - inputs.append(CTxIn(COutPoint(txid, i), nSequence=0)) - double_tx = CTransaction() - double_tx.vin = inputs - double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] + double_tx = self.wallet.create_self_transfer_multi( + utxos_to_spend=splitting_tx_utxos, + sequence=0, + amount_per_output=double_spend_value, + )["tx"] double_tx_hex = double_tx.serialize().hex() # This will raise an exception assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, 0) # If we remove an input, it should pass - double_tx = CTransaction() - double_tx.vin = inputs[0:-1] - double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] + double_tx.vin.pop() double_tx_hex = double_tx.serialize().hex() self.nodes[0].sendrawtransaction(double_tx_hex, 0) @@ -494,20 +472,22 @@ def test_opt_in(self): tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) # Create a non-opting in transaction - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=SEQUENCE_FINAL)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx1a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=SEQUENCE_FINAL, + fee=Decimal("0.1"), + )["new_utxo"] # This transaction isn't shown as replaceable - assert_equal(self.nodes[0].getmempoolentry(tx1a_txid)['bip125-replaceable'], False) + assert_equal(self.nodes[0].getmempoolentry(tx1a_utxo["txid"])['bip125-replaceable'], False) # Shouldn't be able to double-spend - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.9 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.2"), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -515,17 +495,19 @@ def test_opt_in(self): tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) # Create a different non-opting in transaction - tx2a = CTransaction() - tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0xfffffffe)] - tx2a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx2a_hex = tx2a.serialize().hex() - tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, 0) + tx2a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1_outpoint, + sequence=0xfffffffe, + fee=Decimal("0.1"), + )["new_utxo"] # Still shouldn't be able to double-spend - tx2b = CTransaction() - tx2b.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2b.vout = [CTxOut(int(0.9 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx2b_hex = tx2b.serialize().hex() + tx2b_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.2"), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, 0) @@ -534,34 +516,31 @@ def test_opt_in(self): # opt-in on one of the inputs # Transaction should be replaceable on either input - tx1a_txid = int(tx1a_txid, 16) - tx2a_txid = int(tx2a_txid, 16) - - tx3a = CTransaction() - tx3a.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=SEQUENCE_FINAL), - CTxIn(COutPoint(tx2a_txid, 0), nSequence=0xfffffffd)] - tx3a.vout = [CTxOut(int(0.9 * COIN), CScript([b'c'])), CTxOut(int(0.9 * COIN), CScript([b'd']))] - tx3a_hex = tx3a.serialize().hex() - - tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, 0) + tx3a_txid = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[tx1a_utxo, tx2a_utxo], + sequence=[SEQUENCE_FINAL, 0xfffffffd], + fee_per_output=int(0.1 * COIN), + )["txid"] # This transaction is shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx3a_txid)['bip125-replaceable'], True) - tx3b = CTransaction() - tx3b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] - tx3b.vout = [CTxOut(int(0.5 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx3b_hex = tx3b.serialize().hex() - - tx3c = CTransaction() - tx3c.vin = [CTxIn(COutPoint(tx2a_txid, 0), nSequence=0)] - tx3c.vout = [CTxOut(int(0.5 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx3c_hex = tx3c.serialize().hex() + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1a_utxo, + sequence=0, + fee=Decimal("0.4"), + ) - self.nodes[0].sendrawtransaction(tx3b_hex, 0) # If tx3b was accepted, tx3c won't look like a replacement, # but make sure it is accepted anyway - self.nodes[0].sendrawtransaction(tx3c_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx2a_utxo, + sequence=0, + fee=Decimal("0.4"), + ) def test_prioritised_transactions(self): # Ensure that fee deltas used via prioritisetransaction are @@ -570,17 +549,20 @@ def test_prioritised_transactions(self): # 1. Check that feeperkb uses modified fees tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx1a_txid = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.1"), + )["txid"] # Higher fee, but the actual fee per KB is much lower. - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.001 * COIN), CScript([b'a' * 740000]))] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[tx0_outpoint], + sequence=0, + num_outputs=100, + amount_per_output=int(0.00001 * COIN), + )["hex"] # Verify tx1b cannot replace tx1a. assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -596,27 +578,29 @@ def test_prioritised_transactions(self): # 2. Check that absolute fee checks use modified fee. tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx2a = CTransaction() - tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx2a_hex = tx2a.serialize().hex() - self.nodes[0].sendrawtransaction(tx2a_hex, 0) + # tx2a + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.1"), + ) # Lower fee, but we'll prioritise it - tx2b = CTransaction() - tx2b.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2b.vout = [CTxOut(int(1.01 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx2b.rehash() - tx2b_hex = tx2b.serialize().hex() + tx2b = self.wallet.create_self_transfer( + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.09"), + ) # Verify tx2b cannot replace tx2a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, 0) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b["hex"], 0) # Now prioritise tx2b to have a higher modified fee - self.nodes[0].prioritisetransaction(txid=tx2b.hash, fee_delta=int(0.1 * COIN)) + self.nodes[0].prioritisetransaction(txid=tx2b["txid"], fee_delta=int(0.1 * COIN)) # tx2b should now be accepted - tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, 0) + tx2b_txid = self.nodes[0].sendrawtransaction(tx2b["hex"], 0) assert tx2b_txid in self.nodes[0].getrawmempool() From 8d869a7bb55a181358031575211810416aadda70 Mon Sep 17 00:00:00 2001 From: glozow Date: Wed, 6 Jul 2022 11:15:39 +0100 Subject: [PATCH 0098/1090] add glozow builder key --- contrib/builder-keys/keys.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index c70069b440b03..f8377cce33254 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -19,6 +19,7 @@ BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) 6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) +6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C Gloria Zhao (glozow) D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) 2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) From 6eb0909cb7d5883a258f76ad6cf2c989fc6f892f Mon Sep 17 00:00:00 2001 From: chinggg <24590067+chinggg@users.noreply.github.com> Date: Wed, 22 Jun 2022 20:44:43 +0800 Subject: [PATCH 0099/1090] fuzz: add low-level target for txorphanage --- ci/test/06_script_b.sh | 1 + src/Makefile.test.include | 1 + src/test/fuzz/txorphan.cpp | 143 +++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 src/test/fuzz/txorphan.cpp diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index bdb68e0f6f443..32f0ea5e42575 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -46,6 +46,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/policy/settings.cpp"\ " src/rpc/fees.cpp"\ " src/rpc/signmessage.cpp"\ + " src/test/fuzz/txorphan.cpp"\ " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" fi diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 098feacb3d79b..b806b62d5bbc8 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -325,6 +325,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/tx_pool.cpp \ + test/fuzz/txorphan.cpp \ test/fuzz/txrequest.cpp \ test/fuzz/utxo_snapshot.cpp \ test/fuzz/validation_load_mempool.cpp \ diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp new file mode 100644 index 0000000000000..d318baa6a22c6 --- /dev/null +++ b/src/test/fuzz/txorphan.cpp @@ -0,0 +1,143 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include