From e899e2dd86ccde4c7e1dd17b2178c10650a3daa4 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Sat, 5 Mar 2016 16:31:10 -0500 Subject: [PATCH 01/42] main: start of address index Adds a configuration option for addressindex to search for txids by address. Includes an additional rpc method for getting the txids for an address. --- src/init.cpp | 2 ++ src/main.cpp | 62 +++++++++++++++++++++++++++++++++++++++++++ src/main.h | 38 ++++++++++++++++++++++++++ src/rpcmisc.cpp | 36 +++++++++++++++++++++++++ src/rpcserver.cpp | 3 +++ src/rpcserver.h | 2 ++ src/script/script.cpp | 11 ++++++++ src/script/script.h | 2 ++ src/txdb.cpp | 33 +++++++++++++++++++++++ src/txdb.h | 3 +++ 10 files changed, 192 insertions(+) diff --git a/src/init.cpp b/src/init.cpp index 71a9eae861dc2..58411f67bfbe5 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -350,6 +350,8 @@ std::string HelpMessage(HelpMessageMode mode) #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); + strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); + strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); strUsage += HelpMessageOpt("-banscore=", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); diff --git a/src/main.cpp b/src/main.cpp index 5b27698d8b96a..ef451dd45abd1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -65,6 +65,7 @@ int nScriptCheckThreads = 0; bool fImporting = false; bool fReindex = false; bool fTxIndex = false; +bool fAddressIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1232,6 +1233,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } +bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) +{ + if (!fAddressIndex) + return error("%s: address index not enabled"); + + if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex)) + return error("%s: unable to get txids for address"); + + return true; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { @@ -2063,9 +2075,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin std::vector > vPos; vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); + std::vector > addressIndex; + for (unsigned int i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = block.vtx[i]; + const uint256 txhash = tx.GetHash(); nInputs += tx.vin.size(); nSigOps += GetLegacySigOpCount(tx); @@ -2079,6 +2094,22 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return state.DoS(100, error("ConnectBlock(): inputs missing/spent"), REJECT_INVALID, "bad-txns-inputs-missingorspent"); + if (fAddressIndex) + { + for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, j), prevout.nValue * -1)); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, j), prevout.nValue * -1)); + } else { + continue; + } + } + } + if (fStrictPayToScriptHash) { // Add in sigops done by pay-to-script-hash inputs; @@ -2100,6 +2131,24 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin control.Add(vChecks); } + if (fAddressIndex) { + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, k), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, k), out.nValue)); + } else { + continue; + } + + } + } + + CTxUndo undoDummy; if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); @@ -2150,6 +2199,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!pblocktree->WriteTxIndex(vPos)) return AbortNode(state, "Failed to write transaction index"); + if (fAddressIndex) + if (!pblocktree->WriteAddressIndex(addressIndex)) + return AbortNode(state, "Failed to write address index"); + // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3515,6 +3568,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("txindex", fTxIndex); LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); + // Check whether we have an address index + pblocktree->ReadFlag("addressindex", fAddressIndex); + LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -3672,6 +3729,11 @@ bool InitBlockIndex(const CChainParams& chainparams) // Use the provided setting for -txindex in the new database fTxIndex = GetBoolArg("-txindex", DEFAULT_TXINDEX); pblocktree->WriteFlag("txindex", fTxIndex); + + // Use the provided setting for -addressindex in the new database + fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); + pblocktree->WriteFlag("addressindex", fAddressIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 9fd97d2126823..340d033a30031 100644 --- a/src/main.h +++ b/src/main.h @@ -105,6 +105,7 @@ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; +static const bool DEFAULT_ADDRESSINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -281,6 +282,42 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CAddressIndexKey { + uint160 hashBytes; + unsigned int type; + uint256 txhash; + size_t index; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(hashBytes); + READWRITE(type); + READWRITE(txhash); + READWRITE(index); + } + + CAddressIndexKey(uint160 addressHash, unsigned int addressType, uint256 txid, size_t txindex) { + hashBytes = addressHash; + type = addressType; + txhash = txid; + index = txindex; + } + + CAddressIndexKey() { + SetNull(); + } + + void SetNull() { + hashBytes.SetNull(); + type = 0; + txhash.SetNull(); + index = 0; + } + +}; + struct CDiskTxPos : public CDiskBlockPos { unsigned int nTxOffset; // after header @@ -387,6 +424,7 @@ class CScriptCheck ScriptError GetScriptError() const { return error; } }; +bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 9871c3fcc9031..aa762af88f8be 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -396,3 +396,39 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } + +UniValue getaddresstxids(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddresstxids\n" + "\nReturns the txids for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "[\n" + " \"transactionid\" (string) The transaction id\n" + " ,...\n" + "]\n" + ); + + CBitcoinAddress address(params[0].get_str()); + if (!address.IsValid()) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + + CKeyID keyID; + address.GetKeyID(keyID); + + int type = 1; // TODO + std::vector > addressIndex; + + LOCK(cs_main); + + if (!GetAddressIndex(keyID, type, addressIndex)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + + UniValue result(UniValue::VARR); + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) + result.push_back(it->first.txhash.GetHex()); + + return result; + +} diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index b3abeec4a3ce8..67da51ceb75af 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -313,6 +313,9 @@ static const CRPCCommand vRPCCommands[] = { "rawtransactions", "fundrawtransaction", &fundrawtransaction, false }, #endif + /* Address index */ + { "addressindex", "getaddresstxids", &getaddresstxids, false }, + /* Utility functions */ { "util", "createmultisig", &createmultisig, true }, { "util", "validateaddress", &validateaddress, true }, /* uses wallet if enabled */ diff --git a/src/rpcserver.h b/src/rpcserver.h index babf7c8d2e1f2..11f7600511a9c 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,8 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddresstxids(const UniValue& params, bool fHelp); + extern UniValue getpeerinfo(const UniValue& params, bool fHelp); extern UniValue ping(const UniValue& params, bool fHelp); extern UniValue addnode(const UniValue& params, bool fHelp); diff --git a/src/script/script.cpp b/src/script/script.cpp index 9f2809e593756..3e4b72f5d9652 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -201,6 +201,17 @@ unsigned int CScript::GetSigOpCount(const CScript& scriptSig) const return subscript.GetSigOpCount(true); } +bool CScript::IsPayToPublicKeyHash() const +{ + // Extra-fast test for pay-to-pubkey-hash CScripts: + return (this->size() == 25 && + (*this)[0] == OP_DUP && + (*this)[1] == OP_HASH160 && + (*this)[2] == 0x14 && + (*this)[23] == OP_EQUALVERIFY && + (*this)[24] == OP_CHECKSIG); +} + bool CScript::IsPayToScriptHash() const { // Extra-fast test for pay-to-script-hash CScripts: diff --git a/src/script/script.h b/src/script/script.h index 6551eea30d340..8f6b47160a09d 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -596,6 +596,8 @@ class CScript : public CScriptBase */ unsigned int GetSigOpCount(const CScript& scriptSig) const; + bool IsPayToPublicKeyHash() const; + bool IsPayToScriptHash() const; /** Called by IsStandardTx and P2SH/BIP62 VerifyScript (which makes it consensus-critical). */ diff --git a/src/txdb.cpp b/src/txdb.cpp index f99e11f26e3f4..153d7b84c9ad0 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -21,6 +21,7 @@ using namespace std; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; +static const char DB_ADDRESSINDEX = 'a'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -163,6 +164,38 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) + batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); + return WriteBatch(batch); +} + +bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_ADDRESSINDEX, addressHash)); //TODO include type + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_ADDRESSINDEX && key.second.hashBytes == addressHash) { + CAmount nValue; + if (pcursor->GetValue(nValue)) { + addressIndex.push_back(make_pair(key.second, nValue)); + pcursor->Next(); + } else { + return error("failed to get address index value"); + } + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index 22e0c5704cb26..3d2ace581a018 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -17,6 +17,7 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; +struct CAddressIndexKey; class uint256; //! -dbcache default (MiB) @@ -57,6 +58,8 @@ class CBlockTreeDB : public CDBWrapper bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool WriteAddressIndex(const std::vector > &vect); + bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(); From f33a03c653138f1ab05c9eb030aec062c9db64d8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 8 Mar 2016 14:47:12 -0500 Subject: [PATCH 02/42] test: added unit tests for CScript.IsPayToPublicKeyHash --- src/Makefile.test.include | 1 + src/test/script_P2PKH_tests.cpp | 59 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 src/test/script_P2PKH_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d89132f806607..26eccb4ac8595 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -70,6 +70,7 @@ BITCOIN_TESTS =\ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_P2SH_tests.cpp \ + test/script_P2PKH_tests.cpp \ test/script_tests.cpp \ test/scriptnum_tests.cpp \ test/serialize_tests.cpp \ diff --git a/src/test/script_P2PKH_tests.cpp b/src/test/script_P2PKH_tests.cpp new file mode 100644 index 0000000000000..3a7dc16608aaa --- /dev/null +++ b/src/test/script_P2PKH_tests.cpp @@ -0,0 +1,59 @@ +// Copyright (c) 2012-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "script/script.h" +#include "test/test_bitcoin.h" + +#include + +using namespace std; + +BOOST_FIXTURE_TEST_SUITE(script_P2PKH_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(IsPayToPublicKeyHash) +{ + // Test CScript::IsPayToPublicKeyHash() + uint160 dummy; + CScript p2pkh; + p2pkh << OP_DUP << OP_HASH160 << ToByteVector(dummy) << OP_EQUALVERIFY << OP_CHECKSIG; + BOOST_CHECK(p2pkh.IsPayToPublicKeyHash()); + + static const unsigned char direct[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG + }; + BOOST_CHECK(CScript(direct, direct+sizeof(direct)).IsPayToPublicKeyHash()); + + static const unsigned char notp2pkh1[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(notp2pkh1, notp2pkh1+sizeof(notp2pkh1)).IsPayToPublicKeyHash()); + + static const unsigned char p2sh[] = { + OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUAL + }; + BOOST_CHECK(!CScript(p2sh, p2sh+sizeof(p2sh)).IsPayToPublicKeyHash()); + + static const unsigned char extra[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(extra, extra+sizeof(extra)).IsPayToPublicKeyHash()); + + static const unsigned char missing[] = { + OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_RETURN + }; + BOOST_CHECK(!CScript(missing, missing+sizeof(missing)).IsPayToPublicKeyHash()); + + static const unsigned char missing2[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + }; + BOOST_CHECK(!CScript(missing2, missing2+sizeof(missing)).IsPayToPublicKeyHash()); + + static const unsigned char tooshort[] = { + OP_DUP, OP_HASH160, 2, 0,0, OP_EQUALVERIFY, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(tooshort, tooshort+sizeof(direct)).IsPayToPublicKeyHash()); + +} + +BOOST_AUTO_TEST_SUITE_END() From dc02ab75d88e011385501edb349d1ba956ec1be5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 8 Mar 2016 16:15:49 -0500 Subject: [PATCH 03/42] qa: started test for addressindex rpc getaddresstxids --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/addressindex.py | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100755 qa/rpc-tests/addressindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index e7173fda080fe..b092a8cd7e462 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -96,6 +96,7 @@ 'walletbackup.py', 'nodehandling.py', 'reindex.py', + 'addressindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py new file mode 100755 index 0000000000000..c4353d92a6f30 --- /dev/null +++ b/qa/rpc-tests/addressindex.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test addressindex generation and fetching +# + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class AddressIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-addressindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-addressindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + assert_equal(self.nodes[1].getbalance(), 0) + assert_equal(self.nodes[2].getbalance(), 0) + + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + txid3 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + self.nodes[0].generate(1) + self.sync_all() + + txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); + assert_equal(len(txids), 3); + +if __name__ == '__main__': + AddressIndexTest().main() From 8258f57349bef48c09dc752ef65dcf8a4f43d570 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 11:27:30 -0500 Subject: [PATCH 04/42] main: index address index sorted by height --- qa/rpc-tests/addressindex.py | 11 ++++++-- src/main.cpp | 9 +++--- src/main.h | 54 ++++++++++++++++++++++++++++++------ src/txdb.cpp | 4 +-- src/txdb.h | 1 + 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index c4353d92a6f30..f3cbc2ef848e9 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -41,14 +41,19 @@ def run_test(self): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) - txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) - txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) - txid3 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + self.nodes[0].generate(1) + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + self.nodes[0].generate(1) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) self.nodes[0].generate(1) self.sync_all() txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); assert_equal(len(txids), 3); + assert_equal(txids[0], txid0); + assert_equal(txids[1], txid1); + assert_equal(txids[2], txid2); if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/main.cpp b/src/main.cpp index ef451dd45abd1..617b37de8de10 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2100,10 +2100,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); } else { continue; } @@ -2137,10 +2137,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); } else { continue; } @@ -2148,7 +2148,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } } - CTxUndo undoDummy; if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); diff --git a/src/main.h b/src/main.h index 340d033a30031..64286ddc54bef 100644 --- a/src/main.h +++ b/src/main.h @@ -283,26 +283,33 @@ struct CNodeStateStats { }; struct CAddressIndexKey { - uint160 hashBytes; unsigned int type; + uint160 hashBytes; + int blockHeight; + unsigned int txindex; uint256 txhash; - size_t index; + size_t outindex; ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(hashBytes); READWRITE(type); + READWRITE(hashBytes); + READWRITE(blockHeight); + READWRITE(txindex); READWRITE(txhash); - READWRITE(index); + READWRITE(outindex); } - CAddressIndexKey(uint160 addressHash, unsigned int addressType, uint256 txid, size_t txindex) { - hashBytes = addressHash; + CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, + uint256 txid, size_t outputIndex) { type = addressType; + hashBytes = addressHash; + blockHeight = height; + txindex = blockindex; txhash = txid; - index = txindex; + outindex = outputIndex; } CAddressIndexKey() { @@ -310,12 +317,41 @@ struct CAddressIndexKey { } void SetNull() { - hashBytes.SetNull(); type = 0; + hashBytes.SetNull(); + blockHeight = 0; + txindex = 0; txhash.SetNull(); - index = 0; + outindex = 0; + } + +}; + +struct CAddressIndexIteratorKey { + unsigned int type; + uint160 hashBytes; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(type); + READWRITE(hashBytes); + } + + CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { + type = addressType; + hashBytes = addressHash; + } + + CAddressIndexIteratorKey() { + SetNull(); } + void SetNull() { + type = 0; + hashBytes.SetNull(); + } }; struct CDiskTxPos : public CDiskBlockPos diff --git a/src/txdb.cpp b/src/txdb.cpp index 153d7b84c9ad0..e93594b9ed80f 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -167,7 +167,7 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) - batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); + batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); return WriteBatch(batch); } @@ -175,7 +175,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector pcursor(NewIterator()); - pcursor->Seek(make_pair(DB_ADDRESSINDEX, addressHash)); //TODO include type + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); while (pcursor->Valid()) { boost::this_thread::interruption_point(); diff --git a/src/txdb.h b/src/txdb.h index 3d2ace581a018..ca32869ccc37d 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -18,6 +18,7 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; struct CAddressIndexKey; +struct CAddressIndexIteratorKey; class uint256; //! -dbcache default (MiB) From b37ba1cdef256063b7bcd5237c321f6774b0ced6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 17:40:40 -0500 Subject: [PATCH 05/42] rpc: fix issue for querying txids for p2sh addresses --- qa/rpc-tests/addressindex.py | 13 +++++++++++++ src/base58.cpp | 17 +++++++++++++++++ src/base58.h | 1 + src/rpcmisc.cpp | 11 +++++------ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index f3cbc2ef848e9..88533d028fc5a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -42,11 +42,17 @@ def run_test(self): assert_equal(self.nodes[2].getbalance(), 0) txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + txidb1 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 15) self.nodes[0].generate(1) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + txidb2 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 20) self.nodes[0].generate(1) + self.sync_all() txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); @@ -55,5 +61,12 @@ def run_test(self): assert_equal(txids[1], txid1); assert_equal(txids[2], txid2); + txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); + assert_equal(len(txidsb), 3); + assert_equal(txidsb[0], txidb0); + assert_equal(txidsb[1], txidb1); + assert_equal(txidsb[2], txidb2); + + if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/base58.cpp b/src/base58.cpp index 5e26cf8d4738f..80fa99c2ab082 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -262,6 +262,23 @@ CTxDestination CBitcoinAddress::Get() const return CNoDestination(); } +bool CBitcoinAddress::GetIndexKey(uint160& hashBytes, int& type) const +{ + if (!IsValid()) { + return false; + } else if (vchVersion == Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)) { + memcpy(&hashBytes, &vchData[0], 20); + type = 1; + return true; + } else if (vchVersion == Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS)) { + memcpy(&hashBytes, &vchData[0], 20); + type = 2; + return true; + } + + return false; +} + bool CBitcoinAddress::GetKeyID(CKeyID& keyID) const { if (!IsValid() || vchVersion != Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)) diff --git a/src/base58.h b/src/base58.h index a3980118aae34..b8280b1dd621f 100644 --- a/src/base58.h +++ b/src/base58.h @@ -116,6 +116,7 @@ class CBitcoinAddress : public CBase58Data { CTxDestination Get() const; bool GetKeyID(CKeyID &keyID) const; + bool GetIndexKey(uint160& hashBytes, int& type) const; bool IsScript() const; }; diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index aa762af88f8be..0a39d2fbcb3da 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -411,18 +411,17 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) ); CBitcoinAddress address(params[0].get_str()); - if (!address.IsValid()) + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } - CKeyID keyID; - address.GetKeyID(keyID); - - int type = 1; // TODO std::vector > addressIndex; LOCK(cs_main); - if (!GetAddressIndex(keyID, type, addressIndex)) + if (!GetAddressIndex(hashBytes, type, addressIndex)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); UniValue result(UniValue::VARR); From 5bea8c98a0250caf5350c8c4d12e53e6ca7c0056 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 19:45:08 -0500 Subject: [PATCH 06/42] rpc: update getaddresstxids for uniqueness --- qa/rpc-tests/addressindex.py | 28 ++++++++++++++++++++++++++++ src/rpcmisc.cpp | 10 ++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 88533d028fc5a..fd750403f7a11 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -9,6 +9,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii class AddressIndexTest(BitcoinTestFramework): @@ -41,6 +44,9 @@ def run_test(self): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) + # Check p2pkh and p2sh address indexes + print "Testing p2pkh and p2sh address index..." + txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) @@ -67,6 +73,28 @@ def run_test(self): assert_equal(txidsb[1], txidb1); assert_equal(txidsb[2], txidb2); + # Check that outputs with the same address will only return one txid + print "Testing for txid uniqueness..." + addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") + scriptPubKey = CScript([OP_HASH160, addressHash, OP_EQUAL]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(10, scriptPubKey), CTxOut(11, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode('utf-8')) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx['hex'], True) + + self.nodes[0].generate(1) + self.sync_all() + + txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); + assert_equal(len(txidsmany), 4); + assert_equal(txidsmany[3], sent_txid); + + print "Passed\n" + if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 0a39d2fbcb3da..cca8a9896ab2d 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -424,9 +424,15 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) if (!GetAddressIndex(hashBytes, type, addressIndex)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + std::set txids; + UniValue result(UniValue::VARR); - for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) - result.push_back(it->first.txhash.GetHex()); + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + std::string txid = it->first.txhash.GetHex(); + if (txids.insert(txid).second) { + result.push_back(txid); + } + } return result; From f0b6a21761c8d73a37a1e45d1dc7f8e6e891f079 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 15 Mar 2016 15:25:01 -0400 Subject: [PATCH 07/42] rpc: query for multiple addresses txids --- qa/rpc-tests/addressindex.py | 4 ++++ src/rpcmisc.cpp | 46 +++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index fd750403f7a11..a985735f88fd3 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -73,6 +73,10 @@ def run_test(self): assert_equal(txidsb[1], txidb1); assert_equal(txidsb[2], txidb2); + # Check that multiple addresses works + multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); + assert_equal(len(multitxids), 6); + # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index cca8a9896ab2d..a7bf7aad2330c 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -410,20 +410,50 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) "]\n" ); - CBitcoinAddress address(params[0].get_str()); - uint160 hashBytes; - int type = 0; - if (!address.GetIndexKey(hashBytes, type)) { + std::vector > addresses; + + if (params[0].isStr()) { + + CBitcoinAddress address(params[0].get_str()); + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + addresses.push_back(std::make_pair(hashBytes, type)); + + } else if (params[0].isObject()) { + + UniValue addressValues = find_value(params[0].get_obj(), "addresses"); + if (!addressValues.isArray()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Addresses is expected to be an array"); + } + + std::vector values = addressValues.getValues(); + + for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { + + CBitcoinAddress address(it->get_str()); + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + addresses.push_back(std::make_pair(hashBytes, type)); + } + } else { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } std::vector > addressIndex; - LOCK(cs_main); - - if (!GetAddressIndex(hashBytes, type, addressIndex)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); ++it) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + // TODO sort by height std::set txids; UniValue result(UniValue::VARR); From 5b0f9315bfe28f19e40043212de58c8ecf43f5b7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 15 Mar 2016 16:24:55 -0400 Subject: [PATCH 08/42] rpc: sort txids by height for multiple addresses --- qa/rpc-tests/addressindex.py | 12 ++++++++++++ src/rpcmisc.cpp | 13 ++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a985735f88fd3..2e15348d6be1b 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -48,14 +48,20 @@ def run_test(self): print "Testing p2pkh and p2sh address index..." txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + self.nodes[0].generate(1) + txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + self.nodes[0].generate(1) + txidb1 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 15) self.nodes[0].generate(1) txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + self.nodes[0].generate(1) + txidb2 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 20) self.nodes[0].generate(1) @@ -76,6 +82,12 @@ def run_test(self): # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); assert_equal(len(multitxids), 6); + assert_equal(multitxids[0], txid0); + assert_equal(multitxids[1], txidb0); + assert_equal(multitxids[2], txid1); + assert_equal(multitxids[3], txidb1); + assert_equal(multitxids[4], txid2); + assert_equal(multitxids[5], txidb2); # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index a7bf7aad2330c..e04b77d758c8c 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -453,17 +453,24 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - // TODO sort by height std::set txids; + std::vector > vtxids; - UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + int height = it->first.blockHeight; std::string txid = it->first.txhash.GetHex(); if (txids.insert(txid).second) { - result.push_back(txid); + vtxids.push_back(std::make_pair(height, txid)); } } + std::sort(vtxids.begin(), vtxids.end()); + + UniValue result(UniValue::VARR); + for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { + result.push_back(it->second); + } + return result; } From 513c563511daa809e4d19c65fcf3153b78436d55 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 16 Mar 2016 14:00:50 -0400 Subject: [PATCH 09/42] main: serialize height in BE for address index key fixes a sorting issue when iterating over keys --- src/main.h | 49 +++++++++++++++++++++++++++++++++---------------- src/serialize.h | 11 +++++++++++ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/main.h b/src/main.h index 64286ddc54bef..1b7acd54deb2f 100644 --- a/src/main.h +++ b/src/main.h @@ -290,16 +290,27 @@ struct CAddressIndexKey { uint256 txhash; size_t outindex; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(type); - READWRITE(hashBytes); - READWRITE(blockHeight); - READWRITE(txindex); - READWRITE(txhash); - READWRITE(outindex); + size_t GetSerializeSize(int nType, int nVersion) const { + return 65; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + // Heights are stored big-endian for key sorting in LevelDB + ser_writedata32be(s, blockHeight); + ser_writedata32be(s, txindex); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, outindex); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + txindex = ser_readdata32be(s); + txhash.Unserialize(s, nType, nVersion); + outindex = ser_readdata32(s); } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, @@ -331,12 +342,18 @@ struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(type); - READWRITE(hashBytes); + size_t GetSerializeSize(int nType, int nVersion) const { + return 21; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); } CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { diff --git a/src/serialize.h b/src/serialize.h index 5c2db9d332ce3..41a51d237440c 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -91,6 +91,11 @@ template inline void ser_writedata32(Stream &s, uint32_t obj) obj = htole32(obj); s.write((char*)&obj, 4); } +template inline void ser_writedata32be(Stream &s, uint32_t obj) +{ + obj = htobe32(obj); + s.write((char*)&obj, 4); +} template inline void ser_writedata64(Stream &s, uint64_t obj) { obj = htole64(obj); @@ -114,6 +119,12 @@ template inline uint32_t ser_readdata32(Stream &s) s.read((char*)&obj, 4); return le32toh(obj); } +template inline uint32_t ser_readdata32be(Stream &s) +{ + uint32_t obj; + s.read((char*)&obj, 4); + return be32toh(obj); +} template inline uint64_t ser_readdata64(Stream &s) { uint64_t obj; From a7da54bcb97d6be64e2d927cf6e2653ab128f4eb Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 16 Mar 2016 14:50:19 -0400 Subject: [PATCH 10/42] rpc: only sort when combining multiple results It's only necessary to sort when combining results for several addresses as the results are already in order from the database. --- src/rpcmisc.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index e04b77d758c8c..1a86353e27380 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -464,7 +464,9 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - std::sort(vtxids.begin(), vtxids.end()); + if (addresses.size() > 1) { + std::sort(vtxids.begin(), vtxids.end()); + } UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { From ef720cf55022e919d8225f43c4c434a4efc19193 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 17 Mar 2016 16:06:08 -0400 Subject: [PATCH 11/42] rpc: added getaddressbalance method using addressindex --- qa/rpc-tests/addressindex.py | 12 ++++++ src/rpcmisc.cpp | 83 ++++++++++++++++++++++++++++-------- src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 2e15348d6be1b..8f123eaa075f6 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -44,6 +44,10 @@ def run_test(self): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 0); + # Check p2pkh and p2sh address indexes print "Testing p2pkh and p2sh address index..." @@ -89,6 +93,10 @@ def run_test(self): assert_equal(multitxids[4], txid2); assert_equal(multitxids[5], txidb2); + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 45 * 100000000); + # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") @@ -109,6 +117,10 @@ def run_test(self): assert_equal(len(txidsmany), 4); assert_equal(txidsmany[3], sent_txid); + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 45 * 100000000 + 21); + print "Passed\n" diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 1a86353e27380..d4ad29e65581c 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -397,23 +397,9 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } -UniValue getaddresstxids(const UniValue& params, bool fHelp) +bool getAddressesFromParams(const UniValue& params, std::vector > &addresses) { - if (fHelp || params.size() != 1) - throw runtime_error( - "getaddresstxids\n" - "\nReturns the txids for an address (requires addressindex to be enabled).\n" - "\nResult\n" - "[\n" - " \"transactionid\" (string) The transaction id\n" - " ,...\n" - "]\n" - ); - - std::vector > addresses; - if (params[0].isStr()) { - CBitcoinAddress address(params[0].get_str()); uint160 hashBytes; int type = 0; @@ -421,7 +407,6 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } addresses.push_back(std::make_pair(hashBytes, type)); - } else if (params[0].isObject()) { UniValue addressValues = find_value(params[0].get_obj(), "addresses"); @@ -445,9 +430,73 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } + return true; + + +} + +UniValue getaddressbalance(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddressbalance\n" + "\nReturns the balance for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "{\n" + " \"balance\" (string) The current balance\n" + " ,...\n" + "}\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > addressIndex; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + CAmount balance = 0; + + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + balance += it->second; + } + + UniValue result(UniValue::VOBJ); + result.push_back(Pair("balance", balance)); + + return result; + +} + +UniValue getaddresstxids(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddresstxids\n" + "\nReturns the txids for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "[\n" + " \"transactionid\" (string) The transaction id\n" + " ,...\n" + "]\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + std::vector > addressIndex; - for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); ++it) { + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 67da51ceb75af..da66b1c0a1a97 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = /* Address index */ { "addressindex", "getaddresstxids", &getaddresstxids, false }, + { "addressindex", "getaddressbalance", &getaddressbalance, false }, /* Utility functions */ { "util", "createmultisig", &createmultisig, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 11f7600511a9c..8a80fc5ee2abe 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -167,6 +167,7 @@ extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp extern UniValue getaddresstxids(const UniValue& params, bool fHelp); +extern UniValue getaddressbalance(const UniValue& params, bool fHelp); extern UniValue getpeerinfo(const UniValue& params, bool fHelp); extern UniValue ping(const UniValue& params, bool fHelp); From f493604a5122559b5a81e0b2622ff26816cd35c9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 17 Mar 2016 16:40:16 -0400 Subject: [PATCH 12/42] rpc: add receieved to balance --- src/rpcmisc.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index d4ad29e65581c..549e6c7a5b0f5 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -463,13 +463,18 @@ UniValue getaddressbalance(const UniValue& params, bool fHelp) } CAmount balance = 0; + CAmount received = 0; for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + if (it->second > 0) { + received += it->second; + } balance += it->second; } UniValue result(UniValue::VOBJ); result.push_back(Pair("balance", balance)); + result.push_back(Pair("received", received)); return result; From 61826feaff850d23f8fb48ffb836a2dda5918a8d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 22 Mar 2016 13:55:19 -0400 Subject: [PATCH 13/42] rpc: include height in getrawtransaction results --- src/rpcrawtransaction.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index bd51aa0ab018d..6ab1807d46373 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -101,12 +101,14 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) if (mi != mapBlockIndex.end() && (*mi).second) { CBlockIndex* pindex = (*mi).second; if (chainActive.Contains(pindex)) { + entry.push_back(Pair("height", pindex->nHeight)); entry.push_back(Pair("confirmations", 1 + chainActive.Height() - pindex->nHeight)); entry.push_back(Pair("time", pindex->GetBlockTime())); entry.push_back(Pair("blocktime", pindex->GetBlockTime())); - } - else + } else { + entry.push_back(Pair("height", -1)); entry.push_back(Pair("confirmations", 0)); + } } } } From 1ad7ae648e525bf248f85b9be1b5c2e8b5224428 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 22 Mar 2016 18:11:04 -0400 Subject: [PATCH 14/42] main: add block timestamp index --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/timestampindex.py | 51 +++++++++++++++++++++++++++ src/main.cpp | 28 +++++++++++++-- src/main.h | 63 ++++++++++++++++++++++++++++++++++ src/rpcblockchain.cpp | 32 +++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 27 +++++++++++++++ src/txdb.h | 4 +++ 9 files changed, 206 insertions(+), 2 deletions(-) create mode 100755 qa/rpc-tests/timestampindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index b092a8cd7e462..ad7f97ee821eb 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -97,6 +97,7 @@ 'nodehandling.py', 'reindex.py', 'addressindex.py', + 'timestampindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/timestampindex.py b/qa/rpc-tests/timestampindex.py new file mode 100755 index 0000000000000..46ff710bdf37f --- /dev/null +++ b/qa/rpc-tests/timestampindex.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test timestampindex generation and fetching +# + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + + +class TimestampIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-timestampindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-timestampindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining 5 blocks..." + blockhashes = self.nodes[0].generate(5) + low = self.nodes[0].getblock(blockhashes[0])["time"] + high = self.nodes[0].getblock(blockhashes[4])["time"] + self.sync_all() + print "Checking timestamp index..." + hashes = self.nodes[1].getblockhashes(high, low) + assert_equal(len(hashes), 5) + assert_equal(sorted(blockhashes), sorted(hashes)) + print "Passed\n" + + +if __name__ == '__main__': + TimestampIndexTest().main() diff --git a/src/main.cpp b/src/main.cpp index 617b37de8de10..8d84dc03d901d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -66,6 +66,7 @@ bool fImporting = false; bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; +bool fTimestampIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1233,6 +1234,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) +{ + if (!fTimestampIndex) + return error("Timestamp index not enabled"); + + if (!pblocktree->ReadTimestampIndex(high, low, hashes)) + return error("Unable to get hashes for timestamps"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) { if (!fAddressIndex) @@ -2199,8 +2211,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return AbortNode(state, "Failed to write transaction index"); if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) - return AbortNode(state, "Failed to write address index"); + if (!pblocktree->WriteAddressIndex(addressIndex)) + return AbortNode(state, "Failed to write address index"); + + if (fTimestampIndex) + if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + return AbortNode(state, "Failed to write timestamp index"); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3571,6 +3587,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("addressindex", fAddressIndex); LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Check whether we have a timestamp index + pblocktree->ReadFlag("timestampindex", fTimestampIndex); + LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -3733,6 +3753,10 @@ bool InitBlockIndex(const CChainParams& chainparams) fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); pblocktree->WriteFlag("addressindex", fAddressIndex); + // Use the provided setting for -timestampindex in the new database + fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); + pblocktree->WriteFlag("timestampindex", fTimestampIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 1b7acd54deb2f..169444a8f20ed 100644 --- a/src/main.h +++ b/src/main.h @@ -106,6 +106,7 @@ static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; +static const bool DEFAULT_TIMESTAMPINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -282,6 +283,67 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CTimestampIndexIteratorKey { + unsigned int timestamp; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 4; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + } + + CTimestampIndexIteratorKey(unsigned int time) { + timestamp = time; + } + + CTimestampIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + } +}; + +struct CTimestampIndexKey { + unsigned int timestamp; + uint256 blockHash; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 36; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + blockHash.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + blockHash.Unserialize(s, nType, nVersion); + } + + CTimestampIndexKey(unsigned int time, uint256 hash) { + timestamp = time; + blockHash = hash; + } + + CTimestampIndexKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + blockHash.SetNull(); + } +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -477,6 +539,7 @@ class CScriptCheck ScriptError GetScriptError() const { return error; } }; +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); /** Functions for disk access for blocks */ diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index f5b16bc7c49b1..c20f2941d2cc9 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -275,6 +275,38 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) return mempoolToJSON(fVerbose); } +UniValue getblockhashes(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error( + "getblockhashes timestamp\n" + "\nReturns array of hashes of blocks within the timestamp range provided.\n" + "\nArguments:\n" + "1. high (numeric, required) The newer block timestamp\n" + "2. low (numeric, required) The older block timestamp\n" + "\nResult:\n" + "[" + " \"hash\" (string) The block hash\n" + "]" + "\nExamples:\n" + ); + + unsigned int high = params[0].get_int(); + unsigned int low = params[1].get_int(); + std::vector blockHashes; + + if (!GetTimestampIndex(high, low, blockHashes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); + } + + UniValue result(UniValue::VARR); + for (std::vector::const_iterator it=blockHashes.begin(); it!=blockHashes.end(); it++) { + result.push_back(it->GetHex()); + } + + return result; +} + UniValue getblockhash(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index da66b1c0a1a97..51235c184f4df 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -278,6 +278,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "getbestblockhash", &getbestblockhash, true }, { "blockchain", "getblockcount", &getblockcount, true }, { "blockchain", "getblock", &getblock, true }, + { "blockchain", "getblockhashes", &getblockhashes, true }, { "blockchain", "getblockhash", &getblockhash, true }, { "blockchain", "getblockheader", &getblockheader, true }, { "blockchain", "getchaintips", &getchaintips, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 8a80fc5ee2abe..0be81809340be 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -258,6 +258,7 @@ extern UniValue getdifficulty(const UniValue& params, bool fHelp); extern UniValue settxfee(const UniValue& params, bool fHelp); extern UniValue getmempoolinfo(const UniValue& params, bool fHelp); extern UniValue getrawmempool(const UniValue& params, bool fHelp); +extern UniValue getblockhashes(const UniValue& params, bool fHelp); extern UniValue getblockhash(const UniValue& params, bool fHelp); extern UniValue getblockheader(const UniValue& params, bool fHelp); extern UniValue getblock(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index e93594b9ed80f..3aa6860583971 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; +static const char DB_TIMESTAMPINDEX = 's'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -196,6 +197,32 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector &hashes) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_TIMESTAMPINDEX, CTimestampIndexIteratorKey(low))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.timestamp <= high) { + hashes.push_back(key.second.blockHash); + pcursor->Next(); + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index ca32869ccc37d..7fefcaafafefa 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -19,6 +19,8 @@ class CBlockIndex; struct CDiskTxPos; struct CAddressIndexKey; struct CAddressIndexIteratorKey; +struct CTimestampIndexKey; +struct CTimestampIndexIteratorKey; class uint256; //! -dbcache default (MiB) @@ -61,6 +63,8 @@ class CBlockTreeDB : public CDBWrapper bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); + bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); + bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(); From 934d691571d4e8ac08f8cac02413adfbd7131289 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 23 Mar 2016 14:14:36 -0400 Subject: [PATCH 15/42] test: added to for balance after spending --- qa/rpc-tests/addressindex.py | 87 +++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 8f123eaa075f6..59837f7838d97 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -46,7 +46,7 @@ def run_test(self): # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 0); + assert_equal(balance0["balance"], 0) # Check p2pkh and p2sh address indexes print "Testing p2pkh and p2sh address index..." @@ -71,31 +71,31 @@ def run_test(self): self.sync_all() - txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); - assert_equal(len(txids), 3); - assert_equal(txids[0], txid0); - assert_equal(txids[1], txid1); - assert_equal(txids[2], txid2); + txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs") + assert_equal(len(txids), 3) + assert_equal(txids[0], txid0) + assert_equal(txids[1], txid1) + assert_equal(txids[2], txid2) - txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); - assert_equal(len(txidsb), 3); - assert_equal(txidsb[0], txidb0); - assert_equal(txidsb[1], txidb1); - assert_equal(txidsb[2], txidb2); + txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(len(txidsb), 3) + assert_equal(txidsb[0], txidb0) + assert_equal(txidsb[1], txidb1) + assert_equal(txidsb[2], txidb2) # Check that multiple addresses works - multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); - assert_equal(len(multitxids), 6); - assert_equal(multitxids[0], txid0); - assert_equal(multitxids[1], txidb0); - assert_equal(multitxids[2], txid1); - assert_equal(multitxids[3], txidb1); - assert_equal(multitxids[4], txid2); - assert_equal(multitxids[5], txidb2); + multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) + assert_equal(len(multitxids), 6) + assert_equal(multitxids[0], txid0) + assert_equal(multitxids[1], txidb0) + assert_equal(multitxids[2], txid1) + assert_equal(multitxids[3], txidb1) + assert_equal(multitxids[4], txid2) + assert_equal(multitxids[5], txidb2) # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 45 * 100000000); + assert_equal(balance0["balance"], 45 * 100000000) # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." @@ -107,19 +107,54 @@ def run_test(self): tx.vout = [CTxOut(10, scriptPubKey), CTxOut(11, scriptPubKey)] tx.rehash() - signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode('utf-8')) - sent_txid = self.nodes[0].sendrawtransaction(signed_tx['hex'], True) + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) self.nodes[0].generate(1) self.sync_all() - txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); - assert_equal(len(txidsmany), 4); - assert_equal(txidsmany[3], sent_txid); + txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(len(txidsmany), 4) + assert_equal(txidsmany[3], sent_txid) # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 45 * 100000000 + 21); + assert_equal(balance0["balance"], 45 * 100000000 + 21) + + # Check that balances are correct after spending + privkey2 = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address2 = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash2 = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey2 = CScript([OP_DUP, OP_HASH160, addressHash2, OP_EQUALVERIFY, OP_CHECKSIG]) + self.nodes[0].importprivkey(privkey2) + + unspent = self.nodes[0].listunspent() + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + amount = unspent[0]["amount"] * 100000000 + tx.vout = [CTxOut(amount, scriptPubKey2)] + tx.rehash() + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + spending_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + balance1 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance1["balance"], amount) + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(spending_txid, 16), 0))] + send_amount = 1 * 100000000 + 12840 + change_amount = amount - send_amount - 10000 + tx.vout = [CTxOut(send_amount, scriptPubKey), CTxOut(change_amount, scriptPubKey2)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + balance2 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance2["balance"], change_amount) print "Passed\n" From bcb2cde5f2ef0975912c4e73a359084e4b3795c9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 23 Mar 2016 14:41:08 -0400 Subject: [PATCH 16/42] main: fixed bug with overlapping address index keys There was a bug where the spending address index could have the same key as the receiving address index if the input and output indexes matched. This lead to the output always overwriting the input index leading to incorrect balances with missing spent amounts. This patch separates the two so that they have unique keys so balances will be correctly calculated. --- qa/rpc-tests/addressindex.py | 2 +- src/main.cpp | 8 ++++---- src/main.h | 9 ++++++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 59837f7838d97..0ffbcffd77439 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -145,7 +145,7 @@ def run_test(self): tx.vin = [CTxIn(COutPoint(int(spending_txid, 16), 0))] send_amount = 1 * 100000000 + 12840 change_amount = amount - send_amount - 10000 - tx.vout = [CTxOut(send_amount, scriptPubKey), CTxOut(change_amount, scriptPubKey2)] + tx.vout = [CTxOut(change_amount, scriptPubKey2), CTxOut(send_amount, scriptPubKey)] tx.rehash() signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) diff --git a/src/main.cpp b/src/main.cpp index 8d84dc03d901d..467adae380067 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2112,10 +2112,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); } else { continue; } @@ -2149,10 +2149,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); } else { continue; } diff --git a/src/main.h b/src/main.h index 169444a8f20ed..5aadfbea8e7ff 100644 --- a/src/main.h +++ b/src/main.h @@ -351,6 +351,7 @@ struct CAddressIndexKey { unsigned int txindex; uint256 txhash; size_t outindex; + bool spending; size_t GetSerializeSize(int nType, int nVersion) const { return 65; @@ -364,6 +365,8 @@ struct CAddressIndexKey { ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); ser_writedata32(s, outindex); + char f = spending; + ser_writedata8(s, f); } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -373,16 +376,19 @@ struct CAddressIndexKey { txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); outindex = ser_readdata32(s); + char f = ser_readdata8(s); + spending = f; } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t outputIndex) { + uint256 txid, size_t outputIndex, bool isSpending) { type = addressType; hashBytes = addressHash; blockHeight = height; txindex = blockindex; txhash = txid; outindex = outputIndex; + spending = isSpending; } CAddressIndexKey() { @@ -396,6 +402,7 @@ struct CAddressIndexKey { txindex = 0; txhash.SetNull(); outindex = 0; + spending = false; } }; From 738b9bc8503bb64250f44207df5e825deb42f38f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 24 Mar 2016 15:44:23 -0400 Subject: [PATCH 17/42] main: get address deltas between range of block heights --- qa/rpc-tests/addressindex.py | 11 +++++++ src/main.cpp | 9 +++--- src/main.h | 36 +++++++++++++++++----- src/rpcmisc.cpp | 60 ++++++++++++++++++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 13 ++++++-- src/txdb.h | 4 ++- 8 files changed, 120 insertions(+), 15 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 0ffbcffd77439..4fc4cc557e361 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -156,6 +156,17 @@ def run_test(self): balance2 = self.nodes[1].getaddressbalance(address2) assert_equal(balance2["balance"], change_amount) + # Check that deltas are returned correctly + deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 0, "end": 200}) + balance3 = 0; + for delta in deltas: + balance3 += delta["satoshis"] + assert_equal(balance3, change_amount) + + # Check that deltas can be returned from range of block heights + deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) + assert_equal(len(deltas), 1); + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 467adae380067..4f060489cce55 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1245,13 +1245,14 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v return true; } -bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) +bool GetAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, int start, int end) { if (!fAddressIndex) - return error("%s: address index not enabled"); + return error("address index not enabled"); - if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex)) - return error("%s: unable to get txids for address"); + if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex, start, end)) + return error("unable to get txids for address"); return true; } diff --git a/src/main.h b/src/main.h index 5aadfbea8e7ff..06a94be052983 100644 --- a/src/main.h +++ b/src/main.h @@ -350,7 +350,7 @@ struct CAddressIndexKey { int blockHeight; unsigned int txindex; uint256 txhash; - size_t outindex; + size_t index; bool spending; size_t GetSerializeSize(int nType, int nVersion) const { @@ -364,7 +364,7 @@ struct CAddressIndexKey { ser_writedata32be(s, blockHeight); ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); - ser_writedata32(s, outindex); + ser_writedata32(s, index); char f = spending; ser_writedata8(s, f); } @@ -375,19 +375,19 @@ struct CAddressIndexKey { blockHeight = ser_readdata32be(s); txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); - outindex = ser_readdata32(s); + index = ser_readdata32(s); char f = ser_readdata8(s); spending = f; } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t outputIndex, bool isSpending) { + uint256 txid, size_t indexValue, bool isSpending) { type = addressType; hashBytes = addressHash; blockHeight = height; txindex = blockindex; txhash = txid; - outindex = outputIndex; + index = indexValue; spending = isSpending; } @@ -401,7 +401,7 @@ struct CAddressIndexKey { blockHeight = 0; txindex = 0; txhash.SetNull(); - outindex = 0; + index = 0; spending = false; } @@ -410,14 +410,23 @@ struct CAddressIndexKey { struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; + bool includeHeight; + int blockHeight; size_t GetSerializeSize(int nType, int nVersion) const { - return 21; + if (includeHeight) { + return 25; + } else { + return 21; + } } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); + if (includeHeight) { + ser_writedata32be(s, blockHeight); + } } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -428,6 +437,14 @@ struct CAddressIndexIteratorKey { CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { type = addressType; hashBytes = addressHash; + includeHeight = false; + } + + CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash, int height) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + includeHeight = true; } CAddressIndexIteratorKey() { @@ -437,6 +454,7 @@ struct CAddressIndexIteratorKey { void SetNull() { type = 0; hashBytes.SetNull(); + includeHeight = false; } }; @@ -547,7 +565,9 @@ class CScriptCheck }; bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); -bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); +bool GetAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start = 0, int end = 0); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 549e6c7a5b0f5..8685a327aba47 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -435,6 +435,66 @@ bool getAddressesFromParams(const UniValue& params, std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > addressIndex; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + UniValue result(UniValue::VARR); + + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + UniValue delta(UniValue::VOBJ); + delta.push_back(Pair("satoshis", it->second)); + delta.push_back(Pair("txid", it->first.txhash.GetHex())); + delta.push_back(Pair("index", it->first.index)); + delta.push_back(Pair("height", it->first.blockHeight)); + delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); + delta.push_back(Pair("type", (int)it->first.type)); + result.push_back(delta); + } + + return result; +} + UniValue getaddressbalance(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 51235c184f4df..c955484626580 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, { "addressindex", "getaddressbalance", &getaddressbalance, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 0be81809340be..fc57aa16b3f37 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); extern UniValue getaddressbalance(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 3aa6860583971..eb03b30e492b0 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -172,16 +172,25 @@ bool CBlockTreeDB::WriteAddressIndex(const std::vector > &addressIndex) { +bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start, int end) { boost::scoped_ptr pcursor(NewIterator()); - pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); + if (start > 0 && end > 0) { + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash, start))); + } else { + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); + } while (pcursor->Valid()) { boost::this_thread::interruption_point(); std::pair key; if (pcursor->GetKey(key) && key.first == DB_ADDRESSINDEX && key.second.hashBytes == addressHash) { + if (end > 0 && key.second.blockHeight > end) { + break; + } CAmount nValue; if (pcursor->GetValue(nValue)) { addressIndex.push_back(make_pair(key.second, nValue)); diff --git a/src/txdb.h b/src/txdb.h index 7fefcaafafefa..4586d51f898d4 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -62,7 +62,9 @@ class CBlockTreeDB : public CDBWrapper bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); - bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); + bool ReadAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start = 0, int end = 0); bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); From e7e2b09cfb219a787089c8aa9611cc436582b5a1 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 25 Mar 2016 14:52:45 -0400 Subject: [PATCH 18/42] rpc: optimize address txid queries --- src/rpcmisc.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8685a327aba47..8d939ffeaf3f4 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -567,24 +567,26 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - std::set txids; - std::vector > vtxids; + std::set > txids; + UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { int height = it->first.blockHeight; std::string txid = it->first.txhash.GetHex(); - if (txids.insert(txid).second) { - vtxids.push_back(std::make_pair(height, txid)); + + if (addresses.size() > 1) { + txids.insert(std::make_pair(height, txid)); + } else { + if (txids.insert(std::make_pair(height, txid)).second) { + result.push_back(txid); + } } } if (addresses.size() > 1) { - std::sort(vtxids.begin(), vtxids.end()); - } - - UniValue result(UniValue::VARR); - for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { - result.push_back(it->second); + for (std::set >::const_iterator it=txids.begin(); it!=txids.end(); it++) { + result.push_back(it->second); + } } return result; From 6df79b4bbed7356b4c8ee40ebc3bd956e2b9e927 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 28 Mar 2016 16:47:20 -0400 Subject: [PATCH 19/42] main: update address index during reorgs --- qa/rpc-tests/addressindex.py | 19 +++++++++++++-- src/main.cpp | 46 ++++++++++++++++++++++++++++++++++++ src/txdb.cpp | 7 ++++++ src/txdb.h | 1 + 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 4fc4cc557e361..a60c845b34ea7 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -118,10 +118,12 @@ def run_test(self): assert_equal(txidsmany[3], sent_txid) # Check that balances are correct + print "Testing balances..." balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") assert_equal(balance0["balance"], 45 * 100000000 + 21) # Check that balances are correct after spending + print "Testing balances after spending..." privkey2 = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" address2 = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" addressHash2 = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") @@ -158,14 +160,27 @@ def run_test(self): # Check that deltas are returned correctly deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 0, "end": 200}) - balance3 = 0; + balance3 = 0 for delta in deltas: balance3 += delta["satoshis"] assert_equal(balance3, change_amount) # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) - assert_equal(len(deltas), 1); + assert_equal(len(deltas), 1) + + # Check that indexes will be updated with a reorg + print "Testing reorg..." + + best_hash = self.nodes[0].getbestblockhash() + self.nodes[0].invalidateblock(best_hash) + self.nodes[1].invalidateblock(best_hash) + self.nodes[2].invalidateblock(best_hash) + self.nodes[3].invalidateblock(best_hash) + self.sync_all() + + balance4 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance4, balance1) print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 4f060489cce55..b6454e05fcac6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1873,6 +1873,52 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } } + // undo address indexes + if (fAddressIndex) { + std::vector > addressIndex; + + for (unsigned int i = 0; i < block.vtx.size(); i++) { + const CTransaction &tx = block.vtx[i]; + const uint256 txhash = tx.GetHash(); + + if (!tx.IsCoinBase()) { + for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + } else { + continue; + } + } + } + + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + } else { + continue; + } + + } + + } + + if (!pblocktree->EraseAddressIndex(addressIndex)) { + return AbortNode(state, "Failed to delete address index"); + } + } + + // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); diff --git a/src/txdb.cpp b/src/txdb.cpp index eb03b30e492b0..2128c181f27d1 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -172,6 +172,13 @@ bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) + batch.Erase(make_pair(DB_ADDRESSINDEX, it->first)); + return WriteBatch(batch); +} + bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start, int end) { diff --git a/src/txdb.h b/src/txdb.h index 4586d51f898d4..93b5c0f5c14db 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -62,6 +62,7 @@ class CBlockTreeDB : public CDBWrapper bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); + bool EraseAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); From 1e58a292472ac01103382188d0cd0fc9bba1bfaa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 09:13:31 -0400 Subject: [PATCH 20/42] main: fix order of address index when disconnecting block --- src/main.cpp | 59 ++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index b6454e05fcac6..c6f8ef90ae4dd 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1835,11 +1835,32 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) return error("DisconnectBlock(): block and undo data inconsistent"); + std::vector > addressIndex; + // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = block.vtx[i]; uint256 hash = tx.GetHash(); + if (fAddressIndex) { + + for (unsigned int k = tx.vout.size(); k-- > 0;) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + } else { + continue; + } + + } + + } + // Check that all outputs are available and match the outputs in the block itself // exactly. { @@ -1869,56 +1890,30 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI const CTxInUndo &undo = txundo.vprevout[j]; if (!ApplyTxInUndo(undo, view, out)) fClean = false; - } - } - } - - // undo address indexes - if (fAddressIndex) { - std::vector > addressIndex; - - for (unsigned int i = 0; i < block.vtx.size(); i++) { - const CTransaction &tx = block.vtx[i]; - const uint256 txhash = tx.GetHash(); - if (!tx.IsCoinBase()) { - for (size_t j = 0; j < tx.vin.size(); j++) { + if (fAddressIndex) { const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); } else { continue; } } } - - for (unsigned int k = 0; k < tx.vout.size(); k++) { - const CTxOut &out = tx.vout[k]; - - if (out.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); - } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); - } else { - continue; - } - - } - } + } + + if (fAddressIndex) { if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); } } - // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); From 42d71133b176877d7ee0f58acdada354d7662565 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 15:17:30 -0400 Subject: [PATCH 21/42] main: index unspent outputs by address --- qa/rpc-tests/addressindex.py | 6 +++ src/main.cpp | 46 +++++++++++++++++- src/main.h | 91 +++++++++++++++++++++++++++++++++++- src/rpcmisc.cpp | 50 ++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 39 ++++++++++++++++ src/txdb.h | 5 ++ 8 files changed, 236 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a60c845b34ea7..5448af6c6346a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -169,6 +169,12 @@ def run_test(self): deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) assert_equal(len(deltas), 1) + # Check that unspent outputs can be queried + print "Testing utxos..." + utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos), 2) + assert_equal(utxos[0]["satoshis"], 5000000000) + # Check that indexes will be updated with a reorg print "Testing reorg..." diff --git a/src/main.cpp b/src/main.cpp index c6f8ef90ae4dd..9eb5790354fcb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1257,6 +1257,18 @@ bool GetAddressIndex(uint160 addressHash, int type, return true; } +bool GetAddressUnspent(uint160 addressHash, int type, + std::vector > &unspentOutputs) +{ + if (!fAddressIndex) + return error("address index not enabled"); + + if (!pblocktree->ReadAddressUnspentIndex(addressHash, type, unspentOutputs)) + return error("unable to get txids for address"); + + return true; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { @@ -2130,6 +2142,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector > addressIndex; + std::vector > addressUnspentIndex; for (unsigned int i = 0; i < block.vtx.size(); i++) { @@ -2154,10 +2167,21 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // record spending activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // record spending activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + } else { continue; } @@ -2191,10 +2215,22 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + + // record receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + + // record unspent output + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + + // record receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + + // record unspent output + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + } else { continue; } @@ -2252,9 +2288,15 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!pblocktree->WriteTxIndex(vPos)) return AbortNode(state, "Failed to write transaction index"); - if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) + if (fAddressIndex) { + if (!pblocktree->WriteAddressIndex(addressIndex)) { return AbortNode(state, "Failed to write address index"); + } + + if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + return AbortNode(state, "Failed to write address unspent index"); + } + } if (fTimestampIndex) if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) diff --git a/src/main.h b/src/main.h index 06a94be052983..c548c2fdfc31f 100644 --- a/src/main.h +++ b/src/main.h @@ -344,6 +344,93 @@ struct CTimestampIndexKey { } }; +struct CAddressUnspentKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + unsigned int txindex; + uint256 txhash; + size_t index; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 65; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + // Heights are stored big-endian for key sorting in LevelDB + ser_writedata32be(s, blockHeight); + ser_writedata32be(s, txindex); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, index); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + txindex = ser_readdata32be(s); + txhash.Unserialize(s, nType, nVersion); + index = ser_readdata32(s); + } + + CAddressUnspentKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, + uint256 txid, size_t indexValue) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + txindex = blockindex; + txhash = txid; + index = indexValue; + } + + CAddressUnspentKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + blockHeight = 0; + txindex = 0; + txhash.SetNull(); + index = 0; + } +}; + +struct CAddressUnspentValue { + CAmount satoshis; + CScript script; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(satoshis); + READWRITE(*(CScriptBase*)(&script)); + } + + CAddressUnspentValue(CAmount sats, CScript scriptPubKey) { + satoshis = sats; + script = scriptPubKey; + } + + CAddressUnspentValue() { + SetNull(); + } + + void SetNull() { + satoshis = 0; + script = CScript(); + } + + bool IsNull() const { + return (satoshis == 0); + } + +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -354,7 +441,7 @@ struct CAddressIndexKey { bool spending; size_t GetSerializeSize(int nType, int nVersion) const { - return 65; + return 66; } template void Serialize(Stream& s, int nType, int nVersion) const { @@ -568,6 +655,8 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); +bool GetAddressUnspent(uint160 addressHash, int type, + std::vector > &unspentOutputs); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8d939ffeaf3f4..1eeded608ff46 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -435,6 +435,56 @@ bool getAddressesFromParams(const UniValue& params, std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > unspentOutputs; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressUnspent((*it).first, (*it).second, unspentOutputs)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + UniValue result(UniValue::VARR); + + for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { + UniValue output(UniValue::VOBJ); + output.push_back(Pair("addressType", (int)it->first.type)); + output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); + output.push_back(Pair("txid", it->first.txhash.GetHex())); + output.push_back(Pair("height", it->first.blockHeight)); + output.push_back(Pair("outputIndex", it->first.index)); + output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); + output.push_back(Pair("satoshis", it->second.satoshis)); + result.push_back(output); + } + + return result; +} + UniValue getaddressdeltas(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1 || !params[0].isObject()) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index c955484626580..39af0513b5da6 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, { "addressindex", "getaddressbalance", &getaddressbalance, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index fc57aa16b3f37..5ad147dec7f08 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressutxos(const UniValue& params, bool fHelp); extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); extern UniValue getaddressbalance(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 2128c181f27d1..8e485e9eb6620 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; +static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 's'; static const char DB_BLOCK_INDEX = 'b'; @@ -165,6 +166,44 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::UpdateAddressUnspentIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { + if (it->second.IsNull()) { + batch.Erase(make_pair(DB_ADDRESSUNSPENTINDEX, it->first)); + } else { + batch.Write(make_pair(DB_ADDRESSUNSPENTINDEX, it->first), it->second); + } + } + return WriteBatch(batch); +} + +bool CBlockTreeDB::ReadAddressUnspentIndex(uint160 addressHash, int type, + std::vector > &unspentOutputs) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_ADDRESSUNSPENTINDEX, CAddressIndexIteratorKey(type, addressHash))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_ADDRESSUNSPENTINDEX && key.second.hashBytes == addressHash) { + CAddressUnspentValue nValue; + if (pcursor->GetValue(nValue)) { + unspentOutputs.push_back(make_pair(key.second, nValue)); + pcursor->Next(); + } else { + return error("failed to get address unspent value"); + } + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) diff --git a/src/txdb.h b/src/txdb.h index 93b5c0f5c14db..547a48fa8412e 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -17,6 +17,8 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; +struct CAddressUnspentKey; +struct CAddressUnspentValue; struct CAddressIndexKey; struct CAddressIndexIteratorKey; struct CTimestampIndexKey; @@ -61,6 +63,9 @@ class CBlockTreeDB : public CDBWrapper bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool UpdateAddressUnspentIndex(const std::vector >&vect); + bool ReadAddressUnspentIndex(uint160 addressHash, int type, + std::vector > &vect); bool WriteAddressIndex(const std::vector > &vect); bool EraseAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, From 22ee08154dbb44df53e4e391c85e78a174ff8acd Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 17:46:59 -0400 Subject: [PATCH 22/42] rpc: fix argument check for getaddressutxos --- src/rpcmisc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 1eeded608ff46..2c3b7e6d00bb4 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -437,7 +437,7 @@ bool getAddressesFromParams(const UniValue& params, std::vector Date: Tue, 29 Mar 2016 17:56:57 -0400 Subject: [PATCH 23/42] main: update unspent address index during reorgs --- qa/rpc-tests/addressindex.py | 6 ++++++ src/main.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 5448af6c6346a..d051d170a944a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -174,6 +174,7 @@ def run_test(self): utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) assert_equal(len(utxos), 2) assert_equal(utxos[0]["satoshis"], 5000000000) + assert_equal(utxos[1]["satoshis"], 4899977160) # Check that indexes will be updated with a reorg print "Testing reorg..." @@ -188,6 +189,11 @@ def run_test(self): balance4 = self.nodes[1].getaddressbalance(address2) assert_equal(balance4, balance1) + utxos2 = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos2), 2) + assert_equal(utxos2[0]["satoshis"], 5000000000) + assert_equal(utxos2[1]["satoshis"], 5000000000) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 9eb5790354fcb..2278e34f860b3 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1848,6 +1848,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI return error("DisconnectBlock(): block and undo data inconsistent"); std::vector > addressIndex; + std::vector > addressUnspentIndex; // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -1861,10 +1862,22 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + + // undo receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + + // undo unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + + // undo receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + + // undo unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + } else { continue; } @@ -1907,10 +1920,23 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // undo spending activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); + + // restore unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // undo spending activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); + + // restore unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + } else { continue; } @@ -1924,6 +1950,9 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); } + if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + return AbortNode(state, "Failed to write address unspent index"); + } } // move best block pointer to prevout block From cea34b75479db3c747e05d1e1174e8eec4448c37 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 12:50:10 -0400 Subject: [PATCH 24/42] main: don't undo indexes when verifying blocks at startup --- src/main.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 2278e34f860b3..5272a86dbfe84 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1946,6 +1946,14 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } + // move best block pointer to prevout block + view.SetBestBlock(pindex->pprev->GetBlockHash()); + + if (pfClean) { + *pfClean = fClean; + return true; + } + if (fAddressIndex) { if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); @@ -1955,14 +1963,6 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } } - // move best block pointer to prevout block - view.SetBestBlock(pindex->pprev->GetBlockHash()); - - if (pfClean) { - *pfClean = fClean; - return true; - } - return fClean; } From a6e0d582f1699a0ab88b7c9d5856d7db7b2dbd74 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 15:12:19 -0400 Subject: [PATCH 25/42] main: remove spent address utxo indexes --- qa/rpc-tests/addressindex.py | 8 +++----- src/main.cpp | 18 ++++++++++-------- src/main.h | 23 +++++------------------ src/rpcmisc.cpp | 1 - 4 files changed, 18 insertions(+), 32 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index d051d170a944a..703f3029ada91 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -172,9 +172,8 @@ def run_test(self): # Check that unspent outputs can be queried print "Testing utxos..." utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) - assert_equal(len(utxos), 2) - assert_equal(utxos[0]["satoshis"], 5000000000) - assert_equal(utxos[1]["satoshis"], 4899977160) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]["satoshis"], change_amount) # Check that indexes will be updated with a reorg print "Testing reorg..." @@ -190,9 +189,8 @@ def run_test(self): assert_equal(balance4, balance1) utxos2 = self.nodes[1].getaddressutxos({"addresses": [address2]}) - assert_equal(len(utxos2), 2) + assert_equal(len(utxos2), 1) assert_equal(utxos2[0]["satoshis"], 5000000000) - assert_equal(utxos2[1]["satoshis"], 5000000000) print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 5272a86dbfe84..362edf2becc47 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1867,7 +1867,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); // undo unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), hash, k), CAddressUnspentValue())); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -1876,7 +1876,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); // undo unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), hash, k), CAddressUnspentValue())); } else { continue; @@ -1917,6 +1917,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI fClean = false; if (fAddressIndex) { + const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -1925,7 +1926,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { @@ -1935,7 +1936,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); } else { continue; @@ -2193,6 +2194,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fAddressIndex) { for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -2201,7 +2203,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); @@ -2209,7 +2211,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); } else { continue; @@ -2249,7 +2251,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -2258,7 +2260,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); } else { continue; diff --git a/src/main.h b/src/main.h index c548c2fdfc31f..7a6003a813346 100644 --- a/src/main.h +++ b/src/main.h @@ -347,21 +347,16 @@ struct CTimestampIndexKey { struct CAddressUnspentKey { unsigned int type; uint160 hashBytes; - int blockHeight; - unsigned int txindex; uint256 txhash; size_t index; size_t GetSerializeSize(int nType, int nVersion) const { - return 65; + return 57; } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); - // Heights are stored big-endian for key sorting in LevelDB - ser_writedata32be(s, blockHeight); - ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); ser_writedata32(s, index); } @@ -369,18 +364,13 @@ struct CAddressUnspentKey { void Unserialize(Stream& s, int nType, int nVersion) { type = ser_readdata8(s); hashBytes.Unserialize(s, nType, nVersion); - blockHeight = ser_readdata32be(s); - txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); index = ser_readdata32(s); } - CAddressUnspentKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t indexValue) { + CAddressUnspentKey(unsigned int addressType, uint160 addressHash, uint256 txid, size_t indexValue) { type = addressType; hashBytes = addressHash; - blockHeight = height; - txindex = blockindex; txhash = txid; index = indexValue; } @@ -392,8 +382,6 @@ struct CAddressUnspentKey { void SetNull() { type = 0; hashBytes.SetNull(); - blockHeight = 0; - txindex = 0; txhash.SetNull(); index = 0; } @@ -421,14 +409,13 @@ struct CAddressUnspentValue { } void SetNull() { - satoshis = 0; - script = CScript(); + satoshis = -1; + script.clear(); } bool IsNull() const { - return (satoshis == 0); + return (satoshis == -1); } - }; struct CAddressIndexKey { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 2c3b7e6d00bb4..832d55f1af552 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -475,7 +475,6 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("addressType", (int)it->first.type)); output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); output.push_back(Pair("txid", it->first.txhash.GetHex())); - output.push_back(Pair("height", it->first.blockHeight)); output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); From a85c9d14ec773fadbc3b624bd6f94c560fd25e44 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 16:42:37 -0400 Subject: [PATCH 26/42] main: sort address index utxos by height --- qa/rpc-tests/addressindex.py | 15 +++++++++++++++ src/main.cpp | 8 ++++---- src/main.h | 6 +++++- src/rpcmisc.cpp | 8 +++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 703f3029ada91..ad79c741635cc 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -192,6 +192,21 @@ def run_test(self): assert_equal(len(utxos2), 1) assert_equal(utxos2[0]["satoshis"], 5000000000) + # Check sorting of utxos + self.nodes[2].generate(150) + + txidsort1 = self.nodes[2].sendtoaddress(address2, 50) + self.nodes[2].generate(1) + txidsort2 = self.nodes[2].sendtoaddress(address2, 50) + self.nodes[2].generate(1) + self.sync_all() + + utxos3 = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos3), 3) + assert_equal(utxos3[0]["height"], 114) + assert_equal(utxos3[1]["height"], 264) + assert_equal(utxos3[2]["height"], 265) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 362edf2becc47..b604178360217 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1926,7 +1926,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey, undo.nHeight))); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { @@ -1936,7 +1936,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey, undo.nHeight))); } else { continue; @@ -2251,7 +2251,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey, pindex->nHeight))); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -2260,7 +2260,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey, pindex->nHeight))); } else { continue; diff --git a/src/main.h b/src/main.h index 7a6003a813346..887455c39bd69 100644 --- a/src/main.h +++ b/src/main.h @@ -390,6 +390,7 @@ struct CAddressUnspentKey { struct CAddressUnspentValue { CAmount satoshis; CScript script; + int blockHeight; ADD_SERIALIZE_METHODS; @@ -397,11 +398,13 @@ struct CAddressUnspentValue { inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { READWRITE(satoshis); READWRITE(*(CScriptBase*)(&script)); + READWRITE(blockHeight); } - CAddressUnspentValue(CAmount sats, CScript scriptPubKey) { + CAddressUnspentValue(CAmount sats, CScript scriptPubKey, int height) { satoshis = sats; script = scriptPubKey; + blockHeight = height; } CAddressUnspentValue() { @@ -411,6 +414,7 @@ struct CAddressUnspentValue { void SetNull() { satoshis = -1; script.clear(); + blockHeight = 0; } bool IsNull() const { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 832d55f1af552..ec0774660c492 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -431,8 +431,11 @@ bool getAddressesFromParams(const UniValue& params, std::vector a, + std::pair b) { + return a.second.blockHeight < b.second.blockHeight; } UniValue getaddressutxos(const UniValue& params, bool fHelp) @@ -468,6 +471,8 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) } } + std::sort(unspentOutputs.begin(), unspentOutputs.end(), heightSort); + UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { @@ -478,6 +483,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); + output.push_back(Pair("height", it->second.blockHeight)); result.push_back(output); } From e428d035ccfcba69235cea26806c65aec33ca6e5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 4 Apr 2016 16:37:43 -0400 Subject: [PATCH 27/42] main: mempool address index --- qa/rpc-tests/addressindex.py | 40 ++++++++++++++++++- src/addressindex.h | 77 ++++++++++++++++++++++++++++++++++++ src/main.cpp | 3 ++ src/rpcmisc.cpp | 46 +++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txmempool.cpp | 75 +++++++++++++++++++++++++++++++++++ src/txmempool.h | 12 ++++++ 8 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 src/addressindex.h diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index ad79c741635cc..d28cd393fb4b3 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -7,6 +7,7 @@ # Test addressindex generation and fetching # +import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.script import * @@ -25,7 +26,7 @@ def setup_network(self): self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-addressindex"])) # Nodes 2/3 are used for testing - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-addressindex"])) self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-addressindex"])) connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) @@ -207,6 +208,43 @@ def run_test(self): assert_equal(utxos3[1]["height"], 264) assert_equal(utxos3[2]["height"], 265) + # Check mempool indexing + print "Testing mempool indexing..." + + privKey3 = "cVfUn53hAbRrDEuMexyfgDpZPhF7KqXpS8UZevsyTDaugB7HZ3CD" + address3 = "mw4ynwhS7MmrQ27hr82kgqu7zryNDK26JB" + addressHash3 = "aa9872b5bbcdb511d89e0e11aa27da73fd2c3f50".decode("hex") + scriptPubKey3 = CScript([OP_DUP, OP_HASH160, addressHash3, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[2].listunspent() + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + amount = unspent[0]["amount"] * 100000000 + tx.vout = [CTxOut(amount, scriptPubKey3)] + tx.rehash() + signed_tx = self.nodes[2].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + memtxid1 = self.nodes[2].sendrawtransaction(signed_tx["hex"], True) + time.sleep(2) + + tx2 = CTransaction() + tx2.vin = [CTxIn(COutPoint(int(unspent[1]["txid"], 16), unspent[1]["vout"]))] + amount = unspent[1]["amount"] * 100000000 + tx2.vout = [CTxOut(amount, scriptPubKey3)] + tx2.rehash() + signed_tx2 = self.nodes[2].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) + memtxid2 = self.nodes[2].sendrawtransaction(signed_tx2["hex"], True) + time.sleep(2) + + mempool = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool), 2) + assert_equal(mempool[0]["txid"], memtxid1) + assert_equal(mempool[1]["txid"], memtxid2) + + self.nodes[2].generate(1); + self.sync_all(); + mempool2 = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool2), 0) + print "Passed\n" diff --git a/src/addressindex.h b/src/addressindex.h new file mode 100644 index 0000000000000..b6e3587c1f6cf --- /dev/null +++ b/src/addressindex.h @@ -0,0 +1,77 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRESSINDEX_H +#define BITCOIN_ADDRESSINDEX_H + +#include "uint256.h" +#include "amount.h" + +struct CMempoolAddressDelta +{ + int64_t time; + CAmount amount; + uint256 prevhash; + unsigned int prevout; + + CMempoolAddressDelta(int64_t t, CAmount a, uint256 hash, unsigned int out) { + time = t; + amount = a; + prevhash = hash; + prevout = out; + } + + CMempoolAddressDelta(int64_t t, CAmount a) { + time = t; + amount = a; + prevhash.SetNull(); + prevout = 0; + } +}; + +struct CMempoolAddressDeltaKey +{ + int type; + uint160 addressBytes; + uint256 txhash; + unsigned int index; + bool spending; + + CMempoolAddressDeltaKey(int addressType, uint160 addressHash, uint256 hash, unsigned int i, bool s) { + type = addressType; + addressBytes = addressHash; + txhash = hash; + index = i; + spending = s; + } + + CMempoolAddressDeltaKey(int addressType, uint160 addressHash) { + type = addressType; + addressBytes = addressHash; + txhash.SetNull(); + index = 0; + } +}; + +struct CMempoolAddressDeltaKeyCompare +{ + bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) { + if (a.type == b.type) { + if (a.addressBytes == b.addressBytes) { + if (a.txhash == b.txhash) { + return a.index < b.index; + } else { + return a.txhash < b.txhash; + } + } else { + return a.addressBytes < b.addressBytes; + } + } else { + return a.type < b.type; + } + } +}; + +#endif // BITCOIN_ADDRESSINDEX_H diff --git a/src/main.cpp b/src/main.cpp index b604178360217..8046f5bc9c232 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1208,6 +1208,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState &state, const C // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); + if (fAddressIndex) { + pool.addAddressIndex(entry, view); + } // trim mempool and check if tx was trimmed if (!fOverrideMempoolLimit) { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index ec0774660c492..f06baa04278fb 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -11,6 +11,7 @@ #include "netbase.h" #include "rpcserver.h" #include "timedata.h" +#include "txmempool.h" #include "util.h" #include "utilstrencodings.h" #ifdef ENABLE_WALLET @@ -438,6 +439,51 @@ bool heightSort(std::pair a, return a.second.blockHeight < b.second.blockHeight; } +bool timestampSort(std::pair a, + std::pair b) { + return a.second.time < b.second.time; +} + +UniValue getaddressmempool(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddressmempool\n" + "\nReturns all mempool deltas for an address (requires addressindex to be enabled).\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > indexes; + + if (!mempool.getAddressIndex(addresses, indexes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + + std::sort(indexes.begin(), indexes.end(), timestampSort); + + UniValue result(UniValue::VARR); + + for (std::vector >::iterator it = indexes.begin(); + it != indexes.end(); it++) { + + UniValue delta(UniValue::VOBJ); + delta.push_back(Pair("addressType", (int)it->first.type)); + delta.push_back(Pair("addressHash", it->first.addressBytes.GetHex())); + delta.push_back(Pair("txid", it->first.txhash.GetHex())); + delta.push_back(Pair("index", (int)it->first.index)); + delta.push_back(Pair("satoshis", it->second.amount)); + delta.push_back(Pair("timestamp", it->second.time)); + result.push_back(delta); + } + + return result; +} + UniValue getaddressutxos(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 39af0513b5da6..1d5399df39d25 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressmempool", &getaddressmempool, false }, { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 5ad147dec7f08..74ecd11918c87 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressmempool(const UniValue& params, bool fHelp); extern UniValue getaddressutxos(const UniValue& params, bool fHelp); extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index f8e03c25334d3..417636ffac82c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -417,6 +417,80 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, return true; } +void CTxMemPool::addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view) +{ + LOCK(cs); + const CTransaction& tx = entry.GetTx(); + std::vector inserted; + + uint256 txhash = tx.GetHash(); + for (unsigned int j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; + const CTxOut &prevout = view.GetOutputFor(input); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, j, true); + CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); + mapAddress.insert(make_pair(key, delta)); + inserted.push_back(key); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, j, true); + CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); + mapAddress.insert(make_pair(key, delta)); + inserted.push_back(key); + } + } + + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, k, false); + mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); + inserted.push_back(key); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + std::pair ret; + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, k, false); + mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); + inserted.push_back(key); + } + } + + mapAddressInserted.insert(make_pair(txhash, inserted)); +} + +bool CTxMemPool::getAddressIndex(std::vector > &addresses, + std::vector > &results) +{ + LOCK(cs); + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + addressDeltaMap::iterator ait = mapAddress.lower_bound(CMempoolAddressDeltaKey((*it).second, (*it).first)); + while (ait != mapAddress.end() && (*ait).first.addressBytes == (*it).first) { + results.push_back(*ait); + ait++; + } + } + return true; +} + +bool CTxMemPool::removeAddressIndex(const uint256 txhash) +{ + LOCK(cs); + addressDeltaMapInserted::iterator it = mapAddressInserted.find(txhash); + + if (it != mapAddressInserted.end()) { + std::vector keys = (*it).second; + for (std::vector::iterator mit = keys.begin(); mit != keys.end(); mit++) { + mapAddress.erase(*mit); + } + mapAddressInserted.erase(it); + } + + return true; +} + void CTxMemPool::removeUnchecked(txiter it) { const uint256 hash = it->GetTx().GetHash(); @@ -430,6 +504,7 @@ void CTxMemPool::removeUnchecked(txiter it) mapTx.erase(it); nTransactionsUpdated++; minerPolicyEstimator->removeTx(hash); + removeAddressIndex(hash); } // Calculates descendants of entry that are not already in setDescendants, and adds to diff --git a/src/txmempool.h b/src/txmempool.h index 386cb26d2565a..b6ef2e47c557a 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -9,6 +9,7 @@ #include #include +#include "addressindex.h" #include "amount.h" #include "coins.h" #include "primitives/transaction.h" @@ -389,6 +390,12 @@ class CTxMemPool typedef std::map txlinksMap; txlinksMap mapLinks; + typedef std::map addressDeltaMap; + addressDeltaMap mapAddress; + + typedef std::map > addressDeltaMapInserted; + addressDeltaMapInserted mapAddressInserted; + void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); @@ -420,6 +427,11 @@ class CTxMemPool bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); + void addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view); + bool getAddressIndex(std::vector > &addresses, + std::vector > &results); + bool removeAddressIndex(const uint256 txhash); + void remove(const CTransaction &tx, std::list& removed, bool fRecursive = false); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx, std::list& removed); From e37c0e42af58478cdcac285340715d901f9737e7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 10:49:11 -0400 Subject: [PATCH 28/42] rpc: give back base58 encoded address format in utxos --- src/rpcmisc.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index f06baa04278fb..c79a4ccddd402 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -523,8 +523,16 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { UniValue output(UniValue::VOBJ); - output.push_back(Pair("addressType", (int)it->first.type)); - output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); + std::string address; + if (it->first.type == 2) { + address = CBitcoinAddress(CScriptID(it->first.hashBytes)).ToString(); + } else if (it->first.type == 1) { + address = CBitcoinAddress(CKeyID(it->first.hashBytes)).ToString(); + } else { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + + output.push_back(Pair("address", address)); output.push_back(Pair("txid", it->first.txhash.GetHex())); output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); From 9fa2cece5fbd8fe29c9141807f67e61c39c941de Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 15:41:19 -0400 Subject: [PATCH 29/42] main: include timestampindex in help --- src/init.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/init.cpp b/src/init.cpp index 58411f67bfbe5..213437d74fca5 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -351,6 +351,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); + strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); From eab45988aed8f41b8e9dde7c6bb76ffe8e9e0078 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 12:35:25 -0400 Subject: [PATCH 30/42] build: add addressindex.h to make --- src/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Makefile.am b/src/Makefile.am index 4c12e550b4d65..cb6737200568f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -79,6 +79,7 @@ endif .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ + addressindex.h \ addrman.h \ alert.h \ amount.h \ From 7f633d4c84821df64100a6ee77fcfa865bf5c6a7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 13:52:40 -0400 Subject: [PATCH 31/42] rpc: cast indexes to ints --- src/rpcmisc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index c79a4ccddd402..4b0d04ae94056 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -534,7 +534,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("address", address)); output.push_back(Pair("txid", it->first.txhash.GetHex())); - output.push_back(Pair("outputIndex", it->first.index)); + output.push_back(Pair("outputIndex", (int)it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); output.push_back(Pair("height", it->second.blockHeight)); @@ -594,7 +594,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue delta(UniValue::VOBJ); delta.push_back(Pair("satoshis", it->second)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); - delta.push_back(Pair("index", it->first.index)); + delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("height", it->first.blockHeight)); delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); delta.push_back(Pair("type", (int)it->first.type)); From f1c684989494648346429deb42d186da359516dc Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 15:53:38 -0400 Subject: [PATCH 32/42] main: add spentindex option --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/spentindex.py | 73 ++++++++++++++++++++++++++++++++++ src/init.cpp | 1 + src/main.cpp | 78 +++++++++++++++++++++++++++++-------- src/main.h | 65 +++++++++++++++++++++++++++++++ src/rpcmisc.cpp | 39 +++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 17 ++++++++ src/txdb.h | 4 ++ 10 files changed, 263 insertions(+), 17 deletions(-) create mode 100755 qa/rpc-tests/spentindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index ad7f97ee821eb..477d4719d4539 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -98,6 +98,7 @@ 'reindex.py', 'addressindex.py', 'timestampindex.py', + 'spentindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py new file mode 100755 index 0000000000000..0fe98dcbd5e3b --- /dev/null +++ b/qa/rpc-tests/spentindex.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test addressindex generation and fetching +# + +import time +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii + +class SpentIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-spentindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-spentindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + + # Check that + print "Testing spent index..." + + privkey = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey = CScript([OP_DUP, OP_HASH160, addressHash, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + amount = unspent[0]["amount"] * 100000000 + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(amount, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) + assert_equal(info["txid"], txid) + assert_equal(info["index"], 0) + + print "Passed\n" + + +if __name__ == '__main__': + SpentIndexTest().main() diff --git a/src/init.cpp b/src/init.cpp index 213437d74fca5..8476f568cd0e2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -352,6 +352,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); + strUsage += HelpMessageOpt("-spentindex", strprintf(_("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)"), DEFAULT_SPENTINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); diff --git a/src/main.cpp b/src/main.cpp index 8046f5bc9c232..391b1e72c3164 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -67,6 +67,7 @@ bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; bool fTimestampIndex = false; +bool fSpentIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1248,6 +1249,17 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v return true; } +bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) +{ + if (!fSpentIndex) + return error("spent index not enabled"); + + if (!pblocktree->ReadSpentIndex(key, value)) + return error("unable to get spent info"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start, int end) { @@ -1852,6 +1864,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI std::vector > addressIndex; std::vector > addressUnspentIndex; + std::vector > spentIndex; // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -1919,8 +1932,14 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (!ApplyTxInUndo(undo, view, out)) fClean = false; + const CTxIn input = tx.vin[j]; + + if (fSpentIndex) { + // undo and delete the spent index + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue())); + } + if (fAddressIndex) { - const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -1945,6 +1964,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI continue; } } + } } } @@ -2176,6 +2196,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector > addressIndex; std::vector > addressUnspentIndex; + std::vector > spentIndex; for (unsigned int i = 0; i < block.vtx.size(); i++) { @@ -2194,31 +2215,43 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return state.DoS(100, error("ConnectBlock(): inputs missing/spent"), REJECT_INVALID, "bad-txns-inputs-missingorspent"); - if (fAddressIndex) + if (fAddressIndex || fSpentIndex) { for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; - const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); - if (prevout.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + if (fSpentIndex) { + // add the spent index to determine the txid and input that spent an output + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue(txhash, j, pindex->nHeight))); + } - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + if (fAddressIndex) { - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - } else { - continue; + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + + } else { + continue; + } } + } } @@ -2332,6 +2365,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } } + if (fSpentIndex) + if (!pblocktree->UpdateSpentIndex(spentIndex)) + return AbortNode(state, "Failed to write transaction index"); + if (fTimestampIndex) if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) return AbortNode(state, "Failed to write timestamp index"); @@ -3709,6 +3746,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("timestampindex", fTimestampIndex); LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Check whether we have a spent index + pblocktree->ReadFlag("spentindex", fSpentIndex); + LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -3875,6 +3916,9 @@ bool InitBlockIndex(const CChainParams& chainparams) fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); pblocktree->WriteFlag("timestampindex", fTimestampIndex); + fSpentIndex = GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); + pblocktree->WriteFlag("spentindex", fSpentIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 887455c39bd69..c0d8d752492dc 100644 --- a/src/main.h +++ b/src/main.h @@ -107,6 +107,7 @@ static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; static const bool DEFAULT_TIMESTAMPINDEX = false; +static const bool DEFAULT_SPENTINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -283,6 +284,69 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CSpentIndexKey { + uint256 txid; + unsigned int outputIndex; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(outputIndex); + } + + CSpentIndexKey(uint256 t, unsigned int i) { + txid = t; + outputIndex = i; + } + + CSpentIndexKey() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + outputIndex = 0; + } + +}; + +struct CSpentIndexValue { + uint256 txid; + unsigned int inputIndex; + int blockHeight; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(inputIndex); + READWRITE(blockHeight); + } + + CSpentIndexValue(uint256 t, unsigned int i, int h) { + txid = t; + inputIndex = i; + blockHeight = h; + } + + CSpentIndexValue() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + inputIndex = 0; + blockHeight = 0; + } + + bool IsNull() const { + return txid.IsNull(); + } +}; + struct CTimestampIndexIteratorKey { unsigned int timestamp; @@ -643,6 +707,7 @@ class CScriptCheck }; bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); +bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 4b0d04ae94056..b31864993bdcf 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -701,3 +701,42 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) return result; } + +UniValue getspentinfo(const UniValue& params, bool fHelp) +{ + + if (fHelp || params.size() != 1 || !params[0].isObject()) + throw runtime_error( + "getspentinfo\n" + "\nReturns the txid and index where an output is spent.\n" + "\nResult\n" + "{\n" + " \"txid\" (string) The transaction id\n" + " \"index\" (number) The spending input index\n" + " ,...\n" + "}\n" + ); + + UniValue txidValue = find_value(params[0].get_obj(), "txid"); + UniValue indexValue = find_value(params[0].get_obj(), "index"); + + if (!txidValue.isStr() || !indexValue.isNum()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid txid or index"); + } + + uint256 txid = ParseHashV(txidValue, "txid"); + int outputIndex = indexValue.get_int(); + + CSpentIndexKey key(txid, outputIndex); + CSpentIndexValue value; + + if (!GetSpentIndex(key, value)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to get spent info"); + } + + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("txid", value.txid.GetHex())); + obj.push_back(Pair("index", (int)value.inputIndex)); + + return obj; +} diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 1d5399df39d25..6717592fa9e4b 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -290,6 +290,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "verifytxoutproof", &verifytxoutproof, true }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true }, { "blockchain", "verifychain", &verifychain, true }, + { "blockchain", "getspentinfo", &getspentinfo, true }, /* Mining */ { "mining", "getblocktemplate", &getblocktemplate, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 74ecd11918c87..581370c26bf4d 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -271,6 +271,7 @@ extern UniValue verifychain(const UniValue& params, bool fHelp); extern UniValue getchaintips(const UniValue& params, bool fHelp); extern UniValue invalidateblock(const UniValue& params, bool fHelp); extern UniValue reconsiderblock(const UniValue& params, bool fHelp); +extern UniValue getspentinfo(const UniValue& params, bool fHelp); bool StartRPC(); void InterruptRPC(); diff --git a/src/txdb.cpp b/src/txdb.cpp index 8e485e9eb6620..d3a29d5fd2be9 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -24,6 +24,7 @@ static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 's'; +static const char DB_SPENTINDEX = 'p'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -166,6 +167,22 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) { + return Read(make_pair(DB_SPENTINDEX, key), value); +} + +bool CBlockTreeDB::UpdateSpentIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { + if (it->second.IsNull()) { + batch.Erase(make_pair(DB_SPENTINDEX, it->first)); + } else { + batch.Write(make_pair(DB_SPENTINDEX, it->first), it->second); + } + } + return WriteBatch(batch); +} + bool CBlockTreeDB::UpdateAddressUnspentIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { diff --git a/src/txdb.h b/src/txdb.h index 547a48fa8412e..62b115707b236 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -23,6 +23,8 @@ struct CAddressIndexKey; struct CAddressIndexIteratorKey; struct CTimestampIndexKey; struct CTimestampIndexIteratorKey; +struct CSpentIndexKey; +struct CSpentIndexValue; class uint256; //! -dbcache default (MiB) @@ -63,6 +65,8 @@ class CBlockTreeDB : public CDBWrapper bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); + bool UpdateSpentIndex(const std::vector >&vect); bool UpdateAddressUnspentIndex(const std::vector >&vect); bool ReadAddressUnspentIndex(uint160 addressHash, int type, std::vector > &vect); From 0cf87d21cc60eb6f5982aa5e5bc6898ab8502a62 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 12:31:21 -0400 Subject: [PATCH 33/42] rpc: include spent info if spentindex enabled with getrawtransaction verbose --- qa/rpc-tests/spentindex.py | 8 +++++++- src/main.cpp | 2 +- src/rpcrawtransaction.cpp | 12 +++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 0fe98dcbd5e3b..44ba286609d2b 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -27,7 +27,7 @@ def setup_network(self): self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-spentindex"])) # Nodes 2/3 are used for testing self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-spentindex"])) - self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex", "-txindex"])) connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) @@ -62,10 +62,16 @@ def run_test(self): self.nodes[0].generate(1) self.sync_all() + # Check that the spentinfo works standalone info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) assert_equal(info["txid"], txid) assert_equal(info["index"], 0) + # Check that verbose raw transaction includes spent info + txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 391b1e72c3164..5b5aaac924b30 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1252,7 +1252,7 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) { if (!fSpentIndex) - return error("spent index not enabled"); + return false; if (!pblocktree->ReadSpentIndex(key, value)) return error("unable to get spent info"); diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 6ab1807d46373..0e8df7faedadc 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -61,7 +61,8 @@ void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fInclud void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { - entry.push_back(Pair("txid", tx.GetHash().GetHex())); + uint256 txid = tx.GetHash(); + entry.push_back(Pair("txid", txid.GetHex())); entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); entry.push_back(Pair("version", tx.nVersion)); entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); @@ -91,6 +92,15 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) UniValue o(UniValue::VOBJ); ScriptPubKeyToJSON(txout.scriptPubKey, o, true); out.push_back(Pair("scriptPubKey", o)); + + // Add spent information if spentindex is enabled + CSpentIndexValue spentInfo; + CSpentIndexKey spentKey(txid, i); + if (GetSpentIndex(spentKey, spentInfo)) { + out.push_back(Pair("spentTxId", spentInfo.txid.GetHex())); + out.push_back(Pair("spentIndex", (int)spentInfo.inputIndex)); + } + vout.push_back(out); } entry.push_back(Pair("vout", vout)); From c687c42d96c012182c8facba243fed7d6e83d5ef Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 15:31:19 -0400 Subject: [PATCH 34/42] rpc: include height in spentinfo --- qa/rpc-tests/spentindex.py | 2 ++ src/rpcmisc.cpp | 1 + src/rpcrawtransaction.cpp | 1 + 3 files changed, 4 insertions(+) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 44ba286609d2b..6669df462f8de 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -66,11 +66,13 @@ def run_test(self): info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) assert_equal(info["txid"], txid) assert_equal(info["index"], 0) + assert_equal(info["height"], 106) # Check that verbose raw transaction includes spent info txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentHeight"], 106) print "Passed\n" diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index b31864993bdcf..fc55f3df2f969 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -737,6 +737,7 @@ UniValue getspentinfo(const UniValue& params, bool fHelp) UniValue obj(UniValue::VOBJ); obj.push_back(Pair("txid", value.txid.GetHex())); obj.push_back(Pair("index", (int)value.inputIndex)); + obj.push_back(Pair("height", value.blockHeight)); return obj; } diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 0e8df7faedadc..6fbec480edaa5 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -99,6 +99,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) if (GetSpentIndex(spentKey, spentInfo)) { out.push_back(Pair("spentTxId", spentInfo.txid.GetHex())); out.push_back(Pair("spentIndex", (int)spentInfo.inputIndex)); + out.push_back(Pair("spentHeight", spentInfo.blockHeight)); } vout.push_back(out); From b9e0debed43225f42b21413356a8e6f419cc8999 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 16:04:10 -0400 Subject: [PATCH 35/42] rpc: query txids for addresses within block height range --- qa/rpc-tests/addressindex.py | 10 ++++++++++ src/rpcmisc.cpp | 21 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index d28cd393fb4b3..3ef5bcf675be3 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -84,6 +84,16 @@ def run_test(self): assert_equal(txidsb[1], txidb1) assert_equal(txidsb[2], txidb2) + # Check that limiting by height works + chain_height = self.nodes[1].getblockcount() + height_txids = self.nodes[1].getaddresstxids({ + "addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"], + "start": 111, + "end": 111 + }) + assert_equal(len(height_txids), 1) + assert_equal(height_txids[0], txidb2) + # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) assert_equal(len(multitxids), 6) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index fc55f3df2f969..9e5266e31b2aa 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -668,11 +668,28 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } + int start = 0; + int end = 0; + if (params[0].isObject()) { + UniValue startValue = find_value(params[0].get_obj(), "start"); + UniValue endValue = find_value(params[0].get_obj(), "end"); + if (startValue.isNum() && endValue.isNum()) { + start = startValue.get_int(); + end = startValue.get_int(); + } + } + std::vector > addressIndex; for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { - if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + if (start > 0 && end > 0) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } else { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } } } From 9070af69905bfc1cdd3bdcb38e0221a37b10a42f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 20:33:18 -0400 Subject: [PATCH 36/42] rpc: fix issue with querying txids by block heights --- qa/rpc-tests/addressindex.py | 11 ++++---- src/main.h | 50 +++++++++++++++++++++++++----------- src/rpcmisc.cpp | 2 +- src/txdb.cpp | 2 +- src/txdb.h | 1 + 5 files changed, 44 insertions(+), 22 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 3ef5bcf675be3..e45efdbda378b 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -85,14 +85,15 @@ def run_test(self): assert_equal(txidsb[2], txidb2) # Check that limiting by height works - chain_height = self.nodes[1].getblockcount() + print "Testing querying txids by range of block heights.." height_txids = self.nodes[1].getaddresstxids({ "addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"], - "start": 111, - "end": 111 + "start": 105, + "end": 110 }) - assert_equal(len(height_txids), 1) - assert_equal(height_txids[0], txidb2) + assert_equal(len(height_txids), 2) + assert_equal(height_txids[0], txidb0) + assert_equal(height_txids[1], txidb1) # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) diff --git a/src/main.h b/src/main.h index c0d8d752492dc..4f9c85ec365bb 100644 --- a/src/main.h +++ b/src/main.h @@ -552,23 +552,14 @@ struct CAddressIndexKey { struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; - bool includeHeight; - int blockHeight; size_t GetSerializeSize(int nType, int nVersion) const { - if (includeHeight) { - return 25; - } else { - return 21; - } + return 21; } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); - if (includeHeight) { - ser_writedata32be(s, blockHeight); - } } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -579,24 +570,53 @@ struct CAddressIndexIteratorKey { CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { type = addressType; hashBytes = addressHash; - includeHeight = false; } - CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash, int height) { + CAddressIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + } +}; + +struct CAddressIndexIteratorHeightKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 25; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + ser_writedata32be(s, blockHeight); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + } + + CAddressIndexIteratorHeightKey(unsigned int addressType, uint160 addressHash, int height) { type = addressType; hashBytes = addressHash; blockHeight = height; - includeHeight = true; } - CAddressIndexIteratorKey() { + CAddressIndexIteratorHeightKey() { SetNull(); } void SetNull() { type = 0; hashBytes.SetNull(); - includeHeight = false; + blockHeight = 0; } }; diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 9e5266e31b2aa..91a45813a05d6 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -675,7 +675,7 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) UniValue endValue = find_value(params[0].get_obj(), "end"); if (startValue.isNum() && endValue.isNum()) { start = startValue.get_int(); - end = startValue.get_int(); + end = endValue.get_int(); } } diff --git a/src/txdb.cpp b/src/txdb.cpp index d3a29d5fd2be9..8a4d71578eae8 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -242,7 +242,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, boost::scoped_ptr pcursor(NewIterator()); if (start > 0 && end > 0) { - pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash, start))); + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorHeightKey(type, addressHash, start))); } else { pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); } diff --git a/src/txdb.h b/src/txdb.h index 62b115707b236..14d501278f23e 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -21,6 +21,7 @@ struct CAddressUnspentKey; struct CAddressUnspentValue; struct CAddressIndexKey; struct CAddressIndexIteratorKey; +struct CAddressIndexIteratorHeightKey; struct CTimestampIndexKey; struct CTimestampIndexIteratorKey; struct CSpentIndexKey; From 70baccf19323dd9ae0adebd3b54368bec903a827 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 23:25:55 -0400 Subject: [PATCH 37/42] build: fix darwin build --- src/addressindex.h | 2 +- src/txdb.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/addressindex.h b/src/addressindex.h index b6e3587c1f6cf..43b49dca9b1ce 100644 --- a/src/addressindex.h +++ b/src/addressindex.h @@ -57,7 +57,7 @@ struct CMempoolAddressDeltaKey struct CMempoolAddressDeltaKeyCompare { - bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) { + bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) const { if (a.type == b.type) { if (a.addressBytes == b.addressBytes) { if (a.txhash == b.txhash) { diff --git a/src/txdb.cpp b/src/txdb.cpp index 8a4d71578eae8..48150198f3a34 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -271,7 +271,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, bool CBlockTreeDB::WriteTimestampIndex(const CTimestampIndexKey ×tampIndex) { CDBBatch batch(&GetObfuscateKey()); - batch.Write(make_pair(DB_TIMESTAMPINDEX, timestampIndex), NULL); + batch.Write(make_pair(DB_TIMESTAMPINDEX, timestampIndex), 0); return WriteBatch(batch); } From c5305c8b66a398b6a19b12d45740b87db2d34f5a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:07:01 -0400 Subject: [PATCH 38/42] rpc: include prevhash and prevout information for spending deltas --- src/rpcmisc.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 91a45813a05d6..05455fb8bebd9 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -478,6 +478,10 @@ UniValue getaddressmempool(const UniValue& params, bool fHelp) delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("satoshis", it->second.amount)); delta.push_back(Pair("timestamp", it->second.time)); + if (it->second.amount < 0) { + delta.push_back(Pair("prevtxid", it->second.prevhash.GetHex())); + delta.push_back(Pair("prevout", (int)it->second.prevout)); + } result.push_back(delta); } From 51e7b3cd402bb5fa0dd523a9e6440152f739b5fb Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:23:52 -0400 Subject: [PATCH 39/42] test: test for getaddressmempool prevhash and prevout values --- qa/rpc-tests/addressindex.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index e45efdbda378b..c6910cd1ee392 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -256,6 +256,20 @@ def run_test(self): mempool2 = self.nodes[2].getaddressmempool({"addresses": [address3]}) assert_equal(len(mempool2), 0) + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(memtxid2, 16), 0))] + tx.vout = [CTxOut(amount - 10000, scriptPubKey2)] + tx.rehash() + self.nodes[2].importprivkey(privKey3) + signed_tx3 = self.nodes[2].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + memtxid3 = self.nodes[2].sendrawtransaction(signed_tx3["hex"], True) + time.sleep(2) + + mempool3 = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool3), 1) + assert_equal(mempool3[0]["prevtxid"], memtxid2) + assert_equal(mempool3[0]["prevout"], 0) + print "Passed\n" From 754a559398a235169e1c224528c8b42fdf2bf262 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:59:51 -0400 Subject: [PATCH 40/42] rpc: include base58check encoded address in results --- qa/rpc-tests/addressindex.py | 2 ++ src/rpcmisc.cpp | 37 +++++++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index c6910cd1ee392..7ec00cc4f5ba8 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -176,6 +176,7 @@ def run_test(self): for delta in deltas: balance3 += delta["satoshis"] assert_equal(balance3, change_amount) + assert_equal(deltas[0]["address"], address2) # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) @@ -250,6 +251,7 @@ def run_test(self): assert_equal(len(mempool), 2) assert_equal(mempool[0]["txid"], memtxid1) assert_equal(mempool[1]["txid"], memtxid2) + assert_equal(mempool[0]["address"], address3) self.nodes[2].generate(1); self.sync_all(); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 05455fb8bebd9..8f5a741e28a19 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -398,6 +398,18 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } +bool getAddressFromIndex(const int &type, const uint160 &hash, std::string &address) +{ + if (type == 2) { + address = CBitcoinAddress(CScriptID(hash)).ToString(); + } else if (type == 1) { + address = CBitcoinAddress(CKeyID(hash)).ToString(); + } else { + return false; + } + return true; +} + bool getAddressesFromParams(const UniValue& params, std::vector > &addresses) { if (params[0].isStr()) { @@ -471,9 +483,13 @@ UniValue getaddressmempool(const UniValue& params, bool fHelp) for (std::vector >::iterator it = indexes.begin(); it != indexes.end(); it++) { + std::string address; + if (!getAddressFromIndex(it->first.type, it->first.addressBytes, address)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + UniValue delta(UniValue::VOBJ); - delta.push_back(Pair("addressType", (int)it->first.type)); - delta.push_back(Pair("addressHash", it->first.addressBytes.GetHex())); + delta.push_back(Pair("address", address)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("satoshis", it->second.amount)); @@ -528,11 +544,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { UniValue output(UniValue::VOBJ); std::string address; - if (it->first.type == 2) { - address = CBitcoinAddress(CScriptID(it->first.hashBytes)).ToString(); - } else if (it->first.type == 1) { - address = CBitcoinAddress(CKeyID(it->first.hashBytes)).ToString(); - } else { + if (!getAddressFromIndex(it->first.type, it->first.hashBytes, address)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); } @@ -561,8 +573,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) " \"txid\" (string) The related txid\n" " \"index\" (number) The related input or output index\n" " \"height\" (number) The block height\n" - " \"hash\" (string) The address hash\n" - " \"type\" (number) The address type 0 for pubkeyhash 1 for scripthash\n" + " \"address\" (string) The base58check encoded address\n" " }\n" "]\n" ); @@ -595,13 +606,17 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + std::string address; + if (!getAddressFromIndex(it->first.type, it->first.hashBytes, address)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + UniValue delta(UniValue::VOBJ); delta.push_back(Pair("satoshis", it->second)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("height", it->first.blockHeight)); - delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); - delta.push_back(Pair("type", (int)it->first.type)); + delta.push_back(Pair("address", address)); result.push_back(delta); } From be084bd9eb0054fd465cf3fb3847fdf09093a2a7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 16:07:42 -0400 Subject: [PATCH 41/42] rpc: optional "start" and "end" params for getaddressdeltas --- qa/rpc-tests/addressindex.py | 4 ++++ src/rpcmisc.cpp | 24 +++++++++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 7ec00cc4f5ba8..a07d4de1e0a6b 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -178,6 +178,10 @@ def run_test(self): assert_equal(balance3, change_amount) assert_equal(deltas[0]["address"], address2) + # Check that entire range will be queried + deltasAll = self.nodes[1].getaddressdeltas({"addresses": [address2]}) + assert_equal(len(deltasAll), len(deltas)) + # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) assert_equal(len(deltas), 1) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8f5a741e28a19..04eb5eceffbff 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -582,12 +582,16 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue startValue = find_value(params[0].get_obj(), "start"); UniValue endValue = find_value(params[0].get_obj(), "end"); - if (!startValue.isNum() || !endValue.isNum()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Start and end values are expected to be block heights"); - } + int start = 0; + int end = 0; - int start = startValue.get_int(); - int end = startValue.get_int(); + if (startValue.isNum() && endValue.isNum()) { + start = startValue.get_int(); + end = endValue.get_int(); + if (end < start) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "End value is expected to be greater than start"); + } + } std::vector > addresses; @@ -598,8 +602,14 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) std::vector > addressIndex; for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { - if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + if (start > 0 && end > 0) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } else { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } } } From 1e2ffc5364201b0d6e24ca97348ed7f09832bdef Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Apr 2016 13:08:52 -0400 Subject: [PATCH 42/42] rpc: update oksafemode for address commands --- src/rpcserver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 6717592fa9e4b..eb3060c51d96e 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -290,7 +290,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "verifytxoutproof", &verifytxoutproof, true }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true }, { "blockchain", "verifychain", &verifychain, true }, - { "blockchain", "getspentinfo", &getspentinfo, true }, + { "blockchain", "getspentinfo", &getspentinfo, false }, /* Mining */ { "mining", "getblocktemplate", &getblocktemplate, true }, @@ -316,7 +316,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ - { "addressindex", "getaddressmempool", &getaddressmempool, false }, + { "addressindex", "getaddressmempool", &getaddressmempool, true }, { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false },