dcrsqlite: test retrieve BlockHash, PoolInfo, SDiff

dcrsqlite: test RetrieveSDiffRange




fix test name


fix setup


Consolidated debit view (decred#524)

This introduces a new "merged debit" view on the addresses history page that lists only unique transactions.  The Debit DCR amount is the sum of all outpoints corresponding to the given address that are spent in a transaction.

It also fixes a bug with the next button on the same page.
main: always show dcrsqlite sync error without waiting on dcrpg

testutil: fix error message formatting

and ignore test data folder
dcrpg: support side chain tracking, upgrades (decred#552)

This bumps the dcrpg DB table version to 3.3.0 and requires an automatic upgrade.  The upgrade can take hours however.

The table upgrade process has been refactored.  A new TableUpgradeType type is added, and addNewColumnsIfNotFound takes default value.  TableUpgradesRequired always returns a result for each table, ok being one of the results.  Version check succeeds when all tables "ok"

Prior to InsertAddressRows, ensure ValidMainChain is set properly for each row.

Improve documentation for storeTxns.

dbtypes: set Tx.IsValidBlock and .IsMainchainBlock

When processing block transactions in processTransactions, set IsValidBlock and IsMainchainBlock for each Tx.

Improve addresses statements readibility.

dcrsqlite: test GetBestBlockHash

on 'c486442'.

dcrsqlite: test GetBestBlockHeight

explorer: limit logging about websocket updates to zero clients

dcrsqlite: test GetStakeInfoHeight




fix test name


dcrpg: provide error context in handleBlocksTableMainchainUpgrade

ignore test data


Unconfirmed tx in address list and input amounts on unconfirmed tx (decred#489)

This addresses the unconfirmed transactions not visible in address view and input amounts not visible in transaction view.

There were two major areas of work:
* Blending unconfirmed transactions with confirmed transactions in the current address view. Mostly extending what @chappjc already started.
* Compensating for AmountIn values not set in dcrd calls to mempool transactions.
config: add --sync-and-quit flag to only sync (decred#573)

This adds the flag --sync-and-quit to the dcrdata executable that will caus dcrdata to sync to the best block and then exit. It will not start the explorer or API.
Fix bugs in addresses page view (decred#570)

* Improve on logic checking if all transactions were fetched

* Make the default page size to be 20

dcrsqlite: test retrieve BlockHash, PoolInfo, SDiff




dcrsqlite: test RetrieveSDiffRange




dcrsqlite: test RetrieveAllPoolValAndSize



fix test name


dcrsqlite: test retrieve Winners, StakeInfoExtended, BlockHeight


fix test name




dcrsqlite: test RetrieveAllPoolValAndSize



fix test names


dcrsqlite: test RetrievePoolInfoRange




dcrsqlite: test GetBestBlockHash

on 'c486442'.

fix test name


dcrsqlite: test RetrieveBlockSummary


dcrsqlite: test GetBestBlockHeight

dcrsqlite: test RetrieveAllPoolValAndSize


dcrsqlite: add empty DB tests

Add 15.3% coverage for the dcrsqlite-package.
Test default behavior of the sqlite.go/DB for a freshly created (empty) data base.
Push towards the decred#5
delete wrong file


ignore


ignore


ignore

.gitignore

@@ -33,3 +33,6 @@ testnet/
 .DS_Store
 testdata/
 \.project
+dcrdata-testdata
+test-data-repo
+dcrdata-testdata-gitlfs

cmd/rebuilddb2/rebuilddb2.go

@@ -336,8 +336,9 @@ func mainCore() error {
 		}
 
 		var numVins, numVouts int64
+		isValid, isMainchain := true, true
 		numVins, numVouts, err = db.StoreBlock(block.MsgBlock(), winners,
-			true, cfg.AddrSpendInfoOnline, !cfg.TicketSpendInfoBatch)
+			isValid, isMainchain, cfg.AddrSpendInfoOnline, !cfg.TicketSpendInfoBatch)
 		if err != nil {
 			return fmt.Errorf("StoreBlock failed: %v", err)
 		}

config.go

@@ -100,6 +100,7 @@ type config struct {
 	PGPass        string `long:"pgpass" description:"PostgreSQL DB password."`
 	PGHost        string `long:"pghost" description:"PostgreSQL server host:port or UNIX socket (e.g. /run/postgresql)."`
 	NoDevPrefetch bool   `long:"no-dev-prefetch" description:"Disable automatic dev fund balance query on new blocks. When true, the query will still be run on demand, but not automatically after new blocks are connected."`
+	SyncAndQuit   bool   `long:"sync-and-quit" description:"Sync to the best block and exit. Do not start the explorer or API."`
 
 	// WatchAddresses []string `short:"w" long:"watchaddress" description:"Watched address (receiving). One per line."`
 	// SMTPUser     string `long:"smtpuser" description:"SMTP user name"`

db/dbtypes/extraction.go

@@ -37,17 +37,17 @@ func DevSubsidyAddress(params *chaincfg.Params) (string, error) {
 // wire.MsgBlock and returns the processed information in slices of the dbtypes
 // Tx, Vout, and VinTxPropertyARRAY.
 func ExtractBlockTransactions(msgBlock *wire.MsgBlock, txTree int8,
-	chainParams *chaincfg.Params, isValid bool) ([]*Tx, [][]*Vout, []VinTxPropertyARRAY) {
+	chainParams *chaincfg.Params, isValid, isMainchain bool) ([]*Tx, [][]*Vout, []VinTxPropertyARRAY) {
 	dbTxs, dbTxVouts, dbTxVins := processTransactions(msgBlock, txTree,
-		chainParams, isValid)
+		chainParams, isValid, isMainchain)
 	if txTree != wire.TxTreeRegular && txTree != wire.TxTreeStake {
 		fmt.Printf("Invalid transaction tree: %v", txTree)
 	}
 	return dbTxs, dbTxVouts, dbTxVins
 }
 
-func processTransactions(msgBlock *wire.MsgBlock, tree int8,
-	chainParams *chaincfg.Params, isValid bool) ([]*Tx, [][]*Vout, []VinTxPropertyARRAY) {
+func processTransactions(msgBlock *wire.MsgBlock, tree int8, chainParams *chaincfg.Params,
+	isValid, isMainchain bool) ([]*Tx, [][]*Vout, []VinTxPropertyARRAY) {
 
 	var txs []*wire.MsgTx
 	switch tree {
@@ -81,23 +81,25 @@ func processTransactions(msgBlock *wire.MsgBlock, tree int8,
 		}
 		fees := spent - sent
 		dbTx := &Tx{
-			BlockHash:   blockHash.String(),
-			BlockHeight: int64(blockHeight),
-			BlockTime:   blockTime,
-			Time:        blockTime, // TODO, receive time?
-			TxType:      int16(stake.DetermineTxType(tx)),
-			Version:     tx.Version,
-			Tree:        tree,
-			TxID:        tx.TxHash().String(),
-			BlockIndex:  uint32(txIndex),
-			Locktime:    tx.LockTime,
-			Expiry:      tx.Expiry,
-			Size:        uint32(tx.SerializeSize()),
-			Spent:       spent,
-			Sent:        sent,
-			Fees:        fees,
-			NumVin:      uint32(len(tx.TxIn)),
-			NumVout:     uint32(len(tx.TxOut)),
+			BlockHash:        blockHash.String(),
+			BlockHeight:      int64(blockHeight),
+			BlockTime:        blockTime,
+			Time:             blockTime, // TODO, receive time?
+			TxType:           int16(stake.DetermineTxType(tx)),
+			Version:          tx.Version,
+			Tree:             tree,
+			TxID:             tx.TxHash().String(),
+			BlockIndex:       uint32(txIndex),
+			Locktime:         tx.LockTime,
+			Expiry:           tx.Expiry,
+			Size:             uint32(tx.SerializeSize()),
+			Spent:            spent,
+			Sent:             sent,
+			Fees:             fees,
+			NumVin:           uint32(len(tx.TxIn)),
+			NumVout:          uint32(len(tx.TxOut)),
+			IsValidBlock:     isValid,
+			IsMainchainBlock: isMainchain,
 		}
 
 		//dbTx.Vins = make([]VinTxProperty, 0, dbTx.NumVin)
@@ -118,6 +120,7 @@ func processTransactions(msgBlock *wire.MsgBlock, tree int8,
 				BlockIndex:  txin.BlockIndex,
 				ScriptHex:   txin.SignatureScript,
 				IsValid:     isValid,
+				IsMainchain: isMainchain,
 			})
 		}
 

db/dbtypes/types.go

@@ -300,9 +300,10 @@ type Vout struct {
 // AddressRow represents a row in the addresses table
 type AddressRow struct {
 	// id int64
-	Address string
-	// MatchingTxHash that provides the relationship
-	// between spending tx inputs and funding tx outputs
+	Address        string
+	ValidMainChain bool
+	// MatchingTxHash provides the relationship between spending tx inputs and
+	// funding tx outputs.
 	MatchingTxHash   string
 	IsFunding        bool
 	TxBlockTime      uint64
@@ -354,6 +355,7 @@ type VinTxProperty struct {
 	BlockIndex  uint32 `json:"blockindex"`
 	ScriptHex   []byte `json:"scripthex"`
 	IsValid     bool   `json:"is_valid"`
+	IsMainchain bool   `json:"is_mainchain"`
 	Time        int64  `json:"time"`
 }
 
@@ -417,6 +419,8 @@ type Tx struct {
 	VoutDbIds []uint64 `json:"voutdbids"`
 	// NOTE: VoutDbIds may not be needed if there is a vout table since each
 	// vout will have a tx_dbid
+	IsValidBlock     bool `json:"valid_block"`
+	IsMainchainBlock bool `json:"mainchain"`
 }
 
 // Block models a Decred block.

db/dcrpg/internal/addrstmts.go

@@ -2,8 +2,8 @@ package internal
 
 const (
 	insertAddressRow0 = `INSERT INTO addresses (address, matching_tx_hash, tx_hash,
-		tx_vin_vout_index, tx_vin_vout_row_id, value, block_time, is_funding)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8) `
+		tx_vin_vout_index, tx_vin_vout_row_id, value, block_time, is_funding, valid_mainchain)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) `
 
 	InsertAddressRow = insertAddressRow0 + `RETURNING id;`
 
@@ -31,6 +31,7 @@ const (
 		id SERIAL8 PRIMARY KEY,
 		address TEXT,
 		tx_hash TEXT,
+		valid_mainchain BOOLEAN,
 		matching_tx_hash TEXT,
 		value INT8,
 		block_time INT8 NOT NULL,
@@ -39,22 +40,22 @@ const (
 		tx_vin_vout_row_id INT8
 		);`
 
-	SelectAddressAllByAddress = `SELECT * FROM addresses WHERE address=$1 order by block_time desc;`
+	SelectAddressAllByAddress = `SELECT * FROM addresses WHERE address=$1 ORDER BY block_time DESC;`
 	SelectAddressRecvCount    = `SELECT COUNT(*) FROM addresses WHERE address=$1;`
-	SelectAddressesAllTxn     = `SELECT tx_hash, block_time as tx_time, ftxd.block_height as height 
-		from addresses left join transactions as ftxd on funding_tx_row_id=ftxd.id 
-		where address = $1 order by tx_time desc;`
+	SelectAddressesAllTxn     = `SELECT tx_hash, block_time AS tx_time, ftxd.block_height AS height 
+		FROM addresses LEFT JOIN transactions AS ftxd ON funding_tx_row_id=ftxd.id 
+		WHERE address = $1 ORDER BY tx_time desc;`
 
-	SelectAddressesTxnByFundingTx = `SELECT tx_vin_vout_index, tx_hash, tx_vin_vout_index, 
-		block_height FROM addresses LEFT JOIN 
-		transactions on transactions.tx_hash=tx_hash and is_funding = FALSE WHERE 
-		address = $1 and tx_hash=$2;`
+	SelectAddressesTxnByFundingTx = `SELECT tx_vin_vout_index, tx_hash, tx_vin_vout_index, block_height
+		FROM addresses
+		LEFT JOIN transactions ON transactions.tx_hash=tx_hash AND is_funding = FALSE
+		WHERE address = $1 AND tx_hash=$2;`
 
 	SelectAddressUnspentCountAndValue = `SELECT COUNT(*), SUM(value) FROM addresses 
-	    WHERE address = $1 and is_funding = TRUE and matching_tx_hash = '';`
+	    WHERE address = $1 AND is_funding = TRUE AND matching_tx_hash = '' AND valid_mainchain = true;`
 
 	SelectAddressSpentCountAndValue = `SELECT COUNT(*), SUM(value) FROM addresses 
-		WHERE address = $1 and is_funding = FALSE and matching_tx_hash != '';`
+	    WHERE address = $1 AND is_funding = FALSE AND matching_tx_hash != '' AND valid_mainchain = true;`
 
 	SelectAddressesMergedSpentCount = `SELECT COUNT( distinct tx_hash ) FROM addresses
 		WHERE address = $1 and is_funding = false`
@@ -65,10 +66,11 @@ const (
 		JOIN transactions ON 
 			addresses.tx_hash = transactions.tx_hash
 		JOIN vouts on addresses.tx_hash = vouts.tx_hash AND addresses.tx_vin_vout_index=vouts.tx_index
-			WHERE addresses.address=$1 AND addresses.is_funding = FALSE 
+			WHERE addresses.address=$1 AND addresses.is_funding = FALSE AND valid_mainchain = true
 			ORDER BY addresses.block_time DESC;`
 
-	addrsColumnNames = `id, address, matching_tx_hash, tx_hash, tx_vin_vout_index, block_time, tx_vin_vout_row_id, value, is_funding`
+	addrsColumnNames = `id, address, matching_tx_hash, tx_hash, valid_mainchain, 
+		tx_vin_vout_index, block_time, tx_vin_vout_row_id, value, is_funding`
 
 	SelectAddressLimitNByAddress = `SELECT ` + addrsColumnNames + ` FROM addresses
 	    WHERE address=$1 ORDER BY block_time DESC LIMIT $2 OFFSET $3;`
@@ -82,11 +84,11 @@ const (
 		GROUP BY (tx_hash, block_time) ORDER BY block_time DESC LIMIT $2 OFFSET $3;`
 
 	SelectAddressDebitsLimitNByAddress = `SELECT ` + addrsColumnNames + `
-		FROM addresses WHERE address=$1 and is_funding = FALSE
+		FROM addresses WHERE address=$1 AND is_funding = FALSE
 		ORDER BY block_time DESC LIMIT $2 OFFSET $3;`
 
 	SelectAddressCreditsLimitNByAddress = `SELECT ` + addrsColumnNames + `
-		FROM addresses WHERE address=$1 and is_funding = TRUE
+		FROM addresses WHERE address=$1 AND is_funding = TRUE
 		ORDER BY block_time DESC LIMIT $2 OFFSET $3;`
 
 	SelectAddressIDsByFundingOutpoint = `SELECT id, address FROM addresses WHERE tx_hash=$1, 
@@ -96,7 +98,12 @@ const (
 	    tx_vin_vout_row_id=$2 and is_funding = true;`
 
 	SetAddressFundingForMatchingTxHash = `UPDATE addresses SET matching_tx_hash=$1 
-	    WHERE tx_hash=$2 and is_funding = true and tx_vin_vout_index=$3;`
+		WHERE tx_hash=$2 and is_funding = true and tx_vin_vout_index=$3;`
+
+	UpdateValidMainchainFromTransactions = `UPDATE addresses
+		SET valid_mainchain = (tr.is_mainchain::int * tr.is_valid::int)::boolean
+		FROM transactions AS tr
+		WHERE addresses.tx_hash = tr.tx_hash;`
 
 	IndexBlockTimeOnTableAddress   = `CREATE INDEX block_time_index ON addresses (block_time);`
 	DeindexBlockTimeOnTableAddress = `DROP INDEX block_time_index;`

db/dcrpg/internal/blockstmts.go

@@ -7,18 +7,22 @@ import (
 )
 
 const (
-	// Block insert
+	// Block insert. is_valid refers to blocks that have been validated by
+	// stakeholders (voting on the previous block), while is_mainchain
+	// distinguishes blocks that are on the main chain from those that are
+	// on side chains and/or orphaned.
+
 	insertBlockRow0 = `INSERT INTO blocks (
-		hash, height, size, is_valid, version, merkle_root, stake_root,
+		hash, height, size, is_valid, is_mainchain, version, merkle_root, stake_root,
 		numtx, num_rtx, tx, txDbIDs, num_stx, stx, stxDbIDs,
 		time, nonce, vote_bits, final_state, voters,
 		fresh_stake, revocations, pool_size, bits, sbits, 
 		difficulty, extra_data, stake_version, previous_hash)
-	VALUES ($1, $2, $3, $4, $5, $6, $7,
-		$8, $9, %s, %s, $10, %s, %s,
-		$11, $12, $13, $14, $15, 
-		$16, $17, $18, $19, $20,
-		$21, $22, $23, $24) `
+	VALUES ($1, $2, $3, $4, $5, $6, $7, $8,
+		$9, $10, %s, %s, $11, %s, %s,
+		$12, $13, $14, $15, $16, 
+		$17, $18, $19, $20, $21,
+		$22, $23, $24, $25) `
 	insertBlockRow = insertBlockRow0 + `RETURNING id;`
 	// insertBlockRowChecked  = insertBlockRow0 + `ON CONFLICT (hash) DO NOTHING RETURNING id;`
 	upsertBlockRow = insertBlockRow0 + `ON CONFLICT (hash) DO UPDATE 
@@ -50,6 +54,7 @@ const (
 		height INT4,
 		size INT4,
 		is_valid BOOLEAN,
+		is_mainchain BOOLEAN,
 		version INT4,
 		merkle_root TEXT,
 		stake_root TEXT,
@@ -88,6 +93,12 @@ const (
 
 	SelectBlocksBlockSize = `SELECT time, size, numtx, height FROM blocks ORDER BY time;`
 
+	SelectBlocksPreviousHash = `SELECT previous_hash FROM blocks WHERE hash = $1;`
+
+	SelectBlocksHashes = `SELECT hash FROM blocks ORDER BY id;`
+
+	UpdateBlockMainchain = `UPDATE blocks SET is_mainchain = $2 WHERE hash = $1 RETURNING previous_hash;`
+
 	IndexBlocksTableOnHeight = `CREATE INDEX uix_block_height ON blocks(height);`
 
 	DeindexBlocksTableOnHeight = `DROP INDEX uix_block_height;`
@@ -108,7 +119,8 @@ const (
 
 	SelectBlockChainRowIDByHash = `SELECT block_db_id FROM block_chain WHERE this_hash = $1;`
 
-	UpdateBlockNext = `UPDATE block_chain set next_hash = $2 WHERE block_db_id = $1;`
+	UpdateBlockNext       = `UPDATE block_chain SET next_hash = $2 WHERE block_db_id = $1;`
+	UpdateBlockNextByHash = `UPDATE block_chain SET next_hash = $2 WHERE this_hash = $1;`
 )
 
 func MakeBlockInsertStatement(block *dbtypes.Block, checked bool) string {

db/dcrpg/internal/stakestmts.go

@@ -104,20 +104,23 @@ const (
 		ticket_hash TEXT,
 		ticket_tx_db_id INT8,
 		ticket_price FLOAT8,
-		vote_reward FLOAT8
+		vote_reward FLOAT8,
+		is_mainchain BOOLEAN
 	);`
 
 	// Insert
 	insertVoteRow0 = `INSERT INTO votes (
 		height, tx_hash,
 		block_hash, candidate_block_hash,
 		version, vote_bits, block_valid,
-		ticket_hash, ticket_tx_db_id, ticket_price, vote_reward)
+		ticket_hash, ticket_tx_db_id, ticket_price, vote_reward,
+		is_mainchain)
 	VALUES (
 		$1, $2,
 		$3, $4,
 		$5, $6, $7,
-		$8, $9, $10, $11) `
+		$8, $9, $10, $11,
+		$12) `
 	insertVoteRow = insertVoteRow0 + `RETURNING id;`
 	// insertVoteRowChecked = insertVoteRow0 + `ON CONFLICT (tx_hash, block_hash) DO NOTHING RETURNING id;`
 	upsertVoteRow = insertVoteRow0 + `ON CONFLICT (tx_hash, block_hash) DO UPDATE 
@@ -137,6 +140,14 @@ const (
 	SelectAllVoteDbIDsHeightsTicketHashes = `SELECT id, height, ticket_hash FROM votes;`
 	SelectAllVoteDbIDsHeightsTicketDbIDs  = `SELECT id, height, ticket_tx_db_id FROM votes;`
 
+	UpdateVotesMainchainAll = `UPDATE votes
+		SET is_mainchain=b.is_mainchain
+		FROM (
+			SELECT hash, is_mainchain
+			FROM blocks
+		) b
+		WHERE block_hash = b.hash;`
+
 	// Index
 	IndexVotesTableOnHashes = `CREATE UNIQUE INDEX uix_votes_hashes_index
 		ON votes(tx_hash, block_hash);`