Skip to content

Commit

Permalink
blockchain: Remove compression version param.
Browse files Browse the repository at this point in the history 
Over the years it has become increasingly obvious that storing multiple
versioned formats in the database in an attempt to avoid migrations
leads to code that is super hard to reason about and for which it is
also difficult to assert correctness.

This is the case because it results in a combinatorial explosion of
cases that must be handled.  For example, as soon as you have 3
versions, you're already up to 8 variants you have to handle properly
and test, and it only gets exponentially worse with each new version.

Due to this, it is greatly preferred to perform a single migration that
handles the conversion logic once.  This allows the rest of the code,
especially in the critical paths, to work solely with the latest version
and therefore it stays much cleaner, easier to validate for correctness,
and is generally easier to reason about.

With that in mind, this removes the compression version parameter from
the functions that deal with serializing and deserializing compressed
scripts.  Since there is only currently a single version it does not
require any migrations.

However, since the existing migration code for older versions was
passing in the old version parameter, new v1 functions have been added
to the upgrade code to ensure stability there.
  • Loading branch information
@davecgh
davecgh committed Jan 10, 2021
1 parent b606b4c commit cfb953c
Show file tree
Hide file tree
Showing 5 changed files with 195 additions and 91 deletions.
10 changes: 5 additions & 5 deletions blockchain/chain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1917,7 +1917,7 @@ func extractDeploymentIDVersions(params *chaincfg.Params) (map[string]uint32, er
// stxosToScriptSource uses the provided block and spent txo information to
// create a source of previous transaction scripts and versions spent by the
// block.
func stxosToScriptSource(block *dcrutil.Block, stxos []spentTxOut, compressionVersion uint32, isTreasuryEnabled bool, chainParams *chaincfg.Params) scriptSource {
func stxosToScriptSource(block *dcrutil.Block, stxos []spentTxOut, isTreasuryEnabled bool, chainParams *chaincfg.Params) scriptSource {
source := make(scriptSource)
msgBlock := block.MsgBlock()

Expand Down Expand Up @@ -1961,7 +1961,7 @@ func stxosToScriptSource(block *dcrutil.Block, stxos []spentTxOut, compressionVe
prevOut := &txIn.PreviousOutPoint
source[*prevOut] = scriptSourceEntry{
version: stxo.scriptVersion,
script: decompressScript(stxo.pkScript, compressionVersion),
script: decompressScript(stxo.pkScript),
}
}
}
Expand All @@ -1983,7 +1983,7 @@ func stxosToScriptSource(block *dcrutil.Block, stxos []spentTxOut, compressionVe
prevOut := &txIn.PreviousOutPoint
source[*prevOut] = scriptSourceEntry{
version: stxo.scriptVersion,
script: decompressScript(stxo.pkScript, compressionVersion),
script: decompressScript(stxo.pkScript),
}
}
}
Expand Down Expand Up @@ -2033,8 +2033,8 @@ func (q *chainQueryerAdapter) PrevScripts(dbTx database.Tx, block *dcrutil.Block
return nil, err
}

prevScripts := stxosToScriptSource(block, stxos, currentCompressionVersion,
isTreasuryEnabled, q.chainParams)
prevScripts := stxosToScriptSource(block, stxos, isTreasuryEnabled,
q.chainParams)
return prevScripts, nil
}

Expand Down
13 changes: 6 additions & 7 deletions blockchain/chainio.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ func spentTxOutSerializeSize(stxo *spentTxOut) int {

const hasAmount = false
size += compressedTxOutSize(uint64(stxo.amount), stxo.scriptVersion,
stxo.pkScript, currentCompressionVersion, hasAmount)
stxo.pkScript, hasAmount)

if stxo.ticketMinOuts != nil {
size += len(stxo.ticketMinOuts.data)
Expand All @@ -573,7 +573,7 @@ func putSpentTxOut(target []byte, stxo *spentTxOut) int {

const hasAmount = false
offset += putCompressedTxOut(target[offset:], 0, stxo.scriptVersion,
stxo.pkScript, currentCompressionVersion, hasAmount)
stxo.pkScript, hasAmount)

if stxo.ticketMinOuts != nil {
copy(target[offset:], stxo.ticketMinOuts.data)
Expand All @@ -600,7 +600,7 @@ func decodeSpentTxOut(serialized []byte, stxo *spentTxOut, amount int64,
// since in Decred we only need pkScript at most due to fraud proofs
// already storing the decompressed amount.
_, scriptVersion, script, bytesRead, err :=
decodeCompressedTxOut(serialized[offset:], currentCompressionVersion, false)
decodeCompressedTxOut(serialized[offset:], false)
offset += bytesRead
if err != nil {
return offset, errDeserialize(fmt.Sprintf("unable to decode "+
Expand Down Expand Up @@ -937,7 +937,7 @@ func serializeUtxoEntry(entry *UtxoEntry) ([]byte, error) {
serializeSizeVLQ(uint64(entry.blockIndex)) +
serializeSizeVLQ(uint64(flags)) +
compressedTxOutSize(uint64(entry.amount), entry.scriptVersion,
entry.pkScript, currentCompressionVersion, hasAmount)
entry.pkScript, hasAmount)

if entry.ticketMinOuts != nil {
size += len(entry.ticketMinOuts.data)
Expand All @@ -949,7 +949,7 @@ func serializeUtxoEntry(entry *UtxoEntry) ([]byte, error) {
offset += putVLQ(serialized[offset:], uint64(entry.blockIndex))
offset += putVLQ(serialized[offset:], uint64(flags))
offset += putCompressedTxOut(serialized[offset:], uint64(entry.amount),
entry.scriptVersion, entry.pkScript, currentCompressionVersion, hasAmount)
entry.scriptVersion, entry.pkScript, hasAmount)

if entry.ticketMinOuts != nil {
copy(serialized[offset:], entry.ticketMinOuts.data)
Expand Down Expand Up @@ -986,8 +986,7 @@ func deserializeUtxoEntry(serialized []byte, txOutIndex uint32) (*UtxoEntry, err

// Decode the compressed unspent transaction output.
amount, scriptVersion, script, bytesRead, err :=
decodeCompressedTxOut(serialized[offset:], currentCompressionVersion,
true)
decodeCompressedTxOut(serialized[offset:], true)
if err != nil {
return nil, errDeserialize(fmt.Sprintf("unable to decode utxo: %v", err))
}
Expand Down
57 changes: 24 additions & 33 deletions blockchain/compress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,7 @@ func isPubKey(script []byte) (bool, []byte) {

// compressedScriptSize returns the number of bytes the passed script would take
// when encoded with the domain specific compression algorithm described above.
func compressedScriptSize(scriptVersion uint16, pkScript []byte,
compressionVersion uint32) int {
func compressedScriptSize(scriptVersion uint16, pkScript []byte) int {
// Pay-to-pubkey-hash or pay-to-script-hash script.
if isPubKeyHash(pkScript) || isScriptHash(pkScript) {
return 21
Expand All @@ -286,7 +285,7 @@ func compressedScriptSize(scriptVersion uint16, pkScript []byte,
// script, possibly followed by other data, and returns the number of bytes it
// occupies taking into account the special encoding of the script size by the
// domain specific compression algorithm described above.
func decodeCompressedScriptSize(serialized []byte, compressionVersion uint32) int {
func decodeCompressedScriptSize(serialized []byte) int {
scriptSize, bytesRead := deserializeVLQ(serialized)
if bytesRead == 0 {
return 0
Expand Down Expand Up @@ -314,8 +313,7 @@ func decodeCompressedScriptSize(serialized []byte, compressionVersion uint32) in
// target byte slice. The target byte slice must be at least large enough to
// handle the number of bytes returned by the compressedScriptSize function or
// it will panic.
func putCompressedScript(target []byte, scriptVersion uint16, pkScript []byte,
compressionVersion uint32) int {
func putCompressedScript(target []byte, scriptVersion uint16, pkScript []byte) int {
if len(target) == 0 {
target[0] = 0x00
return 1
Expand Down Expand Up @@ -376,8 +374,7 @@ func putCompressedScript(target []byte, scriptVersion uint16, pkScript []byte,
// NOTE: The script parameter must already have been proven to be long enough
// to contain the number of bytes returned by decodeCompressedScriptSize or it
// will panic. This is acceptable since it is only an internal function.
func decompressScript(compressedPkScript []byte,
compressionVersion uint32) []byte {
func decompressScript(compressedPkScript []byte) []byte {
// Empty scripts, specified by 0x00, are considered nil.
if len(compressedPkScript) == 0 {
return nil
Expand Down Expand Up @@ -466,8 +463,8 @@ func decompressScript(compressedPkScript []byte,
// While this is simply exchanging one uint64 for another, the resulting value
// for typical amounts has a much smaller magnitude which results in fewer bytes
// when encoded as variable length quantity. For example, consider the amount
// of 0.1 DCR which is 10000000 atoms. Encoding 10000000 as a VarInt would take
// 4 bytes while encoding the compressed value of 8 as a VarInt only takes 1 byte.
// of 0.1 DCR which is 10000000 atoms. Encoding 10000000 as a VLQ would take
// 4 bytes while encoding the compressed value of 8 as a VLQ only takes 1 byte.
//
// Essentially the compression is achieved by splitting the value into an
// exponent in the range [0-9] and a digit in the range [1-9], when possible,
Expand All @@ -484,15 +481,15 @@ func decompressScript(compressedPkScript []byte,
// 1 + 10*(n-1) + e == 10 + 10*(n-1)
//
// Example encodings:
// (The numbers in parenthesis are the number of bytes when serialized as a VarInt)
// 0 (1) -> 0 (1) * 0.00000000 BTC
// 1000 (2) -> 4 (1) * 0.00001000 BTC
// 10000 (2) -> 5 (1) * 0.00010000 BTC
// 12345678 (4) -> 111111101(4) * 0.12345678 BTC
// 50000000 (4) -> 47 (1) * 0.50000000 BTC
// 100000000 (4) -> 9 (1) * 1.00000000 BTC
// 500000000 (5) -> 49 (1) * 5.00000000 BTC
// 1000000000 (5) -> 10 (1) * 10.00000000 BTC
// (The numbers in parenthesis are the number of bytes when serialized as a VLQ)
// 0 (1) -> 0 (1) * 0.00000000 DCR
// 1000 (2) -> 4 (1) * 0.00001000 DCR
// 10000 (2) -> 5 (1) * 0.00010000 DCR
// 12345678 (4) -> 111111101(4) * 0.12345678 DCR
// 50000000 (4) -> 48 (1) * 0.50000000 DCR
// 100000000 (4) -> 9 (1) * 1.00000000 DCR
// 500000000 (5) -> 49 (1) * 5.00000000 DCR
// 1000000000 (5) -> 10 (1) * 10.00000000 DCR
// -----------------------------------------------------------------------------

// compressTxOutAmount compresses the passed amount according to the domain
Expand Down Expand Up @@ -581,16 +578,16 @@ func decompressTxOutAmount(amount uint64) uint64 {
// compressedTxOutSize returns the number of bytes the passed transaction output
// fields would take when encoded with the format described above.
func compressedTxOutSize(amount uint64, scriptVersion uint16, pkScript []byte,
compressionVersion uint32, hasAmount bool) int {
hasAmount bool) int {

scriptVersionSize := serializeSizeVLQ(uint64(scriptVersion))
if !hasAmount {
return scriptVersionSize + compressedScriptSize(scriptVersion,
pkScript, compressionVersion)
pkScript)
}

return scriptVersionSize + serializeSizeVLQ(compressTxOutAmount(amount)) +
compressedScriptSize(scriptVersion, pkScript, compressionVersion)
compressedScriptSize(scriptVersion, pkScript)
}

// putCompressedTxOut compresses the passed amount and script according to their
Expand All @@ -599,28 +596,24 @@ func compressedTxOutSize(amount uint64, scriptVersion uint16, pkScript []byte,
// slice must be at least large enough to handle the number of bytes returned by
// the compressedTxOutSize function or it will panic.
func putCompressedTxOut(target []byte, amount uint64, scriptVersion uint16,
pkScript []byte, compressionVersion uint32, hasAmount bool) int {
pkScript []byte, hasAmount bool) int {

if !hasAmount {
offset := putVLQ(target, uint64(scriptVersion))
offset += putCompressedScript(target[offset:], scriptVersion, pkScript,
compressionVersion)
offset += putCompressedScript(target[offset:], scriptVersion, pkScript)
return offset
}

offset := putVLQ(target, compressTxOutAmount(amount))
offset += putVLQ(target[offset:], uint64(scriptVersion))
offset += putCompressedScript(target[offset:], scriptVersion, pkScript,
compressionVersion)
offset += putCompressedScript(target[offset:], scriptVersion, pkScript)
return offset
}

// decodeCompressedTxOut decodes the passed compressed txout, possibly followed
// by other data, into its uncompressed amount and script and returns them along
// with the number of bytes they occupied prior to decompression.
func decodeCompressedTxOut(serialized []byte, compressionVersion uint32,
hasAmount bool) (int64, uint16, []byte, int, error) {

func decodeCompressedTxOut(serialized []byte, hasAmount bool) (int64, uint16, []byte, int, error) {
var amount int64
var bytesRead int
var offset int
Expand Down Expand Up @@ -648,8 +641,7 @@ func decodeCompressedTxOut(serialized []byte, compressionVersion uint32,

// Decode the compressed script size and ensure there are enough bytes
// left in the slice for it.
scriptSize := decodeCompressedScriptSize(serialized[offset:],
compressionVersion)
scriptSize := decodeCompressedScriptSize(serialized[offset:])
// Note: scriptSize == 0 is OK (an empty compressed script is valid)
if scriptSize < 0 {
return 0, 0, nil, offset, errDeserialize("negative script size")
Expand All @@ -661,8 +653,7 @@ func decodeCompressedTxOut(serialized []byte, compressionVersion uint32,
}

// Decompress the script.
script := decompressScript(serialized[offset:offset+scriptSize],
compressionVersion)
script := decompressScript(serialized[offset : offset+scriptSize])

return amount, uint16(scriptVersion), script, offset + scriptSize, nil
}
Expand Down
Loading

0 comments on commit cfb953c

Please sign in to comment.