walletdb: Write time with blockmeta record. Update wdb version to 3.

This requires full wdb block entry wipe and rescan. That is handled by
PR #889. `layout.h` is looked up by height, so only missing data was
time. Now we can implement walletdb only median time past calculation.

lib/wallet/client.js

@@ -11,6 +11,7 @@ const NodeClient = require('../client/node');
 const TX = require('../primitives/tx');
 const Coin = require('../primitives/coin');
 const NameState = require('../covenants/namestate');
+const {encoding} = require('bufio');
 
 const parsers = {
   'block connect': (entry, txs) => parseBlock(entry, txs),
@@ -62,6 +63,18 @@ class WalletClient extends NodeClient {
     return parseEntry(await super.getEntry(block));
   }
 
+  /**
+   * Get entries.
+   * @param {Number} [start=-1]
+   * @param {Number} [end=-1]
+   * @returns {Promise<Object[]>}
+   */
+
+  async getEntries(start, end) {
+    const entries = await super.getEntries(start, end);
+    return entries.map(parseEntry);
+  }
+
   async send(tx) {
     return super.send(tx.encode());
   }
@@ -94,6 +107,9 @@ class WalletClient extends NodeClient {
  */
 
 function parseEntry(data) {
+  if (!data)
+    return null;
+
   // 32  hash
   // 4   height
   // 4   nonce
@@ -112,12 +128,17 @@ function parseEntry(data) {
 
   assert(Buffer.isBuffer(data));
   // Just enough to read the three data below
-  assert(data.length >= 44);
+  assert(data.length >= 48);
+
+  const hash = data.slice(0, 32);
+  const height = encoding.readU32(data, 32);
+  // skip nonce 4.
+  const time = encoding.readU64(data, 40);
 
   return {
-    hash: data.slice(0, 32),
-    height: data.readUInt32LE(32),
-    time: data.readUInt32LE(40)
+    hash,
+    height,
+    time
   };
 }
 

lib/wallet/nodeclient.js

@@ -233,6 +233,17 @@ class NodeClient extends AsyncEmitter {
     return this.node.chain.getHashes(start, end);
   }
 
+  /**
+   * Get entries range.
+   * @param {Number} start
+   * @param {Number} end
+   * @returns {Promise<ChainEntry[]>}
+   */
+
+  async getEntries(start = -1, end = -1) {
+    return this.node.chain.getEntries(start, end);
+  }
+
   /**
    * Rescan for any missed transactions.
    * @param {Number|Hash} start - Start block.

lib/wallet/nullclient.js

@@ -9,6 +9,7 @@
 const assert = require('bsert');
 const EventEmitter = require('events');
 const NameState = require('../covenants/namestate');
+const Block = require('../primitives/block');
 const util = require('../utils/util');
 
 /**
@@ -173,6 +174,23 @@ class NullClient extends EventEmitter {
     return [this.network.genesis.hash];
   }
 
+  /**
+   * Get entries.
+   * @param {Number} [start=-1]
+   * @param {Number} [end=-1]
+   * @returns {Promise}
+   */
+
+  async getEntries(start = -1, end = -1) {
+    const genesisBlock = Block.decode(this.network.genesisBlock);
+    const entry = {
+      hash: genesisBlock.hash(),
+      height: 0,
+      time: genesisBlock.time
+    };
+    return [entry];
+  }
+
   /**
    * Rescan for any missed transactions.
    * @param {Number|Hash} start - Start block.

lib/wallet/records.js

@@ -118,12 +118,40 @@ class BlockMeta extends bio.Struct {
   }
 
   /**
-   * Get block meta hash as a buffer.
+   * Encode hash and time.
    * @returns {Buffer}
    */
 
-  toHash() {
-    return this.hash;
+  toHashAndTime() {
+    const data = Buffer.allocUnsafe(32 + 8);
+    bio.writeBytes(data, this.hash, 0);
+    bio.writeU64(data, this.time, 32);
+    return data;
+  }
+
+  /**
+   * Decode hash and time.
+   * @param {Buffer} data
+   * @param {Number} height
+   * @returns {BlockMeta}
+   */
+
+  fromHashAndTime(data, height) {
+    this.hash = data.slice(0, 32);
+    this.time = bio.readU64(data, 32);
+    this.height = height;
+    return this;
+  }
+
+  /**
+   * Instantiate block meta from hash and time.
+   * @param {Buffer} data
+   * @param {Number} height
+   * @returns {BlockMeta}
+   */
+
+  static fromHashAndTime(data, height) {
+    return new this().fromHashAndTime(data, height);
   }
 
   /**
@@ -139,6 +167,16 @@ class BlockMeta extends bio.Struct {
     return this;
   }
 
+  /**
+   * Instantiate block meta from chain entry.
+   * @param {ChainEntry} entry
+   * @returns {BlockMeta}
+   */
+
+  static fromEntry(entry) {
+    return new this().fromEntry(entry);
+  }
+
   /**
    * Instantiate block meta from serialized tip data.
    * @private
@@ -148,27 +186,17 @@ class BlockMeta extends bio.Struct {
   read(br) {
     this.hash = br.readHash();
     this.height = br.readU32();
-    this.time = br.readU32();
+    this.time = br.readU64();
     return this;
   }
 
-  /**
-   * Instantiate block meta from chain entry.
-   * @param {ChainEntry} entry
-   * @returns {BlockMeta}
-   */
-
-  static fromEntry(entry) {
-    return new this().fromEntry(entry);
-  }
-
   /**
    * Calculate size.
    * @returns {Number}
    */
 
   getSize() {
-    return 40;
+    return 44;
   }
 
   /**
@@ -179,7 +207,7 @@ class BlockMeta extends bio.Struct {
   write(bw) {
     bw.writeHash(this.hash);
     bw.writeU32(this.height);
-    bw.writeU32(this.time);
+    bw.writeU64(this.time);
     return bw;
   }
 
@@ -457,5 +485,3 @@ exports.ChainState = ChainState;
 exports.BlockMeta = BlockMeta;
 exports.TXRecord = TXRecord;
 exports.MapRecord = MapRecord;
-
-module.exports = exports;

lib/wallet/walletdb.js

@@ -17,10 +17,12 @@ const Logger = require('blgr');
 const {safeEqual} = require('bcrypto/lib/safe');
 const aes = require('bcrypto/lib/aes');
 const Network = require('../protocol/network');
+const consensus = require('../protocol/consensus');
 const Path = require('./path');
 const common = require('./common');
 const Wallet = require('./wallet');
 const Account = require('./account');
+const Block = require('../primitives/block');
 const Outpoint = require('../primitives/outpoint');
 const layouts = require('./layout');
 const records = require('./records');
@@ -62,7 +64,7 @@ class WalletDB extends EventEmitter {
     this.feeRate = this.options.feeRate;
     this.db = bdb.create(this.options);
     this.name = 'wallet';
-    this.version = 2;
+    this.version = 3;
 
     // chain state.
     this.hasStateCache = false;
@@ -305,6 +307,25 @@ class WalletDB extends EventEmitter {
     return undefined;
   }
 
+  /**
+   * Add genesis block.
+   * @returns {Promise}
+   */
+
+  async saveGenesis() {
+    // Write genesis block.
+    const network = this.network;
+    const block = Block.decode(network.genesisBlock);
+    const entry = {
+      hash: block.hash(),
+      height: 0,
+      time: block.time,
+      prevBlock: consensus.ZERO_HASH
+    };
+
+    await this.addBlock(entry, []);
+  }
+
   /**
    * Close the walletdb, wait for the database to close.
    * @returns {Promise}
@@ -416,6 +437,7 @@ class WalletDB extends EventEmitter {
       await this.syncInitState();
       await this.syncFilter();
       await this.syncChain();
+      this.rescanning = false;
       await this.resend();
     } finally {
       this.rescanning = false;
@@ -431,8 +453,10 @@ class WalletDB extends EventEmitter {
   async loadState() {
     const cache = await this.getState();
 
-    if (!cache)
+    if (!cache) {
+      await this.saveGenesis();
       return;
+    }
 
     this.logger.info('Initialized chain state from the database.');
     this.hasStateCache = true;
@@ -453,14 +477,15 @@ class WalletDB extends EventEmitter {
     this.logger.info('Initializing database state from server.');
 
     const b = this.db.batch();
-    const hashes = await this.client.getHashes();
+    const entries = await this.client.getEntries();
 
     let tip = null;
 
-    for (let height = 0; height < hashes.length; height++) {
-      const hash = hashes[height];
-      const meta = new BlockMeta(hash, height);
-      b.put(layout.h.encode(height), meta.toHash());
+    for (let height = 0; height < entries.length; height++) {
+      const entry = entries[height];
+      assert(entry.height === height);
+      const meta = new BlockMeta(entry.hash, entry.height, entry.time);
+      b.put(layout.h.encode(height), meta.toHashAndTime());
       tip = meta;
     }
 
@@ -1894,7 +1919,7 @@ class WalletDB extends EventEmitter {
     }
 
     // Save tip and state.
-    b.put(layout.h.encode(tip.height), tip.toHash());
+    b.put(layout.h.encode(tip.height), tip.toHashAndTime());
     b.put(layout.R.encode(), state.encode());
 
     await b.write();
@@ -2177,17 +2202,17 @@ class WalletDB extends EventEmitter {
 
   /**
    * Get a wallet block meta.
-   * @param {Hash} hash
+   * @param {Number} height
    * @returns {Promise<BlockMeta?>}
    */
 
   async getBlock(height) {
-    const hash = await this.db.get(layout.h.encode(height));
+    const data = await this.db.get(layout.h.encode(height));
 
-    if (!hash)
+    if (!data)
       return null;
 
-    return new BlockMeta(hash, height);
+    return BlockMeta.fromHashAndTime(data, height);
   }
 
   /**