diff --git a/src/common/api_helper.ts b/src/common/api_helper.ts index 00e4a60..e859585 100644 --- a/src/common/api_helper.ts +++ b/src/common/api_helper.ts @@ -23,9 +23,11 @@ function cleanReceipt(r: string) { return trimPrefix(trimPrefix(r, '0x'), '02') } +// TODO: merge filteredRawReceiptList and filteredRawReceiptIndexList (only keep index list) export function rlpDecodeAndEventFilter(rawreceiptList: any, srcAddrList: any, srcEsigsList: any) { const filteredRawReceiptList = [] const filteredEventsList = [] + const filteredRawReceiptIndexList = [] for (const i in rawreceiptList) { const es = TxReceipt.fromRawStr(rawreceiptList[i]).filter( @@ -35,9 +37,10 @@ export function rlpDecodeAndEventFilter(rawreceiptList: any, srcAddrList: any, s if (es.length > 0) { filteredRawReceiptList.push(rawreceiptList[i]) filteredEventsList.push(es) + filteredRawReceiptIndexList.push(i) } } - return [filteredRawReceiptList, filteredEventsList] + return [filteredRawReceiptList, filteredEventsList, filteredRawReceiptIndexList] } export function genStreamAndMatchedEventOffsets(rawreceiptList: any[], eventList: any[]): [Uint8Array, any[]] { @@ -88,9 +91,9 @@ export function formatVarLenInput(input: string) { return formatted } -export function filterEvents(eventDSAddrList: any[], eventDSEsigsList: any[], rawreceiptList: string | any[]): [Uint8Array, Uint32Array] { +export function filterEvents(eventDSAddrList: any[], eventDSEsigsList: any[], rawreceiptList: string | any[]): [Uint8Array, Uint32Array, number[]] { // RLP Decode and Filter - const [filteredRawReceiptList, filteredEventList] = rlpDecodeAndEventFilter( + const [filteredRawReceiptList, filteredEventList, filteredRawReceiptIndexList] = rlpDecodeAndEventFilter( rawreceiptList, eventDSAddrList.map(addr => fromHexString(addr)), eventDSEsigsList.map(esigList => esigList.map((esig: string) => fromHexString(esig))), @@ -113,5 +116,5 @@ export function filterEvents(eventDSAddrList: any[], eventDSEsigsList: any[], ra // may remove const matchedEventOffsets = Uint32Array.from(_matchedEventOffsets) as any - return [rawReceipts, matchedEventOffsets] + return [rawReceipts, matchedEventOffsets, filteredRawReceiptIndexList] } diff --git a/src/dsp/ethereum.unsafe/fill.ts b/src/dsp/ethereum.unsafe/fill.ts index 4a646d8..89a8567 100644 --- a/src/dsp/ethereum.unsafe/fill.ts +++ b/src/dsp/ethereum.unsafe/fill.ts @@ -3,7 +3,7 @@ import { toHexString } from '../../common/utils' import type { BlockPrep } from '../ethereum/blockprep' export function unsafeFillInputEvents(input: any, blockPrep: BlockPrep, eventDSAddrList: string[], eventDSEsigsList: string[][]) { - const rawreceiptList = blockPrep?.getRLPReceipts() + const rawreceiptList = blockPrep?.getReceiptRLPs() // TODO: return list rather than appending string. // NODE: rm `matchedEventOffsets` already. please add it yourself. diff --git a/src/dsp/ethereum.unsafe/prepare.ts b/src/dsp/ethereum.unsafe/prepare.ts index 89f9742..182e3dd 100644 --- a/src/dsp/ethereum.unsafe/prepare.ts +++ b/src/dsp/ethereum.unsafe/prepare.ts @@ -51,7 +51,7 @@ export async function unsafePrepareOneBlock(provider: providers.JsonRpcProvider, }, ) - block.addRLPReceipts(rawreceiptList) + block.setReceiptRLPs(rawreceiptList) } if (needTransactions) { diff --git a/src/dsp/ethereum/blockprep.ts b/src/dsp/ethereum/blockprep.ts index 7572ddc..d727b83 100644 --- a/src/dsp/ethereum/blockprep.ts +++ b/src/dsp/ethereum/blockprep.ts @@ -2,6 +2,7 @@ import type { providers } from 'ethers' import { RLP } from '@ethereumjs/rlp' import { DataPrep } from '../interface' import { safeHex, uint8ArrayToHex } from '../../common/utils' +import type { MPTTrie } from './trie' // includes both exec & prove params export class EthereumDataPrep extends DataPrep { @@ -38,7 +39,7 @@ export class AccountPrep { rlpNode: any storageHash: any accountProof: any - slots: Map + slots: Map constructor( address: any, rlpNode: any, @@ -88,8 +89,10 @@ export class BlockPrep { receiptsRoot: string transactionsRoot: string accounts: Map - rlpreceipts: any[] + rlpreceipts: string[] transactions: providers.TransactionResponse[] + // tries + receiptTrie: MPTTrie // constructor(blocknum: number | bigint | BytesLike | Hexable, hash: string, stateRoot: string, receiptsRoot: string, transactionsRoot: string) { constructor(rawblock: Record) { // console.log('rawblock:', rawblock) @@ -134,23 +137,22 @@ export class BlockPrep { return nestedList } - addAccount(address: string, rlpAccount: string, storageHash: string, accountProof: any) { + setAccount(address: string, rlpAccount: string, storageHash: string, accountProof: any) { this.accounts.set( - address, + address.toLowerCase(), new AccountPrep(address, rlpAccount, storageHash, accountProof), ) } - getAccount(address: string) { - if (!this.hasAccount(address)) + getAccount(address: string): AccountPrep { + const acctPrep = this.accounts.get(address.toLowerCase()) + if (!acctPrep) throw new Error(`Lack data in blockPrep: account (${address})`) - - return this.accounts.get(address) + return acctPrep } hasAccount(address: string) { - const addressLowercase = address.toLowerCase() - return this.accounts.has(addressLowercase) + return this.accounts.has(address.toLowerCase()) } addFromGetProofResult(ethproof: { address: any; accountProof: any; storageHash: any; storageProof: any }, accountRLP: string | null = null) { @@ -161,23 +163,29 @@ export class BlockPrep { if (accountRLP == null) throw new Error('lack of accountRLP when new Account') - this.addAccount(accountAddress, accountRLP, ethproof.storageHash, ethproof.accountProof) + this.setAccount(accountAddress, accountRLP, ethproof.storageHash, ethproof.accountProof) } this.getAccount(accountAddress)?.addFromStorageProofList(ethproof.storageProof) } - addRLPReceipts(rlpReceiptList: any[]) { - rlpReceiptList.forEach((rlpRcpt: any) => { + setReceiptRLPs(rlpReceiptList: string[]) { + this.rlpreceipts = [] + // deep copy + rlpReceiptList.forEach((rlpRcpt: string) => { this.rlpreceipts.push(rlpRcpt) }) } - setTransactions(transactions: providers.TransactionResponse[]) { - this.transactions = transactions + getReceiptRLPs(): string[] { + return this.rlpreceipts } - getRLPReceipts() { - return this.rlpreceipts + setReceiptTrie(trie: MPTTrie) { + this.receiptTrie = trie + } + + setTransactions(transactions: providers.TransactionResponse[]): void { + this.transactions = transactions } } diff --git a/src/dsp/ethereum/fill_blocks.ts b/src/dsp/ethereum/fill_blocks.ts index f13b104..63db87a 100644 --- a/src/dsp/ethereum/fill_blocks.ts +++ b/src/dsp/ethereum/fill_blocks.ts @@ -163,21 +163,34 @@ export function fillInputStorage(input: any, blockPrep: BlockPrep, stateDSAddrLi } export function fillInputEvents(input: any, blockPrep: BlockPrep, eventDSAddrList: string[], eventDSEsigsList: string[][]) { - const rawreceiptList = blockPrep?.getRLPReceipts() + const rawreceiptList = blockPrep?.getReceiptRLPs() // TODO: return list rather than appending string. // NODE: rm `matchedEventOffsets` already. please add it yourself. - const [rawReceipts] = filterEvents( + const [rawReceipts, , filteredRawReceiptIndexList] = filterEvents( eventDSAddrList, eventDSEsigsList, rawreceiptList as any, ) - // TODO: calc receipt count from filterEvents - const receiptCount = (rawReceipts.length > 0 ? rawreceiptList?.length : 0) || 0 + const receiptCount = filteredRawReceiptIndexList.length input.addInt(receiptCount, false) // receipt count (tmp) if (receiptCount > 0) { + const trie = blockPrep.receiptTrie + for (let i = 0; i < receiptCount; i++) { + const idx = filteredRawReceiptIndexList[i] + // fill mpt key + input.addVarLenHexString(trie.keys[idx], 0) + // fill mpt lastNodeHash + const prf = trie.proof.get(idx) + if (!prf) + throw new Error(`missing receipt proof in Trie cache (idx: ${idx})`) + input.addHexString(trie.lastNodeRlpHash(prf), 0) + // fill receipt rlp len + input.addInt(rawreceiptList[idx].length, 0) + } + // fill raw receipts input.addVarLenHexString(toHexString(rawReceipts), false) } diff --git a/src/dsp/ethereum/prepare_blocks.ts b/src/dsp/ethereum/prepare_blocks.ts index db4752f..93bb033 100644 --- a/src/dsp/ethereum/prepare_blocks.ts +++ b/src/dsp/ethereum/prepare_blocks.ts @@ -4,7 +4,9 @@ import { RLP } from '@ethereumjs/rlp' import { safeHex, uint8ArrayToHex } from '../../common/utils' import type { CLEYaml, EthereumDataSource } from '../../types' import { dspHooks } from '../hooks' +import { filterEvents } from '../../common/api_helper' import { BlockPrep, EthereumDataPrep } from './blockprep' +import { MPTTrie } from './trie' export async function prepareBlocksByYaml(provider: providers.JsonRpcProvider, contextBlocknumber: number, expectedStateStr: string, cleYaml: CLEYaml) { const blockPrepMap = new Map() @@ -29,21 +31,27 @@ export function setPrePareOneBlockFunc(_func: any) { } export async function prepareOneBlockByYaml(provider: providers.JsonRpcProvider, blockNumber: any, cleYaml: CLEYaml) { - let stateDSAddrList, stateDSSlotsList + // let stateDSAddrList, stateDSSlotsList const ds = cleYaml.getFilteredSourcesByKind('ethereum')[0] as unknown as EthereumDataSource - if (ds.storage) - [stateDSAddrList, stateDSSlotsList] = ds.getStorageLists() - else - [stateDSAddrList, stateDSSlotsList] = [[], []] + const [stateDSAddrList, stateDSSlotsList] = ds.storage ? ds.getStorageLists() : [[], []] + const [eventDSAddrList, eventDSEsigsList] = ds.event ? ds.getEventLists() : [[], []] - const needRLPReceiptList = ds.event != null const needTransactions = ds.transaction != null - return await prepareOneBlockFunc(provider, blockNumber, stateDSAddrList, stateDSSlotsList, needRLPReceiptList, needTransactions) + // return await prepareOneBlockFunc(provider, blockNumber, stateDSAddrList, stateDSSlotsList, needRLPReceiptList, needTransactions) + return await prepareOneBlockFunc(provider, blockNumber, stateDSAddrList, stateDSSlotsList, eventDSAddrList, eventDSEsigsList, needTransactions) } -export async function prepareOneBlock(provider: providers.JsonRpcProvider, blockNumber: number, stateDSAddrList: any[], stateDSSlotsList: any[][], needRLPReceiptList: boolean, needTransactions: boolean) { +// export async function prepareOneBlock(provider: providers.JsonRpcProvider, blockNumber: number, stateDSAddrList: any[], stateDSSlotsList: any[][], needRLPReceiptList: boolean, needTransactions: boolean) { +export async function prepareOneBlock( + provider: providers.JsonRpcProvider, + blockNumber: number, + stateDSAddrList: string[], + stateDSSlotsList: string[][], + eventDSAddrList: string[], + eventDSEsigsList: string[][], + needTransactions: boolean) { // let [stateDSAddrList, stateDSSlotsList] = [stateDSAddrList, stateDSSlotsList] const rawblock = await dspHooks.getBlock(provider, blockNumber) const block = new BlockPrep(rawblock) @@ -82,10 +90,24 @@ export async function prepareOneBlock(provider: providers.JsonRpcProvider, block /** * prepare raw receipts data */ - if (needRLPReceiptList) { + if (eventDSAddrList.length > 0) { const rawreceiptList = await dspHooks.getRawReceipts(provider, blockNumber) - block.addRLPReceipts(rawreceiptList) + // get filtered receipt index list + const [, , filteredRawReceiptIndexList] = filterEvents( + eventDSAddrList, + eventDSEsigsList, + rawreceiptList as any, + ) + // cache trie proof for receipts since prove is an async process + const trie = await new MPTTrie().build(rawreceiptList) + for (let i = 0; i < filteredRawReceiptIndexList.length; i++) { + const idx = filteredRawReceiptIndexList[i] + await trie.prove(idx, true) // cache proof in trie + } + // cache all receipt rlp + block.setReceiptRLPs(rawreceiptList) + block.setReceiptTrie(trie) } // TODO: improve this, reduce getBlock times diff --git a/src/dsp/ethereum/trie.ts b/src/dsp/ethereum/trie.ts new file mode 100644 index 0000000..60db472 --- /dev/null +++ b/src/dsp/ethereum/trie.ts @@ -0,0 +1,76 @@ +import { Trie } from '@ethereumjs/trie' +import { MapDB } from '@ethereumjs/util' +import { RLP } from '@ethereumjs/rlp' +import { utils } from 'ethers' +import { fromHexString, safeHex, uint8ArrayToHex } from '../../common/utils' + +// TODO: move this to rek +export class MPTTrie { + trie: Trie + keys: string[] + values: string[] + proof = new Map() // cache, only fill with prove() + constructor() { + this.keys = [] + this.values = [] + } + + /** + * + * @param leafValues adapt to receipt trie, can pass in receiptRlp as leafValue + * @returns this + */ + async build(leafValues: string[]) { + const leafCount = leafValues.length + if (leafCount === 0) + throw new Error('empty leaf value list, can\'t create mpt trie') + this.trie = await Trie.create({ db: new MapDB() }) + this.keys = [] + this.values = [] + for (let leafIdx = 0; leafIdx < leafCount; leafIdx++) { + const key = uint8ArrayToHex(RLP.encode(leafIdx)) + const rlp = safeHex(leafValues[leafIdx]) + this.keys.push(key) + this.values.push(rlp) + + // console.log('key: ', key) + // console.log('rlp: ', rlp) + await this.trie.put(fromHexString(key), fromHexString(rlp)) + } + return this + } + + root(): Uint8Array { + return this.trie.root() + } + + async prove(idx: number, cache = false): Promise { + const prf = await this._prove(this.keys[idx]) + if (cache) + this.proof.set(idx, prf) + return prf + // return this._proof(this.keys[idx]); + } + + async _prove(key: string): Promise { + // _proof(key: string): string[] { + const { stack } = await this.trie.findPath(fromHexString(key)) + const proof_paths = [] + for (let i = 0; i < stack.length; i++) + proof_paths.push(uint8ArrayToHex(stack[i].serialize())) + + // console.log('proof paths: ', proof_paths) + return proof_paths + } + + lastNodeRlp(proof: string[]) { + const last = proof.pop() + if (!last) + throw new Error('can\'t find last node.') + return last + } + + lastNodeRlpHash(proof: string[]) { + return utils.keccak256(fromHexString(this.lastNodeRlp(proof))) + } +}