Skip to content

Commit

Permalink
feat!:add receipt mpt to private input for context (safe-eth)
Browse files Browse the repository at this point in the history
  • Loading branch information
nom4dv3 committed Jun 6, 2024
1 parent 8da835b commit e01336f
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 37 deletions.
11 changes: 7 additions & 4 deletions src/common/api_helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ function cleanReceipt(r: string) {
return trimPrefix(trimPrefix(r, '0x'), '02')
}

// TODO: merge filteredRawReceiptList and filteredRawReceiptIndexList (only keep index list)
export function rlpDecodeAndEventFilter(rawreceiptList: any, srcAddrList: any, srcEsigsList: any) {
const filteredRawReceiptList = []
const filteredEventsList = []
const filteredRawReceiptIndexList = []

for (const i in rawreceiptList) {
const es = TxReceipt.fromRawStr(rawreceiptList[i]).filter(
Expand All @@ -35,9 +37,10 @@ export function rlpDecodeAndEventFilter(rawreceiptList: any, srcAddrList: any, s
if (es.length > 0) {
filteredRawReceiptList.push(rawreceiptList[i])
filteredEventsList.push(es)
filteredRawReceiptIndexList.push(i)
}
}
return [filteredRawReceiptList, filteredEventsList]
return [filteredRawReceiptList, filteredEventsList, filteredRawReceiptIndexList]
}

export function genStreamAndMatchedEventOffsets(rawreceiptList: any[], eventList: any[]): [Uint8Array, any[]] {
Expand Down Expand Up @@ -88,9 +91,9 @@ export function formatVarLenInput(input: string) {
return formatted
}

export function filterEvents(eventDSAddrList: any[], eventDSEsigsList: any[], rawreceiptList: string | any[]): [Uint8Array, Uint32Array] {
export function filterEvents(eventDSAddrList: any[], eventDSEsigsList: any[], rawreceiptList: string | any[]): [Uint8Array, Uint32Array, number[]] {
// RLP Decode and Filter
const [filteredRawReceiptList, filteredEventList] = rlpDecodeAndEventFilter(
const [filteredRawReceiptList, filteredEventList, filteredRawReceiptIndexList] = rlpDecodeAndEventFilter(
rawreceiptList,
eventDSAddrList.map(addr => fromHexString(addr)),
eventDSEsigsList.map(esigList => esigList.map((esig: string) => fromHexString(esig))),
Expand All @@ -113,5 +116,5 @@ export function filterEvents(eventDSAddrList: any[], eventDSEsigsList: any[], ra
// may remove
const matchedEventOffsets = Uint32Array.from(_matchedEventOffsets) as any

return [rawReceipts, matchedEventOffsets]
return [rawReceipts, matchedEventOffsets, filteredRawReceiptIndexList]
}
2 changes: 1 addition & 1 deletion src/dsp/ethereum.unsafe/fill.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { toHexString } from '../../common/utils'
import type { BlockPrep } from '../ethereum/blockprep'

export function unsafeFillInputEvents(input: any, blockPrep: BlockPrep, eventDSAddrList: string[], eventDSEsigsList: string[][]) {
const rawreceiptList = blockPrep?.getRLPReceipts()
const rawreceiptList = blockPrep?.getReceiptRLPs()

// TODO: return list rather than appending string.
// NODE: rm `matchedEventOffsets` already. please add it yourself.
Expand Down
2 changes: 1 addition & 1 deletion src/dsp/ethereum.unsafe/prepare.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ export async function unsafePrepareOneBlock(provider: providers.JsonRpcProvider,
},
)

block.addRLPReceipts(rawreceiptList)
block.setReceiptRLPs(rawreceiptList)
}

if (needTransactions) {
Expand Down
42 changes: 25 additions & 17 deletions src/dsp/ethereum/blockprep.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { providers } from 'ethers'
import { RLP } from '@ethereumjs/rlp'
import { DataPrep } from '../interface'
import { safeHex, uint8ArrayToHex } from '../../common/utils'
import type { MPTTrie } from './trie'

// includes both exec & prove params
export class EthereumDataPrep extends DataPrep {
Expand Down Expand Up @@ -38,7 +39,7 @@ export class AccountPrep {
rlpNode: any
storageHash: any
accountProof: any
slots: Map<any, any>
slots: Map<string, SlotPrep>
constructor(
address: any,
rlpNode: any,
Expand Down Expand Up @@ -88,8 +89,10 @@ export class BlockPrep {
receiptsRoot: string
transactionsRoot: string
accounts: Map<string, AccountPrep>
rlpreceipts: any[]
rlpreceipts: string[]
transactions: providers.TransactionResponse[]
// tries
receiptTrie: MPTTrie
// constructor(blocknum: number | bigint | BytesLike | Hexable, hash: string, stateRoot: string, receiptsRoot: string, transactionsRoot: string) {
constructor(rawblock: Record<string, string>) {
// console.log('rawblock:', rawblock)
Expand Down Expand Up @@ -134,23 +137,22 @@ export class BlockPrep {
return nestedList
}

addAccount(address: string, rlpAccount: string, storageHash: string, accountProof: any) {
setAccount(address: string, rlpAccount: string, storageHash: string, accountProof: any) {
this.accounts.set(
address,
address.toLowerCase(),
new AccountPrep(address, rlpAccount, storageHash, accountProof),
)
}

getAccount(address: string) {
if (!this.hasAccount(address))
getAccount(address: string): AccountPrep {
const acctPrep = this.accounts.get(address.toLowerCase())
if (!acctPrep)
throw new Error(`Lack data in blockPrep: account (${address})`)

return this.accounts.get(address)
return acctPrep
}

hasAccount(address: string) {
const addressLowercase = address.toLowerCase()
return this.accounts.has(addressLowercase)
return this.accounts.has(address.toLowerCase())
}

addFromGetProofResult(ethproof: { address: any; accountProof: any; storageHash: any; storageProof: any }, accountRLP: string | null = null) {
Expand All @@ -161,23 +163,29 @@ export class BlockPrep {
if (accountRLP == null)
throw new Error('lack of accountRLP when new Account')

this.addAccount(accountAddress, accountRLP, ethproof.storageHash, ethproof.accountProof)
this.setAccount(accountAddress, accountRLP, ethproof.storageHash, ethproof.accountProof)
}

this.getAccount(accountAddress)?.addFromStorageProofList(ethproof.storageProof)
}

addRLPReceipts(rlpReceiptList: any[]) {
rlpReceiptList.forEach((rlpRcpt: any) => {
setReceiptRLPs(rlpReceiptList: string[]) {
this.rlpreceipts = []
// deep copy
rlpReceiptList.forEach((rlpRcpt: string) => {
this.rlpreceipts.push(rlpRcpt)
})
}

setTransactions(transactions: providers.TransactionResponse[]) {
this.transactions = transactions
getReceiptRLPs(): string[] {
return this.rlpreceipts
}

getRLPReceipts() {
return this.rlpreceipts
setReceiptTrie(trie: MPTTrie) {
this.receiptTrie = trie
}

setTransactions(transactions: providers.TransactionResponse[]): void {
this.transactions = transactions
}
}
21 changes: 17 additions & 4 deletions src/dsp/ethereum/fill_blocks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,21 +163,34 @@ export function fillInputStorage(input: any, blockPrep: BlockPrep, stateDSAddrLi
}

export function fillInputEvents(input: any, blockPrep: BlockPrep, eventDSAddrList: string[], eventDSEsigsList: string[][]) {
const rawreceiptList = blockPrep?.getRLPReceipts()
const rawreceiptList = blockPrep?.getReceiptRLPs()

// TODO: return list rather than appending string.
// NODE: rm `matchedEventOffsets` already. please add it yourself.
const [rawReceipts] = filterEvents(
const [rawReceipts, , filteredRawReceiptIndexList] = filterEvents(
eventDSAddrList,
eventDSEsigsList,
rawreceiptList as any,
)

// TODO: calc receipt count from filterEvents
const receiptCount = (rawReceipts.length > 0 ? rawreceiptList?.length : 0) || 0
const receiptCount = filteredRawReceiptIndexList.length
input.addInt(receiptCount, false) // receipt count (tmp)

if (receiptCount > 0) {
const trie = blockPrep.receiptTrie
for (let i = 0; i < receiptCount; i++) {
const idx = filteredRawReceiptIndexList[i]
// fill mpt key
input.addVarLenHexString(trie.keys[idx], 0)
// fill mpt lastNodeHash
const prf = trie.proof.get(idx)
if (!prf)
throw new Error(`missing receipt proof in Trie cache (idx: ${idx})`)
input.addHexString(trie.lastNodeRlpHash(prf), 0)
// fill receipt rlp len
input.addInt(rawreceiptList[idx].length, 0)
}

// fill raw receipts
input.addVarLenHexString(toHexString(rawReceipts), false)
}
Expand Down
42 changes: 32 additions & 10 deletions src/dsp/ethereum/prepare_blocks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import { RLP } from '@ethereumjs/rlp'
import { safeHex, uint8ArrayToHex } from '../../common/utils'
import type { CLEYaml, EthereumDataSource } from '../../types'
import { dspHooks } from '../hooks'
import { filterEvents } from '../../common/api_helper'
import { BlockPrep, EthereumDataPrep } from './blockprep'
import { MPTTrie } from './trie'

export async function prepareBlocksByYaml(provider: providers.JsonRpcProvider, contextBlocknumber: number, expectedStateStr: string, cleYaml: CLEYaml) {
const blockPrepMap = new Map()
Expand All @@ -29,21 +31,27 @@ export function setPrePareOneBlockFunc(_func: any) {
}

export async function prepareOneBlockByYaml(provider: providers.JsonRpcProvider, blockNumber: any, cleYaml: CLEYaml) {
let stateDSAddrList, stateDSSlotsList
// let stateDSAddrList, stateDSSlotsList
const ds = cleYaml.getFilteredSourcesByKind('ethereum')[0] as unknown as EthereumDataSource
if (ds.storage)
[stateDSAddrList, stateDSSlotsList] = ds.getStorageLists()

else
[stateDSAddrList, stateDSSlotsList] = [[], []]
const [stateDSAddrList, stateDSSlotsList] = ds.storage ? ds.getStorageLists() : [[], []]
const [eventDSAddrList, eventDSEsigsList] = ds.event ? ds.getEventLists() : [[], []]

const needRLPReceiptList = ds.event != null
const needTransactions = ds.transaction != null

return await prepareOneBlockFunc(provider, blockNumber, stateDSAddrList, stateDSSlotsList, needRLPReceiptList, needTransactions)
// return await prepareOneBlockFunc(provider, blockNumber, stateDSAddrList, stateDSSlotsList, needRLPReceiptList, needTransactions)
return await prepareOneBlockFunc(provider, blockNumber, stateDSAddrList, stateDSSlotsList, eventDSAddrList, eventDSEsigsList, needTransactions)
}

export async function prepareOneBlock(provider: providers.JsonRpcProvider, blockNumber: number, stateDSAddrList: any[], stateDSSlotsList: any[][], needRLPReceiptList: boolean, needTransactions: boolean) {
// export async function prepareOneBlock(provider: providers.JsonRpcProvider, blockNumber: number, stateDSAddrList: any[], stateDSSlotsList: any[][], needRLPReceiptList: boolean, needTransactions: boolean) {
export async function prepareOneBlock(
provider: providers.JsonRpcProvider,
blockNumber: number,
stateDSAddrList: string[],
stateDSSlotsList: string[][],
eventDSAddrList: string[],
eventDSEsigsList: string[][],
needTransactions: boolean) {
// let [stateDSAddrList, stateDSSlotsList] = [stateDSAddrList, stateDSSlotsList]
const rawblock = await dspHooks.getBlock(provider, blockNumber)
const block = new BlockPrep(rawblock)
Expand Down Expand Up @@ -82,10 +90,24 @@ export async function prepareOneBlock(provider: providers.JsonRpcProvider, block
/**
* prepare raw receipts data
*/
if (needRLPReceiptList) {
if (eventDSAddrList.length > 0) {
const rawreceiptList = await dspHooks.getRawReceipts(provider, blockNumber)

block.addRLPReceipts(rawreceiptList)
// get filtered receipt index list
const [, , filteredRawReceiptIndexList] = filterEvents(
eventDSAddrList,
eventDSEsigsList,
rawreceiptList as any,
)
// cache trie proof for receipts since prove is an async process
const trie = await new MPTTrie().build(rawreceiptList)
for (let i = 0; i < filteredRawReceiptIndexList.length; i++) {
const idx = filteredRawReceiptIndexList[i]
await trie.prove(idx, true) // cache proof in trie
}
// cache all receipt rlp
block.setReceiptRLPs(rawreceiptList)
block.setReceiptTrie(trie)
}

// TODO: improve this, reduce getBlock times
Expand Down
76 changes: 76 additions & 0 deletions src/dsp/ethereum/trie.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { Trie } from '@ethereumjs/trie'
import { MapDB } from '@ethereumjs/util'
import { RLP } from '@ethereumjs/rlp'
import { utils } from 'ethers'
import { fromHexString, safeHex, uint8ArrayToHex } from '../../common/utils'

// TODO: move this to rek
export class MPTTrie {
trie: Trie
keys: string[]
values: string[]
proof = new Map<number, string[]>() // cache, only fill with prove()
constructor() {
this.keys = []
this.values = []
}

/**
*
* @param leafValues adapt to receipt trie, can pass in receiptRlp as leafValue
* @returns this
*/
async build(leafValues: string[]) {
const leafCount = leafValues.length
if (leafCount === 0)
throw new Error('empty leaf value list, can\'t create mpt trie')
this.trie = await Trie.create({ db: new MapDB() })
this.keys = []
this.values = []
for (let leafIdx = 0; leafIdx < leafCount; leafIdx++) {
const key = uint8ArrayToHex(RLP.encode(leafIdx))
const rlp = safeHex(leafValues[leafIdx])
this.keys.push(key)
this.values.push(rlp)

// console.log('key: ', key)
// console.log('rlp: ', rlp)
await this.trie.put(fromHexString(key), fromHexString(rlp))
}
return this
}

root(): Uint8Array {
return this.trie.root()
}

async prove(idx: number, cache = false): Promise<string[]> {
const prf = await this._prove(this.keys[idx])
if (cache)
this.proof.set(idx, prf)
return prf
// return this._proof(this.keys[idx]);
}

async _prove(key: string): Promise<string[]> {
// _proof(key: string): string[] {
const { stack } = await this.trie.findPath(fromHexString(key))
const proof_paths = []
for (let i = 0; i < stack.length; i++)
proof_paths.push(uint8ArrayToHex(stack[i].serialize()))

// console.log('proof paths: ', proof_paths)
return proof_paths
}

lastNodeRlp(proof: string[]) {
const last = proof.pop()
if (!last)
throw new Error('can\'t find last node.')
return last
}

lastNodeRlpHash(proof: string[]) {
return utils.keccak256(fromHexString(this.lastNodeRlp(proof)))
}
}

0 comments on commit e01336f

Please sign in to comment.