Merge pull request #80 from streamingfast/feature/detail_level

improve rpc poller, more details in protobuf

CHANGELOG.md

@@ -206,6 +206,10 @@ start:
 
 * The default value for `merger-grpc-listen-addr` changed from `:15012` to `:10012`. If you didn't had this flag defined and wish to keep the old default, define `merger-grpc-listen-addr: :15012`.
 
+### Protobuf model changes
+
+* Added field `DetailLevel` (Base, Extended(default)) to `sf.ethereum.type.v2.Block` to distinguish the new blocks produced from polling RPC (base) from the blocks normally produced with firehose instrumentation (extended)
+
 ### Removed
 
 * Transform `sf.ethereum.transform.v1.LightBlock` is not supported, this has been deprecated for a long time and should not be used anywhere.

cmd/fireeth/tools_compare_blocks_rpc.go

@@ -180,21 +180,37 @@ func bigIntFromEthUint256(in *eth.Uint256) *pbeth.BigInt {
 	return pbeth.BigIntFromBytes(slice)
 }
 
+func bigIntFromEthUint256Padded32(in *eth.Uint256) *pbeth.BigInt {
+	if in == nil {
+		return &pbeth.BigInt{}
+	}
+
+	in32 := (*uint256.Int)(in).Bytes32()
+
+	if in32 == [32]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} {
+		return &pbeth.BigInt{}
+	}
+	return pbeth.BigIntFromBytes(in32[:])
+}
+
 func toFirehoseBlock(in *rpc.Block, logs []*rpc.LogEntry) (*pbeth.Block, map[string]bool) {
 
 	trx, hashesWithoutTo := toFirehoseTraces(in.Transactions, logs)
 
 	out := &pbeth.Block{
+		DetailLevel:       pbeth.Block_DETAILLEVEL_BASE,
 		Hash:              in.Hash.Bytes(),
 		Number:            uint64(in.Number),
 		Ver:               3,
 		Size:              uint64(in.BlockSize),
 		Uncles:            toFirehoseUncles(in.Uncles),
 		TransactionTraces: trx,
+		BalanceChanges:    nil, // not available
+		CodeChanges:       nil, // not available
 		Header: &pbeth.BlockHeader{
-			ParentHash: in.ParentHash.Bytes(),
-			// Coinbase:         nil, // FIXME
-			// UncleHash:        nil,
+			ParentHash:       in.ParentHash.Bytes(),
+			Coinbase:         in.Miner,
+			UncleHash:        in.UnclesSHA3,
 			StateRoot:        in.StateRoot.Bytes(),
 			TransactionsRoot: in.TransactionsRoot.Bytes(),
 			ReceiptRoot:      in.ReceiptsRoot.Bytes(),
@@ -210,8 +226,8 @@ func toFirehoseBlock(in *rpc.Block, logs []*rpc.LogEntry) (*pbeth.Block, map[str
 			Hash:             in.Hash.Bytes(),
 			MixHash:          in.MixHash.Bytes(),
 			BaseFeePerGas:    bigIntFromEthUint256(in.BaseFeePerGas),
-			// WithdrawalsRoot: in.WithdrawalsRoot, // FIXME
-			// TxDependency: in.TxDependency // FIXME
+			WithdrawalsRoot:  nil, // not available
+			TxDependency:     nil, // not available
 		},
 	}
 	return out, hashesWithoutTo
@@ -227,8 +243,27 @@ func toFirehoseUncles(in []eth.Hash) []*pbeth.BlockHeader {
 	return out
 }
 
+func toAccessList(in rpc.AccessList) []*pbeth.AccessTuple {
+	out := make([]*pbeth.AccessTuple, len(in))
+	for i, v := range in {
+		out[i] = &pbeth.AccessTuple{
+			Address: v.Address,
+		}
+		if v.StorageKeys != nil {
+			out[i].StorageKeys = make([][]byte, len(v.StorageKeys))
+			for ii, vv := range v.StorageKeys {
+				out[i].StorageKeys[ii] = []byte(vv)
+			}
+		}
+	}
+
+	return out
+}
+
 func toFirehoseTraces(in *rpc.BlockTransactions, logs []*rpc.LogEntry) (traces []*pbeth.TransactionTrace, hashesWithoutTo map[string]bool) {
 
+	ordinal := uint64(0)
+
 	receipts, _ := in.Receipts()
 	out := make([]*pbeth.TransactionTrace, len(receipts))
 	hashesWithoutTo = make(map[string]bool)
@@ -252,12 +287,27 @@ func toFirehoseTraces(in *rpc.BlockTransactions, logs []*rpc.LogEntry) (traces [
 			From:     receipts[i].From.Bytes(),
 			Index:    uint32(receipts[i].TransactionIndex),
 			Receipt:  &pbeth.TransactionReceipt{
-				// filled next
+				// Logs: ,            // filled below
+				// CumulativeGasUsed: // only available on getTransactionReceipt
+				// StateRoot:         // only available on getTransactionReceipt
+				// LogsBloom:         // only available on getTransactionReceipt
 			},
-			V: pbeth.NewBigInt(int64(receipts[i].V)).Bytes,
-			//R: bigIntFromEthUint256(receipts[i].R).Bytes,
-			//S: bigIntFromEthUint256(receipts[i].S).Bytes,
+			V:            pbeth.NewBigInt(int64(receipts[i].V)).Bytes,
+			R:            bigIntFromEthUint256Padded32(receipts[i].R).Bytes,
+			S:            bigIntFromEthUint256Padded32(receipts[i].S).Bytes,
+			AccessList:   toAccessList(receipts[i].AccessList),
+			BeginOrdinal: ordinal,
+
+			// Status:  // only available on getTransactionReceipt
+			// Type:    // only available on getTransactionReceipt
+			// GasUsed: // only available on getTransactionReceipt
+			// MaxFeePerGas:            // not available on RPC
+			// MaxPriorityFeePerGas:    // not available on RPC
+			// ReturnData:              // not available on RPC
+			// PublicKey:               // not available on RPC
+			// Calls:                   // not available on RPC
 		}
+		ordinal++
 
 		for _, log := range logs {
 			if eth.Hash(log.TransactionHash).String() == txHash {
@@ -266,9 +316,13 @@ func toFirehoseTraces(in *rpc.BlockTransactions, logs []*rpc.LogEntry) (traces [
 					Topics:     hashesToBytes(log.Topics),      //[][]byte `protobuf:"bytes,2,rep,name=topics,proto3" json:"topics,omitempty"`
 					Data:       log.Data.Bytes(),               //[]byte   `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"`
 					BlockIndex: uint32(log.ToLog().BlockIndex), //uint32 `protobuf:"varint,6,opt,name=blockIndex,proto3" json:"blockIndex,omitempty"`
+					Ordinal:    ordinal,
 				})
+				ordinal++
 			}
 		}
+		out[i].EndOrdinal = ordinal
+		ordinal++
 
 	}
 	return out, hashesWithoutTo
@@ -308,10 +362,7 @@ func stripFirehoseUncles(in []*pbeth.BlockHeader) {
 }
 
 func stripFirehoseHeader(in *pbeth.BlockHeader) {
-	in.Coinbase = nil
-	in.Timestamp = nil
 	in.TxDependency = nil
-	in.UncleHash = nil
 	in.WithdrawalsRoot = nil
 
 	if in.BaseFeePerGas == nil {
@@ -324,22 +375,22 @@ func stripFirehoseHeader(in *pbeth.BlockHeader) {
 }
 
 func stripFirehoseBlock(in *pbeth.Block, hashesWithoutTo map[string]bool) {
+	in.DetailLevel = pbeth.Block_DETAILLEVEL_BASE
 	// clean up internal values
 	msg := in.ProtoReflect()
 	msg.SetUnknown(nil)
 	in = msg.Interface().(*pbeth.Block)
 
-	in.Ver = 0
+	in.Ver = 3
 	stripFirehoseHeader(in.Header)
 	stripFirehoseUncles(in.Uncles)
 	stripFirehoseTransactionTraces(in.TransactionTraces, hashesWithoutTo)
 
 	// ARB-ONE FIX
-	if in.Header.TotalDifficulty.Uint64() == 2 {
+	if in.Header.TotalDifficulty.Uint64() == 2 { // arb-one-specific
 		in.Header.TotalDifficulty = pbeth.NewBigInt(int64(in.Number) - 22207816)
 	}
 
-	// FIXME temp
 	in.BalanceChanges = nil
 	in.CodeChanges = nil
 }
@@ -349,34 +400,26 @@ func stripFirehoseTransactionTraces(in []*pbeth.TransactionTrace, hashesWithoutT
 	for _, trace := range in {
 
 		if hashesWithoutTo[eth.Hash(trace.Hash).String()] {
-			trace.To = nil // FIXME: we could compute this from nonce+address
+			trace.To = nil // only available on getTransactionReceipt
 		}
-
 		trace.BeginOrdinal = 0
 		trace.EndOrdinal = 0
-		trace.AccessList = nil
-
-		trace.GasUsed = 0 // FIXME receipt?
 
+		trace.GasUsed = 0 // only available on getTransactionReceipt
 		if trace.GasPrice == nil {
 			trace.GasPrice = &pbeth.BigInt{}
 		}
 
-		// FIXME ...
-		trace.R = nil
-		trace.S = nil
-
-		trace.Type = 0
-		trace.AccessList = nil
-		trace.MaxFeePerGas = nil
-		trace.MaxPriorityFeePerGas = nil
+		trace.Type = 0 // only available on getTransactionReceipt
 
-		trace.ReturnData = nil
-		trace.PublicKey = nil
+		trace.MaxFeePerGas = nil         // not available on RPC
+		trace.MaxPriorityFeePerGas = nil // not available on RPC
+		trace.ReturnData = nil           // not available on RPC
+		trace.PublicKey = nil            // not available on RPC
 
-		trace.Status = 0
+		trace.Status = 0 // only available on getTransactionReceipt
 		stripFirehoseTrxReceipt(trace.Receipt)
-		trace.Calls = nil
+		trace.Calls = nil // not available on RPC
 
 		if trace.Value == nil {
 			trace.Value = &pbeth.BigInt{}
@@ -390,9 +433,9 @@ func stripFirehoseTrxReceipt(in *pbeth.TransactionReceipt) {
 		log.Ordinal = 0
 		log.Index = 0 // index inside transaction is a pbeth construct, it doesn't exist in RPC interface and we can't reconstruct exactly the same from RPC because the pbeth ones are increased even when a call is reverted.
 	}
-	in.LogsBloom = nil
-	in.StateRoot = nil
-	in.CumulativeGasUsed = 0
+	in.LogsBloom = nil       // only available on getTransactionReceipt
+	in.StateRoot = nil       // only available on getTransactionReceipt
+	in.CumulativeGasUsed = 0 // only available on getTransactionReceipt
 }
 
 func CompareFirehoseToRPC(fhBlock *pbeth.Block, rpcBlock *rpc.Block, logs []*rpc.LogEntry) (isEqual bool, differences []string) {
@@ -403,6 +446,15 @@ func CompareFirehoseToRPC(fhBlock *pbeth.Block, rpcBlock *rpc.Block, logs []*rpc
 	rpcAsPBEth, hashesWithoutTo := toFirehoseBlock(rpcBlock, logs)
 	stripFirehoseBlock(fhBlock, hashesWithoutTo)
 
+	// tweak that new block for comparison
+	for _, tx := range rpcAsPBEth.TransactionTraces {
+		tx.BeginOrdinal = 0
+		tx.EndOrdinal = 0
+		for _, log := range tx.Receipt.Logs {
+			log.Ordinal = 0
+		}
+	}
+
 	if !proto.Equal(fhBlock, rpcAsPBEth) {
 		fh, err := rpc.MarshalJSONRPCIndent(fhBlock, "", " ")
 		cli.NoError(err, "cannot marshal Firehose block to JSON")

proto/sf/ethereum/type/v2/type.proto

@@ -34,8 +34,23 @@ message Block {
   // reward for the block mined, the transfer of ETH to the miner happens outside the normal
   // transaction flow of the chain and is recorded as a `BalanceChange` here since we cannot
   // attached it to any transaction.
+  //
+  // Only available in DetailLevel: EXTENDED
   repeated BalanceChange balance_changes = 11;
 
+  enum DetailLevel{
+    DETAILLEVEL_EXTENDED = 0;
+    // DETAILLEVEL_TRACE = 1; // TBD
+    DETAILLEVEL_BASE = 2;
+  }
+
+  // DetailLevel affects the data available in this block.
+  //
+  // EXTENDED describes the most complete block, with traces, balance changes, storage changes. It is extracted during the execution of the block.
+  // BASE describes a block that contains only the block header, transaction receipts and event logs: everything that can be extracted using the base JSON-RPC interface (https://ethereum.org/en/developers/docs/apis/json-rpc/#json-rpc-methods) 
+  //      Furthermore, the eth_getTransactionReceipt call has been avoided because it brings only minimal improvements at the cost of requiring an archive node or a full node with complete transaction index.
+  DetailLevel detail_level = 12;
+
   // CodeChanges here is the array of smart code change that happened that happened at the block level
   // outside of the normal transaction flow of a block. Some Ethereum's fork like BSC and Polygon
   // has some capabilities to upgrade internal smart contracts used usually to track the validator
@@ -44,6 +59,7 @@ message Block {
   // On hard fork, some procedure runs to upgrade the smart contract code to a new version. In those
   // network, a `CodeChange` for each modified smart contract on upgrade would be present here. Note
   // that this happen rarely, so the vast majority of block will have an empty list here.
+  // Only available in DetailLevel: EXTENDED
   repeated CodeChange code_changes = 20;
 
   reserved 40; // bool filtering_applied = 40 [deprecated = true];
@@ -137,8 +153,11 @@ message BlockHeader {
   BigInt base_fee_per_gas = 18;
 
   // Withdrawals root hash according to EIP-4895 (e.g. Shangai Fork) rules, only set if Shangai is present/active on the chain.
+  //
+  // Only available in DetailLevel: EXTENDED
   bytes withdrawals_root = 19;
 
+  // Only available in DetailLevel: EXTENDED
   Uint64NestedArray tx_dependency = 20;
 }
 
@@ -189,10 +208,14 @@ message TransactionTrace {
   bytes s = 9;
 
   // GasUsed is the total amount of gas unit used for the whole execution of the transaction.
+  //
+  // Only available in DetailLevel: EXTENDED
   uint64 gas_used = 10;
 
   // Type represents the Ethereum transaction type, available only since EIP-2718 & EIP-2930 activation which happened on Berlin fork.
   // The value is always set even for transaction before Berlin fork because those before the fork are still legacy transactions.
+  //
+  // Only available in DetailLevel: EXTENDED
   Type type = 12;
 
   enum Type {
@@ -232,20 +255,26 @@ message TransactionTrace {
   //
   // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only
   // if Londong fork is active on the chain.
+  //
+  // Only available in DetailLevel: EXTENDED
   BigInt max_fee_per_gas = 11;
 
   // MaxPriorityFeePerGas is priority fee per gas the user to pay in extra to the miner on top of the block's
   // base fee.
   //
   // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only
-  // if Londong fork is active on the chain.
+  // if London fork is active on the chain.
+  //
+  // Only available in DetailLevel: EXTENDED
   BigInt max_priority_fee_per_gas = 13;
 
   // meta
   uint32 index = 20;
   bytes hash = 21;
   bytes from = 22;
+  // Only available in DetailLevel: EXTENDED
   bytes return_data = 23;
+  // Only available in DetailLevel: EXTENDED
   bytes public_key = 24;
   uint64 begin_ordinal = 25;
   uint64 end_ordinal = 26;
@@ -274,8 +303,13 @@ message TransactionTrace {
   // balance changes you process those where `reason` is either `REASON_GAS_BUY`, `REASON_GAS_REFUND` or
   // `REASON_REWARD_TRANSACTION_FEE` and for nonce change, still on the root call, you pick the nonce change which the
   // smallest ordinal (if more than one).
+  //
+  // Only available in DetailLevel: EXTENDED
   TransactionTraceStatus status = 30;
+
   TransactionReceipt receipt = 31;
+
+  // Only available in DetailLevel: EXTENDED
   repeated Call calls = 32;
 }
 
@@ -305,8 +339,14 @@ message TransactionReceipt {
   // field, following `EIP-658`.
   //
   // Before Byzantinium hard fork, this field is always empty.
+  //
+  // Only available in DetailLevel: EXTENDED
   bytes state_root = 1;
+  //
+  // Only available in DetailLevel: EXTENDED
   uint64 cumulative_gas_used = 2;
+  //
+  // Only available in DetailLevel: EXTENDED
   bytes logs_bloom = 3;
   repeated Log logs = 4;
 }
@@ -319,6 +359,8 @@ message Log {
   // Index is the index of the log relative to the transaction. This index
   // is always populated regardless of the state revertion of the the call
   // that emitted this log.
+  //
+  // Only available in DetailLevel: EXTENDED
   uint32 index = 4;
 
   // BlockIndex represents the index of the log relative to the Block.

types/pb/last_generate.txt

@@ -1,3 +1,3 @@
-generate.sh - Tue Sep 12 14:03:13 EDT 2023 - maoueh
-streamingfast/proto revision: 43ccc05e2374e18dee0ce81f8634f078770fe94e
-streamingfast/firehose-ethereum/proto revision: ecc5a5e
+generate.sh - Fri 27 Oct 2023 16:39:57 EDT - stepd
+streamingfast/proto revision: 086dc4643579c0102b2a198bc40b89854dc50111
+streamingfast/firehose-ethereum/proto revision: debe10f