[block] blocksync to use block.WithBlobSidecars()

action/blob_data.go

@@ -110,9 +110,11 @@ func fromProtoBlobTxData(pb *iotextypes.BlobTxData) (*BlobTxData, error) {
 			blob.blobHashes[i] = common.BytesToHash(bh[i])
 		}
 	}
-	var err error
-	if blob.sidecar, err = FromProtoBlobTxSideCar(pb.GetBlobTxSidecar()); err != nil {
-		return nil, err
+	if sc := pb.GetBlobTxSidecar(); sc != nil {
+		var err error
+		if blob.sidecar, err = FromProtoBlobTxSideCar(sc); err != nil {
+			return nil, err
+		}
 	}
 	return &blob, nil
 }

blockchain/block/block.go

@@ -8,6 +8,7 @@ package block
 import (
 	"time"
 
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/iotexproject/go-pkgs/hash"
 	"github.com/iotexproject/iotex-proto/golang/iotextypes"
 	"github.com/pkg/errors"
@@ -132,3 +133,33 @@ func (b *Block) HasBlob() bool {
 	}
 	return false
 }
+
+func (b *Block) WithBlobSidecars(sidecars []*types.BlobTxSidecar, txhash []string, deser *action.Deserializer) (*Block, error) {
+	scMap := make(map[hash.Hash256]*types.BlobTxSidecar)
+	for i := range txhash {
+		h, err := hash.HexStringToHash256(txhash[i])
+		if err != nil {
+			return nil, err
+		}
+		scMap[h] = sidecars[i]
+	}
+	for i, act := range b.Actions {
+		h, _ := act.Hash()
+		if sc, ok := scMap[h]; ok {
+			// add the sidecar to this action
+			pb := act.Proto()
+			blobData := pb.GetCore().GetBlobTxData()
+			if blobData == nil {
+				// this is not a blob tx, something's wrong
+				return nil, errors.Wrap(action.ErrInvalidAct, "tx is not blob type")
+			}
+			blobData.BlobTxSidecar = action.ToProtoSideCar(sc)
+			actWithBlob, err := deser.ActionToSealedEnvelope(pb)
+			if err != nil {
+				return nil, err
+			}
+			b.Actions[i] = actWithBlob
+		}
+	}
+	return b, nil
+}

blockchain/block/block_test.go

@@ -13,19 +13,19 @@ import (
 	"testing"
 	"time"
 
+	"github.com/iotexproject/go-pkgs/hash"
+	"github.com/iotexproject/iotex-proto/golang/iotextypes"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
-	"github.com/iotexproject/go-pkgs/hash"
 	"github.com/iotexproject/iotex-core/action"
 	"github.com/iotexproject/iotex-core/pkg/compress"
 	"github.com/iotexproject/iotex-core/pkg/log"
 	"github.com/iotexproject/iotex-core/pkg/unit"
 	"github.com/iotexproject/iotex-core/pkg/version"
 	"github.com/iotexproject/iotex-core/test/identityset"
 	"github.com/iotexproject/iotex-core/testutil"
-	"github.com/iotexproject/iotex-proto/golang/iotextypes"
 )
 
 func TestMerkle(t *testing.T) {
@@ -207,21 +207,9 @@ func makeBlock(tb testing.TB, n int) *Block {
 func TestVerifyBlock(t *testing.T) {
 	require := require.New(t)
 
-	tsf1, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), 1, big.NewInt(20), []byte{}, 100000, big.NewInt(10))
-	require.NoError(err)
-
-	tsf2, err := action.SignedTransfer(identityset.Address(29).String(), identityset.PrivateKey(27), 1, big.NewInt(30), []byte{}, 100000, big.NewInt(10))
-	require.NoError(err)
-
-	blkhash, err := tsf1.Hash()
-	require.NoError(err)
-	blk, err := NewTestingBuilder().
-		SetHeight(1).
-		SetPrevBlockHash(blkhash).
-		SetTimeStamp(testutil.TimestampNow()).
-		AddActions(tsf1, tsf2).
-		SignAndBuild(identityset.PrivateKey(27))
+	b, err := CreateTestBlockWithBlob(1, 1)
 	require.NoError(err)
+	blk := b[0]
 	t.Run("success", func(t *testing.T) {
 		require.True(blk.Header.VerifySignature())
 		require.NoError(blk.VerifyTxRoot())

blockchain/block/blockstore.go

@@ -32,7 +32,18 @@ func (in *Store) ToProto() *iotextypes.BlockStore {
 	for _, r := range in.Receipts {
 		receipts = append(receipts, r.ConvertToReceiptPb())
 	}
-	// blob sidecar data are stored separately
+	return &iotextypes.BlockStore{
+		Block:    in.Block.ConvertToBlockPb(),
+		Receipts: receipts,
+	}
+}
+
+// ToProto converts to proto message
+func (in *Store) ToProtoWithoutSidecar() *iotextypes.BlockStore {
+	receipts := []*iotextypes.Receipt{}
+	for _, r := range in.Receipts {
+		receipts = append(receipts, r.ConvertToReceiptPb())
+	}
 	return &iotextypes.BlockStore{
 		Block:    in.Block.ProtoWithoutSidecar(),
 		Receipts: receipts,

blockchain/block/testing.go

@@ -6,16 +6,23 @@
 package block
 
 import (
+	"math/big"
 	"time"
 
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
+	"github.com/holiman/uint256"
 	"github.com/iotexproject/go-pkgs/crypto"
 	"github.com/iotexproject/go-pkgs/hash"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
 	"github.com/iotexproject/iotex-core/action"
 	"github.com/iotexproject/iotex-core/pkg/log"
+	. "github.com/iotexproject/iotex-core/pkg/util/assertions"
 	"github.com/iotexproject/iotex-core/pkg/version"
+	"github.com/iotexproject/iotex-core/test/identityset"
+	"github.com/iotexproject/iotex-core/testutil"
 )
 
 // TestingBuilder is used to construct Block.
@@ -122,3 +129,71 @@ func NewBlockDeprecated(
 	}
 	return block
 }
+
+func createTestBlobSidecar(m, n int) *types.BlobTxSidecar {
+	testBlob := kzg4844.Blob{byte(m), byte(n)}
+	testBlobCommit := MustNoErrorV(kzg4844.BlobToCommitment(testBlob))
+	testBlobProof := MustNoErrorV(kzg4844.ComputeBlobProof(testBlob, testBlobCommit))
+	return &types.BlobTxSidecar{
+		Blobs:       []kzg4844.Blob{testBlob},
+		Commitments: []kzg4844.Commitment{testBlobCommit},
+		Proofs:      []kzg4844.Proof{testBlobProof},
+	}
+}
+
+func CreateTestBlockWithBlob(start, n int) ([]*Block, error) {
+	var (
+		amount = big.NewInt(20)
+		price  = big.NewInt(10)
+	)
+	tsf1, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), 1, amount, nil, 100000, price)
+	if err != nil {
+		return nil, err
+	}
+	tsf2, err := action.SignedTransfer(identityset.Address(29).String(), identityset.PrivateKey(27), 1, amount, nil, 100000, price)
+	if err != nil {
+		return nil, err
+	}
+	blks := make([]*Block, n)
+	for i := start; i < start+n; i++ {
+		sc3 := createTestBlobSidecar(i, i+1)
+		act3 := (&action.EnvelopeBuilder{}).SetVersion(action.BlobTxType).SetChainID(1).SetNonce(uint64(i)).
+			SetGasLimit(20000).SetDynamicGas(big.NewInt(100), big.NewInt(200)).
+			SetBlobTxData(uint256.NewInt(15), sc3.BlobHashes(), sc3).
+			SetAction(action.NewTransfer(amount, "", nil)).Build()
+		tsf3, err := action.Sign(act3, identityset.PrivateKey(25))
+		if err != nil {
+			return nil, err
+		}
+		if tsf3.Version() != action.BlobTxType {
+			return nil, action.ErrInvalidAct
+		}
+		sc4 := createTestBlobSidecar(i+1, i)
+		act4 := (&action.EnvelopeBuilder{}).SetVersion(action.BlobTxType).SetChainID(1).SetNonce(uint64(i)).
+			SetGasLimit(20000).SetDynamicGas(big.NewInt(100), big.NewInt(200)).
+			SetBlobTxData(uint256.NewInt(15), sc4.BlobHashes(), sc4).
+			SetAction(action.NewTransfer(amount, "", nil)).Build()
+		tsf4, err := action.Sign(act4, identityset.PrivateKey(26))
+		if err != nil {
+			return nil, err
+		}
+		if tsf4.Version() != action.BlobTxType {
+			return nil, action.ErrInvalidAct
+		}
+		blkhash, err := tsf1.Hash()
+		if err != nil {
+			return nil, err
+		}
+		blk, err := NewTestingBuilder().
+			SetHeight(uint64(i)).
+			SetPrevBlockHash(blkhash).
+			SetTimeStamp(testutil.TimestampNow()).
+			AddActions(tsf1, tsf3, tsf2, tsf4).
+			SignAndBuild(identityset.PrivateKey(27))
+		if err != nil {
+			return nil, err
+		}
+		blks[i-start] = &blk
+	}
+	return blks, nil
+}

blockchain/blockdao/blob_store_test.go

@@ -11,13 +11,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/iotexproject/go-pkgs/crypto"
 	"github.com/iotexproject/go-pkgs/hash"
 	"github.com/stretchr/testify/require"
 
+	"github.com/iotexproject/iotex-core/blockchain/block"
 	"github.com/iotexproject/iotex-core/db"
+	"github.com/iotexproject/iotex-core/pkg/compress"
 	. "github.com/iotexproject/iotex-core/pkg/util/assertions"
 	"github.com/iotexproject/iotex-core/testutil"
 )
@@ -118,16 +118,74 @@ func TestBlobStore(t *testing.T) {
 		ctx := context.Background()
 		testPath, err := testutil.PathOfTempFile("test-blob-store")
 		r.NoError(err)
+		defer func() {
+			testutil.CleanupPath(testPath)
+		}()
 		cfg := db.DefaultConfig
 		cfg.DbPath = testPath
 		kvs := db.NewBoltDB(cfg)
 		bs := NewBlobStore(kvs, 24, time.Second)
-		r.NoError(bs.Start(ctx))
+		testPath1, err := testutil.PathOfTempFile("test-blob-store")
+		r.NoError(err)
+		cfg.DbPath = testPath1
+		fd, err := createFileDAO(false, false, compress.Snappy, cfg)
+		r.NoError(err)
+		r.NotNil(fd)
+		dao := NewBlockDAOWithIndexersAndCache(fd, nil, 10, WithBlobStore(bs))
+		r.NoError(err)
+		r.NoError(dao.Start(ctx))
 		defer func() {
-			r.NoError(bs.Stop(ctx))
-			testutil.CleanupPath(testPath)
+			r.NoError(dao.Stop(ctx))
+			testutil.CleanupPath(testPath1)
 		}()
+
+		blks, err := block.CreateTestBlockWithBlob(1, cfg.BlockStoreBatchSize+7)
+		r.NoError(err)
+		for _, blk := range blks {
+			r.True(blk.HasBlob())
+			r.NoError(dao.PutBlock(ctx, blk))
+		}
 		// cannot store blocks less than tip height
+		err = bs.PutBlock(blks[len(blks)-1])
+		r.ErrorContains(err, "block height 23 is less than current tip height")
+		for i := 0; i < cfg.BlockStoreBatchSize+7; i++ {
+			blk, err := dao.GetBlockByHeight(1 + uint64(i))
+			r.NoError(err)
+			if i < cfg.BlockStoreBatchSize {
+				// blocks written to disk has sidecar removed
+				r.False(blk.HasBlob())
+				r.Equal(4, len(blk.Actions))
+				// verify sidecar
+				sc, hashes, err := dao.GetBlobsByHeight(1 + uint64(i))
+				r.NoError(err)
+				r.Equal(blks[i].Actions[1].BlobTxSidecar(), sc[0])
+				h := MustNoErrorV(blks[i].Actions[1].Hash())
+				r.Equal(hex.EncodeToString(h[:]), hashes[0][2:])
+				sc1, h1, err := dao.GetBlob(h)
+				r.NoError(err)
+				r.Equal(hashes[0], h1)
+				r.Equal(sc[0], sc1)
+				r.Equal(blks[i].Actions[3].BlobTxSidecar(), sc[1])
+				h = MustNoErrorV(blks[i].Actions[3].Hash())
+				r.Equal(hex.EncodeToString(h[:]), hashes[1][2:])
+				sc3, h3, err := dao.GetBlob(h)
+				r.NoError(err)
+				r.Equal(hashes[1], h3)
+				r.Equal(sc[1], sc3)
+
+			} else {
+				// blocks in the staging buffer still has sidecar attached
+				r.True(blk.HasBlob())
+				r.Equal(blks[i], blk)
+			}
+			h := blk.HashBlock()
+			height, err := dao.GetBlockHeight(h)
+			r.NoError(err)
+			r.Equal(1+uint64(i), height)
+			hash, err := dao.GetBlockHash(height)
+			r.NoError(err)
+			r.Equal(h, hash)
+		}
 	})
 }
 
@@ -136,14 +194,3 @@ func createTestHash(i int, height uint64) [][]byte {
 	h2 := hash.BytesToHash256([]byte{byte(height)})
 	return [][]byte{h1[:], h2[:]}
 }
-
-func createTestBlobTxData(n uint64) *types.BlobTxSidecar {
-	testBlob := kzg4844.Blob{byte(n)}
-	testBlobCommit := MustNoErrorV(kzg4844.BlobToCommitment(testBlob))
-	testBlobProof := MustNoErrorV(kzg4844.ComputeBlobProof(testBlob, testBlobCommit))
-	return &types.BlobTxSidecar{
-		Blobs:       []kzg4844.Blob{testBlob},
-		Commitments: []kzg4844.Commitment{testBlobCommit},
-		Proofs:      []kzg4844.Proof{testBlobProof},
-	}
-}

blockchain/filedao/filedao_v2_test.go

@@ -349,3 +349,70 @@ func TestNewFdStart(t *testing.T) {
 		}
 	}
 }
+
+func TestBlockWithSidecar(t *testing.T) {
+	testBlockWithSidecar := func(cfg db.Config, start uint64, r *require.Assertions) {
+		testutil.CleanupPath(cfg.DbPath)
+		r.Equal(5, cfg.BlockStoreBatchSize)
+		deser := block.NewDeserializer(_defaultEVMNetworkID)
+		fd, err := newFileDAOv2(start, cfg, deser)
+		r.NoError(err)
+		ctx := context.Background()
+		r.NoError(fd.Start(ctx))
+		defer func() {
+			r.NoError(fd.Stop(ctx))
+		}()
+
+		blks, err := block.CreateTestBlockWithBlob(int(start), cfg.BlockStoreBatchSize+2)
+		r.NoError(err)
+		for _, blk := range blks {
+			r.True(blk.HasBlob())
+			r.NoError(fd.PutBlock(ctx, blk))
+		}
+		for i := 0; i < cfg.BlockStoreBatchSize+2; i++ {
+			blk, err := fd.GetBlockByHeight(start + uint64(i))
+			r.NoError(err)
+			if i < cfg.BlockStoreBatchSize {
+				// blocks written to disk has sidecar removed
+				r.False(blk.HasBlob())
+				r.Equal(4, len(blk.Actions))
+				blk.Actions[0].Hash()
+				r.Equal(blks[i].Actions[0], blk.Actions[0])
+				r.NotEqual(blks[i].Actions[1], blk.Actions[1])
+				blk.Actions[2].Hash()
+				r.Equal(blks[i].Actions[2], blk.Actions[2])
+				r.NotEqual(blks[i].Actions[3], blk.Actions[3])
+			} else {
+				// blocks in the staging buffer still has sidecar attached
+				r.True(blk.HasBlob())
+				r.Equal(blks[i], blk)
+			}
+			h := blk.HashBlock()
+			height, err := fd.GetBlockHeight(h)
+			r.NoError(err)
+			r.Equal(start+uint64(i), height)
+			hash, err := fd.GetBlockHash(height)
+			r.NoError(err)
+			r.Equal(h, hash)
+		}
+	}
+
+	r := require.New(t)
+	testPath, err := testutil.PathOfTempFile("test-sidecar")
+	r.NoError(err)
+	defer func() {
+		testutil.CleanupPath(testPath)
+	}()
+
+	cfg := db.DefaultConfig
+	cfg.BlockStoreBatchSize = 5
+	cfg.DbPath = testPath
+	for _, compress := range []string{"", compress.Snappy} {
+		for _, start := range []uint64{1, 4, uint64(cfg.BlockStoreBatchSize) + 3, 3 * uint64(cfg.BlockStoreBatchSize)} {
+			cfg.Compressor = compress
+			t.Run("test block with sidecar", func(t *testing.T) {
+				testBlockWithSidecar(cfg, start, r)
+			})
+		}
+	}
+}