Merge branch 'main' into celestiaorg#2189-win-compatibility

.github/workflows/ci_release.yml

@@ -25,15 +25,15 @@ on:
 jobs:
   # Dockerfile Linting
   hadolint:
-    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.0 # yamllint disable-line rule:line-length
+    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.2 # yamllint disable-line rule:line-length
     with:
       dockerfile: Dockerfile
 
   yamllint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.0
+      - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.2
 
   markdown-lint:
     name: Markdown Lint
@@ -59,7 +59,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Version Release
-        uses: celestiaorg/.github/.github/actions/version-release@v0.2.0
+        uses: celestiaorg/.github/.github/actions/version-release@v0.2.2
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           version-bump: ${{inputs.version}}

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/celestiaorg/go-fraud v0.1.2
 	github.com/celestiaorg/go-header v0.2.12
 	github.com/celestiaorg/go-libp2p-messenger v0.2.0
-	github.com/celestiaorg/nmt v0.17.0
+	github.com/celestiaorg/nmt v0.18.1
 	github.com/celestiaorg/rsmt2d v0.10.0
 	github.com/cosmos/cosmos-sdk v0.46.13
 	github.com/cosmos/cosmos-sdk/api v0.1.0

go.sum

@@ -362,8 +362,8 @@ github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan
 github.com/celestiaorg/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ=
 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc=
 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA=
-github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSLU=
-github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A=
+github.com/celestiaorg/nmt v0.18.1 h1:zU3apzW4y0fs0ilQA74XnEYW8FvRv0CUK2LXK66L3rA=
+github.com/celestiaorg/nmt v0.18.1/go.mod h1:0l8q6UYRju1xNrxtvV6NwPdW3lfsN6KuZ0htRnModdc=
 github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0=
 github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI=
 github.com/celestiaorg/rsmt2d v0.10.0 h1:8dprr6CW5mCk5YPnbiLdirojw9YsJOE+XB+GORb8sT0=

nodebuilder/node/metrics.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 )
 
@@ -33,6 +34,14 @@ func WithMetrics() error {
 		return err
 	}
 
+	buildInfoGauge, err := meter.Float64ObservableGauge(
+		"build_info",
+		metric.WithDescription("Celestia Node build information"),
+	)
+	if err != nil {
+		return err
+	}
+
 	callback := func(ctx context.Context, observer metric.Observer) error {
 		if !nodeStarted {
 			// Observe node start timestamp
@@ -42,9 +51,22 @@ func WithMetrics() error {
 		}
 
 		observer.ObserveFloat64(totalNodeRunTime, time.Since(timeStarted).Seconds())
+
+		// Observe build info with labels
+		labels := metric.WithAttributes(
+			attribute.String("build_time", buildTime),
+			attribute.String("last_commit", lastCommit),
+			attribute.String("semantic_version", semanticVersion),
+			attribute.String("system_version", systemVersion),
+			attribute.String("golang_version", golangVersion),
+		)
+
+		observer.ObserveFloat64(buildInfoGauge, 1, labels)
+
 		return nil
 	}
 
-	_, err = meter.RegisterCallback(callback, nodeStartTS, totalNodeRunTime)
+	_, err = meter.RegisterCallback(callback, nodeStartTS, totalNodeRunTime, buildInfoGauge)
+
 	return err
 }

nodebuilder/tests/swamp/swamp.go

@@ -164,7 +164,7 @@ func (s *Swamp) createPeer(ks keystore.Keystore) host.Host {
 // setupGenesis sets up genesis Header.
 // This is required to initialize and start correctly.
 func (s *Swamp) setupGenesis() {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 
 	// ensure core has surpassed genesis block

share/availability/full/availability.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 
-	ipldFormat "github.com/ipfs/go-ipld-format"
 	logging "github.com/ipfs/go-log/v2"
 
 	"github.com/celestiaorg/celestia-node/share"
@@ -77,7 +76,7 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Ro
 		}
 		log.Errorw("availability validation failed", "root", root.String(), "err", err.Error())
 		var byzantineErr *byzantine.ErrByzantine
-		if ipldFormat.IsNotFound(err) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) {
+		if errors.Is(err, share.ErrNotFound) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) {
 			return share.ErrNotAvailable
 		}
 	}

share/availability/full/availability_test.go

@@ -4,9 +4,16 @@ import (
 	"context"
 	"testing"
 
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
+	"github.com/celestiaorg/celestia-app/pkg/da"
+
+	"github.com/celestiaorg/celestia-node/share"
 	availability_test "github.com/celestiaorg/celestia-node/share/availability/test"
+	"github.com/celestiaorg/celestia-node/share/eds/edstest"
+	"github.com/celestiaorg/celestia-node/share/mocks"
 )
 
 func TestShareAvailableOverMocknet_Full(t *testing.T) {
@@ -32,3 +39,22 @@ func TestSharesAvailable_Full(t *testing.T) {
 	err := avail.SharesAvailable(ctx, dah)
 	assert.NoError(t, err)
 }
+
+func TestSharesAvailable_Full_ErrNotAvailable(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	getter := mocks.NewMockGetter(ctrl)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	eds := edstest.RandEDS(t, 4)
+	dah, err := da.NewDataAvailabilityHeader(eds)
+	require.NoError(t, err)
+	avail := TestAvailability(getter)
+
+	errors := []error{share.ErrNotFound, context.DeadlineExceeded}
+	for _, getterErr := range errors {
+		getter.EXPECT().GetEDS(gomock.Any(), gomock.Any()).Return(nil, getterErr)
+		err := avail.SharesAvailable(ctx, &dah)
+		require.ErrorIs(t, err, share.ErrNotAvailable)
+	}
+}

share/eds/byzantine/bad_encoding.go

@@ -2,6 +2,7 @@ package byzantine
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 
 	"github.com/celestiaorg/celestia-app/pkg/wrapper"
@@ -114,6 +115,7 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error {
 	if !ok {
 		panic(fmt.Sprintf("invalid header type received during BEFP validation: expected %T, got %T", header, hdr))
 	}
+
 	if header.Height() != int64(p.BlockHeight) {
 		return fmt.Errorf("incorrect block height during BEFP validation: expected %d, got %d",
 			p.BlockHeight, header.Height(),
@@ -137,11 +139,13 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error {
 	if p.Axis == rsmt2d.Row {
 		merkleRoots = header.DAH.ColumnRoots
 	}
+
 	if int(p.Index) >= len(merkleRoots) {
 		return fmt.Errorf("invalid %s proof: index out of bounds (%d >= %d)",
 			BadEncoding, int(p.Index), len(merkleRoots),
 		)
 	}
+
 	if len(p.Shares) != len(merkleRoots) {
 		// Since p.Shares should contain all the shares from either a row or a
 		// column, it should exactly match the number of row roots. In this
@@ -152,6 +156,22 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error {
 		)
 	}
 
+	odsWidth := uint64(len(merkleRoots) / 2)
+	amount := uint64(0)
+	for _, share := range p.Shares {
+		if share == nil {
+			continue
+		}
+		amount++
+		if amount == odsWidth {
+			break
+		}
+	}
+
+	if amount < odsWidth {
+		return errors.New("fraud: invalid proof: not enough shares provided to reconstruct row/col")
+	}
+
 	// verify that Merkle proofs correspond to particular shares.
 	shares := make([][]byte, len(merkleRoots))
 	for index, shr := range p.Shares {
@@ -167,31 +187,44 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error {
 		shares[index] = share.GetData(shr.Share)
 	}
 
-	odsWidth := uint64(len(merkleRoots) / 2)
 	codec := share.DefaultRSMT2DCodec()
 
-	// rebuild a row or col.
+	// We can conclude that the proof is valid in case we proved the inclusion of `Shares` but
+	// the row/col can't be reconstructed, or the building of NMTree fails.
 	rebuiltShares, err := codec.Decode(shares)
 	if err != nil {
-		return err
+		log.Infow("failed to decode shares at height",
+			"height", header.Height(), "err", err,
+		)
+		return nil
 	}
+
 	rebuiltExtendedShares, err := codec.Encode(rebuiltShares[0:odsWidth])
 	if err != nil {
-		return err
+		log.Infow("failed to encode shares at height",
+			"height", header.Height(), "err", err,
+		)
+		return nil
 	}
 	copy(rebuiltShares[odsWidth:], rebuiltExtendedShares)
 
 	tree := wrapper.NewErasuredNamespacedMerkleTree(odsWidth, uint(p.Index))
 	for _, share := range rebuiltShares {
 		err = tree.Push(share)
 		if err != nil {
-			return err
+			log.Infow("failed to build a tree from the reconstructed shares at height",
+				"height", header.Height(), "err", err,
+			)
+			return nil
 		}
 	}
 
 	expectedRoot, err := tree.Root()
 	if err != nil {
-		return err
+		log.Infow("failed to build a tree root at height",
+			"height", header.Height(), "err", err,
+		)
+		return nil
 	}
 
 	// root is a merkle root of the row/col where ErrByzantine occurred

share/eds/retriever.go

@@ -64,7 +64,6 @@ func (r *Retriever) Retrieve(ctx context.Context, dah *da.DataAvailabilityHeader
 	defer span.End()
 	span.SetAttributes(
 		attribute.Int("size", len(dah.RowRoots)),
-		attribute.String("data_hash", dah.String()),
 	)
 
 	log.Debugw("retrieving data square", "data_hash", dah.String(), "size", len(dah.RowRoots))
@@ -248,7 +247,6 @@ func (rs *retrievalSession) doRequest(ctx context.Context, q *quadrant) {
 			nd, err := ipld.GetNode(ctx, rs.bget, root)
 			if err != nil {
 				rs.span.RecordError(err, trace.WithAttributes(
-					attribute.String("requesting-root", root.String()),
 					attribute.Int("root-index", i),
 				))
 				return

share/eds/store.go

@@ -168,7 +168,6 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext
 	}
 
 	ctx, span := tracer.Start(ctx, "store/put", trace.WithAttributes(
-		attribute.String("root", root.String()),
 		attribute.Int("width", int(square.Width())),
 	))
 	defer func() {
@@ -214,7 +213,7 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext
 // The shard is cached in the Store, so subsequent calls to GetCAR with the same root will use the
 // same reader. The cache is responsible for closing the underlying reader.
 func (s *Store) GetCAR(ctx context.Context, root share.DataHash) (io.Reader, error) {
-	ctx, span := tracer.Start(ctx, "store/get-car", trace.WithAttributes(attribute.String("root", root.String())))
+	ctx, span := tracer.Start(ctx, "store/get-car")
 	defer span.End()
 
 	key := root.String()
@@ -251,7 +250,7 @@ func (s *Store) CARBlockstore(
 
 // GetDAH returns the DataAvailabilityHeader for the EDS identified by DataHash.
 func (s *Store) GetDAH(ctx context.Context, root share.DataHash) (*share.Root, error) {
-	ctx, span := tracer.Start(ctx, "store/get-dah", trace.WithAttributes(attribute.String("root", root.String())))
+	ctx, span := tracer.Start(ctx, "store/get-dah")
 	defer span.End()
 
 	key := shard.KeyFromString(root.String())
@@ -330,7 +329,7 @@ func (s *Store) getCachedAccessor(ctx context.Context, key shard.Key) (*accessor
 // Remove removes EDS from Store by the given share.Root hash and cleans up all
 // the indexing.
 func (s *Store) Remove(ctx context.Context, root share.DataHash) (err error) {
-	ctx, span := tracer.Start(ctx, "store/remove", trace.WithAttributes(attribute.String("root", root.String())))
+	ctx, span := tracer.Start(ctx, "store/remove")
 	defer func() {
 		utils.SetStatusAndEnd(span, err)
 	}()
@@ -371,7 +370,7 @@ func (s *Store) Remove(ctx context.Context, root share.DataHash) (err error) {
 // It reads only one quadrant(1/4) of the EDS and verifies the integrity of the stored data by
 // recomputing it.
 func (s *Store) Get(ctx context.Context, root share.DataHash) (eds *rsmt2d.ExtendedDataSquare, err error) {
-	ctx, span := tracer.Start(ctx, "store/get", trace.WithAttributes(attribute.String("root", root.String())))
+	ctx, span := tracer.Start(ctx, "store/get")
 	defer func() {
 		utils.SetStatusAndEnd(span, err)
 	}()
@@ -389,7 +388,7 @@ func (s *Store) Get(ctx context.Context, root share.DataHash) (eds *rsmt2d.Exten
 
 // Has checks if EDS exists by the given share.Root hash.
 func (s *Store) Has(ctx context.Context, root share.DataHash) (bool, error) {
-	_, span := tracer.Start(ctx, "store/has", trace.WithAttributes(attribute.String("root", root.String())))
+	_, span := tracer.Start(ctx, "store/has")
 	defer span.End()
 
 	key := root.String()

share/getters/cascade.go

@@ -33,7 +33,6 @@ func NewCascadeGetter(getters []share.Getter) *CascadeGetter {
 // GetShare gets a share from any of registered share.Getters in cascading order.
 func (cg *CascadeGetter) GetShare(ctx context.Context, root *share.Root, row, col int) (share.Share, error) {
 	ctx, span := tracer.Start(ctx, "cascade/get-share", trace.WithAttributes(
-		attribute.String("root", root.String()),
 		attribute.Int("row", row),
 		attribute.Int("col", col),
 	))
@@ -48,9 +47,7 @@ func (cg *CascadeGetter) GetShare(ctx context.Context, root *share.Root, row, co
 
 // GetEDS gets a full EDS from any of registered share.Getters in cascading order.
 func (cg *CascadeGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) {
-	ctx, span := tracer.Start(ctx, "cascade/get-eds", trace.WithAttributes(
-		attribute.String("root", root.String()),
-	))
+	ctx, span := tracer.Start(ctx, "cascade/get-eds")
 	defer span.End()
 
 	get := func(ctx context.Context, get share.Getter) (*rsmt2d.ExtendedDataSquare, error) {
@@ -68,7 +65,6 @@ func (cg *CascadeGetter) GetSharesByNamespace(
 	namespace share.Namespace,
 ) (share.NamespacedShares, error) {
 	ctx, span := tracer.Start(ctx, "cascade/get-shares-by-namespace", trace.WithAttributes(
-		attribute.String("root", root.String()),
 		attribute.String("namespace", namespace.String()),
 	))
 	defer span.End()

share/getters/ipld.go

@@ -41,7 +41,6 @@ func NewIPLDGetter(bServ blockservice.BlockService) *IPLDGetter {
 func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) {
 	var err error
 	ctx, span := tracer.Start(ctx, "ipld/get-share", trace.WithAttributes(
-		attribute.String("root", dah.String()),
 		attribute.Int("row", row),
 		attribute.Int("col", col),
 	))
@@ -66,9 +65,7 @@ func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col in
 }
 
 func (ig *IPLDGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d.ExtendedDataSquare, err error) {
-	ctx, span := tracer.Start(ctx, "ipld/get-eds", trace.WithAttributes(
-		attribute.String("root", root.String()),
-	))
+	ctx, span := tracer.Start(ctx, "ipld/get-eds")
 	defer func() {
 		utils.SetStatusAndEnd(span, err)
 	}()
@@ -91,7 +88,6 @@ func (ig *IPLDGetter) GetSharesByNamespace(
 	namespace share.Namespace,
 ) (shares share.NamespacedShares, err error) {
 	ctx, span := tracer.Start(ctx, "ipld/get-shares-by-namespace", trace.WithAttributes(
-		attribute.String("root", root.String()),
 		attribute.String("namespace", namespace.String()),
 	))
 	defer func() {