bypass default 1048576 limit for persisting activeset in proposal bui…

…lder (#5767)

2024-03-22T15:10:00.904Z        INFO    node.proposalBuilder    prepared activeset      {"epoch_id": 18, "size": 1846733, "weight": 237002437775, "elapsed": "640.12701ms"}
panic: encode struct slice: too many elements to encode in collection with scale limit set: 1048576

it was introduced in #5552  , i didn't check that pr with newer epochs

sql/localsql/activeset/activeset.go

@@ -1,9 +1,12 @@
 package activeset
 
 import (
+	"bytes"
 	"fmt"
+	"math"
+
+	"github.com/spacemeshos/go-scale"
 
-	"github.com/spacemeshos/go-spacemesh/codec"
 	"github.com/spacemeshos/go-spacemesh/common/types"
 	"github.com/spacemeshos/go-spacemesh/sql"
 )
@@ -15,6 +18,9 @@ const (
 	Hare
 )
 
+// disable verification for max number of atxs as it is performed before calling functions in this module.
+const maxAtxs = math.MaxUint32
+
 // Add adds an activeset with weight to the database.
 // It is expected to return error if activeset is not unique per epoch.
 func Add(
@@ -25,7 +31,11 @@ func Add(
 	weight uint64,
 	set []types.ATXID,
 ) error {
-	buf := codec.MustEncodeSlice(set)
+	var b bytes.Buffer
+	_, err := scale.EncodeStructSlice(scale.NewEncoder(&b, scale.WithEncodeMaxElements(maxAtxs)), set)
+	if err != nil {
+		panic(fmt.Sprintf("failed to encode activeset %s for epoch %d: %v", id.ShortString(), epoch, err))
+	}
 	if _, err := db.Exec(`
 		insert into prepared_activeset (kind, epoch, id, weight, data)
 		values (?1, ?2, ?3, ?4, ?5);`,
@@ -34,7 +44,7 @@ func Add(
 			stmt.BindInt64(2, int64(epoch))
 			stmt.BindBytes(3, id.Bytes())
 			stmt.BindInt64(4, int64(weight))
-			stmt.BindBytes(5, buf)
+			stmt.BindBytes(5, b.Bytes())
 		}, nil,
 	); err != nil {
 		return fmt.Errorf("failed to save activeset %s for epoch %d: %w", id.ShortString(), epoch, err)
@@ -55,7 +65,13 @@ func Get(db sql.Executor, kind Kind, epoch types.EpochID) (types.Hash32, uint64,
 		}, func(stmt *sql.Statement) bool {
 			stmt.ColumnBytes(0, id[:])
 			weight = uint64(stmt.ColumnInt64(1))
-			set = codec.MustDecodeSliceFromReader[types.ATXID](stmt.ColumnReader(2))
+			v, _, err := scale.DecodeStructSlice[types.ATXID](
+				scale.NewDecoder(stmt.ColumnReader(2), scale.WithDecodeMaxElements(maxAtxs)),
+			)
+			if err != nil {
+				panic(fmt.Sprintf("failed to decode activeset %v for epoch %d: %v", kind, epoch, err))
+			}
+			set = v
 			return true
 		})
 	if err != nil {

sql/localsql/activeset/activeset_test.go

@@ -41,3 +41,12 @@ func TestGet(t *testing.T) {
 	require.Equal(t, expectWeight, weight)
 	require.Equal(t, expectSet, set)
 }
+
+func TestLarge(t *testing.T) {
+	db := localsql.InMemory()
+	expect := make([]types.ATXID, 5_000_000)
+	require.NoError(t, Add(db, Tortoise, 1, types.Hash32{1}, 10, expect))
+	_, _, set, err := Get(db, Tortoise, 1)
+	require.NoError(t, err)
+	require.Equal(t, expect, set)
+}