Add versioning to PoST data directory via postdata_metadata.json file

Makefile

@@ -90,3 +90,7 @@ postcli: get-postrs-lib
 bench:
 	@$(ULIMIT) CGO_LDFLAGS="$(CGO_TEST_LDFLAGS)" go test -benchmem -run='^$$' -bench 'BenchmarkVerifying|BenchmarkProving' github.com/spacemeshos/post/proving github.com/spacemeshos/post/verifying
 .PHONY: bench
+
+fuzz:
+	@$(ULIMIT) CGO_LDFLAGS="$(CGO_TEST_LDFLAGS)" ./scripts/fuzz.sh $(FUZZTIME)
+.PHONY: fuzz

config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"math"
 	"os"
@@ -118,7 +117,7 @@ type InitOpts struct {
 	MaxFileSize uint64
 	ProviderID  *uint32
 	Throttle    bool
-	Scrypt      ScryptParams
+	Scrypt      shared.ScryptParams
 	// ComputeBatchSize must be greater than 0
 	ComputeBatchSize uint64
 
@@ -140,25 +139,8 @@ func (o *InitOpts) TotalFiles(labelsPerUnit uint64) int {
 	return int(math.Ceil(float64(o.TotalLabels(labelsPerUnit)) / float64(o.MaxFileNumLabels())))
 }
 
-type ScryptParams struct {
-	N, R, P uint
-}
-
-func (p *ScryptParams) Validate() error {
-	if p.N == 0 {
-		return errors.New("scrypt parameter N cannot be 0")
-	}
-	if p.R == 0 {
-		return errors.New("scrypt parameter r cannot be 0")
-	}
-	if p.P == 0 {
-		return errors.New("scrypt parameter p cannot be 0")
-	}
-	return nil
-}
-
-func DefaultLabelParams() ScryptParams {
-	return ScryptParams{
+func DefaultLabelParams() shared.ScryptParams {
+	return shared.ScryptParams{
 		N: 8192,
 		R: 1,
 		P: 1,

initialization/initialization.go

@@ -172,8 +172,8 @@
 
 	// these values are atomics so they can be read from multiple other goroutines safely
 	// write is protected by mtx
-	nonceValue       atomic.Pointer[[]byte]
 	nonce            atomic.Pointer[uint64]
+	nonceValue       atomic.Pointer[[]byte]
 	lastPosition     atomic.Pointer[uint64]
 	numLabelsWritten atomic.Uint64
 
@@ -229,39 +229,10 @@
 			return nil, err
 		}
 		init.nonce.Store(m.Nonce)
+		nonceValue := make([]byte, postrs.LabelLength)
+		copy(nonceValue, m.NonceValue)
+		init.nonceValue.Store(&nonceValue)
 		init.lastPosition.Store(m.LastPosition)
-
-		switch {
-		case m.NonceValue != nil:
-			// there is already a nonce value in the metadata
-			nonceValue := make([]byte, postrs.LabelLength)
-			copy(nonceValue, m.NonceValue)
-			init.nonceValue.Store(&nonceValue)
-		case m.Nonce != nil:
-			// there is a nonce in the metadata but no nonce value
-			cpuProviderID := CPUProviderID()
-			wo, err := oracle.New(
-				oracle.WithProviderID(&cpuProviderID),
-				oracle.WithCommitment(init.commitment),
-				oracle.WithVRFDifficulty(make([]byte, 32)), // we are not looking for it, so set difficulty to 0
-				oracle.WithScryptParams(init.opts.Scrypt),
-				oracle.WithLogger(init.logger),
-			)
-			if err != nil {
-				return nil, fmt.Errorf("failed to create work oracle: %w", err)
-			}
-			defer wo.Close()
-
-			result, err := wo.Position(*m.Nonce)
-			if err != nil {
-				return nil, fmt.Errorf("failed to compute nonce value: %w", err)
-			}
-			nonceValue := make([]byte, postrs.LabelLength)
-			copy(nonceValue, result.Output)
-			init.nonceValue.Store(&nonceValue)
-		default:
-			// no nonce in the metadata
-		}
 	}
 
 	if err := init.saveMetadata(); err != nil {
@@ -686,13 +657,18 @@
 
 func (init *Initializer) saveMetadata() error {
 	v := shared.PostMetadata{
+		Version: 1,
+
 		NodeId:          init.nodeId,
 		CommitmentAtxId: init.commitmentAtxId,
-		LabelsPerUnit:   init.cfg.LabelsPerUnit,
-		NumUnits:        init.opts.NumUnits,
-		MaxFileSize:     init.opts.MaxFileSize,
-		Nonce:           init.nonce.Load(),
-		LastPosition:    init.lastPosition.Load(),
+
+		LabelsPerUnit: init.cfg.LabelsPerUnit,
+		NumUnits:      init.opts.NumUnits,
+		MaxFileSize:   init.opts.MaxFileSize,
+		Scrypt:        init.opts.Scrypt,
+
+		Nonce:        init.nonce.Load(),
+		LastPosition: init.lastPosition.Load(),
 	}
 	if init.nonceValue.Load() != nil {
 		v.NonceValue = *init.nonceValue.Load()
@@ -701,5 +677,8 @@
 }
 
 func (init *Initializer) loadMetadata() (*shared.PostMetadata, error) {
+	if err := MigratePoST(init.opts.DataDir, init.logger); err != nil {
+		return nil, err
+	}
 	return LoadMetadata(init.opts.DataDir)
 }

initialization/initialization_test.go

@@ -3,6 +3,7 @@ package initialization
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/fs"
@@ -76,8 +77,9 @@ func TestInitialize(t *testing.T) {
 	require.NoError(t, verifying.VerifyVRFNonce(init.Nonce(), m, verifying.WithLabelScryptParams(opts.Scrypt)))
 }
 
-func TestInitialize_BeforeNonceValue(t *testing.T) {
+func TestInitialize_Migrate_Metadata(t *testing.T) {
 	cfg, opts := getTestConfig(t)
+	opts.Scrypt.N = 8192 // use default scrypt params
 
 	init, err := NewInitializer(
 		WithNodeId(nodeId),
@@ -99,10 +101,27 @@ func TestInitialize_BeforeNonceValue(t *testing.T) {
 	require.NotNil(t, meta.Nonce)
 	require.NotNil(t, meta.NonceValue)
 	nonceValue := meta.NonceValue
+	nonce := meta.Nonce
+
+	old := postMetadataV0{
+		NodeId:          nodeId,
+		CommitmentAtxId: commitmentAtxId,
+		LabelsPerUnit:   meta.LabelsPerUnit,
+		NumUnits:        meta.NumUnits,
+		MaxFileSize:     meta.MaxFileSize,
+		Nonce:           meta.Nonce,
+		NonceValue:      meta.NonceValue,
+		LastPosition:    meta.LastPosition,
+	}
 
 	// delete nonce value
-	meta.NonceValue = nil
-	require.NoError(t, SaveMetadata(opts.DataDir, meta))
+	old.NonceValue = nil
+
+	// store in old metadata format
+	f, err := os.Create(filepath.Join(opts.DataDir, MetadataFileName))
+	require.NoError(t, err)
+	require.NoError(t, json.NewEncoder(f).Encode(old))
+	require.NoError(t, f.Close())
 
 	// just creating a new initializer should update the metadata
 	init, err = NewInitializer(
@@ -117,8 +136,10 @@ func TestInitialize_BeforeNonceValue(t *testing.T) {
 
 	meta, err = LoadMetadata(opts.DataDir)
 	require.NoError(t, err)
+	require.Equal(t, 1, meta.Version)
 	require.NotNil(t, meta.Nonce)
 	require.NotNil(t, meta.NonceValue)
+	require.Equal(t, *nonce, *meta.Nonce)
 	require.Equal(t, nonceValue, meta.NonceValue)
 }
 
@@ -1125,3 +1146,33 @@ func TestRemoveRedundantFiles(t *testing.T) {
 		require.NoError(t, err)
 	}
 }
+
+func Test_Initialize_Migrates_Metadata(t *testing.T) {
+	cfg := config.DefaultConfig()
+
+	opts := config.DefaultInitOpts()
+	opts.DataDir = t.TempDir()
+	opts.NumUnits = 3
+	opts.MaxFileSize = 2 * cfg.UnitSize()
+
+	expectedFilesCount := opts.TotalFiles(cfg.LabelsPerUnit)
+	// Create 2 redundant files
+	for i := 0; i < expectedFilesCount+2; i++ {
+		f, err := os.Create(filepath.Join(opts.DataDir, shared.InitFileName(i)))
+		require.NoError(t, err)
+		_, err = f.Write([]byte("test"))
+		require.NoError(t, err)
+		require.NoError(t, f.Close())
+	}
+
+	removeRedundantFiles(cfg, opts, zap.NewNop())
+
+	files, err := os.ReadDir(opts.DataDir)
+	require.NoError(t, err)
+	require.Len(t, files, expectedFilesCount)
+
+	for i := 0; i < expectedFilesCount; i++ {
+		_, err := os.Stat(filepath.Join(opts.DataDir, shared.InitFileName(i)))
+		require.NoError(t, err)
+	}
+}