From c84196ea13434ed145360ed487f631bbe38fd607 Mon Sep 17 00:00:00 2001 From: aeddi Date: Mon, 9 Sep 2024 15:19:50 +0200 Subject: [PATCH] test(backup): edit unit test for batch feature --- backup/backup_test.go | 372 ++++++++++++++++++++++++++---------------- backup/mock_test.go | 20 ++- 2 files changed, 242 insertions(+), 150 deletions(-) diff --git a/backup/backup_test.go b/backup/backup_test.go index 46ab293..cecadb4 100644 --- a/backup/backup_test.go +++ b/backup/backup_test.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "errors" + "fmt" "os" "testing" "time" @@ -81,177 +82,264 @@ func TestBackup_DetermineRightBound(t *testing.T) { }) } -func TestBackup_ExecuteBackup_FixedRange(t *testing.T) { - t.Parallel() - - var ( - tempFile = createTempFile(t) +func generateMemo(blockNum, txNum uint64) string { + return fmt.Sprintf("example transaction %d:%d", blockNum, txNum) +} - fromBlock uint64 = 10 - toBlock = fromBlock + 10 +func generateBlocksTransactions(t *testing.T, fromBlock, toBlock, nTxs uint64) [][]*types.TxData { + t.Helper() - exampleTx = std.Tx{ - Memo: "example transaction", - } + // generateBlocksTransactions return only blocks containing transaction + if nTxs == 0 { + return nil + } - cfg = DefaultConfig() + gen := make([][]*types.TxData, toBlock-fromBlock+1) - mockClient = &mockClient{ - getLatestBlockNumberFn: func() (uint64, error) { - return toBlock, nil - }, - getBlockTransactionsFn: func(blockNum uint64) ([]std.Tx, error) { - // Sanity check - if blockNum < fromBlock && blockNum > toBlock { - t.Fatal("invalid block number requested") - } + for i := range gen { + txs := make([]*types.TxData, nTxs) - return []std.Tx{exampleTx}, nil // 1 tx per block - }, + for j := range txs { + txs[j] = &types.TxData{ + BlockNum: fromBlock + uint64(i), + Tx: std.Tx{ + Memo: generateMemo(fromBlock+uint64(i), uint64(j)), + }, + } } - ) - - // Temp file cleanup - t.Cleanup(func() { - require.NoError(t, tempFile.Close()) - require.NoError(t, os.Remove(tempFile.Name())) - }) - - // Set the config - cfg.FromBlock = fromBlock - cfg.ToBlock = &toBlock - s := NewService(mockClient, standard.NewWriter(tempFile), WithLogger(noop.New())) + gen[i] = txs + } - // Run the backup procedure - require.NoError( - t, - s.ExecuteBackup( - context.Background(), - cfg, - ), - ) + return gen +} - // Read the output file - fileRaw, err := os.Open(tempFile.Name()) - require.NoError(t, err) +type testCase struct { + name string + batchSize uint + fromBlock uint64 + toBlock uint64 + txsPerBlock uint64 +} - // Set up a line-by-line scanner - scanner := bufio.NewScanner(fileRaw) +var testCases = []testCase{ + // Batch 0 (should be forced to fetch by 1 by config) + {name: "batch 0/10 blocks/3 txs", batchSize: 0, fromBlock: 1, toBlock: 10, txsPerBlock: 3}, + {name: "batch 0/10 blocks/1 tx", batchSize: 0, fromBlock: 1, toBlock: 10, txsPerBlock: 1}, + {name: "batch 0/10 blocks/0 tx", batchSize: 0, fromBlock: 1, toBlock: 10, txsPerBlock: 0}, + // Batch 1 (fetch 1 by 1) + {name: "batch 1/10 blocks/3 txs", batchSize: 1, fromBlock: 1, toBlock: 10, txsPerBlock: 3}, + {name: "batch 1/10 blocks/1 tx", batchSize: 1, fromBlock: 1, toBlock: 10, txsPerBlock: 1}, + {name: "batch 1/10 blocks/0 tx", batchSize: 1, fromBlock: 1, toBlock: 10, txsPerBlock: 0}, + // Batch 6 (first fetch 6, then 4) + {name: "batch 6/10 blocks/3 txs", batchSize: 6, fromBlock: 1, toBlock: 10, txsPerBlock: 3}, + {name: "batch 6/10 blocks/1 tx", batchSize: 6, fromBlock: 1, toBlock: 10, txsPerBlock: 1}, + {name: "batch 6/10 blocks/0 tx", batchSize: 6, fromBlock: 1, toBlock: 10, txsPerBlock: 0}, + // Batch 10 (fetch all blocks in 1 batch) + {name: "batch 10/10 blocks/3 txs", batchSize: 10, fromBlock: 1, toBlock: 10, txsPerBlock: 3}, + {name: "batch 10/10 blocks/1 tx", batchSize: 10, fromBlock: 1, toBlock: 10, txsPerBlock: 1}, + {name: "batch 10/10 blocks/0 tx", batchSize: 10, fromBlock: 1, toBlock: 10, txsPerBlock: 0}, + // Batch 11 (batch size (11) bigger than block count within range (10)) + {name: "batch 11/10 blocks/3 txs", batchSize: 11, fromBlock: 1, toBlock: 10, txsPerBlock: 3}, + {name: "batch 11/10 blocks/1 tx", batchSize: 11, fromBlock: 1, toBlock: 10, txsPerBlock: 1}, + {name: "batch 11/10 blocks/0 tx", batchSize: 11, fromBlock: 1, toBlock: 10, txsPerBlock: 0}, +} - expectedBlock := fromBlock +func TestBackup_ExecuteBackup_FixedRange(t *testing.T) { + t.Parallel() - // Iterate over each line in the file - for scanner.Scan() { - var txData types.TxData + //nolint:all + testFunc := func(t *testing.T, tCase testCase) { + t.Run(tCase.name, func(t *testing.T) { + t.Parallel() + + var ( + tempFile = createTempFile(t) + cfg = DefaultConfig() + + mockClient = &mockClient{ + getLatestBlockNumberFn: func() (uint64, error) { + return tCase.toBlock, nil + }, + getBlocksTransactionsFn: func(ctx context.Context, start, stop uint64) ([][]*types.TxData, error) { + // Sanity check + if start > stop { + t.Fatal("invalid block number requested") + } + + return generateBlocksTransactions(t, start, stop, tCase.txsPerBlock), nil + }, + } + ) + + // Temp file cleanup + t.Cleanup(func() { + require.NoError(t, tempFile.Close()) + require.NoError(t, os.Remove(tempFile.Name())) + }) + + // Set the config + cfg.FromBlock = tCase.fromBlock + cfg.ToBlock = &tCase.toBlock + + s := NewService(mockClient, standard.NewWriter(tempFile), WithLogger(noop.New()), WithBatchSize(tCase.batchSize)) + + // Run the backup procedure + require.NoError( + t, + s.ExecuteBackup( + context.Background(), + cfg, + ), + ) + + // Read the output file + fileRaw, err := os.Open(tempFile.Name()) + require.NoError(t, err) + + // Set up a line-by-line scanner + scanner := bufio.NewScanner(fileRaw) + lineCount := uint64(0) + + // Iterate over each line in the file + for ; scanner.Scan(); lineCount++ { + var txData types.TxData + + // Unmarshal the JSON data into the Person struct + if err := amino.UnmarshalJSON(scanner.Bytes(), &txData); err != nil { + t.Fatalf("unable to unmarshal JSON line, %v", err) + } - // Unmarshal the JSON data into the Person struct - if err := amino.UnmarshalJSON(scanner.Bytes(), &txData); err != nil { - t.Fatalf("unable to unmarshal JSON line, %v", err) - } + expectedBlock := tCase.fromBlock + lineCount/tCase.txsPerBlock + expectedTx := lineCount % tCase.txsPerBlock + expectedTxMemo := generateMemo(expectedBlock, expectedTx) + assert.Equal(t, expectedBlock, txData.BlockNum) + assert.Equal(t, expectedTxMemo, txData.Tx.Memo) + } - assert.Equal(t, expectedBlock, txData.BlockNum) - assert.Equal(t, exampleTx, txData.Tx) + // Check for errors during scanning + if err := scanner.Err(); err != nil { + t.Fatalf("error encountered during scan, %v", err) + } - expectedBlock++ + // Ensure we found 1 line by expected transaction + expectedTxCount := (tCase.toBlock - tCase.fromBlock + 1) * tCase.txsPerBlock + assert.Equal(t, expectedTxCount, lineCount) + }) } - // Check for errors during scanning - if err := scanner.Err(); err != nil { - t.Fatalf("error encountered during scan, %v", err) + for _, tCase := range testCases { + testFunc(t, tCase) } } func TestBackup_ExecuteBackup_Watch(t *testing.T) { t.Parallel() - // Set up the context that is controlled by the test - ctx, cancelFn := context.WithCancel(context.Background()) - defer cancelFn() - - var ( - tempFile = createTempFile(t) - - fromBlock uint64 = 10 - toBlock = fromBlock + 10 - - requestToBlock = toBlock / 2 - - exampleTx = std.Tx{ - Memo: "example transaction", - } - - cfg = DefaultConfig() - - mockClient = &mockClient{ - getLatestBlockNumberFn: func() (uint64, error) { - return toBlock, nil - }, - getBlockTransactionsFn: func(blockNum uint64) ([]std.Tx, error) { - // Sanity check - if blockNum < fromBlock && blockNum > toBlock { - t.Fatal("invalid block number requested") + //nolint:thelper + testFunc := func(t *testing.T, tCase testCase) { + t.Run(tCase.name, func(t *testing.T) { + t.Parallel() + + // Set up the context that is controlled by the test + ctx, cancelFn := context.WithCancel(context.Background()) + defer cancelFn() + + var ( + tempFile = createTempFile(t) + cfg = DefaultConfig() + watchStart = tCase.toBlock / 2 + latest = watchStart + + mockClient = &mockClient{ + getLatestBlockNumberFn: func() (uint64, error) { + if latest == tCase.toBlock { // Simulate last block incrementing while in watch mode + cancelFn() + } else { + latest++ + } + + return latest, nil + }, + getBlocksTransactionsFn: func(ctx context.Context, start, stop uint64) ([][]*types.TxData, error) { + // Sanity check + if start > stop { + t.Fatal("invalid block number requested") + } + + switch { + case start > tCase.toBlock: + return nil, nil + case start > watchStart: // Watch mode, return blocks 1 by 1 + return generateBlocksTransactions(t, start, start, tCase.txsPerBlock), nil + case stop > latest: + return generateBlocksTransactions(t, start, latest, tCase.txsPerBlock), nil + } + + return generateBlocksTransactions(t, start, stop, tCase.txsPerBlock), nil + }, } - - if blockNum == toBlock { - // End of the road, close the watch process - cancelFn() + ) + + // Temp file cleanup + t.Cleanup(func() { + require.NoError(t, tempFile.Close()) + require.NoError(t, os.Remove(tempFile.Name())) + }) + + // Set the config + cfg.FromBlock = tCase.fromBlock + cfg.ToBlock = &tCase.toBlock + cfg.Watch = true + + s := NewService(mockClient, standard.NewWriter(tempFile), WithLogger(noop.New()), WithBatchSize(tCase.batchSize)) + s.watchInterval = 10 * time.Millisecond // make the interval almost instant for the test + + // Run the backup procedure + require.NoError( + t, + s.ExecuteBackup( + ctx, + cfg, + ), + ) + + // Read the output file + fileRaw, err := os.Open(tempFile.Name()) + require.NoError(t, err) + + // Set up a line-by-line scanner + scanner := bufio.NewScanner(fileRaw) + lineCount := uint64(0) + + // Iterate over each line in the file + for ; scanner.Scan(); lineCount++ { + var txData types.TxData + + // Unmarshal the JSON data into the Person struct + if err := amino.UnmarshalJSON(scanner.Bytes(), &txData); err != nil { + t.Fatalf("unable to unmarshal JSON line, %v", err) } - return []std.Tx{exampleTx}, nil // 1 tx per block - }, - } - ) - - // Temp file cleanup - t.Cleanup(func() { - require.NoError(t, tempFile.Close()) - require.NoError(t, os.Remove(tempFile.Name())) - }) - - // Set the config - cfg.FromBlock = fromBlock - cfg.ToBlock = &requestToBlock - cfg.Watch = true - - s := NewService(mockClient, standard.NewWriter(tempFile), WithLogger(noop.New())) - s.watchInterval = 10 * time.Millisecond // make the interval almost instant for the test - - // Run the backup procedure - require.NoError( - t, - s.ExecuteBackup( - ctx, - cfg, - ), - ) - - // Read the output file - fileRaw, err := os.Open(tempFile.Name()) - require.NoError(t, err) - - // Set up a line-by-line scanner - scanner := bufio.NewScanner(fileRaw) - - expectedBlock := fromBlock - - // Iterate over each line in the file - for scanner.Scan() { - var txData types.TxData - - // Unmarshal the JSON data into the Person struct - if err := amino.UnmarshalJSON(scanner.Bytes(), &txData); err != nil { - t.Fatalf("unable to unmarshal JSON line, %v", err) - } + expectedBlock := tCase.fromBlock + lineCount/tCase.txsPerBlock + expectedTx := lineCount % tCase.txsPerBlock + expectedTxMemo := generateMemo(expectedBlock, expectedTx) + assert.Equal(t, expectedBlock, txData.BlockNum) + assert.Equal(t, expectedTxMemo, txData.Tx.Memo) + } - assert.Equal(t, expectedBlock, txData.BlockNum) - assert.Equal(t, exampleTx, txData.Tx) + // Check for errors during scanning + if err := scanner.Err(); err != nil { + t.Fatalf("error encountered during scan, %v", err) + } - expectedBlock++ + // Ensure we found 1 line by expected transaction + expectedTxCount := (tCase.toBlock - tCase.fromBlock + 1) * tCase.txsPerBlock + assert.Equal(t, expectedTxCount, lineCount) + }) } - // Check for errors during scanning - if err := scanner.Err(); err != nil { - t.Fatalf("error encountered during scan, %v", err) + for _, tCase := range testCases { + testFunc(t, tCase) } } diff --git a/backup/mock_test.go b/backup/mock_test.go index 25fda87..a289d38 100644 --- a/backup/mock_test.go +++ b/backup/mock_test.go @@ -1,15 +1,19 @@ package backup -import "github.com/gnolang/gno/tm2/pkg/std" +import ( + "context" + + "github.com/gnolang/tx-archive/types" +) type ( - getLatestBlockNumberDelegate func() (uint64, error) - getBlockTransactionsDelegate func(uint64) ([]std.Tx, error) + getLatestBlockNumberDelegate func() (uint64, error) + getBlocksTransactionsDelegate func(context.Context, uint64, uint64) ([][]*types.TxData, error) ) type mockClient struct { - getLatestBlockNumberFn getLatestBlockNumberDelegate - getBlockTransactionsFn getBlockTransactionsDelegate + getLatestBlockNumberFn getLatestBlockNumberDelegate + getBlocksTransactionsFn getBlocksTransactionsDelegate } func (m *mockClient) GetLatestBlockNumber() (uint64, error) { @@ -20,9 +24,9 @@ func (m *mockClient) GetLatestBlockNumber() (uint64, error) { return 0, nil } -func (m *mockClient) GetBlockTransactions(blockNum uint64) ([]std.Tx, error) { - if m.getBlockTransactionsFn != nil { - return m.getBlockTransactionsFn(blockNum) +func (m *mockClient) GetBlocksTransactions(ctx context.Context, fromBlock, toBlock uint64) ([][]*types.TxData, error) { + if m.getBlocksTransactionsFn != nil { + return m.getBlocksTransactionsFn(ctx, fromBlock, toBlock) } return nil, nil