diff --git a/api/bfgapi/bfgapi.go b/api/bfgapi/bfgapi.go index ea2221bb..9b4cde23 100644 --- a/api/bfgapi/bfgapi.go +++ b/api/bfgapi/bfgapi.go @@ -150,6 +150,8 @@ type BTCFinalityByRecentKeystonesResponse struct { type BTCFinalityByKeystonesRequest struct { L2Keystones []hemi.L2Keystone `json:"l2_keystones"` + Page uint32 `json:"page,omitempty"` + Limit uint32 `json:"limit,omitempty"` } type BTCFinalityByKeystonesResponse struct { diff --git a/api/bssapi/bssapi.go b/api/bssapi/bssapi.go index fe98a5a9..e9bc3e14 100644 --- a/api/bssapi/bssapi.go +++ b/api/bssapi/bssapi.go @@ -82,6 +82,8 @@ type BTCFinalityByRecentKeystonesResponse struct { type BTCFinalityByKeystonesRequest struct { L2Keystones []hemi.L2Keystone `json:"l2_keystones"` + Page uint32 `json:"page,omitempty"` + Limit uint32 `json:"limit,omitempty"` } type BTCFinalityByKeystonesResponse struct { diff --git a/database/bfgd/database.go b/database/bfgd/database.go index ef0ed5e9..6d5b4681 100644 --- a/database/bfgd/database.go +++ b/database/bfgd/database.go @@ -34,7 +34,7 @@ type Database interface { PopBasisUpdateBTCFields(ctx context.Context, pb *PopBasis) (int64, error) L2BTCFinalityMostRecent(ctx context.Context, limit uint32) ([]L2BTCFinality, error) - L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2KeystoneAbrevHashes []database.ByteArray) ([]L2BTCFinality, error) + L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2KeystoneAbrevHashes []database.ByteArray, page uint32, limit uint32) ([]L2BTCFinality, error) BtcBlockCanonicalHeight(ctx context.Context) (uint64, error) diff --git a/database/bfgd/database_ext_test.go b/database/bfgd/database_ext_test.go index 51fc5865..c2c0a1b7 100644 --- a/database/bfgd/database_ext_test.go +++ b/database/bfgd/database_ext_test.go @@ -1598,6 +1598,8 @@ func TestL2BtcFinalitiesByL2Keystone(t *testing.T) { finalities, err := db.L2BTCFinalityByL2KeystoneAbrevHash( ctx, []database.ByteArray{firstKeystone.Hash}, + 0, + 100, ) if err != nil { t.Fatal(err) @@ -1640,6 +1642,8 @@ func TestL2BtcFinalitiesByL2KeystoneNotPublishedHeight(t *testing.T) { finalities, err := db.L2BTCFinalityByL2KeystoneAbrevHash( ctx, []database.ByteArray{firstKeystone.Hash}, + 0, + 100, ) if err != nil { t.Fatal(err) diff --git a/database/bfgd/postgres/postgres.go b/database/bfgd/postgres/postgres.go index 0ff86ac9..b4990f15 100644 --- a/database/bfgd/postgres/postgres.go +++ b/database/bfgd/postgres/postgres.go @@ -799,7 +799,7 @@ func (p *pgdb) L2BTCFinalityMostRecent(ctx context.Context, limit uint32) ([]bfg // L2BTCFinalityByL2KeystoneAbrevHash queries for finalities by L2KeystoneAbrevHash // and returns them descending by l2_block_number -func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2KeystoneAbrevHashes []database.ByteArray) ([]bfgd.L2BTCFinality, error) { +func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2KeystoneAbrevHashes []database.ByteArray, page uint32, limit uint32) ([]bfgd.L2BTCFinality, error) { log.Tracef("L2BTCFinalityByL2KeystoneAbrevHash") defer log.Tracef("L2BTCFinalityByL2KeystoneAbrevHash exit") @@ -807,6 +807,13 @@ func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2Keyston return nil, errors.New("l2KeystoneAbrevHashes cannot be longer than 100") } + // don't let users query for more than 100 at a time, default 0 to 100 for + // backwards compatibility + if limit > 100 || limit == 0 { + log.Tracef("limit was set to %d, defaulting to 100", limit) + limit = 100 + } + sql := fmt.Sprintf(` SELECT btc_blocks_can.hash, @@ -831,6 +838,10 @@ func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2Keyston WHERE l2_keystones.l2_keystone_abrev_hash = ANY($1) ORDER BY l2_keystones.l2_block_number DESC + + OFFSET $2 + + LIMIT $3 `, effectiveHeightSql) l2KeystoneAbrevHashesStr := [][]byte{} @@ -841,7 +852,7 @@ func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2Keyston // XXX this doesn't go here log.Infof("the hashes are %v", l2KeystoneAbrevHashesStr) - rows, err := p.db.QueryContext(ctx, sql, pq.Array(l2KeystoneAbrevHashesStr)) + rows, err := p.db.QueryContext(ctx, sql, pq.Array(l2KeystoneAbrevHashesStr), page*limit, limit) if err != nil { return nil, err } diff --git a/e2e/e2e_ext_test.go b/e2e/e2e_ext_test.go index 26ca63ee..298ba8df 100644 --- a/e2e/e2e_ext_test.go +++ b/e2e/e2e_ext_test.go @@ -2654,6 +2654,210 @@ func TestGetFinalitiesByL2KeystoneBSS(t *testing.T) { } } +func TestGetFinalitiesByL2KeystoneBSSWithPagination(t *testing.T) { + db, pgUri, sdb, cleanup := createTestDB(context.Background(), t) + defer func() { + db.Close() + sdb.Close() + cleanup() + }() + + ctx, cancel := defaultTestContext() + defer cancel() + + _, _, bfgWsurl, _ := createBfgServer(ctx, t, pgUri, "", 1000) + + _, _, bssWsurl := createBssServer(ctx, t, bfgWsurl) + + btcBlock := createBtcBlock(ctx, t, db, 1, 998, []byte{}, 1) // finality should be 1000 - 998 - 9 + 1 = -6 + createBtcBlock(ctx, t, db, 1, -1, []byte{}, 2) // finality should be 1000 - 1000 - 9 + 1 = -8 (unpublished) + createBtcBlock(ctx, t, db, 1, 1000, btcBlock.Hash, 3) // finality should be 1000 - 1000 - 9 + 1 = -8 + expectedFinalitiesDesc := []int32{-8, -6} + + c, _, err := websocket.Dial(ctx, bssWsurl, nil) + if err != nil { + t.Fatal(err) + } + defer c.CloseNow() + + assertPing(ctx, t, c, bssapi.CmdPingRequest) + + bws := &bssWs{ + conn: protocol.NewWSConn(c), + } + + // first and second btcBlocks + recentFinalities, err := db.L2BTCFinalityMostRecent(ctx, 100) + if err != nil { + t.Fatal(err) + } + + l2Keystones := []hemi.L2Keystone{} + for _, r := range recentFinalities[1:] { + l, err := hemi.L2BTCFinalityFromBfgd(&r, 0, 0) + if err != nil { + t.Fatal(err) + } + l2Keystones = append(l2Keystones, l.L2Keystone) + } + + receivedFinalities := []hemi.L2BTCFinality{} + + for i := range 2 { + finalityRequest := bssapi.BTCFinalityByKeystonesRequest{ + L2Keystones: l2Keystones, + Page: uint32(i), + Limit: 1, + } + + err = bssapi.Write(ctx, bws.conn, "someid", finalityRequest) + if err != nil { + t.Fatal(err) + } + + var v protocol.Message + err = wsjson.Read(ctx, c, &v) + if err != nil { + t.Fatal(err) + } + + if v.Header.Command != bssapi.CmdBTCFinalityByKeystonesResponse { + t.Fatalf("received unexpected command: %s", v.Header.Command) + } + + finalityResponse := bssapi.BTCFinalityByRecentKeystonesResponse{} + err = json.Unmarshal(v.Payload, &finalityResponse) + if err != nil { + t.Fatal(err) + } + + t.Logf("length is %d", len(finalityResponse.L2BTCFinalities)) + + if len(finalityResponse.L2BTCFinalities) != 1 { + t.Fatalf("unexpected length %v", len(finalityResponse.L2BTCFinalities)) + } + + receivedFinalities = append(receivedFinalities, finalityResponse.L2BTCFinalities[0]) + } + + expectedResponse := []hemi.L2BTCFinality{} + for i, r := range recentFinalities[1:] { + f, err := hemi.L2BTCFinalityFromBfgd(&r, 0, 0) + if err != nil { + t.Fatal(err) + } + + f.BTCFinality = expectedFinalitiesDesc[i] + expectedResponse = append(expectedResponse, *f) + } + + diff := deep.Equal(expectedResponse, receivedFinalities) + if len(diff) > 0 { + t.Fatalf("unexpected diff %s", diff) + } +} + +func TestGetFinalitiesByL2KeystoneBFGWithPagination(t *testing.T) { + db, pgUri, sdb, cleanup := createTestDB(context.Background(), t) + defer func() { + db.Close() + sdb.Close() + cleanup() + }() + + ctx, cancel := defaultTestContext() + defer cancel() + + _, _, bfgWsurl, _ := createBfgServer(ctx, t, pgUri, "", 1000) + + btcBlock := createBtcBlock(ctx, t, db, 1, 998, []byte{}, 1) // finality should be 1000 - 998 - 9 + 1 = -6 + createBtcBlock(ctx, t, db, 1, -1, []byte{}, 2) // finality should be 1000 - 1000 - 9 + 1 = -8 (unpublished) + createBtcBlock(ctx, t, db, 1, 1000, btcBlock.Hash, 3) // finality should be 1000 - 1000 - 9 + 1 = -8 + expectedFinalitiesDesc := []int32{-8, -6} + + c, _, err := websocket.Dial(ctx, bfgWsurl, nil) + if err != nil { + t.Fatal(err) + } + defer c.CloseNow() + + assertPing(ctx, t, c, bfgapi.CmdPingRequest) + + bws := &bssWs{ + conn: protocol.NewWSConn(c), + } + + // first and second btcBlocks + recentFinalities, err := db.L2BTCFinalityMostRecent(ctx, 100) + if err != nil { + t.Fatal(err) + } + + l2Keystones := []hemi.L2Keystone{} + for _, r := range recentFinalities[1:] { + l, err := hemi.L2BTCFinalityFromBfgd(&r, 0, 0) + if err != nil { + t.Fatal(err) + } + l2Keystones = append(l2Keystones, l.L2Keystone) + } + + receivedFinalities := []hemi.L2BTCFinality{} + + for i := range 2 { + finalityRequest := bfgapi.BTCFinalityByKeystonesRequest{ + L2Keystones: l2Keystones, + Page: uint32(i), + Limit: 1, + } + + err = bfgapi.Write(ctx, bws.conn, "someid", finalityRequest) + if err != nil { + t.Fatal(err) + } + + var v protocol.Message + err = wsjson.Read(ctx, c, &v) + if err != nil { + t.Fatal(err) + } + + if v.Header.Command != bfgapi.CmdBTCFinalityByKeystonesResponse { + t.Fatalf("received unexpected command: %s", v.Header.Command) + } + + finalityResponse := bfgapi.BTCFinalityByRecentKeystonesResponse{} + err = json.Unmarshal(v.Payload, &finalityResponse) + if err != nil { + t.Fatal(err) + } + + t.Logf("length is %d", len(finalityResponse.L2BTCFinalities)) + + if len(finalityResponse.L2BTCFinalities) != 1 { + t.Fatalf("unexpected length %v", len(finalityResponse.L2BTCFinalities)) + } + + receivedFinalities = append(receivedFinalities, finalityResponse.L2BTCFinalities[0]) + } + + expectedResponse := []hemi.L2BTCFinality{} + for i, r := range recentFinalities[1:] { + f, err := hemi.L2BTCFinalityFromBfgd(&r, 0, 0) + if err != nil { + t.Fatal(err) + } + + f.BTCFinality = expectedFinalitiesDesc[i] + expectedResponse = append(expectedResponse, *f) + } + + diff := deep.Equal(expectedResponse, receivedFinalities) + if len(diff) > 0 { + t.Fatalf("unexpected diff %s", diff) + } +} + func TestGetFinalitiesByL2KeystoneBSSLowerServerHeight(t *testing.T) { db, pgUri, sdb, cleanup := createTestDB(context.Background(), t) defer func() { diff --git a/service/bfg/bfg.go b/service/bfg/bfg.go index 0977ce12..ced38496 100644 --- a/service/bfg/bfg.go +++ b/service/bfg/bfg.go @@ -1275,6 +1275,8 @@ func (s *Server) handleBtcFinalityByKeystonesRequest(ctx context.Context, bfkr * finalities, err := s.db.L2BTCFinalityByL2KeystoneAbrevHash( ctx, l2KeystoneAbrevHashes, + bfkr.Page, + bfkr.Limit, ) if err != nil { e := protocol.NewInternalErrorf("l2 keystones: %w", err) diff --git a/service/bss/bss.go b/service/bss/bss.go index 090d2383..6d27e75c 100644 --- a/service/bss/bss.go +++ b/service/bss/bss.go @@ -290,6 +290,8 @@ func (s *Server) handleBtcFinalityByKeystonesRequest(ctx context.Context, msg *b response, err := s.callBFG(ctx, &bfgapi.BTCFinalityByKeystonesRequest{ L2Keystones: msg.L2Keystones, + Limit: msg.Limit, + Page: msg.Page, }) if err != nil { e := protocol.NewInternalErrorf("btc finality keystones: %w", err)