From 6f4642ce6191ab808397fa6b04ba33f1c4ec31a8 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 21 Aug 2024 14:14:11 +0200 Subject: [PATCH 1/5] api: hotfix legacy endpoints, restoring their old behaviour rather than a "404 page not found" all the legacy endpoints had different behaviours endpoints that return an empty list * GET /accounts/{accountId}/fees/page/{page} * GET /accounts/page/{page} * GET /chain/organizations/page/{page} * POST /chain/organizations/filter/page/{page} * GET /chain/transactions/page/{page} * GET /chain/blocks/{height}/transactions/page/{page} * GET /chain/fees/page/{page} * GET /chain/fees/reference/{reference}/page/{page} * GET /chain/fees/type/{type}/page/{page} * POST /elections/filter/page/{page} * POST /elections/filter * GET /elections/page/{page} * GET /elections/{electionId}/votes/page/{page} odd endpoints that return {} * GET /accounts/{organizationId}/elections/page/{page} * GET /accounts/{organizationId}/elections/status/{status}/page/{page} odd endpoint that returns {"transfers":{"received":[],"sent":[]}} * GET /accounts/{accountId}/transfers/page/{page} the new endpoints are of course unaffected, they still return "page not found": * GET /accounts * GET /chain/organizations * GET /chain/transactions * GET /chain/fees * GET /chain/transfers * GET /elections * GET /votes --- api/accounts.go | 75 ++++++++++++++++---- api/chain.go | 155 ++++++++++++++++++++++++++++++++---------- api/elections.go | 71 +++++++++++++++---- api/legacy.go | 61 +++++++++++++++++ api/vote.go | 21 +++--- test/apierror_test.go | 4 -- 6 files changed, 312 insertions(+), 75 deletions(-) create mode 100644 api/legacy.go diff --git a/api/accounts.go b/api/accounts.go index 7de92ac00..ad3322719 100644 --- a/api/accounts.go +++ b/api/accounts.go @@ -367,7 +367,16 @@ func (a *API) accountElectionsListByPageHandler(_ *apirest.APIdata, ctx *httprou return ErrMissingParameter } - return a.sendElectionList(ctx, params) + list, err := a.electionList(params) + if err != nil { + // keep the odd legacy behaviour of sending an empty json "{}"" rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, struct{}{}) + } + return err + } + + return marshalAndSend(ctx, list) } // accountElectionsListByStatusAndPageHandler @@ -398,7 +407,16 @@ func (a *API) accountElectionsListByStatusAndPageHandler(_ *apirest.APIdata, ctx return ErrMissingParameter } - return a.sendElectionList(ctx, params) + list, err := a.electionList(params) + if err != nil { + // keep the odd legacy behaviour of sending an empty json "{}"" rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, struct{}{}) + } + return err + } + + return marshalAndSend(ctx, list) } // accountElectionsCountHandler @@ -458,7 +476,16 @@ func (a *API) tokenTransfersListHandler(_ *apirest.APIdata, ctx *httprouter.HTTP return err } - return a.sendTransfersList(ctx, params) + list, err := a.transfersList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyTransfersList()) + } + return err + } + + return marshalAndSend(ctx, list) } // tokenFeesHandler @@ -490,7 +517,16 @@ func (a *API) tokenFeesHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) return ErrMissingParameter } - return a.sendFeesList(ctx, params) + list, err := a.feesList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyFeesList()) + } + return err + } + + return marshalAndSend(ctx, list) } // tokenTransfersCountHandler @@ -546,7 +582,17 @@ func (a *API) accountListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPC if err != nil { return err } - return a.sendAccountList(ctx, params) + + list, err := a.accountList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyAccountsList()) + } + return err + } + + return marshalAndSend(ctx, list) } // accountListHandler @@ -570,33 +616,38 @@ func (a *API) accountListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext if err != nil { return err } - return a.sendAccountList(ctx, params) + + list, err := a.accountList(params) + if err != nil { + return err + } + + return marshalAndSend(ctx, list) } -// sendAccountList produces a paginated AccountsList, -// and sends it marshalled over ctx.Send +// accountList produces a paginated AccountsList. // // Errors returned are always of type APIerror. -func (a *API) sendAccountList(ctx *httprouter.HTTPContext, params *AccountParams) error { +func (a *API) accountList(params *AccountParams) (*AccountsList, error) { accounts, total, err := a.indexer.AccountList( params.Limit, params.Page*params.Limit, params.AccountID, ) if err != nil { - return ErrIndexerQueryFailed.WithErr(err) + return nil, ErrIndexerQueryFailed.WithErr(err) } pagination, err := calculatePagination(params.Page, params.Limit, total) if err != nil { - return err + return nil, err } list := &AccountsList{ Accounts: accounts, Pagination: pagination, } - return marshalAndSend(ctx, list) + return list, nil } // parseAccountParams returns an AccountParams filled with the passed params diff --git a/api/chain.go b/api/chain.go index 85c88801c..13fd80127 100644 --- a/api/chain.go +++ b/api/chain.go @@ -267,7 +267,13 @@ func (a *API) organizationListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - return a.sendOrganizationList(ctx, params) + + list, err := a.organizationList(params) + if err != nil { + return err + } + + return marshalAndSend(ctx, list) } // organizationListByPageHandler @@ -291,7 +297,17 @@ func (a *API) organizationListByPageHandler(_ *apirest.APIdata, ctx *httprouter. if err != nil { return err } - return a.sendOrganizationList(ctx, params) + + list, err := a.organizationList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyOrganizationsList()) + } + return err + } + + return marshalAndSend(ctx, list) } // organizationListByFilterAndPageHandler @@ -325,26 +341,34 @@ func (a *API) organizationListByFilterAndPageHandler(msg *apirest.APIdata, ctx * return ErrMissingParameter } - return a.sendOrganizationList(ctx, params) + list, err := a.organizationList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyOrganizationsList()) + } + return err + } + + return marshalAndSend(ctx, list) } -// sendOrganizationList produces a filtered, paginated OrganizationsList, -// and sends it marshalled over ctx.Send +// organizationList produces a filtered, paginated OrganizationsList. // // Errors returned are always of type APIerror. -func (a *API) sendOrganizationList(ctx *httprouter.HTTPContext, params *OrganizationParams) error { +func (a *API) organizationList(params *OrganizationParams) (*OrganizationsList, error) { orgs, total, err := a.indexer.EntityList( params.Limit, params.Page*params.Limit, params.OrganizationID, ) if err != nil { - return ErrIndexerQueryFailed.WithErr(err) + return nil, ErrIndexerQueryFailed.WithErr(err) } pagination, err := calculatePagination(params.Page, params.Limit, total) if err != nil { - return err + return nil, err } list := &OrganizationsList{ @@ -357,7 +381,7 @@ func (a *API) sendOrganizationList(ctx *httprouter.HTTPContext, params *Organiza ElectionCount: uint64(org.ProcessCount), }) } - return marshalAndSend(ctx, list) + return list, nil } // organizationCountHandler @@ -738,7 +762,12 @@ func (a *API) chainTxListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext return err } - return a.sendTransactionList(ctx, params) + list, err := a.transactionList(params) + if err != nil { + return err + } + + return marshalAndSend(ctx, list) } // chainTxListByPageHandler @@ -764,7 +793,16 @@ func (a *API) chainTxListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPC return err } - return a.sendTransactionList(ctx, params) + list, err := a.transactionList(params) + if err != nil { + // keep the odd legacy behaviour of sending a 204 rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return ErrTransactionNotFound + } + return err + } + + return marshalAndSend(ctx, list) } // chainTxListByHeightAndPageHandler @@ -791,14 +829,22 @@ func (a *API) chainTxListByHeightAndPageHandler(_ *apirest.APIdata, ctx *httprou return err } - return a.sendTransactionList(ctx, params) + list, err := a.transactionList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyTransactionsList()) + } + return err + } + + return marshalAndSend(ctx, list) } -// sendTransactionList produces a filtered, paginated TransactionList, -// and sends it marshalled over ctx.Send +// transactionList produces a filtered, paginated TransactionList. // // Errors returned are always of type APIerror. -func (a *API) sendTransactionList(ctx *httprouter.HTTPContext, params *TransactionParams) error { +func (a *API) transactionList(params *TransactionParams) (*TransactionsList, error) { txs, total, err := a.indexer.SearchTransactions( params.Limit, params.Page*params.Limit, @@ -806,19 +852,19 @@ func (a *API) sendTransactionList(ctx *httprouter.HTTPContext, params *Transacti params.Type, ) if err != nil { - return ErrIndexerQueryFailed.WithErr(err) + return nil, ErrIndexerQueryFailed.WithErr(err) } pagination, err := calculatePagination(params.Page, params.Limit, total) if err != nil { - return err + return nil, err } list := &TransactionsList{ Transactions: txs, Pagination: pagination, } - return marshalAndSend(ctx, list) + return list, nil } // chainValidatorsHandler @@ -971,7 +1017,12 @@ func (a *API) chainFeesListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPConte return err } - return a.sendFeesList(ctx, params) + list, err := a.feesList(params) + if err != nil { + return err + } + + return marshalAndSend(ctx, list) } // chainFeesListByPageHandler @@ -998,7 +1049,16 @@ func (a *API) chainFeesListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTT return err } - return a.sendFeesList(ctx, params) + list, err := a.feesList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyFeesList()) + } + return err + } + + return marshalAndSend(ctx, list) } // chainFeesListByReferenceAndPageHandler @@ -1030,7 +1090,16 @@ func (a *API) chainFeesListByReferenceAndPageHandler(_ *apirest.APIdata, ctx *ht return ErrMissingParameter } - return a.sendFeesList(ctx, params) + list, err := a.feesList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyFeesList()) + } + return err + } + + return marshalAndSend(ctx, list) } // chainFeesListByTypeAndPageHandler @@ -1062,16 +1131,24 @@ func (a *API) chainFeesListByTypeAndPageHandler(_ *apirest.APIdata, ctx *httprou return ErrMissingParameter } - return a.sendFeesList(ctx, params) + list, err := a.feesList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyFeesList()) + } + return err + } + + return marshalAndSend(ctx, list) } -// sendFeesList produces a filtered, paginated FeesList, -// and sends it marshalled over ctx.Send +// feesList produces a filtered, paginated FeesList. // // Errors returned are always of type APIerror. -func (a *API) sendFeesList(ctx *httprouter.HTTPContext, params *FeesParams) error { +func (a *API) feesList(params *FeesParams) (*FeesList, error) { if params.AccountID != "" && !a.indexer.AccountExists(params.AccountID) { - return ErrAccountNotFound + return nil, ErrAccountNotFound } fees, total, err := a.indexer.TokenFeesList( @@ -1082,19 +1159,19 @@ func (a *API) sendFeesList(ctx *httprouter.HTTPContext, params *FeesParams) erro params.AccountID, ) if err != nil { - return ErrIndexerQueryFailed.WithErr(err) + return nil, ErrIndexerQueryFailed.WithErr(err) } pagination, err := calculatePagination(params.Page, params.Limit, total) if err != nil { - return err + return nil, err } list := &FeesList{ Fees: fees, Pagination: pagination, } - return marshalAndSend(ctx, list) + return list, nil } // chainTransfersListHandler @@ -1123,17 +1200,21 @@ func (a *API) chainTransfersListHandler(_ *apirest.APIdata, ctx *httprouter.HTTP return err } - return a.sendTransfersList(ctx, params) + list, err := a.transfersList(params) + if err != nil { + return err + } + + return marshalAndSend(ctx, list) } -// sendTransfersList produces a filtered, paginated TokenTransfersList, -// and sends it marshalled over ctx.Send +// transfersList produces a filtered, paginated TransfersList. // // Errors returned are always of type APIerror. -func (a *API) sendTransfersList(ctx *httprouter.HTTPContext, params *TransfersParams) error { +func (a *API) transfersList(params *TransfersParams) (*TransfersList, error) { for _, param := range []string{params.AccountID, params.AccountIDFrom, params.AccountIDTo} { if param != "" && !a.indexer.AccountExists(param) { - return ErrAccountNotFound.With(param) + return nil, ErrAccountNotFound.With(param) } } @@ -1145,19 +1226,19 @@ func (a *API) sendTransfersList(ctx *httprouter.HTTPContext, params *TransfersPa params.AccountIDTo, ) if err != nil { - return ErrIndexerQueryFailed.WithErr(err) + return nil, ErrIndexerQueryFailed.WithErr(err) } pagination, err := calculatePagination(params.Page, params.Limit, total) if err != nil { - return err + return nil, err } list := &TransfersList{ Transfers: transfers, Pagination: pagination, } - return marshalAndSend(ctx, list) + return list, nil } // chainIndexerExportHandler diff --git a/api/elections.go b/api/elections.go index 89b4d7411..9e921f616 100644 --- a/api/elections.go +++ b/api/elections.go @@ -169,7 +169,16 @@ func (a *API) electionListByFilterAndPageHandler(msg *apirest.APIdata, ctx *http return ErrMissingParameter } - return a.sendElectionList(ctx, params) + list, err := a.electionList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyElectionsList()) + } + return err + } + + return marshalAndSend(ctx, list) } // electionListByFilterHandler @@ -191,7 +200,16 @@ func (a *API) electionListByFilterHandler(msg *apirest.APIdata, ctx *httprouter. return ErrCantParseDataAsJSON.WithErr(err) } - return a.sendElectionList(ctx, params) + list, err := a.electionList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyElectionsList()) + } + return err + } + + return marshalAndSend(ctx, list) } // electionListByPageHandler @@ -213,7 +231,17 @@ func (a *API) electionListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTP if err != nil { return err } - return a.sendElectionList(ctx, params) + + list, err := a.electionList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyElectionsList()) + } + return err + } + + return marshalAndSend(ctx, list) } // electionListHandler @@ -251,21 +279,26 @@ func (a *API) electionListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContex if err != nil { return err } - return a.sendElectionList(ctx, params) + + list, err := a.electionList(params) + if err != nil { + return err + } + + return marshalAndSend(ctx, list) } -// sendElectionList produces a filtered, paginated ElectionsList, -// and sends it marshalled over ctx.Send +// electionList produces a filtered, paginated ElectionsList. // // Errors returned are always of type APIerror. -func (a *API) sendElectionList(ctx *httprouter.HTTPContext, params *ElectionParams) error { +func (a *API) electionList(params *ElectionParams) (*ElectionsList, error) { if params.OrganizationID != "" && !a.indexer.EntityExists(params.OrganizationID) { - return ErrOrgNotFound + return nil, ErrOrgNotFound } status, err := parseStatus(params.Status) if err != nil { - return err + return nil, err } eids, total, err := a.indexer.ProcessList( @@ -285,12 +318,12 @@ func (a *API) sendElectionList(ctx *httprouter.HTTPContext, params *ElectionPara params.EndDateBefore, ) if err != nil { - return ErrIndexerQueryFailed.WithErr(err) + return nil, ErrIndexerQueryFailed.WithErr(err) } pagination, err := calculatePagination(params.Page, params.Limit, total) if err != nil { - return err + return nil, err } list := &ElectionsList{ @@ -300,11 +333,11 @@ func (a *API) sendElectionList(ctx *httprouter.HTTPContext, params *ElectionPara for _, eid := range eids { e, err := a.indexer.ProcessInfo(eid) if err != nil { - return ErrCantFetchElection.Withf("(%x): %v", eid, err) + return nil, ErrCantFetchElection.Withf("(%x): %v", eid, err) } list.Elections = append(list.Elections, a.electionSummary(e)) } - return marshalAndSend(ctx, list) + return list, nil } // electionHandler @@ -483,7 +516,17 @@ func (a *API) electionVotesListByPageHandler(_ *apirest.APIdata, ctx *httprouter if err != nil { return err } - return a.sendVotesList(ctx, params) + + list, err := a.votesList(params) + if err != nil { + // keep legacy behaviour of sending an empty list rather than a 404 + if errors.Is(err, ErrPageNotFound) { + return marshalAndSend(ctx, emptyVotesList()) + } + return err + } + + return marshalAndSend(ctx, list) } // electionScrutinyHandler diff --git a/api/legacy.go b/api/legacy.go new file mode 100644 index 000000000..436ea1531 --- /dev/null +++ b/api/legacy.go @@ -0,0 +1,61 @@ +package api + +// +// Legacy lists used to return an empty list (instead of null or an error) +// + +func emptyElectionsList() any { + return struct { + List []any `json:"elections"` + }{ + List: []any{}, + } +} + +func emptyOrganizationsList() any { + return struct { + List []any `json:"organizations"` + }{ + List: []any{}, + } +} + +func emptyVotesList() any { + return struct { + List []any `json:"votes"` + }{ + List: []any{}, + } +} + +func emptyTransactionsList() any { + return struct { + List []any `json:"transactions"` + }{ + List: []any{}, + } +} + +func emptyFeesList() any { + return struct { + List []any `json:"fees"` + }{ + List: []any{}, + } +} + +func emptyTransfersList() any { + return struct { + List []any `json:"transfers"` + }{ + List: []any{}, + } +} + +func emptyAccountsList() any { + return struct { + List []any `json:"accounts"` + }{ + List: []any{}, + } +} diff --git a/api/vote.go b/api/vote.go index 33322b103..aeda3a862 100644 --- a/api/vote.go +++ b/api/vote.go @@ -218,16 +218,21 @@ func (a *API) votesListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) if err != nil { return err } - return a.sendVotesList(ctx, params) + + list, err := a.votesList(params) + if err != nil { + return err + } + + return marshalAndSend(ctx, list) } -// sendVotesList produces a filtered, paginated VotesList, -// and sends it marshalled over ctx.Send +// votesList produces a filtered, paginated VotesList. // // Errors returned are always of type APIerror. -func (a *API) sendVotesList(ctx *httprouter.HTTPContext, params *VoteParams) error { +func (a *API) votesList(params *VoteParams) (*VotesList, error) { if params.ElectionID != "" && !a.indexer.ProcessExists(params.ElectionID) { - return ErrElectionNotFound + return nil, ErrElectionNotFound } votes, total, err := a.indexer.VoteList( @@ -237,12 +242,12 @@ func (a *API) sendVotesList(ctx *httprouter.HTTPContext, params *VoteParams) err "", ) if err != nil { - return ErrIndexerQueryFailed.WithErr(err) + return nil, ErrIndexerQueryFailed.WithErr(err) } pagination, err := calculatePagination(params.Page, params.Limit, total) if err != nil { - return err + return nil, err } list := &VotesList{ @@ -259,7 +264,7 @@ func (a *API) sendVotesList(ctx *httprouter.HTTPContext, params *VoteParams) err TransactionIndex: &vote.TxIndex, }) } - return marshalAndSend(ctx, list) + return list, nil } // parseVoteParams returns an VoteParams filled with the passed params diff --git a/test/apierror_test.go b/test/apierror_test.go index 989f30962..659f8f4e7 100644 --- a/test/apierror_test.go +++ b/test/apierror_test.go @@ -120,10 +120,6 @@ func TestAPIerror(t *testing.T) { args: args{"GET", nil, []string{"elections", "page", "thisIsTotallyNotAnInt"}}, want: api.ErrCantParseNumber, }, - { - args: args{"GET", nil, []string{"elections", "page", "1"}}, - want: api.ErrPageNotFound, - }, { args: args{"GET", nil, []string{"elections", "page", "-1"}}, want: api.ErrPageNotFound, From 5bde6d9c4f585c9e2b7046133114a3bff84cd2fa Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Tue, 20 Aug 2024 12:17:19 +0200 Subject: [PATCH 2/5] api: new endpoint /chain/blocks accepts QueryParams: * GET /chain/blocks * page * limit * chainId * hash * proposerAddress * api: add structs BlockList, BlockParams * api: add consts ParamChainId, ParamHash, ParamProposerAddress * indexer: add indexertypes.Block --- api/api.go | 3 + api/api_types.go | 14 +++++ api/chain.go | 80 +++++++++++++++++++++++++++ vochain/indexer/indexertypes/block.go | 20 +++++++ 4 files changed, 117 insertions(+) create mode 100644 vochain/indexer/indexertypes/block.go diff --git a/api/api.go b/api/api.go index 6dcacc5f6..33fcba409 100644 --- a/api/api.go +++ b/api/api.go @@ -71,6 +71,9 @@ const ( ParamWithResults = "withResults" ParamFinalResults = "finalResults" ParamManuallyEnded = "manuallyEnded" + ParamChainId = "chainId" + ParamHash = "hash" + ParamProposerAddress = "proposerAddress" ParamHeight = "height" ParamReference = "reference" ParamType = "type" diff --git a/api/api_types.go b/api/api_types.go index d948b7207..efb390503 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -54,6 +54,14 @@ type TransactionParams struct { Type string `json:"type,omitempty"` } +// BlockParams allows the client to filter blocks +type BlockParams struct { + PaginationParams + ChainID string `json:"chainId,omitempty"` + Hash string `json:"hash,omitempty"` + ProposerAddress string `json:"proposerAddress,omitempty"` +} + // FeesParams allows the client to filter fees type FeesParams struct { PaginationParams @@ -439,3 +447,9 @@ type Block struct { comettypes.Block `json:",inline"` Hash types.HexBytes `json:"hash" ` } + +// BlockList is used to return a paginated list to the client +type BlockList struct { + Blocks []*indexertypes.Block `json:"blocks"` + Pagination *Pagination `json:"pagination"` +} diff --git a/api/chain.go b/api/chain.go index 13fd80127..79f0405d8 100644 --- a/api/chain.go +++ b/api/chain.go @@ -178,6 +178,14 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } + if err := a.Endpoint.RegisterMethod( + "/chain/blocks", + "GET", + apirest.MethodAccessTypePublic, + a.chainBlockListHandler, + ); err != nil { + return err + } if err := a.Endpoint.RegisterMethod( "/chain/organizations/filter/page/{page}", "POST", @@ -972,6 +980,63 @@ func (a *API) chainBlockByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo return ctx.Send(convertKeysToCamel(data), apirest.HTTPstatusOK) } +// chainBlockListHandler +// +// @Summary List all blocks +// @Description Returns the list of blocks, ordered by descending height. +// @Tags Chain +// @Accept json +// @Produce json +// @Param page query number false "Page" +// @Param limit query number false "Items per page" +// @Param chainId query string false "Filter by exact chainId" +// @Param hash query string false "Filter by partial hash" +// @Param proposerAddress query string false "Filter by exact proposerAddress" +// @Success 200 {object} BlockList +// @Router /chain/blocks [get] +func (a *API) chainBlockListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + params, err := parseBlockParams( + ctx.QueryParam(ParamPage), + ctx.QueryParam(ParamLimit), + ctx.QueryParam(ParamChainId), + ctx.QueryParam(ParamHash), + ctx.QueryParam(ParamProposerAddress), + ) + if err != nil { + return err + } + + return a.sendBlockList(ctx, params) +} + +// sendBlockList produces a filtered, paginated BlockList, +// and sends it marshalled over ctx.Send +// +// Errors returned are always of type APIerror. +func (a *API) sendBlockList(ctx *httprouter.HTTPContext, params *BlockParams) error { + blocks, total, err := a.indexer.BlockList( + params.Limit, + params.Page*params.Limit, + params.ChainID, + params.Hash, + params.ProposerAddress, + ) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) + } + + pagination, err := calculatePagination(params.Page, params.Limit, total) + if err != nil { + return err + } + + list := &BlockList{ + Blocks: blocks, + Pagination: pagination, + } + return marshalAndSend(ctx, list) +} + // chainTransactionCountHandler // // @Summary Transactions count @@ -1320,3 +1385,18 @@ func parseTransactionParams(paramPage, paramLimit, paramHeight, paramType string Type: paramType, }, nil } + +// parseBlockParams returns an BlockParams filled with the passed params +func parseBlockParams(paramPage, paramLimit, paramChainId, paramHash, paramProposerAddress string) (*BlockParams, error) { + pagination, err := parsePaginationParams(paramPage, paramLimit) + if err != nil { + return nil, err + } + + return &BlockParams{ + PaginationParams: pagination, + ChainID: paramChainId, + Hash: util.TrimHex(paramHash), + ProposerAddress: util.TrimHex(paramProposerAddress), + }, nil +} diff --git a/vochain/indexer/indexertypes/block.go b/vochain/indexer/indexertypes/block.go new file mode 100644 index 000000000..4954dbc22 --- /dev/null +++ b/vochain/indexer/indexertypes/block.go @@ -0,0 +1,20 @@ +package indexertypes + +import ( + "time" + + "go.vocdoni.io/dvote/types" +) + +// Block represents a block handled by the Vochain. +// The indexer Block data type is different from the vochain state data type +// since it is optimized for querying purposes and not for keeping a shared consensus state. +type Block struct { + ChainID string `json:"chainId"` + Height int64 `json:"height"` + Time time.Time `json:"time"` + Hash types.HexBytes `json:"hash"` + ProposerAddress types.HexBytes `json:"proposer"` + LastBlockHash types.HexBytes `json:"lastBlockHash"` + TxCount int64 `json:"txCount"` +} From 50c3f988683cca6a360de4da043af8a15e312e80 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Fri, 23 Aug 2024 08:46:31 +0200 Subject: [PATCH 3/5] api: endpoint /chain/blocks fetches from a.vocapp.GetBlockByHeight --- api/chain.go | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/api/chain.go b/api/chain.go index 79f0405d8..7907d5d99 100644 --- a/api/chain.go +++ b/api/chain.go @@ -18,6 +18,7 @@ import ( "go.vocdoni.io/dvote/vochain" "go.vocdoni.io/dvote/vochain/genesis" "go.vocdoni.io/dvote/vochain/indexer" + "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/state" ) @@ -1014,7 +1015,39 @@ func (a *API) chainBlockListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCont // // Errors returned are always of type APIerror. func (a *API) sendBlockList(ctx *httprouter.HTTPContext, params *BlockParams) error { - blocks, total, err := a.indexer.BlockList( + // TODO: replace this by a.indexer.BlockList when it's available + blockList := func(limit, offset int, _, _, _ string) ([]*indexertypes.Block, uint64, error) { + if offset < 0 { + return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) + } + if limit <= 0 { + return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) + } + height := a.vocapp.Height() + total := uint64(height) - uint64(a.vocapp.Node.BlockStore().Base()) + start := height - uint32(params.Page*params.Limit) + end := start - uint32(params.Limit) + list := []*indexertypes.Block{} + for h := start; h > end; h-- { + tmblock := a.vocapp.GetBlockByHeight(int64(h)) + if tmblock == nil { + break + } + list = append(list, &indexertypes.Block{ + ChainID: tmblock.ChainID, + Height: tmblock.Height, + Time: tmblock.Time, + Hash: types.HexBytes(tmblock.Hash()), + ProposerAddress: tmblock.ProposerAddress.Bytes(), + LastBlockHash: tmblock.LastBlockID.Hash.Bytes(), + TxCount: int64(len(tmblock.Txs)), + }) + } + + return list, uint64(total), nil + } + + blocks, total, err := blockList( params.Limit, params.Page*params.Limit, params.ChainID, From ff4aaa1632e931247f3eae38c9032df3029142b1 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 10 Jun 2024 11:18:11 +0200 Subject: [PATCH 4/5] indexer: CreateBlock during Commit instead of OnBeginBlock since the CreateBlock now fetches the whole block, we need app.NodeClient initialized in all nodes (including seeds), else seeds panic on first Commit --- vochain/app.go | 4 ---- vochain/appsetup.go | 3 --- vochain/indexer/block.go | 17 ----------------- vochain/indexer/indexer.go | 11 +++++++++++ vochain/keykeeper/keykeeper.go | 3 --- .../offchaindatahandler/offchaindatahandler.go | 1 - vochain/state/eventlistener.go | 15 --------------- vochain/state/state_test.go | 1 - 8 files changed, 11 insertions(+), 44 deletions(-) diff --git a/vochain/app.go b/vochain/app.go index 7e0107537..f6691be61 100644 --- a/vochain/app.go +++ b/vochain/app.go @@ -290,10 +290,6 @@ func (app *BaseApplication) beginBlock(t time.Time, height uint32) { app.State.SetHeight(height) go app.State.CachePurge(height) - app.State.OnBeginBlock(vstate.BeginBlock{ - Height: int64(height), - Time: t, - }) } // endBlock is called at the end of every block. diff --git a/vochain/appsetup.go b/vochain/appsetup.go index fb52f66ed..6726b8b21 100644 --- a/vochain/appsetup.go +++ b/vochain/appsetup.go @@ -25,9 +25,6 @@ func (app *BaseApplication) SetNode(vochaincfg *config.VochainCfg) error { if app.Node, err = newTendermint(app, vochaincfg); err != nil { return fmt.Errorf("could not set tendermint node service: %s", err) } - if vochaincfg.IsSeedNode { - return nil - } // Note that cometcli.New logs any error rather than returning it. app.NodeClient = cometcli.New(app.Node) return nil diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index e40a5fdfb..8c9d76a22 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -6,28 +6,11 @@ import ( "errors" "fmt" "time" - - "go.vocdoni.io/dvote/log" - indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" - "go.vocdoni.io/dvote/vochain/state" ) // ErrBlockNotFound is returned if the block is not found in the indexer database. var ErrBlockNotFound = fmt.Errorf("block not found") -func (idx *Indexer) OnBeginBlock(bb state.BeginBlock) { - idx.blockMu.Lock() - defer idx.blockMu.Unlock() - queries := idx.blockTxQueries() - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - Height: bb.Height, - Time: bb.Time, - DataHash: nonNullBytes(bb.DataHash), - }); err != nil { - log.Errorw(err, "cannot index new block") - } -} - // BlockTimestamp returns the timestamp of the block at the given height func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 43b59323e..769b76720 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -414,6 +414,17 @@ func (idx *Indexer) Commit(height uint32) error { queries := idx.blockTxQueries() ctx := context.TODO() + // index the new block + bb := idx.App.GetBlockByHeight(int64(height)) + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + Height: bb.Height, + Time: bb.Time, + DataHash: nonNullBytes(bb.DataHash), + // TODO: ProposerAddress, Hash, AppHash, ChainID, LastBlockHash?, ValidatorSignatures? + }); err != nil { + log.Errorw(err, "cannot index new block") + } + for _, pidStr := range updateProcs { pid := types.ProcessID(pidStr) if err := idx.updateProcess(ctx, queries, pid); err != nil { diff --git a/vochain/keykeeper/keykeeper.go b/vochain/keykeeper/keykeeper.go index 7f46f654d..4a4425cd1 100644 --- a/vochain/keykeeper/keykeeper.go +++ b/vochain/keykeeper/keykeeper.go @@ -268,9 +268,6 @@ func (*KeyKeeper) OnVote(_ *state.Vote, _ int32) {} // OnNewTx is not used by the KeyKeeper func (*KeyKeeper) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -// OnBeginBlock is not used by the KeyKeeper -func (*KeyKeeper) OnBeginBlock(_ state.BeginBlock) {} - // OnCensusUpdate is not used by the KeyKeeper func (*KeyKeeper) OnCensusUpdate(_, _ []byte, _ string, _ uint64) {} diff --git a/vochain/offchaindatahandler/offchaindatahandler.go b/vochain/offchaindatahandler/offchaindatahandler.go index d6e97bd9b..4bc3be6b4 100644 --- a/vochain/offchaindatahandler/offchaindatahandler.go +++ b/vochain/offchaindatahandler/offchaindatahandler.go @@ -166,7 +166,6 @@ func (d *OffChainDataHandler) OnSetAccount(_ []byte, account *state.Account) { func (*OffChainDataHandler) OnCancel(_ []byte, _ int32) {} func (*OffChainDataHandler) OnVote(_ *state.Vote, _ int32) {} func (*OffChainDataHandler) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -func (*OffChainDataHandler) OnBeginBlock(state.BeginBlock) {} func (*OffChainDataHandler) OnProcessKeys(_ []byte, _ string, _ int32) {} func (*OffChainDataHandler) OnRevealKeys(_ []byte, _ string, _ int32) {} func (*OffChainDataHandler) OnProcessStatusChange(_ []byte, _ models.ProcessStatus, _ int32) {} diff --git a/vochain/state/eventlistener.go b/vochain/state/eventlistener.go index 1c2d05281..7b599702b 100644 --- a/vochain/state/eventlistener.go +++ b/vochain/state/eventlistener.go @@ -1,8 +1,6 @@ package state import ( - "time" - "go.vocdoni.io/dvote/vochain/transaction/vochaintx" "go.vocdoni.io/proto/build/go/models" ) @@ -32,7 +30,6 @@ type EventListener interface { OnSpendTokens(addr []byte, txType models.TxType, cost uint64, reference string) OnCensusUpdate(pid, censusRoot []byte, censusURI string, censusSize uint64) Commit(height uint32) (err error) - OnBeginBlock(BeginBlock) Rollback() } @@ -46,15 +43,3 @@ func (v *State) AddEventListener(l EventListener) { func (v *State) CleanEventListeners() { v.eventListeners = nil } - -type BeginBlock struct { - Height int64 - Time time.Time - DataHash []byte -} - -func (v *State) OnBeginBlock(bb BeginBlock) { - for _, l := range v.eventListeners { - l.OnBeginBlock(bb) - } -} diff --git a/vochain/state/state_test.go b/vochain/state/state_test.go index e2b76c685..b36dc7722 100644 --- a/vochain/state/state_test.go +++ b/vochain/state/state_test.go @@ -182,7 +182,6 @@ type Listener struct { func (*Listener) OnVote(_ *Vote, _ int32) {} func (*Listener) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -func (*Listener) OnBeginBlock(BeginBlock) {} func (*Listener) OnProcess(_ *models.Process, _ int32) {} func (*Listener) OnProcessStatusChange(_ []byte, _ models.ProcessStatus, _ int32) {} func (*Listener) OnProcessDurationChange(_ []byte, _ uint32, _ int32) {} From e27cf8ab7424ab55685005a3be393d537c551cb4 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 10 Jun 2024 11:18:11 +0200 Subject: [PATCH 5/5] indexer: index more details about blocks and transactions * indexerdb: add block details chain_id, proposer_address and last_block_hash * indexerdb: rename method GetBlock -> GetBlockByHeight * indexerdb: new method GetBlockByHash * indexerdb: new method SearchBlocks * indexer: new method BlockByHeight * indexer: new method SearchBlocks transactions: * indexerdb: add raw_tx and signature in transactions table * indexer: add CountTransactionsByHeight and SearchTransactions * indexer: rename GetTransaction* methods -> GetTxMetadata* --- api/api_types.go | 4 +- api/chain.go | 10 +- vochain/indexer/bench_test.go | 13 +- vochain/indexer/block.go | 62 +++++- vochain/indexer/db/blocks.sql.go | 141 +++++++++++- vochain/indexer/db/db.go | 202 ++++++++++-------- vochain/indexer/db/models.go | 11 +- vochain/indexer/db/transactions.sql.go | 48 ++++- vochain/indexer/indexer.go | 19 +- vochain/indexer/indexer_test.go | 6 +- vochain/indexer/indexertypes/block.go | 26 +++ vochain/indexer/indexertypes/types.go | 42 ++-- .../0013_alter_columns_table_blocks.sql | 13 ++ .../0014_alter_columns_table_transactions.sql | 7 + vochain/indexer/queries/blocks.sql | 35 ++- vochain/indexer/queries/transactions.sql | 10 +- vochain/indexer/transaction.go | 51 +++-- 17 files changed, 528 insertions(+), 172 deletions(-) create mode 100644 vochain/indexer/migrations/0013_alter_columns_table_blocks.sql create mode 100644 vochain/indexer/migrations/0014_alter_columns_table_transactions.sql diff --git a/api/api_types.go b/api/api_types.go index efb390503..b1169db8d 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -275,8 +275,8 @@ type TransactionReference struct { // TransactionsList is used to return a paginated list to the client type TransactionsList struct { - Transactions []*indexertypes.Transaction `json:"transactions"` - Pagination *Pagination `json:"pagination"` + Transactions []*indexertypes.TransactionMetadata `json:"transactions"` + Pagination *Pagination `json:"pagination"` } // FeesList is used to return a paginated list to the client diff --git a/api/chain.go b/api/chain.go index 7907d5d99..e638de421 100644 --- a/api/chain.go +++ b/api/chain.go @@ -654,7 +654,7 @@ func (a *API) chainTxRefByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - ref, err := a.indexer.GetTxHashReference(hash) + ref, err := a.indexer.GetTxMetadataByHash(hash) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound @@ -698,7 +698,7 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ErrVochainGetTxFailed.WithErr(err) } - ref, err := a.indexer.GetTxReferenceByBlockHeightAndBlockIndex(height, index) + ref, err := a.indexer.GetTxByBlockHeightAndBlockIndex(height, index) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound @@ -719,8 +719,8 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er // chainTxRefByIndexHandler // -// @Summary Transaction by index -// @Description Get transaction by its index. This is not transaction reference (hash), and neither the block height and block index. The transaction index is an incremental counter for each transaction. You could use the transaction `block` and `index` to retrieve full info using [transaction by block and index](transaction-by-block-index). +// @Summary Transaction metadata (by db index) +// @Description Get transaction by its internal index. This is not the transaction hash, and neither the block height and block index. The transaction index is an incremental counter for each transaction. You could use the transaction `block` and `index` to retrieve full info using [transaction by block and index](transaction-by-block-index). // @Tags Chain // @Accept json // @Produce json @@ -733,7 +733,7 @@ func (a *API) chainTxRefByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPC if err != nil { return err } - ref, err := a.indexer.GetTransaction(index) + ref, err := a.indexer.GetTxMetadataByID(index) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 631e83205..f73645bd9 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -85,6 +85,7 @@ func BenchmarkIndexer(b *testing.B) { tx := &vochaintx.Tx{ TxID: rnd.Random32(), TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, } idx.OnNewTx(tx, height, txBlockIndex) curTxs = append(curTxs, tx) @@ -112,7 +113,7 @@ func BenchmarkIndexer(b *testing.B) { qt.Check(b, bytes.Equal(voteRef.Meta.TxHash, tx.TxID[:]), qt.IsTrue) } - txRef, err := idx.GetTxHashReference(tx.TxID[:]) + txRef, err := idx.GetTxMetadataByHash(tx.TxID[:]) qt.Check(b, err, qt.IsNil) if err == nil { qt.Check(b, txRef.BlockHeight, qt.Equals, vote.Height) @@ -138,7 +139,11 @@ func BenchmarkFetchTx(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { for j := 0; j < numTxs; j++ { - idx.OnNewTx(&vochaintx.Tx{TxID: util.Random32()}, uint32(i), int32(j)) + idx.OnNewTx(&vochaintx.Tx{ + TxID: util.Random32(), + TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, + }, uint32(i), int32(j)) } err := idx.Commit(uint32(i)) qt.Assert(b, err, qt.IsNil) @@ -147,14 +152,14 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTransaction(uint64((i * numTxs) + j + 1)) + _, err = idx.GetTxMetadataByID(uint64((i * numTxs) + j + 1)) qt.Assert(b, err, qt.IsNil) } log.Infof("fetched %d transactions (out of %d total) by index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTxHashReference([]byte(fmt.Sprintf("hash%d%d", i, j))) + _, err = idx.GetTxMetadataByHash([]byte(fmt.Sprintf("hash%d%d", i, j))) qt.Assert(b, err, qt.IsNil) } log.Infof("fetched %d transactions (out of %d total) by hash, took %s", diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index 8c9d76a22..5d2118ef4 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -6,6 +6,9 @@ import ( "errors" "fmt" "time" + + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" + "go.vocdoni.io/dvote/vochain/indexer/indexertypes" ) // ErrBlockNotFound is returned if the block is not found in the indexer database. @@ -13,12 +16,63 @@ var ErrBlockNotFound = fmt.Errorf("block not found") // BlockTimestamp returns the timestamp of the block at the given height func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { - block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) + block, err := idx.BlockByHeight(height) if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return time.Time{}, ErrBlockNotFound - } return time.Time{}, err } return block.Time, nil } + +// BlockByHeight returns the available information of the block at the given height +func (idx *Indexer) BlockByHeight(height int64) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), height) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrBlockNotFound + } + return nil, err + } + return indexertypes.BlockFromDB(&block), nil +} + +// BlockByHash returns the available information of the block with the given hash +func (idx *Indexer) BlockByHash(hash []byte) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHash(context.TODO(), hash) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrBlockNotFound + } + return nil, err + } + return indexertypes.BlockFromDB(&block), nil +} + +// BlockList returns the list of blocks indexed. +// chainID, hash, proposerAddress are optional, if declared as zero-value will be ignored. +// The first one returned is the newest, so they are in descending order. +func (idx *Indexer) BlockList(limit, offset int, chainID, hash, proposerAddress string) ([]*indexertypes.Block, uint64, error) { + if offset < 0 { + return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) + } + if limit <= 0 { + return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) + } + results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{ + Limit: int64(limit), + Offset: int64(offset), + ChainID: chainID, + HashSubstr: hash, + ProposerAddress: proposerAddress, + }) + if err != nil { + return nil, 0, err + } + list := []*indexertypes.Block{} + for _, row := range results { + list = append(list, indexertypes.BlockFromDBRow(&row)) + } + if len(results) == 0 { + return list, 0, nil + } + return list, uint64(results[0].TotalCount), nil +} diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index ffa3fa896..01e6ba54a 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -13,31 +13,150 @@ import ( const createBlock = `-- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? + ?, ?, ?, ?, ?, ? ) ` type CreateBlockParams struct { - Height int64 - Time time.Time - DataHash []byte + ChainID string + Height int64 + Time time.Time + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } func (q *Queries) CreateBlock(ctx context.Context, arg CreateBlockParams) (sql.Result, error) { - return q.exec(ctx, q.createBlockStmt, createBlock, arg.Height, arg.Time, arg.DataHash) + return q.exec(ctx, q.createBlockStmt, createBlock, + arg.ChainID, + arg.Height, + arg.Time, + arg.Hash, + arg.ProposerAddress, + arg.LastBlockHash, + ) } -const getBlock = `-- name: GetBlock :one -SELECT height, time, data_hash FROM blocks +const getBlockByHash = `-- name: GetBlockByHash :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks +WHERE hash = ? +LIMIT 1 +` + +func (q *Queries) GetBlockByHash(ctx context.Context, hash []byte) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHashStmt, getBlockByHash, hash) + var i Block + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) + return i, err +} + +const getBlockByHeight = `-- name: GetBlockByHeight :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks WHERE height = ? LIMIT 1 ` -func (q *Queries) GetBlock(ctx context.Context, height int64) (Block, error) { - row := q.queryRow(ctx, q.getBlockStmt, getBlock, height) +func (q *Queries) GetBlockByHeight(ctx context.Context, height int64) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHeightStmt, getBlockByHeight, height) var i Block - err := row.Scan(&i.Height, &i.Time, &i.DataHash) + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) return i, err } + +const searchBlocks = `-- name: SearchBlocks :many +SELECT + b.height, b.time, b.chain_id, b.hash, b.proposer_address, b.last_block_hash, + COUNT(t.id) AS tx_count, + COUNT(*) OVER() AS total_count +FROM blocks AS b +LEFT JOIN transactions AS t + ON b.height = t.block_height +WHERE ( + (?1 = '' OR b.chain_id = ?1) + AND LENGTH(?2) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?2 = '' + OR (LENGTH(?2) = 64 AND LOWER(HEX(b.hash)) = LOWER(?2)) + OR (LENGTH(?2) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(?2)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (?3 = '' OR LOWER(HEX(b.proposer_address)) = LOWER(?3)) +) +GROUP BY b.height +ORDER BY b.height DESC +LIMIT ?5 +OFFSET ?4 +` + +type SearchBlocksParams struct { + ChainID interface{} + HashSubstr interface{} + ProposerAddress interface{} + Offset int64 + Limit int64 +} + +type SearchBlocksRow struct { + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte + TxCount int64 + TotalCount int64 +} + +func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]SearchBlocksRow, error) { + rows, err := q.query(ctx, q.searchBlocksStmt, searchBlocks, + arg.ChainID, + arg.HashSubstr, + arg.ProposerAddress, + arg.Offset, + arg.Limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SearchBlocksRow + for rows.Next() { + var i SearchBlocksRow + if err := rows.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + &i.TxCount, + &i.TotalCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 3695e8b73..786977651 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -36,6 +36,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.countTransactionsStmt, err = db.PrepareContext(ctx, countTransactions); err != nil { return nil, fmt.Errorf("error preparing query CountTransactions: %w", err) } + if q.countTransactionsByHeightStmt, err = db.PrepareContext(ctx, countTransactionsByHeight); err != nil { + return nil, fmt.Errorf("error preparing query CountTransactionsByHeight: %w", err) + } if q.countVotesStmt, err = db.PrepareContext(ctx, countVotes); err != nil { return nil, fmt.Errorf("error preparing query CountVotes: %w", err) } @@ -60,8 +63,11 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.createVoteStmt, err = db.PrepareContext(ctx, createVote); err != nil { return nil, fmt.Errorf("error preparing query CreateVote: %w", err) } - if q.getBlockStmt, err = db.PrepareContext(ctx, getBlock); err != nil { - return nil, fmt.Errorf("error preparing query GetBlock: %w", err) + if q.getBlockByHashStmt, err = db.PrepareContext(ctx, getBlockByHash); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHash: %w", err) + } + if q.getBlockByHeightStmt, err = db.PrepareContext(ctx, getBlockByHeight); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHeight: %w", err) } if q.getEntityCountStmt, err = db.PrepareContext(ctx, getEntityCount); err != nil { return nil, fmt.Errorf("error preparing query GetEntityCount: %w", err) @@ -87,8 +93,8 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } - if q.getTxReferenceByBlockHeightAndBlockIndexStmt, err = db.PrepareContext(ctx, getTxReferenceByBlockHeightAndBlockIndex); err != nil { - return nil, fmt.Errorf("error preparing query GetTxReferenceByBlockHeightAndBlockIndex: %w", err) + if q.getTransactionByHeightAndIndexStmt, err = db.PrepareContext(ctx, getTransactionByHeightAndIndex); err != nil { + return nil, fmt.Errorf("error preparing query GetTransactionByHeightAndIndex: %w", err) } if q.getVoteStmt, err = db.PrepareContext(ctx, getVote); err != nil { return nil, fmt.Errorf("error preparing query GetVote: %w", err) @@ -96,6 +102,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.searchAccountsStmt, err = db.PrepareContext(ctx, searchAccounts); err != nil { return nil, fmt.Errorf("error preparing query SearchAccounts: %w", err) } + if q.searchBlocksStmt, err = db.PrepareContext(ctx, searchBlocks); err != nil { + return nil, fmt.Errorf("error preparing query SearchBlocks: %w", err) + } if q.searchEntitiesStmt, err = db.PrepareContext(ctx, searchEntities); err != nil { return nil, fmt.Errorf("error preparing query SearchEntities: %w", err) } @@ -157,6 +166,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing countTransactionsStmt: %w", cerr) } } + if q.countTransactionsByHeightStmt != nil { + if cerr := q.countTransactionsByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countTransactionsByHeightStmt: %w", cerr) + } + } if q.countVotesStmt != nil { if cerr := q.countVotesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing countVotesStmt: %w", cerr) @@ -197,9 +211,14 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing createVoteStmt: %w", cerr) } } - if q.getBlockStmt != nil { - if cerr := q.getBlockStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getBlockStmt: %w", cerr) + if q.getBlockByHashStmt != nil { + if cerr := q.getBlockByHashStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHashStmt: %w", cerr) + } + } + if q.getBlockByHeightStmt != nil { + if cerr := q.getBlockByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHeightStmt: %w", cerr) } } if q.getEntityCountStmt != nil { @@ -242,9 +261,9 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) } } - if q.getTxReferenceByBlockHeightAndBlockIndexStmt != nil { - if cerr := q.getTxReferenceByBlockHeightAndBlockIndexStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTxReferenceByBlockHeightAndBlockIndexStmt: %w", cerr) + if q.getTransactionByHeightAndIndexStmt != nil { + if cerr := q.getTransactionByHeightAndIndexStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getTransactionByHeightAndIndexStmt: %w", cerr) } } if q.getVoteStmt != nil { @@ -257,6 +276,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing searchAccountsStmt: %w", cerr) } } + if q.searchBlocksStmt != nil { + if cerr := q.searchBlocksStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing searchBlocksStmt: %w", cerr) + } + } if q.searchEntitiesStmt != nil { if cerr := q.searchEntitiesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing searchEntitiesStmt: %w", cerr) @@ -354,85 +378,91 @@ func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, ar } type Queries struct { - db DBTX - tx *sql.Tx - computeProcessVoteCountStmt *sql.Stmt - countAccountsStmt *sql.Stmt - countTokenTransfersByAccountStmt *sql.Stmt - countTransactionsStmt *sql.Stmt - countVotesStmt *sql.Stmt - createAccountStmt *sql.Stmt - createBlockStmt *sql.Stmt - createProcessStmt *sql.Stmt - createTokenFeeStmt *sql.Stmt - createTokenTransferStmt *sql.Stmt - createTransactionStmt *sql.Stmt - createVoteStmt *sql.Stmt - getBlockStmt *sql.Stmt - getEntityCountStmt *sql.Stmt - getProcessStmt *sql.Stmt - getProcessCountStmt *sql.Stmt - getProcessIDsByFinalResultsStmt *sql.Stmt - getProcessStatusStmt *sql.Stmt - getTokenTransferStmt *sql.Stmt - getTransactionStmt *sql.Stmt - getTransactionByHashStmt *sql.Stmt - getTxReferenceByBlockHeightAndBlockIndexStmt *sql.Stmt - getVoteStmt *sql.Stmt - searchAccountsStmt *sql.Stmt - searchEntitiesStmt *sql.Stmt - searchProcessesStmt *sql.Stmt - searchTokenFeesStmt *sql.Stmt - searchTokenTransfersStmt *sql.Stmt - searchTransactionsStmt *sql.Stmt - searchVotesStmt *sql.Stmt - setProcessResultsCancelledStmt *sql.Stmt - setProcessResultsReadyStmt *sql.Stmt - updateProcessEndDateStmt *sql.Stmt - updateProcessFromStateStmt *sql.Stmt - updateProcessResultByIDStmt *sql.Stmt - updateProcessResultsStmt *sql.Stmt + db DBTX + tx *sql.Tx + computeProcessVoteCountStmt *sql.Stmt + countAccountsStmt *sql.Stmt + countTokenTransfersByAccountStmt *sql.Stmt + countTransactionsStmt *sql.Stmt + countTransactionsByHeightStmt *sql.Stmt + countVotesStmt *sql.Stmt + createAccountStmt *sql.Stmt + createBlockStmt *sql.Stmt + createProcessStmt *sql.Stmt + createTokenFeeStmt *sql.Stmt + createTokenTransferStmt *sql.Stmt + createTransactionStmt *sql.Stmt + createVoteStmt *sql.Stmt + getBlockByHashStmt *sql.Stmt + getBlockByHeightStmt *sql.Stmt + getEntityCountStmt *sql.Stmt + getProcessStmt *sql.Stmt + getProcessCountStmt *sql.Stmt + getProcessIDsByFinalResultsStmt *sql.Stmt + getProcessStatusStmt *sql.Stmt + getTokenTransferStmt *sql.Stmt + getTransactionStmt *sql.Stmt + getTransactionByHashStmt *sql.Stmt + getTransactionByHeightAndIndexStmt *sql.Stmt + getVoteStmt *sql.Stmt + searchAccountsStmt *sql.Stmt + searchBlocksStmt *sql.Stmt + searchEntitiesStmt *sql.Stmt + searchProcessesStmt *sql.Stmt + searchTokenFeesStmt *sql.Stmt + searchTokenTransfersStmt *sql.Stmt + searchTransactionsStmt *sql.Stmt + searchVotesStmt *sql.Stmt + setProcessResultsCancelledStmt *sql.Stmt + setProcessResultsReadyStmt *sql.Stmt + updateProcessEndDateStmt *sql.Stmt + updateProcessFromStateStmt *sql.Stmt + updateProcessResultByIDStmt *sql.Stmt + updateProcessResultsStmt *sql.Stmt } func (q *Queries) WithTx(tx *sql.Tx) *Queries { return &Queries{ - db: tx, - tx: tx, - computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, - countAccountsStmt: q.countAccountsStmt, - countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, - countTransactionsStmt: q.countTransactionsStmt, - countVotesStmt: q.countVotesStmt, - createAccountStmt: q.createAccountStmt, - createBlockStmt: q.createBlockStmt, - createProcessStmt: q.createProcessStmt, - createTokenFeeStmt: q.createTokenFeeStmt, - createTokenTransferStmt: q.createTokenTransferStmt, - createTransactionStmt: q.createTransactionStmt, - createVoteStmt: q.createVoteStmt, - getBlockStmt: q.getBlockStmt, - getEntityCountStmt: q.getEntityCountStmt, - getProcessStmt: q.getProcessStmt, - getProcessCountStmt: q.getProcessCountStmt, - getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, - getProcessStatusStmt: q.getProcessStatusStmt, - getTokenTransferStmt: q.getTokenTransferStmt, - getTransactionStmt: q.getTransactionStmt, - getTransactionByHashStmt: q.getTransactionByHashStmt, - getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, - getVoteStmt: q.getVoteStmt, - searchAccountsStmt: q.searchAccountsStmt, - searchEntitiesStmt: q.searchEntitiesStmt, - searchProcessesStmt: q.searchProcessesStmt, - searchTokenFeesStmt: q.searchTokenFeesStmt, - searchTokenTransfersStmt: q.searchTokenTransfersStmt, - searchTransactionsStmt: q.searchTransactionsStmt, - searchVotesStmt: q.searchVotesStmt, - setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, - setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, - updateProcessEndDateStmt: q.updateProcessEndDateStmt, - updateProcessFromStateStmt: q.updateProcessFromStateStmt, - updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, - updateProcessResultsStmt: q.updateProcessResultsStmt, + db: tx, + tx: tx, + computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, + countAccountsStmt: q.countAccountsStmt, + countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, + countTransactionsStmt: q.countTransactionsStmt, + countTransactionsByHeightStmt: q.countTransactionsByHeightStmt, + countVotesStmt: q.countVotesStmt, + createAccountStmt: q.createAccountStmt, + createBlockStmt: q.createBlockStmt, + createProcessStmt: q.createProcessStmt, + createTokenFeeStmt: q.createTokenFeeStmt, + createTokenTransferStmt: q.createTokenTransferStmt, + createTransactionStmt: q.createTransactionStmt, + createVoteStmt: q.createVoteStmt, + getBlockByHashStmt: q.getBlockByHashStmt, + getBlockByHeightStmt: q.getBlockByHeightStmt, + getEntityCountStmt: q.getEntityCountStmt, + getProcessStmt: q.getProcessStmt, + getProcessCountStmt: q.getProcessCountStmt, + getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, + getProcessStatusStmt: q.getProcessStatusStmt, + getTokenTransferStmt: q.getTokenTransferStmt, + getTransactionStmt: q.getTransactionStmt, + getTransactionByHashStmt: q.getTransactionByHashStmt, + getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, + getVoteStmt: q.getVoteStmt, + searchAccountsStmt: q.searchAccountsStmt, + searchBlocksStmt: q.searchBlocksStmt, + searchEntitiesStmt: q.searchEntitiesStmt, + searchProcessesStmt: q.searchProcessesStmt, + searchTokenFeesStmt: q.searchTokenFeesStmt, + searchTokenTransfersStmt: q.searchTokenTransfersStmt, + searchTransactionsStmt: q.searchTransactionsStmt, + searchVotesStmt: q.searchVotesStmt, + setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, + setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, + updateProcessEndDateStmt: q.updateProcessEndDateStmt, + updateProcessFromStateStmt: q.updateProcessFromStateStmt, + updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, + updateProcessResultsStmt: q.updateProcessResultsStmt, } } diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index 08a6e0e2b..db1778b1c 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -11,9 +11,12 @@ import ( ) type Block struct { - Height int64 - Time time.Time - DataHash []byte + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } type Process struct { @@ -62,4 +65,6 @@ type Transaction struct { BlockHeight int64 BlockIndex int64 Type string + RawTx []byte + Signature []byte } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index 2d06f135a..a38c1a81a 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -23,11 +23,23 @@ func (q *Queries) CountTransactions(ctx context.Context) (int64, error) { return count, err } +const countTransactionsByHeight = `-- name: CountTransactionsByHeight :one +SELECT COUNT(*) FROM transactions +WHERE block_height = ? +` + +func (q *Queries) CountTransactionsByHeight(ctx context.Context, blockHeight int64) (int64, error) { + row := q.queryRow(ctx, q.countTransactionsByHeightStmt, countTransactionsByHeight, blockHeight) + var count int64 + err := row.Scan(&count) + return count, err +} + const createTransaction = `-- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, raw_tx, signature ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ?, ? ) ` @@ -36,6 +48,8 @@ type CreateTransactionParams struct { BlockHeight int64 BlockIndex int64 Type string + RawTx []byte + Signature []byte } func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (sql.Result, error) { @@ -44,11 +58,13 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa arg.BlockHeight, arg.BlockIndex, arg.Type, + arg.RawTx, + arg.Signature, ) } const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE id = ? LIMIT 1 ` @@ -62,12 +78,14 @@ func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, er &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.RawTx, + &i.Signature, ) return i, err } const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE hash = ? LIMIT 1 ` @@ -81,23 +99,25 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.RawTx, + &i.Signature, ) return i, err } -const getTxReferenceByBlockHeightAndBlockIndex = `-- name: GetTxReferenceByBlockHeightAndBlockIndex :one -SELECT id, hash, block_height, block_index, type FROM transactions +const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one +SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` -type GetTxReferenceByBlockHeightAndBlockIndexParams struct { +type GetTransactionByHeightAndIndexParams struct { BlockHeight int64 BlockIndex int64 } -func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, arg GetTxReferenceByBlockHeightAndBlockIndexParams) (Transaction, error) { - row := q.queryRow(ctx, q.getTxReferenceByBlockHeightAndBlockIndexStmt, getTxReferenceByBlockHeightAndBlockIndex, arg.BlockHeight, arg.BlockIndex) +func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTransactionByHeightAndIndexParams) (Transaction, error) { + row := q.queryRow(ctx, q.getTransactionByHeightAndIndexStmt, getTransactionByHeightAndIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( &i.ID, @@ -105,20 +125,22 @@ func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.RawTx, + &i.Signature, ) return i, err } const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT id, hash, block_height, block_index, type + SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) AND (?4 = '' OR LOWER(type) = LOWER(?4)) ) ) -SELECT id, hash, block_height, block_index, type, COUNT(*) OVER() AS total_count +SELECT id, hash, block_height, block_index, type, raw_tx, signature, COUNT(*) OVER() AS total_count FROM results ORDER BY id DESC LIMIT ?2 @@ -138,6 +160,8 @@ type SearchTransactionsRow struct { BlockHeight int64 BlockIndex int64 Type string + RawTx []byte + Signature []byte TotalCount int64 } @@ -161,6 +185,8 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.RawTx, + &i.Signature, &i.TotalCount, ); err != nil { return nil, err diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 769b76720..d66ce1f07 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -415,14 +415,17 @@ func (idx *Indexer) Commit(height uint32) error { ctx := context.TODO() // index the new block - bb := idx.App.GetBlockByHeight(int64(height)) - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - Height: bb.Height, - Time: bb.Time, - DataHash: nonNullBytes(bb.DataHash), - // TODO: ProposerAddress, Hash, AppHash, ChainID, LastBlockHash?, ValidatorSignatures? - }); err != nil { - log.Errorw(err, "cannot index new block") + if b := idx.App.GetBlockByHeight(int64(height)); b != nil { + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: b.ChainID, + Height: b.Height, + Time: b.Time, + Hash: nonNullBytes(b.Hash()), + ProposerAddress: nonNullBytes(b.ProposerAddress), + LastBlockHash: nonNullBytes(b.LastBlockID.Hash), + }); err != nil { + log.Errorw(err, "cannot index new block") + } } for _, pidStr := range updateProcs { diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index fad19d248..b780f3b87 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1392,6 +1392,8 @@ func TestTxIndexer(t *testing.T) { idx.OnNewTx(&vochaintx.Tx{ TxID: getTxID(i, j), TxModelType: "setAccount", + Tx: &models.Tx{Payload: &models.Tx_SetAccount{}}, + Signature: []byte("cafe"), }, uint32(i), int32(j)) } } @@ -1404,7 +1406,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTransaction(uint64(i*txsPerBlock + j + 1)) + ref, err := idx.GetTxMetadataByID(uint64(i*txsPerBlock + j + 1)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1412,7 +1414,7 @@ func TestTxIndexer(t *testing.T) { h := make([]byte, 32) id := getTxID(i, j) copy(h, id[:]) - hashRef, err := idx.GetTxHashReference(h) + hashRef, err := idx.GetTxMetadataByHash(h) qt.Assert(t, err, qt.IsNil) qt.Assert(t, hashRef.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, hashRef.TxBlockIndex, qt.Equals, int32(j)) diff --git a/vochain/indexer/indexertypes/block.go b/vochain/indexer/indexertypes/block.go index 4954dbc22..4b711bdbc 100644 --- a/vochain/indexer/indexertypes/block.go +++ b/vochain/indexer/indexertypes/block.go @@ -4,6 +4,7 @@ import ( "time" "go.vocdoni.io/dvote/types" + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" ) // Block represents a block handled by the Vochain. @@ -18,3 +19,28 @@ type Block struct { LastBlockHash types.HexBytes `json:"lastBlockHash"` TxCount int64 `json:"txCount"` } + +// BlockFromDB converts the indexerdb.Block into a Block +func BlockFromDB(dbblock *indexerdb.Block) *Block { + return &Block{ + ChainID: dbblock.ChainID, + Height: dbblock.Height, + Time: dbblock.Time, + Hash: nonEmptyBytes(dbblock.Hash), + ProposerAddress: nonEmptyBytes(dbblock.ProposerAddress), + LastBlockHash: nonEmptyBytes(dbblock.LastBlockHash), + } +} + +// BlockFromDBRow converts the indexerdb.SearchBlocksRow into a Block +func BlockFromDBRow(row *indexerdb.SearchBlocksRow) *Block { + return &Block{ + ChainID: row.ChainID, + Height: row.Height, + Time: row.Time, + Hash: nonEmptyBytes(row.Hash), + ProposerAddress: nonEmptyBytes(row.ProposerAddress), + LastBlockHash: nonEmptyBytes(row.LastBlockHash), + TxCount: row.TxCount, + } +} diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index d0d2b29aa..1f6270596 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -176,16 +176,8 @@ type TxPackage struct { Signature types.HexBytes `json:"signature"` } -// TxMetadata contains tx information for the TransactionList api -type TxMetadata struct { - Type string `json:"type"` - BlockHeight uint32 `json:"blockHeight,omitempty"` - Index int32 `json:"index"` - Hash types.HexBytes `json:"hash"` -} - -// Transaction holds the db reference for a single transaction -type Transaction struct { +// TransactionMetadata contains tx information for the TransactionList api +type TransactionMetadata struct { Index uint64 `json:"transactionNumber" format:"int64" example:"944"` Hash types.HexBytes `json:"transactionHash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` @@ -193,8 +185,8 @@ type Transaction struct { TxType string `json:"transactionType" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` } -func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { - return &Transaction{ +func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { + return &TransactionMetadata{ Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), @@ -203,6 +195,32 @@ func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { } } +func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *TransactionMetadata { + return &TransactionMetadata{ + Index: uint64(dbtx.ID), + Hash: dbtx.Hash, + BlockHeight: uint32(dbtx.BlockHeight), + TxBlockIndex: int32(dbtx.BlockIndex), + TxType: dbtx.Type, + } +} + +// Transaction holds a single transaction +type Transaction struct { + *TransactionMetadata + RawTx types.HexBytes `json:"-"` + Signature types.HexBytes `json:"-"` +} + +// TransactionFromDB converts an indexerdb.Transaction into a Transaction +func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { + return &Transaction{ + TransactionMetadata: TransactionMetadataFromDB(dbtx), + RawTx: dbtx.RawTx, + Signature: dbtx.Signature, + } +} + // TokenTransferMeta contains the information of a token transfer and some extra useful information. // The types are compatible with the SQL defined schema. type TokenTransferMeta struct { diff --git a/vochain/indexer/migrations/0013_alter_columns_table_blocks.sql b/vochain/indexer/migrations/0013_alter_columns_table_blocks.sql new file mode 100644 index 000000000..c83c32c06 --- /dev/null +++ b/vochain/indexer/migrations/0013_alter_columns_table_blocks.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE blocks DROP COLUMN data_hash; +ALTER TABLE blocks ADD COLUMN chain_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE blocks ADD COLUMN hash BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN proposer_address BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN last_block_hash BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE blocks ADD COLUMN data_hash BLOB NOT NULL; +ALTER TABLE blocks DROP COLUMN chain_id; +ALTER TABLE blocks DROP COLUMN hash; +ALTER TABLE blocks DROP COLUMN proposer_address; +ALTER TABLE blocks DROP COLUMN last_block_hash; diff --git a/vochain/indexer/migrations/0014_alter_columns_table_transactions.sql b/vochain/indexer/migrations/0014_alter_columns_table_transactions.sql new file mode 100644 index 000000000..555b6506a --- /dev/null +++ b/vochain/indexer/migrations/0014_alter_columns_table_transactions.sql @@ -0,0 +1,7 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN raw_tx BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signature BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE transactions DROP COLUMN signature; +ALTER TABLE transactions DROP COLUMN raw_tx; diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index 577e875b5..38380967b 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -1,11 +1,40 @@ -- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? + ?, ?, ?, ?, ?, ? ); --- name: GetBlock :one +-- name: GetBlockByHeight :one SELECT * FROM blocks WHERE height = ? LIMIT 1; + +-- name: GetBlockByHash :one +SELECT * FROM blocks +WHERE hash = ? +LIMIT 1; + +-- name: SearchBlocks :many +SELECT + b.*, + COUNT(t.id) AS tx_count, + COUNT(*) OVER() AS total_count +FROM blocks AS b +LEFT JOIN transactions AS t + ON b.height = t.block_height +WHERE ( + (sqlc.arg(chain_id) = '' OR b.chain_id = sqlc.arg(chain_id)) + AND LENGTH(sqlc.arg(hash_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(b.hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (sqlc.arg(proposer_address) = '' OR LOWER(HEX(b.proposer_address)) = LOWER(sqlc.arg(proposer_address))) +) +GROUP BY b.height +ORDER BY b.height DESC +LIMIT sqlc.arg(limit) +OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 0e625a197..98307ea4c 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -1,8 +1,8 @@ -- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, raw_tx, signature ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ?, ? ); -- name: GetTransaction :one @@ -18,7 +18,11 @@ LIMIT 1; -- name: CountTransactions :one SELECT COUNT(*) FROM transactions; --- name: GetTxReferenceByBlockHeightAndBlockIndex :one +-- name: CountTransactionsByHeight :one +SELECT COUNT(*) FROM transactions +WHERE block_height = ?; + +-- name: GetTransactionByHeightAndIndex :one SELECT * FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1; diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 4b8c9fe21..4b8fe7b83 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -11,6 +11,7 @@ import ( indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" + "google.golang.org/protobuf/proto" ) // ErrTransactionNotFound is returned if the transaction is not found. @@ -22,8 +23,13 @@ func (idx *Indexer) CountTotalTransactions() (uint64, error) { return uint64(count), err } -// GetTransaction fetches the txReference for the given tx height -func (idx *Indexer) GetTransaction(id uint64) (*indexertypes.Transaction, error) { +// CountTransactionsByHeight returns the number of transactions indexed for a given height +func (idx *Indexer) CountTransactionsByHeight(height int64) (int64, error) { + return idx.readOnlyQuery.CountTransactionsByHeight(context.TODO(), height) +} + +// GetTxMetadataByID fetches the tx metadata for the given tx height +func (idx *Indexer) GetTxMetadataByID(id uint64) (*indexertypes.TransactionMetadata, error) { sqlTxRef, err := idx.readOnlyQuery.GetTransaction(context.TODO(), int64(id)) if err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -31,32 +37,32 @@ func (idx *Indexer) GetTransaction(id uint64) (*indexertypes.Transaction, error) } return nil, fmt.Errorf("tx with id %d not found: %v", id, err) } - return indexertypes.TransactionFromDB(&sqlTxRef), nil + return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil } -// GetTxReferenceByBlockHeightAndBlockIndex fetches the txReference for the given tx height and block tx index -func (idx *Indexer) GetTxReferenceByBlockHeightAndBlockIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTxReferenceByBlockHeightAndBlockIndex(context.TODO(), indexerdb.GetTxReferenceByBlockHeightAndBlockIndexParams{ - BlockHeight: blockHeight, - BlockIndex: blockIndex, - }) +// GetTxMetadataByHash fetches the tx metadata for the given tx hash +func (idx *Indexer) GetTxMetadataByHash(hash types.HexBytes) (*indexertypes.TransactionMetadata, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrTransactionNotFound } - return nil, fmt.Errorf("tx at block %d and index %d not found: %v", blockHeight, blockIndex, err) + return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) } - return indexertypes.TransactionFromDB(&sqlTxRef), nil + return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil } -// GetTxHashReference fetches the txReference for the given tx hash -func (idx *Indexer) GetTxHashReference(hash types.HexBytes) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) +// GetTransactionByHeightAndIndex fetches the full tx for the given tx height and block tx index +func (idx *Indexer) GetTransactionByHeightAndIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHeightAndIndex(context.TODO(), indexerdb.GetTransactionByHeightAndIndexParams{ + BlockHeight: blockHeight, + BlockIndex: blockIndex, + }) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrTransactionNotFound } - return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) + return nil, fmt.Errorf("tx at block %d and index %d not found: %v", blockHeight, blockIndex, err) } return indexertypes.TransactionFromDB(&sqlTxRef), nil } @@ -64,7 +70,7 @@ func (idx *Indexer) GetTxHashReference(hash types.HexBytes) (*indexertypes.Trans // SearchTransactions returns the list of transactions indexed. // height and txType are optional, if declared as zero-value will be ignored. // The first one returned is the newest, so they are in descending order. -func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType string) ([]*indexertypes.Transaction, uint64, error) { +func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType string) ([]*indexertypes.TransactionMetadata, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) } @@ -80,9 +86,9 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx if err != nil { return nil, 0, err } - list := []*indexertypes.Transaction{} + list := []*indexertypes.TransactionMetadata{} for _, row := range results { - list = append(list, &indexertypes.Transaction{ + list = append(list, &indexertypes.TransactionMetadata{ Index: uint64(row.ID), Hash: row.Hash, BlockHeight: uint32(row.BlockHeight), @@ -99,12 +105,21 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) { idx.blockMu.Lock() defer idx.blockMu.Unlock() + + rawtx, err := proto.Marshal(tx.Tx) + if err != nil { + log.Errorw(err, "indexer cannot marshal new transaction") + return + } + queries := idx.blockTxQueries() if _, err := queries.CreateTransaction(context.TODO(), indexerdb.CreateTransactionParams{ Hash: tx.TxID[:], BlockHeight: int64(blockHeight), BlockIndex: int64(txIndex), Type: tx.TxModelType, + RawTx: rawtx, + Signature: tx.Signature, }); err != nil { log.Errorw(err, "cannot index new transaction") }