chore: add transaction indexes, make fees non-nullable

api/transactions.go

@@ -63,10 +63,6 @@ func (api *api) SendPayment(ctx context.Context, invoice string) (*SendPaymentRe
 }
 
 func toApiTransaction(transaction *transactions.Transaction) *Transaction {
-	fee := uint64(0)
-	if transaction.FeeMsat != nil {
-		fee = *transaction.FeeMsat
-	}
 
 	createdAt := transaction.CreatedAt.Format(time.RFC3339)
 	var settledAt *string
@@ -97,7 +93,7 @@ func toApiTransaction(transaction *transactions.Transaction) *Transaction {
 		PaymentHash:     transaction.PaymentHash,
 		Amount:          transaction.AmountMsat,
 		AppId:           transaction.AppId,
-		FeesPaid:        fee,
+		FeesPaid:        transaction.FeeMsat,
 		CreatedAt:       createdAt,
 		SettledAt:       settledAt,
 		Metadata:        metadata,

db/migrations/202407201604_transactions_indexes.go

@@ -0,0 +1,81 @@
+package migrations
+
+import (
+	_ "embed"
+
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// This migration
+// - Sets NULL fees to 0 instead
+// - Adds indexes that should speed up queries related to transactions
+// - (basic indexes)
+// - index for budget query
+// - index for isolated balance queries
+// - index for list_transactions / lookup transaciton queries
+
+var _202407201604_transactions_indexes = &gormigrate.Migration{
+	ID: "202407201604_transactions_indexes",
+	Migrate: func(db *gorm.DB) error {
+
+		if err := db.Transaction(func(tx *gorm.DB) error {
+
+			// make transaction fees non-nullable
+			err := tx.Exec(`
+UPDATE transactions set fee_msat = 0 where fee_msat is NULL;
+UPDATE transactions set fee_reserve_msat = 0 where fee_reserve_msat is NULL;
+		`).Error
+			if err != nil {
+				return err
+			}
+
+			// basic transaction indexes
+			err = tx.Exec(`
+CREATE INDEX idx_transactions_app_id ON transactions(app_id);
+CREATE INDEX idx_transactions_request_event_id ON transactions(request_event_id);
+CREATE INDEX idx_transactions_state ON transactions(state);
+CREATE INDEX idx_transactions_type ON transactions(type);
+CREATE INDEX idx_transactions_payment_hash ON transactions(payment_hash);
+CREATE INDEX idx_transactions_created_at ON transactions(created_at);
+CREATE INDEX idx_transactions_settled_at ON transactions(settled_at);
+		`).Error
+			if err != nil {
+				return err
+			}
+
+			// budgets
+			err = tx.Exec(`
+CREATE INDEX idx_transactions_app_id_type_state_created_at ON transactions(app_id, type, state, created_at);
+		`).Error
+			if err != nil {
+				return err
+			}
+
+			// isolated balance
+			err = tx.Exec(`
+CREATE INDEX idx_transactions_app_id_type_state ON transactions(app_id, type, state);
+		`).Error
+			if err != nil {
+				return err
+			}
+
+			// list transactions / lookup transaction variations
+			err = tx.Exec(`
+CREATE INDEX idx_transactions_app_id_type_state_created_at_settled_at_payment_hash ON transactions(app_id, type, state, created_at, settled_at, payment_hash);
+		`).Error
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		return nil
+	},
+	Rollback: func(tx *gorm.DB) error {
+		return nil
+	},
+}

db/migrations/migrate.go

@@ -17,6 +17,7 @@ func Migrate(gormDB *gorm.DB) error {
 		_202406301207_rename_request_methods,
 		_202407012100_transactions,
 		_202407151352_autoincrement,
+		_202407201604_transactions_indexes,
 	})
 
 	return m.Migrate()

db/models.go

@@ -64,8 +64,8 @@ type Transaction struct {
 	Type            string
 	State           string
 	AmountMsat      uint64
-	FeeMsat         *uint64
-	FeeReserveMsat  *uint64 // non-zero for unsettled outgoing payments only
+	FeeMsat         uint64
+	FeeReserveMsat  uint64
 	PaymentRequest  string
 	PaymentHash     string
 	Description     string

db/queries/get_budget_usage.go

@@ -14,7 +14,7 @@ func GetBudgetUsageSat(tx *gorm.DB, appPermission *db.AppPermission) uint64 {
 	}
 	tx.
 		Table("transactions").
-		Select("SUM(amount_msat + coalesce(fee_msat, 0) + coalesce(fee_reserve_msat, 0)) as sum").
+		Select("SUM(amount_msat + fee_msat + fee_reserve_msat) as sum").
 		Where("app_id = ? AND type = ? AND (state = ? OR state = ?) AND created_at > ?", appPermission.AppId, constants.TRANSACTION_TYPE_OUTGOING, constants.TRANSACTION_STATE_SETTLED, constants.TRANSACTION_STATE_PENDING, getStartOfBudget(appPermission.BudgetRenewal)).Scan(&result)
 	return result.Sum / 1000
 }

db/queries/get_isolated_balance.go

@@ -20,7 +20,7 @@ func GetIsolatedBalance(tx *gorm.DB, appId uint) uint64 {
 
 	tx.
 		Table("transactions").
-		Select("SUM(amount_msat + coalesce(fee_msat, 0) + coalesce(fee_reserve_msat, 0)) as sum").
+		Select("SUM(amount_msat + fee_msat + fee_reserve_msat) as sum").
 		Where("app_id = ? AND type = ? AND (state = ? OR state = ?)", appId, constants.TRANSACTION_TYPE_OUTGOING, constants.TRANSACTION_STATE_SETTLED, constants.TRANSACTION_STATE_PENDING).Scan(&spent)
 
 	return received.Sum - spent.Sum

lnclient/cashu/cashu.go

@@ -85,12 +85,12 @@ func (cs *CashuService) SendPaymentSync(ctx context.Context, invoice string) (re
 
 	return &lnclient.PayInvoiceResponse{
 		Preimage: meltResponse.Preimage,
-		Fee:      &fee,
+		Fee:      fee,
 	}, nil
 }
 
 func (cs *CashuService) SendKeysend(ctx context.Context, amount uint64, destination string, custom_records []lnclient.TLVRecord) (paymentHash string, preimage string, fee uint64, err error) {
-	return "", "", 0, errors.New("Keysend not supported")
+	return "", "", 0, errors.New("keysend not supported")
 }
 
 func (cs *CashuService) GetBalance(ctx context.Context) (balance int64, err error) {

lnclient/ldk/ldk.go

@@ -523,7 +523,7 @@ func (ls *LDKService) SendPaymentSync(ctx context.Context, invoice string) (*lnc
 
 	return &lnclient.PayInvoiceResponse{
 		Preimage: preimage,
-		Fee:      &fee,
+		Fee:      fee,
 	}, nil
 }
 

lnclient/lnd/lnd.go

@@ -286,7 +286,7 @@ func (svc *LNDService) SendPaymentSync(ctx context.Context, payReq string) (*lnc
 	}
 
 	if resp.PaymentPreimage == nil {
-		return nil, errors.New("No preimage in response")
+		return nil, errors.New("no preimage in response")
 	}
 
 	var fee uint64 = 0
@@ -296,7 +296,7 @@ func (svc *LNDService) SendPaymentSync(ctx context.Context, payReq string) (*lnc
 
 	return &lnclient.PayInvoiceResponse{
 		Preimage: hex.EncodeToString(resp.PaymentPreimage),
-		Fee:      &fee,
+		Fee:      fee,
 	}, nil
 }
 

lnclient/models.go

@@ -153,8 +153,8 @@ type LightningBalanceResponse struct {
 }
 
 type PayInvoiceResponse struct {
-	Preimage string  `json:"preimage"`
-	Fee      *uint64 `json:"fee"`
+	Preimage string `json:"preimage"`
+	Fee      uint64 `json:"fee"`
 }
 
 type BalancesResponse struct {

lnclient/phoenixd/phoenixd.go

@@ -425,10 +425,9 @@ func (svc *PhoenixService) SendPaymentSync(ctx context.Context, payReq string) (
 		return nil, err
 	}
 
-	fee := uint64(payRes.RoutingFeeSat) * 1000
 	return &lnclient.PayInvoiceResponse{
 		Preimage: payRes.PaymentPreimage,
-		Fee:      &fee,
+		Fee:      uint64(payRes.RoutingFeeSat) * 1000,
 	}, nil
 }
 

nip47/controllers/list_transactions_controller_test.go

@@ -50,7 +50,6 @@ func TestHandleListTransactionsEvent(t *testing.T) {
 	assert.NoError(t, err)
 
 	for i, _ := range tests.MockLNClientTransactions {
-		feesPaid := uint64(tests.MockLNClientTransactions[i].FeesPaid)
 		settledAt := time.Unix(*tests.MockLNClientTransactions[i].SettledAt, 0)
 		err = svc.DB.Create(&db.Transaction{
 			Type:            tests.MockLNClientTransactions[i].Type,
@@ -60,7 +59,7 @@ func TestHandleListTransactionsEvent(t *testing.T) {
 			Preimage:        &tests.MockLNClientTransactions[i].Preimage,
 			PaymentHash:     tests.MockLNClientTransactions[i].PaymentHash,
 			AmountMsat:      uint64(tests.MockLNClientTransactions[i].Amount),
-			FeeMsat:         &feesPaid,
+			FeeMsat:         uint64(tests.MockLNClientTransactions[i].FeesPaid),
 			SettledAt:       &settledAt,
 			State:           constants.TRANSACTION_STATE_SETTLED,
 			AppId:           &app.ID,

nip47/controllers/lookup_invoice_controller_test.go

@@ -44,7 +44,6 @@ func TestHandleLookupInvoiceEvent(t *testing.T) {
 	err = svc.DB.Create(&dbRequestEvent).Error
 	assert.NoError(t, err)
 
-	feesPaid := uint64(tests.MockLNClientTransaction.FeesPaid)
 	settledAt := time.Unix(*tests.MockLNClientTransaction.SettledAt, 0)
 	err = svc.DB.Create(&db.Transaction{
 		Type:            tests.MockLNClientTransaction.Type,
@@ -54,7 +53,7 @@ func TestHandleLookupInvoiceEvent(t *testing.T) {
 		Preimage:        &tests.MockLNClientTransaction.Preimage,
 		PaymentHash:     tests.MockLNClientTransaction.PaymentHash,
 		AmountMsat:      uint64(tests.MockLNClientTransaction.Amount),
-		FeeMsat:         &feesPaid,
+		FeeMsat:         uint64(tests.MockLNClientTransaction.FeesPaid),
 		SettledAt:       &settledAt,
 		AppId:           &app.ID,
 	}).Error

nip47/controllers/models.go

@@ -8,6 +8,6 @@ import (
 type publishFunc = func(*models.Response, nostr.Tags)
 
 type payResponse struct {
-	Preimage string  `json:"preimage"`
-	FeesPaid *uint64 `json:"fees_paid"`
+	Preimage string `json:"preimage"`
+	FeesPaid uint64 `json:"fees_paid"`
 }

nip47/models/transactions.go

@@ -9,11 +9,6 @@ import (
 )
 
 func ToNip47Transaction(transaction *transactions.Transaction) *Transaction {
-	fees := int64(0)
-	if transaction.FeeMsat != nil {
-		fees = int64(*transaction.FeeMsat)
-	}
-
 	var expiresAt *int64
 	if transaction.ExpiresAt != nil {
 		expiresAtUnix := transaction.ExpiresAt.Unix()
@@ -47,7 +42,7 @@ func ToNip47Transaction(transaction *transactions.Transaction) *Transaction {
 		Preimage:        preimage,
 		PaymentHash:     transaction.PaymentHash,
 		Amount:          int64(transaction.AmountMsat),
-		FeesPaid:        fees,
+		FeesPaid:        int64(transaction.FeeMsat),
 		CreatedAt:       transaction.CreatedAt.Unix(),
 		ExpiresAt:       expiresAt,
 		SettledAt:       settledAt,

nip47/notifications/nip47_notifier_test.go

@@ -48,7 +48,6 @@ func TestSendNotification_PaymentReceived(t *testing.T) {
 	err = svc.DB.Create(appPermission).Error
 	assert.NoError(t, err)
 
-	feesPaid := uint64(tests.MockLNClientTransaction.FeesPaid)
 	settledAt := time.Unix(*tests.MockLNClientTransaction.SettledAt, 0)
 	err = svc.DB.Create(&db.Transaction{
 		Type:            constants.TRANSACTION_TYPE_INCOMING,
@@ -58,7 +57,7 @@ func TestSendNotification_PaymentReceived(t *testing.T) {
 		Preimage:        &tests.MockLNClientTransaction.Preimage,
 		PaymentHash:     tests.MockLNClientTransaction.PaymentHash,
 		AmountMsat:      uint64(tests.MockLNClientTransaction.Amount),
-		FeeMsat:         &feesPaid,
+		FeeMsat:         uint64(tests.MockLNClientTransaction.FeesPaid),
 		SettledAt:       &settledAt,
 		AppId:           &app.ID,
 	}).Error
@@ -129,7 +128,6 @@ func TestSendNotification_PaymentSent(t *testing.T) {
 	err = svc.DB.Create(appPermission).Error
 	assert.NoError(t, err)
 
-	feesPaid := uint64(tests.MockLNClientTransaction.FeesPaid)
 	settledAt := time.Unix(*tests.MockLNClientTransaction.SettledAt, 0)
 	err = svc.DB.Create(&db.Transaction{
 		Type:            constants.TRANSACTION_TYPE_OUTGOING,
@@ -139,7 +137,7 @@ func TestSendNotification_PaymentSent(t *testing.T) {
 		Preimage:        &tests.MockLNClientTransaction.Preimage,
 		PaymentHash:     tests.MockLNClientTransaction.PaymentHash,
 		AmountMsat:      uint64(tests.MockLNClientTransaction.Amount),
-		FeeMsat:         &feesPaid,
+		FeeMsat:         uint64(tests.MockLNClientTransaction.FeesPaid),
 		SettledAt:       &settledAt,
 		AppId:           &app.ID,
 	}).Error

transactions/keysend_test.go

@@ -26,7 +26,7 @@ func TestSendKeysend(t *testing.T) {
 	assert.Equal(t, uint64(1000), transaction.AmountMsat)
 	assert.Equal(t, constants.TRANSACTION_TYPE_OUTGOING, transaction.Type)
 	assert.Equal(t, constants.TRANSACTION_STATE_SETTLED, transaction.State)
-	assert.Zero(t, *transaction.FeeReserveMsat)
+	assert.Zero(t, transaction.FeeReserveMsat)
 }
 
 func TestSendKeysend_App_NoPermission(t *testing.T) {
@@ -83,7 +83,7 @@ func TestSendKeysend_App_WithPermission(t *testing.T) {
 	assert.Equal(t, constants.TRANSACTION_STATE_SETTLED, transaction.State)
 	assert.Equal(t, app.ID, *transaction.AppId)
 	assert.Equal(t, dbRequestEvent.ID, *transaction.RequestEventId)
-	assert.Zero(t, *transaction.FeeReserveMsat)
+	assert.Zero(t, transaction.FeeReserveMsat)
 }
 
 func TestSendKeysend_App_BudgetExceeded(t *testing.T) {
@@ -148,7 +148,7 @@ func TestSendKeysend_App_BudgetNotExceeded(t *testing.T) {
 	assert.Equal(t, constants.TRANSACTION_STATE_SETTLED, transaction.State)
 	assert.Equal(t, app.ID, *transaction.AppId)
 	assert.Equal(t, dbRequestEvent.ID, *transaction.RequestEventId)
-	assert.Zero(t, *transaction.FeeReserveMsat)
+	assert.Zero(t, transaction.FeeReserveMsat)
 }
 
 func TestSendKeysend_App_BalanceExceeded(t *testing.T) {
@@ -230,7 +230,7 @@ func TestSendKeysend_App_BalanceSufficient(t *testing.T) {
 	assert.Equal(t, constants.TRANSACTION_STATE_SETTLED, transaction.State)
 	assert.Equal(t, app.ID, *transaction.AppId)
 	assert.Equal(t, dbRequestEvent.ID, *transaction.RequestEventId)
-	assert.Zero(t, *transaction.FeeReserveMsat)
+	assert.Zero(t, transaction.FeeReserveMsat)
 }
 
 func TestSendKeysend_TLVs(t *testing.T) {
@@ -253,5 +253,5 @@ func TestSendKeysend_TLVs(t *testing.T) {
 	assert.Equal(t, uint64(1000), transaction.AmountMsat)
 	assert.Equal(t, constants.TRANSACTION_TYPE_OUTGOING, transaction.Type)
 	assert.Equal(t, constants.TRANSACTION_STATE_SETTLED, transaction.State)
-	assert.Zero(t, *transaction.FeeReserveMsat)
+	assert.Zero(t, transaction.FeeReserveMsat)
 }

transactions/list_transactions_test.go

@@ -44,7 +44,7 @@ func TestListTransactions(t *testing.T) {
 	assert.Equal(t, uint64(123000), incomingTransactions[0].AmountMsat)
 	assert.Equal(t, constants.TRANSACTION_STATE_SETTLED, incomingTransactions[0].State)
 	assert.Equal(t, tests.MockLNClientTransaction.Preimage, *incomingTransactions[0].Preimage)
-	assert.Nil(t, incomingTransactions[0].FeeReserveMsat)
+	assert.Zero(t, incomingTransactions[0].FeeReserveMsat)
 }
 
 func TestListTransactions_Unsettled(t *testing.T) {

transactions/lookup_transaction_test.go

@@ -34,7 +34,7 @@ func TestLookupTransaction_IncomingPayment(t *testing.T) {
 	assert.Equal(t, uint64(123000), incomingTransaction.AmountMsat)
 	assert.Equal(t, constants.TRANSACTION_STATE_PENDING, incomingTransaction.State)
 	assert.Equal(t, tests.MockLNClientTransaction.Preimage, *incomingTransaction.Preimage)
-	assert.Nil(t, incomingTransaction.FeeReserveMsat)
+	assert.Zero(t, incomingTransaction.FeeReserveMsat)
 }
 
 func TestLookupTransaction_OutgoingPayment(t *testing.T) {
@@ -61,5 +61,5 @@ func TestLookupTransaction_OutgoingPayment(t *testing.T) {
 	assert.Equal(t, uint64(123000), outgoingTransaction.AmountMsat)
 	assert.Equal(t, constants.TRANSACTION_STATE_PENDING, outgoingTransaction.State)
 	assert.Equal(t, tests.MockLNClientTransaction.Preimage, *outgoingTransaction.Preimage)
-	assert.Nil(t, outgoingTransaction.FeeReserveMsat)
+	assert.Zero(t, outgoingTransaction.FeeReserveMsat)
 }