From a1bd8cf386b931d99bc698a6e92afec2bbc4c8ca Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 14 Jun 2024 14:07:37 +1000 Subject: [PATCH 1/4] fix: events: sqlite db improvements * fix unclosed multi-row query * tune options to limit wal growth Ref: https://github.com/filecoin-project/lotus/issues/12089 --- chain/events/filter/index.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chain/events/filter/index.go b/chain/events/filter/index.go index 98cc54d2029..dee4e5bf7e1 100644 --- a/chain/events/filter/index.go +++ b/chain/events/filter/index.go @@ -26,10 +26,11 @@ var pragmas = []string{ "PRAGMA temp_store = memory", "PRAGMA mmap_size = 30000000000", "PRAGMA page_size = 32768", - "PRAGMA auto_vacuum = NONE", // not useful until we implement GC + "PRAGMA auto_vacuum = NONE", "PRAGMA automatic_index = OFF", "PRAGMA journal_mode = WAL", - "PRAGMA read_uncommitted = ON", + "PRAGMA wal_autocheckpoint = 256", // checkpoint @ 256 pages + "PRAGMA journal_size_limit = 0", // always reset journal and wal files } // Any changes to this schema should be matched for the `lotus-shed indexes backfill-events` command @@ -751,6 +752,7 @@ func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, exclude } return xerrors.Errorf("exec prefill query: %w", err) } + defer q.Close() var ces []*CollectedEvent var currentID int64 = -1 @@ -839,7 +841,6 @@ func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, exclude Codec: row.codec, Value: row.value, }) - } if ce != nil { From 4ace38d36eaa604ae7db68f7057916f2335524b3 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 14 Jun 2024 14:27:28 +1000 Subject: [PATCH 2/4] fix: events: use correct context for CollectEvents transaction --- chain/events/filter/index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/events/filter/index.go b/chain/events/filter/index.go index dee4e5bf7e1..9085eb81c34 100644 --- a/chain/events/filter/index.go +++ b/chain/events/filter/index.go @@ -522,7 +522,7 @@ func (ei *EventIndex) Close() error { } func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, revert bool, resolver func(ctx context.Context, emitter abi.ActorID, ts *types.TipSet) (address.Address, bool)) error { - tx, err := ei.db.Begin() + tx, err := ei.db.BeginTx(ctx, nil) if err != nil { return xerrors.Errorf("begin transaction: %w", err) } From c76f83a2bc417aaee066785f30ab24b7fd8b6997 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 14 Jun 2024 14:29:04 +1000 Subject: [PATCH 3/4] fix: events: close prepared read statement --- chain/events/filter/index.go | 1 + 1 file changed, 1 insertion(+) diff --git a/chain/events/filter/index.go b/chain/events/filter/index.go index 9085eb81c34..bfa6a182735 100644 --- a/chain/events/filter/index.go +++ b/chain/events/filter/index.go @@ -744,6 +744,7 @@ func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, exclude if err != nil { return xerrors.Errorf("prepare prefill query: %w", err) } + defer stmt.Close() q, err := stmt.QueryContext(ctx, values...) if err != nil { From 2003bdbf13734fd48ed6252a228b0d21cc93b3c1 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 14 Jun 2024 14:41:54 +1000 Subject: [PATCH 4/4] fix: events: close initial query; handle lint failures --- chain/events/filter/index.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/chain/events/filter/index.go b/chain/events/filter/index.go index bfa6a182735..5ebb8fb580c 100644 --- a/chain/events/filter/index.go +++ b/chain/events/filter/index.go @@ -439,6 +439,9 @@ func NewEventIndex(ctx context.Context, path string, chainStore *store.ChainStor eventIndex := EventIndex{db: db} q, err := db.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type='table' AND name='_meta';") + if q != nil { + defer func() { _ = q.Close() }() + } if errors.Is(err, sql.ErrNoRows) || !q.Next() { // empty database, create the schema for _, ddl := range ddls { @@ -744,7 +747,7 @@ func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, exclude if err != nil { return xerrors.Errorf("prepare prefill query: %w", err) } - defer stmt.Close() + defer func() { _ = stmt.Close() }() q, err := stmt.QueryContext(ctx, values...) if err != nil { @@ -753,7 +756,7 @@ func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, exclude } return xerrors.Errorf("exec prefill query: %w", err) } - defer q.Close() + defer func() { _ = q.Close() }() var ces []*CollectedEvent var currentID int64 = -1