Replace crawshaw/sqlite with go-llsqlite/llsqlite (#4554)

## Motivation
Closes #4541 

## Changes
- Changes library used for sqlite
- Use `fs.WalkDir` and avoid loading `*.sql` files into a buffer before processing them during migrations to speed them up

## Test Plan
Existing tests pass

## TODO
<!-- This section should be removed when all items are complete -->
- [x] Explain motivation or link existing issue(s)
- [x] Test changes and document test plan
- [x] Update documentation as needed

## DevOps Notes
<!-- Please uncheck these items as applicable to make DevOps aware of changes that may affect releases -->
- [x] This PR does not require configuration changes (e.g., environment variables, GitHub secrets, VM resources)
- [x] This PR does not affect public APIs
- [x] This PR does not rely on a new version of external services (PoET, elasticsearch, etc.)
- [x] This PR does not make changes to log messages (which monitoring infrastructure may rely on)

go.mod

@@ -4,11 +4,11 @@ go 1.19
 
 require (
 	cloud.google.com/go/storage v1.30.1
-	crawshaw.io/sqlite v0.3.3-0.20211227050848-2cdb5c1a86a1
 	github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924
 	github.com/benbjohnson/clock v1.3.5
 	github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230531032220-a757362a12e1
 	github.com/cosmos/btcutil v1.0.5
+	github.com/go-llsqlite/llsqlite v0.0.0-20230612031458-a9e271fe723a
 	github.com/gofrs/flock v0.8.1
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.3

go.sum

@@ -50,8 +50,6 @@ cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/o
 cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
 crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797 h1:yDf7ARQc637HoxDho7xjqdvO5ZA2Yb+xzv/fOnnvZzw=
 crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
-crawshaw.io/sqlite v0.3.3-0.20211227050848-2cdb5c1a86a1 h1:Ee5qKxzDpjFjX2tdxl8NlPCTr3PDpagnyf0jL5ow+PY=
-crawshaw.io/sqlite v0.3.3-0.20211227050848-2cdb5c1a86a1/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
 dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
@@ -168,6 +166,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-llsqlite/llsqlite v0.0.0-20230612031458-a9e271fe723a h1:2GgRlm6BrV7CIOjjE/o7WJs6foe33AqCQC8bnl1RJQc=
+github.com/go-llsqlite/llsqlite v0.0.0-20230612031458-a9e271fe723a/go.mod h1:suaTfGNQ00ObHGOoHxPb8pkAki7jm0/ZkR2rcY9yF1s=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=

sql/database.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"time"
 
-	"crawshaw.io/sqlite"
-	"crawshaw.io/sqlite/sqlitex"
+	"github.com/go-llsqlite/llsqlite"
+	"github.com/go-llsqlite/llsqlite/sqlitex"
 	"github.com/prometheus/client_golang/prometheus"
 )
 

sql/database_test.go

@@ -24,7 +24,7 @@ func TestTransactionIsolation(t *testing.T) {
 		WithLatencyMetering(true),
 	)
 
-	tx, err := db.Tx(context.TODO())
+	tx, err := db.Tx(context.Background())
 	require.NoError(t, err)
 
 	key := "dsada"

sql/functions.go

@@ -3,7 +3,7 @@ package sql
 import (
 	"fmt"
 
-	"crawshaw.io/sqlite"
+	"github.com/go-llsqlite/llsqlite"
 )
 
 func registerFunctions(conn *sqlite.Conn) error {

sql/migrations.go

@@ -5,7 +5,7 @@ import (
 	"bytes"
 	"embed"
 	"fmt"
-	"path"
+	"io/fs"
 	"sort"
 	"strconv"
 	"strings"
@@ -24,26 +24,27 @@ type migration struct {
 type Migrations func(Executor) error
 
 func embeddedMigrations(db Executor) error {
-	files, err := embedded.ReadDir("migrations")
-	if err != nil {
-		return fmt.Errorf("readdir migrations: %w", err)
-	}
 	var migrations []migration
-	for _, file := range files {
-		parts := strings.Split(file.Name(), "_")
+	fs.WalkDir(embedded, "migrations", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return fmt.Errorf("walkdir %s: %w", path, err)
+		}
+		if d.IsDir() {
+			return nil
+		}
+		parts := strings.Split(d.Name(), "_")
 		if len(parts) < 1 {
-			return fmt.Errorf("invalid migration %s", file.Name())
+			return fmt.Errorf("invalid migration %s", d.Name())
 		}
 		order, err := strconv.Atoi(parts[0])
 		if err != nil {
-			return fmt.Errorf("invalid migration %s: %w", file.Name(), err)
+			return fmt.Errorf("invalid migration %s: %w", d.Name(), err)
 		}
-		fpath := path.Join("migrations", file.Name())
-		content, err := embedded.ReadFile(fpath)
+		f, err := embedded.Open(path)
 		if err != nil {
-			return fmt.Errorf("readfile %s: %w", fpath, err)
+			return fmt.Errorf("readfile %s: %w", path, err)
 		}
-		scanner := bufio.NewScanner(bytes.NewBuffer(content))
+		scanner := bufio.NewScanner(f)
 		scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
 			if i := bytes.Index(data, []byte(";")); i >= 0 {
 				return i + 1, data[0 : i+1], nil
@@ -52,10 +53,11 @@ func embeddedMigrations(db Executor) error {
 		})
 		migrations = append(migrations, migration{
 			order:   order,
-			name:    file.Name(),
+			name:    d.Name(),
 			content: scanner,
 		})
-	}
+		return nil
+	})
 	sort.Slice(migrations, func(i, j int) bool {
 		return migrations[i].order < migrations[j].order
 	})