diff --git a/go.mod b/go.mod index d424dee5821..f4421137fe9 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/google/certificate-transparency-go v1.1.6 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/jmhodges/clock v1.2.0 - github.com/letsencrypt/borp v0.0.0-20230707160741-6cc6ce580243 + github.com/letsencrypt/borp v0.0.0-20240620175310-a78493c6e2bd github.com/letsencrypt/challtestsrv v1.2.1 github.com/letsencrypt/pkcs11key/v4 v4.0.0 github.com/letsencrypt/validator/v10 v10.0.0-20230215210743-a0c7dfc17158 diff --git a/go.sum b/go.sum index 6046ee73061..a545959477c 100644 --- a/go.sum +++ b/go.sum @@ -150,14 +150,16 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/letsencrypt/borp v0.0.0-20230707160741-6cc6ce580243 h1:xS2U6PQYRURk61YN4Y5xvyLbQVyAP/8fpE6hJZdwEWs= -github.com/letsencrypt/borp v0.0.0-20230707160741-6cc6ce580243/go.mod h1:podMDq5wDu2ZO6JMKYQcjD3QdqOfNLWtP2RDSy8CHUU= +github.com/letsencrypt/borp v0.0.0-20240620175310-a78493c6e2bd h1:3c+LdlAOEcW1qmG8gtkMCyAEoslmj6XCmniB+926kMM= +github.com/letsencrypt/borp v0.0.0-20240620175310-a78493c6e2bd/go.mod h1:gMSMCNKhxox/ccR923EJsIvHeVVYfCABGbirqa0EwuM= github.com/letsencrypt/challtestsrv v1.2.1 h1:Lzv4jM+wSgVMCeO5a/F/IzSanhClstFMnX6SfrAJXjI= github.com/letsencrypt/challtestsrv v1.2.1/go.mod h1:Ur4e4FvELUXLGhkMztHOsPIsvGxD/kzSJninOrkM+zc= github.com/letsencrypt/pkcs11key/v4 v4.0.0 h1:qLc/OznH7xMr5ARJgkZCCWk+EomQkiNTOoOF5LAgagc= github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag= github.com/letsencrypt/validator/v10 v10.0.0-20230215210743-a0c7dfc17158 h1:HGFsIltYMUiB5eoFSowFzSoXkocM2k9ctmJ57QMGjys= github.com/letsencrypt/validator/v10 v10.0.0-20230215210743-a0c7dfc17158/go.mod h1:ZFNBS3H6OEsprCRjscty6GCBe5ZiX44x6qY4s7+bDX0= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= diff --git a/sa/type-converter.go b/sa/type-converter.go index 2ffb5bc1bc1..d7d92eb7942 100644 --- a/sa/type-converter.go +++ b/sa/type-converter.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "time" "github.com/go-jose/go-jose/v4" @@ -35,6 +36,18 @@ func (tc BoulderTypeConverter) ToDb(val interface{}) (interface{}, error) { return string(t), nil case core.OCSPStatus: return string(t), nil + // Time types get truncated to the nearest second. Given our DB schema, + // only seconds are stored anyhow. Avoiding sending queries with sub-second + // precision may help the query planner avoid pathological cases when + // querying against indexes on time fields (#5437). + case time.Time: + return t.Truncate(time.Second), nil + case *time.Time: + if t == nil { + return nil, nil + } + newT := t.Truncate(time.Second) + return &newT, nil default: return val, nil } diff --git a/sa/type-converter_test.go b/sa/type-converter_test.go index c0849e759e2..8ca7d35d199 100644 --- a/sa/type-converter_test.go +++ b/sa/type-converter_test.go @@ -3,6 +3,7 @@ package sa import ( "encoding/json" "testing" + "time" "github.com/letsencrypt/boulder/core" "github.com/letsencrypt/boulder/identifier" @@ -151,3 +152,26 @@ func TestStringSlice(t *testing.T) { test.AssertNotError(t, err, "failed to scanner.Binder") test.AssertMarshaledEquals(t, au, out) } + +func TestTimeTruncate(t *testing.T) { + tc := BoulderTypeConverter{} + preciseTime := time.Date(2024, 06, 20, 00, 00, 00, 999999999, time.UTC) + dbTime, err := tc.ToDb(preciseTime) + test.AssertNotError(t, err, "Could not ToDb") + dbTimeT, ok := dbTime.(time.Time) + test.Assert(t, ok, "Could not convert dbTime to time.Time") + test.Assert(t, dbTimeT.Nanosecond() == 0, "Nanosecond not truncated") + + dbTimePtr, err := tc.ToDb(&preciseTime) + test.AssertNotError(t, err, "Could not ToDb") + dbTimePtrT, ok := dbTimePtr.(*time.Time) + test.Assert(t, ok, "Could not convert dbTimePtr to *time.Time") + test.Assert(t, dbTimePtrT.Nanosecond() == 0, "Nanosecond not truncated") + + var dbTimePtrNil *time.Time + shouldBeNil, err := tc.ToDb(dbTimePtrNil) + test.AssertNotError(t, err, "Could not ToDb") + if shouldBeNil != nil { + t.Errorf("Expected nil, got %v", shouldBeNil) + } +} diff --git a/vendor/github.com/letsencrypt/borp/README.md b/vendor/github.com/letsencrypt/borp/README.md index f140696f846..34115fd1337 100644 --- a/vendor/github.com/letsencrypt/borp/README.md +++ b/vendor/github.com/letsencrypt/borp/README.md @@ -55,6 +55,9 @@ func main() { // fetch one row - note use of "post_id" instead of "Id" since column is aliased // + // Postgres users should use $1 instead of ? placeholders + // See 'Known Issues' below + err = dbmap.SelectOne(&p2, "select * from posts where post_id=?", p2.Id) checkErr(err, "SelectOne failed") log.Println("p2 row:", p2) @@ -368,7 +371,7 @@ if reflect.DeepEqual(list[0], expected) { Borp provides a few convenience methods for selecting a single string or int64. ```go -// select single int64 from db +// select single int64 from db (use $1 instead of ? for postgresql) i64, err := dbmap.SelectInt("select count(*) from foo where blah=?", blahVal) // select single string from db: @@ -579,6 +582,7 @@ interface that should be implemented per database vendor. Dialects are provided for: * MySQL +* PostgreSQL * sqlite3 Each of these three databases pass the test suite. See `borp_test.go` @@ -612,6 +616,41 @@ func customDriver() (*sql.DB, error) { ## Known Issues +### SQL placeholder portability + +Different databases use different strings to indicate variable +placeholders in prepared SQL statements. Unlike some database +abstraction layers (such as JDBC), Go's `database/sql` does not +standardize this. + +SQL generated by borp in the `Insert`, `Update`, `Delete`, and `Get` +methods delegates to a Dialect implementation for each database, and +will generate portable SQL. + +Raw SQL strings passed to `Exec`, `Select`, `SelectOne`, `SelectInt`, +etc will not be parsed. Consequently you may have portability issues +if you write a query like this: + +```go +// works on MySQL and Sqlite3, but not with Postgresql err := +dbmap.SelectOne(&val, "select * from foo where id = ?", 30) +``` + +In `Select` and `SelectOne` you can use named parameters to work +around this. The following is portable: + +```go +err := dbmap.SelectOne(&val, "select * from foo where id = :id", +map[string]interface{} { "id": 30}) +``` + +Additionally, when using Postgres as your database, you should utilize +`$1` instead of `?` placeholders as utilizing `?` placeholders when +querying Postgres will result in `pq: operator does not exist` +errors. Alternatively, use `dbMap.Dialect.BindVar(varIdx)` to get the +proper variable binding for your dialect. + + ### time.Time and time zones borp will pass `time.Time` fields through to the `database/sql` @@ -630,7 +669,7 @@ To avoid any potential issues with timezone/DST, consider: ## Running the tests -The included tests may be run against MySQL or sqlite3. +The included tests may be run against MySQL, Postgres, or sqlite3. You must set two environment variables so the test code knows which driver to use, and how to connect to your database. @@ -647,7 +686,7 @@ go test -bench="Bench" -benchtime 10 ``` Valid `GORP_TEST_DIALECT` values are: "mysql"(for mymysql), -"gomysql"(for go-sql-driver), or "sqlite" See the +"gomysql"(for go-sql-driver), "postgres", or "sqlite" See the `test_all.sh` script for examples of all 3 databases. This is the script I run locally to test the library. diff --git a/vendor/github.com/letsencrypt/borp/db.go b/vendor/github.com/letsencrypt/borp/db.go index 539941464dd..c29bfca52a4 100644 --- a/vendor/github.com/letsencrypt/borp/db.go +++ b/vendor/github.com/letsencrypt/borp/db.go @@ -102,6 +102,9 @@ func (m *DbMap) createIndexImpl(ctx context.Context, dialect reflect.Type, } s.WriteString(" index") s.WriteString(fmt.Sprintf(" %s on %s", index.IndexName, table.TableName)) + if dname := dialect.Name(); dname == "PostgresDialect" && index.IndexType != "" { + s.WriteString(fmt.Sprintf(" %s %s", m.Dialect.CreateIndexSuffix(), index.IndexType)) + } s.WriteString(" (") for x, col := range index.columns { if x > 0 { @@ -594,6 +597,11 @@ func (m *DbMap) Select(ctx context.Context, i interface{}, query string, args .. expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return nil, err + } + return hookedselect(ctx, m, m, i, query, args...) } @@ -604,6 +612,11 @@ func (m *DbMap) ExecContext(ctx context.Context, query string, args ...interface expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return nil, err + } + if m.logger != nil { now := time.Now() defer m.trace(now, query, args...) @@ -617,6 +630,11 @@ func (m *DbMap) SelectInt(ctx context.Context, query string, args ...interface{} expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return 0, err + } + return SelectInt(ctx, m, query, args...) } @@ -626,6 +644,11 @@ func (m *DbMap) SelectNullInt(ctx context.Context, query string, args ...interfa expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return sql.NullInt64{}, err + } + return SelectNullInt(ctx, m, query, args...) } @@ -635,6 +658,11 @@ func (m *DbMap) SelectFloat(ctx context.Context, query string, args ...interface expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return 0, err + } + return SelectFloat(ctx, m, query, args...) } @@ -644,6 +672,11 @@ func (m *DbMap) SelectNullFloat(ctx context.Context, query string, args ...inter expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return sql.NullFloat64{}, err + } + return SelectNullFloat(ctx, m, query, args...) } @@ -653,6 +686,11 @@ func (m *DbMap) SelectStr(ctx context.Context, query string, args ...interface{} expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return "", err + } + return SelectStr(ctx, m, query, args...) } @@ -662,6 +700,11 @@ func (m *DbMap) SelectNullStr(ctx context.Context, query string, args ...interfa expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return sql.NullString{}, err + } + return SelectNullStr(ctx, m, query, args...) } @@ -671,6 +714,11 @@ func (m *DbMap) SelectOne(ctx context.Context, holder interface{}, query string, expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return err + } + return SelectOne(ctx, m, m, holder, query, args...) } @@ -795,6 +843,11 @@ func (m *DbMap) QueryRowContext(ctx context.Context, query string, args ...inter expandSliceArgs(&query, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return nil + } + if m.logger != nil { now := time.Now() defer m.trace(now, query, args...) @@ -808,6 +861,11 @@ func (m *DbMap) QueryContext(ctx context.Context, q string, args ...interface{}) expandSliceArgs(&q, args...) } + args, err := m.convertArgs(args...) + if err != nil { + return nil, err + } + if m.logger != nil { now := time.Now() defer m.trace(now, q, args...) @@ -826,6 +884,22 @@ func (m *DbMap) trace(started time.Time, query string, args ...interface{}) { } } +// convertArgs passes each argument through the TypeConverter, if any, +// and returns the result (which may be identical to the input). +func (m *DbMap) convertArgs(args ...interface{}) ([]interface{}, error) { + if m.TypeConverter == nil { + return args, nil + } + for i, arg := range args { + converted, err := m.TypeConverter.ToDb(arg) + if err != nil { + return nil, err + } + args[i] = converted + } + return args, nil +} + type stringer interface { ToStringSlice() []string } diff --git a/vendor/github.com/letsencrypt/borp/dialect.go b/vendor/github.com/letsencrypt/borp/dialect.go index 2d48e06d246..a2591ab0103 100644 --- a/vendor/github.com/letsencrypt/borp/dialect.go +++ b/vendor/github.com/letsencrypt/borp/dialect.go @@ -45,7 +45,7 @@ type Dialect interface { TruncateClause() string // Bind variable string to use when forming SQL statements - // in many dbs it is "?". + // in many dbs it is "?", but Postgres appears to use $1 // // i is a zero based index of the bind variable in this statement // diff --git a/vendor/github.com/letsencrypt/borp/dialect_postgres.go b/vendor/github.com/letsencrypt/borp/dialect_postgres.go new file mode 100644 index 00000000000..937f81e6192 --- /dev/null +++ b/vendor/github.com/letsencrypt/borp/dialect_postgres.go @@ -0,0 +1,150 @@ +// Copyright 2012 James Cooper. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package borp + +import ( + "context" + "fmt" + "reflect" + "strings" + "time" +) + +type PostgresDialect struct { + suffix string + LowercaseFields bool +} + +func (d PostgresDialect) QuerySuffix() string { return ";" } + +func (d PostgresDialect) ToSqlType(val reflect.Type, maxsize int, isAutoIncr bool) string { + switch val.Kind() { + case reflect.Ptr: + return d.ToSqlType(val.Elem(), maxsize, isAutoIncr) + case reflect.Bool: + return "boolean" + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: + if isAutoIncr { + return "serial" + } + return "integer" + case reflect.Int64, reflect.Uint64: + if isAutoIncr { + return "bigserial" + } + return "bigint" + case reflect.Float64: + return "double precision" + case reflect.Float32: + return "real" + case reflect.Slice: + if val.Elem().Kind() == reflect.Uint8 { + return "bytea" + } + } + + switch val.Name() { + case "NullInt64": + return "bigint" + case "NullFloat64": + return "double precision" + case "NullBool": + return "boolean" + case "Time", "NullTime": + return "timestamp with time zone" + } + + if maxsize > 0 { + return fmt.Sprintf("varchar(%d)", maxsize) + } else { + return "text" + } + +} + +// Returns empty string +func (d PostgresDialect) AutoIncrStr() string { + return "" +} + +func (d PostgresDialect) AutoIncrBindValue() string { + return "default" +} + +func (d PostgresDialect) AutoIncrInsertSuffix(col *ColumnMap) string { + return " returning " + d.QuoteField(col.ColumnName) +} + +// Returns suffix +func (d PostgresDialect) CreateTableSuffix() string { + return d.suffix +} + +func (d PostgresDialect) CreateIndexSuffix() string { + return "using" +} + +func (d PostgresDialect) DropIndexSuffix() string { + return "" +} + +func (d PostgresDialect) TruncateClause() string { + return "truncate" +} + +func (d PostgresDialect) SleepClause(s time.Duration) string { + return fmt.Sprintf("pg_sleep(%f)", s.Seconds()) +} + +// Returns "$(i+1)" +func (d PostgresDialect) BindVar(i int) string { + return fmt.Sprintf("$%d", i+1) +} + +func (d PostgresDialect) InsertAutoIncrToTarget(ctx context.Context, exec SqlExecutor, insertSql string, target interface{}, params ...interface{}) error { + rows, err := exec.QueryContext(ctx, insertSql, params...) + if err != nil { + return err + } + defer rows.Close() + + if !rows.Next() { + return fmt.Errorf("No serial value returned for insert: %s Encountered error: %s", insertSql, rows.Err()) + } + if err := rows.Scan(target); err != nil { + return err + } + if rows.Next() { + return fmt.Errorf("more than two serial value returned for insert: %s", insertSql) + } + return rows.Err() +} + +func (d PostgresDialect) QuoteField(f string) string { + if d.LowercaseFields { + return `"` + strings.ToLower(f) + `"` + } + return `"` + f + `"` +} + +func (d PostgresDialect) QuotedTableForQuery(schema string, table string) string { + if strings.TrimSpace(schema) == "" { + return d.QuoteField(table) + } + + return schema + "." + d.QuoteField(table) +} + +func (d PostgresDialect) IfSchemaNotExists(command, schema string) string { + return fmt.Sprintf("%s if not exists", command) +} + +func (d PostgresDialect) IfTableExists(command, schema, table string) string { + return fmt.Sprintf("%s if exists", command) +} + +func (d PostgresDialect) IfTableNotExists(command, schema, table string) string { + return fmt.Sprintf("%s if not exists", command) +} diff --git a/vendor/github.com/letsencrypt/borp/gorp.go b/vendor/github.com/letsencrypt/borp/gorp.go index a78caa7d991..c5c32d8bc9b 100644 --- a/vendor/github.com/letsencrypt/borp/gorp.go +++ b/vendor/github.com/letsencrypt/borp/gorp.go @@ -281,8 +281,8 @@ func fieldByName(val reflect.Value, fieldName string) *reflect.Value { return &f } - // try to find by case insensitive match in the case where columns are - // aliased in the sql + // try to find by case insensitive match - only the Postgres driver + // seems to require this fieldNameL := strings.ToLower(fieldName) fieldCount := val.NumField() t := val.Type() diff --git a/vendor/github.com/letsencrypt/borp/test_all.sh b/vendor/github.com/letsencrypt/borp/test_all.sh index c3dd4240d71..91007d64540 100644 --- a/vendor/github.com/letsencrypt/borp/test_all.sh +++ b/vendor/github.com/letsencrypt/borp/test_all.sh @@ -6,6 +6,11 @@ echo "Running unit tests" go test -race +echo "Testing against postgres" +export GORP_TEST_DSN="host=postgres user=gorptest password=gorptest dbname=gorptest sslmode=disable" +export GORP_TEST_DIALECT=postgres +go test -tags integration $GOBUILDFLAG $@ . + echo "Testing against sqlite" export GORP_TEST_DSN=/tmp/gorptest.bin export GORP_TEST_DIALECT=sqlite diff --git a/vendor/modules.txt b/vendor/modules.txt index 061d2b26d24..691db79b7e2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -193,7 +193,7 @@ github.com/grpc-ecosystem/grpc-gateway/v2/utilities # github.com/jmhodges/clock v1.2.0 ## explicit; go 1.17 github.com/jmhodges/clock -# github.com/letsencrypt/borp v0.0.0-20230707160741-6cc6ce580243 +# github.com/letsencrypt/borp v0.0.0-20240620175310-a78493c6e2bd ## explicit; go 1.20 github.com/letsencrypt/borp # github.com/letsencrypt/challtestsrv v1.2.1