From d3fb6e00da0c7bc7fce3b22a895a0fb72e39322e Mon Sep 17 00:00:00 2001 From: Kirill Mironov <22131753+KirillMironov@users.noreply.github.com> Date: Sat, 4 Nov 2023 14:00:41 +0300 Subject: [PATCH 1/7] implement json.Marshaler and json.Unmarshaler for Float4, Float8 --- pgtype/float4.go | 24 ++++++++++++++++++++++++ pgtype/float4_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ pgtype/float8.go | 30 +++++++++++++++++++++++------- pgtype/float8_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 7 deletions(-) diff --git a/pgtype/float4.go b/pgtype/float4.go index 2540f9e51..91ca01473 100644 --- a/pgtype/float4.go +++ b/pgtype/float4.go @@ -3,6 +3,7 @@ package pgtype import ( "database/sql/driver" "encoding/binary" + "encoding/json" "fmt" "math" "strconv" @@ -65,6 +66,29 @@ func (f Float4) Value() (driver.Value, error) { return float64(f.Float32), nil } +func (f Float4) MarshalJSON() ([]byte, error) { + if !f.Valid { + return []byte("null"), nil + } + return json.Marshal(f.Float32) +} + +func (f *Float4) UnmarshalJSON(b []byte) error { + var n *float32 + err := json.Unmarshal(b, &n) + if err != nil { + return err + } + + if n == nil { + *f = Float4{} + } else { + *f = Float4{Float32: *n, Valid: true} + } + + return nil +} + type Float4Codec struct{} func (Float4Codec) FormatSupported(format int16) bool { diff --git a/pgtype/float4_test.go b/pgtype/float4_test.go index f155ed976..bc74921cf 100644 --- a/pgtype/float4_test.go +++ b/pgtype/float4_test.go @@ -21,3 +21,44 @@ func TestFloat4Codec(t *testing.T) { {nil, new(*float32), isExpectedEq((*float32)(nil))}, }) } + +func TestFloat4MarshalJSON(t *testing.T) { + successfulTests := []struct { + source pgtype.Float4 + result string + }{ + {source: pgtype.Float4{Float32: 0}, result: "null"}, + {source: pgtype.Float4{Float32: 1.23, Valid: true}, result: "1.23"}, + } + for i, tt := range successfulTests { + r, err := tt.source.MarshalJSON() + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if string(r) != tt.result { + t.Errorf("%d: expected %v to convert to %v, but it was %v", i, tt.source, tt.result, string(r)) + } + } +} + +func TestFloat4UnmarshalJSON(t *testing.T) { + successfulTests := []struct { + source string + result pgtype.Float4 + }{ + {source: "null", result: pgtype.Float4{Float32: 0}}, + {source: "1.23", result: pgtype.Float4{Float32: 1.23, Valid: true}}, + } + for i, tt := range successfulTests { + var r pgtype.Float4 + err := r.UnmarshalJSON([]byte(tt.source)) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if r != tt.result { + t.Errorf("%d: expected %v to convert to %v, but it was %v", i, tt.source, tt.result, r) + } + } +} diff --git a/pgtype/float8.go b/pgtype/float8.go index 65e5d8b32..9c923c9a3 100644 --- a/pgtype/float8.go +++ b/pgtype/float8.go @@ -74,6 +74,29 @@ func (f Float8) Value() (driver.Value, error) { return f.Float64, nil } +func (f Float8) MarshalJSON() ([]byte, error) { + if !f.Valid { + return []byte("null"), nil + } + return json.Marshal(f.Float64) +} + +func (f *Float8) UnmarshalJSON(b []byte) error { + var n *float64 + err := json.Unmarshal(b, &n) + if err != nil { + return err + } + + if n == nil { + *f = Float8{} + } else { + *f = Float8{Float64: *n, Valid: true} + } + + return nil +} + type Float8Codec struct{} func (Float8Codec) FormatSupported(format int16) bool { @@ -109,13 +132,6 @@ func (Float8Codec) PlanEncode(m *Map, oid uint32, format int16, value any) Encod return nil } -func (f *Float8) MarshalJSON() ([]byte, error) { - if !f.Valid { - return []byte("null"), nil - } - return json.Marshal(f.Float64) -} - type encodePlanFloat8CodecBinaryFloat64 struct{} func (encodePlanFloat8CodecBinaryFloat64) Encode(value any, buf []byte) (newBuf []byte, err error) { diff --git a/pgtype/float8_test.go b/pgtype/float8_test.go index 496b718b3..64593d97c 100644 --- a/pgtype/float8_test.go +++ b/pgtype/float8_test.go @@ -21,3 +21,44 @@ func TestFloat8Codec(t *testing.T) { {nil, new(*float64), isExpectedEq((*float64)(nil))}, }) } + +func TestFloat8MarshalJSON(t *testing.T) { + successfulTests := []struct { + source pgtype.Float8 + result string + }{ + {source: pgtype.Float8{Float64: 0}, result: "null"}, + {source: pgtype.Float8{Float64: 1.23, Valid: true}, result: "1.23"}, + } + for i, tt := range successfulTests { + r, err := tt.source.MarshalJSON() + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if string(r) != tt.result { + t.Errorf("%d: expected %v to convert to %v, but it was %v", i, tt.source, tt.result, string(r)) + } + } +} + +func TestFloat8UnmarshalJSON(t *testing.T) { + successfulTests := []struct { + source string + result pgtype.Float8 + }{ + {source: "null", result: pgtype.Float8{Float64: 0}}, + {source: "1.23", result: pgtype.Float8{Float64: 1.23, Valid: true}}, + } + for i, tt := range successfulTests { + var r pgtype.Float8 + err := r.UnmarshalJSON([]byte(tt.source)) + if err != nil { + t.Errorf("%d: %v", i, err) + } + + if r != tt.result { + t.Errorf("%d: expected %v to convert to %v, but it was %v", i, tt.source, tt.result, r) + } + } +} From 96f5f9cd952eccb28ce575089953c8ba7d99d77b Mon Sep 17 00:00:00 2001 From: Jack Christensen Date: Sat, 4 Nov 2023 10:27:32 -0500 Subject: [PATCH 2/7] Release v5.5.0 --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb2304a2f..63ee30d55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# 5.5.0 (November 4, 2023) + +* Add CollectExactlyOneRow. (Julien GOTTELAND) +* Add OpenDBFromPool to create *database/sql.DB from *pgxpool.Pool. (Lev Zakharov) +* Prepare can automatically choose statement name based on sql. This makes it easier to explicitly manage prepared statements. +* Statement cache now uses deterministic, stable statement names. +* database/sql prepared statement names are deterministically generated. +* Fix: SendBatch wasn't respecting context cancellation. +* Fix: Timeout error from pipeline is now normalized. +* Fix: database/sql encoding json.RawMessage to []byte. +* CancelRequest: Wait for the cancel request to be acknowledged by the server. This should improve PgBouncer compatibility. (Anton Levakin) +* stdlib: Use Ping instead of CheckConn in ResetSession +* Add json.Marshaler and json.Unmarshaler for Float4, Float8 (Kirill Mironov) + # 5.4.3 (August 5, 2023) * Fix: QCharArrayOID was defined with the wrong OID (Christoph Engelbert) From ccdd85a5ebfbeaf2037d81cfd72f7f74e5a9e181 Mon Sep 17 00:00:00 2001 From: robford Date: Fri, 20 Oct 2023 12:02:50 +0200 Subject: [PATCH 3/7] added ChopyFromCh --- copy_from.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/copy_from.go b/copy_from.go index a2c227fd4..b15a0ae16 100644 --- a/copy_from.go +++ b/copy_from.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "reflect" "github.com/jackc/pgx/v5/internal/pgio" "github.com/jackc/pgx/v5/pgconn" @@ -64,6 +65,51 @@ func (cts *copyFromSlice) Err() error { return cts.err } +// CopyFromCh returns a CopyFromSource interface over the provided channel. +// FieldNames is an ordered list of field names to copy from the struct, which +// order must match the order of the columns. +func CopyFromCh[T any](ch chan T, fieldNames []string) CopyFromSource { + return ©FromCh[T]{c: ch, fieldNames: fieldNames} +} + +type copyFromCh[T any] struct { + c chan T + fieldNames []string + valueRow []interface{} + err error +} + +func (g *copyFromCh[T]) Next() bool { + g.valueRow = g.valueRow[:0] // Clear buffer + val, ok := <-g.c + if !ok { + return false + } + // Handle both pointer to struct and struct + s := reflect.ValueOf(val) + if s.Kind() == reflect.Ptr { + s = s.Elem() + } + + for i := 0; i < len(g.fieldNames); i++ { + f := s.FieldByName(g.fieldNames[i]) + if !f.IsValid() { + g.err = fmt.Errorf("'%v' field not found in %#v", g.fieldNames[i], s.Interface()) + return false + } + g.valueRow = append(g.valueRow, f.Interface()) + } + return true +} + +func (g *copyFromCh[T]) Values() ([]interface{}, error) { + return g.valueRow, nil +} + +func (g *copyFromCh[T]) Err() error { + return g.err +} + // CopyFromSource is the interface used by *Conn.CopyFrom as the source for copy data. type CopyFromSource interface { // Next returns true if there is another row and makes the next row data From b4d72d4fce1335169a02e889d3278ec1d4b7d15d Mon Sep 17 00:00:00 2001 From: robford Date: Mon, 23 Oct 2023 13:22:04 +0200 Subject: [PATCH 4/7] copyFromFunc --- copy_from.go | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/copy_from.go b/copy_from.go index b15a0ae16..d26a69dd0 100644 --- a/copy_from.go +++ b/copy_from.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "reflect" "github.com/jackc/pgx/v5/internal/pgio" "github.com/jackc/pgx/v5/pgconn" @@ -68,45 +67,26 @@ func (cts *copyFromSlice) Err() error { // CopyFromCh returns a CopyFromSource interface over the provided channel. // FieldNames is an ordered list of field names to copy from the struct, which // order must match the order of the columns. -func CopyFromCh[T any](ch chan T, fieldNames []string) CopyFromSource { - return ©FromCh[T]{c: ch, fieldNames: fieldNames} +func CopyFromFunc(nxtf func() ([]any, error)) CopyFromSource { + return ©FromFunc{next: nxtf} } -type copyFromCh[T any] struct { - c chan T - fieldNames []string - valueRow []interface{} - err error +type copyFromFunc struct { + next func() ([]any, error) + valueRow []any + err error } -func (g *copyFromCh[T]) Next() bool { - g.valueRow = g.valueRow[:0] // Clear buffer - val, ok := <-g.c - if !ok { - return false - } - // Handle both pointer to struct and struct - s := reflect.ValueOf(val) - if s.Kind() == reflect.Ptr { - s = s.Elem() - } - - for i := 0; i < len(g.fieldNames); i++ { - f := s.FieldByName(g.fieldNames[i]) - if !f.IsValid() { - g.err = fmt.Errorf("'%v' field not found in %#v", g.fieldNames[i], s.Interface()) - return false - } - g.valueRow = append(g.valueRow, f.Interface()) - } - return true +func (g *copyFromFunc) Next() bool { + g.valueRow, g.err = g.next() + return g.err == nil } -func (g *copyFromCh[T]) Values() ([]interface{}, error) { +func (g *copyFromFunc) Values() ([]any, error) { return g.valueRow, nil } -func (g *copyFromCh[T]) Err() error { +func (g *copyFromFunc) Err() error { return g.err } From 9b6d3809d6e577e2b33253c06de4aba0f02157e4 Mon Sep 17 00:00:00 2001 From: robford Date: Fri, 27 Oct 2023 15:07:22 +0200 Subject: [PATCH 5/7] added tests --- copy_from_test.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/copy_from_test.go b/copy_from_test.go index ac2ccaabd..faed1d461 100644 --- a/copy_from_test.go +++ b/copy_from_test.go @@ -2,6 +2,7 @@ package pgx_test import ( "context" + "errors" "fmt" "os" "reflect" @@ -802,3 +803,47 @@ func TestConnCopyFromAutomaticStringConversion(t *testing.T) { ensureConnValid(t, conn) } + +func TestCopyFromFunc(t *testing.T) { + t.Parallel() + + conn := mustConnectString(t, os.Getenv("PGX_TEST_DATABASE")) + defer closeConn(t, conn) + + mustExec(t, conn, `create temporary table foo( + a int + )`) + + dataCh := make(chan int, 1) + closeChanErr := errors.New("closed channel") + + const channelItems = 10 + go func() { + for i := 0; i < channelItems; i++ { + dataCh <- i + } + close(dataCh) + }() + + copyCount, err := conn.CopyFrom(context.Background(), pgx.Identifier{"foo"}, []string{"a"}, + pgx.CopyFromFunc(func() ([]any, error) { + v, ok := <-dataCh + if !ok { + return nil, closeChanErr + } + return []any{v}, nil + })) + + fmt.Print(copyCount, err, "\n") + + require.ErrorIs(t, err, closeChanErr) + require.EqualValues(t, channelItems, copyCount) + + rows, err := conn.Query(context.Background(), "select * from foo order by a") + require.NoError(t, err) + nums, err := pgx.CollectRows(rows, pgx.RowTo[int64]) + require.NoError(t, err) + require.Equal(t, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, nums) + + ensureConnValid(t, conn) +} From d38dd857565046d05fb5d1ef73d50364f36e388b Mon Sep 17 00:00:00 2001 From: robford Date: Tue, 7 Nov 2023 09:19:16 +0100 Subject: [PATCH 6/7] Allowed nxtf to signal end of data by returning nil,nil Added some test Improved documentation --- copy_from.go | 13 +++++++------ copy_from_test.go | 23 +++++++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/copy_from.go b/copy_from.go index d26a69dd0..abcd22396 100644 --- a/copy_from.go +++ b/copy_from.go @@ -64,10 +64,10 @@ func (cts *copyFromSlice) Err() error { return cts.err } -// CopyFromCh returns a CopyFromSource interface over the provided channel. -// FieldNames is an ordered list of field names to copy from the struct, which -// order must match the order of the columns. -func CopyFromFunc(nxtf func() ([]any, error)) CopyFromSource { +// CopyFromFunc returns a CopyFromSource interface that relies on nxtf for values. +// nxtf returns rows until it either signals an 'end of data' by returning row=nil and err=nil, +// or it returns an error. If nxtf returns an error, the copy is aborted. +func CopyFromFunc(nxtf func() (row []any, err error)) CopyFromSource { return ©FromFunc{next: nxtf} } @@ -79,11 +79,12 @@ type copyFromFunc struct { func (g *copyFromFunc) Next() bool { g.valueRow, g.err = g.next() - return g.err == nil + // only return true if valueRow exists and no error + return g.valueRow != nil && g.err == nil } func (g *copyFromFunc) Values() ([]any, error) { - return g.valueRow, nil + return g.valueRow, g.err } func (g *copyFromFunc) Err() error { diff --git a/copy_from_test.go b/copy_from_test.go index faed1d461..9da23c04f 100644 --- a/copy_from_test.go +++ b/copy_from_test.go @@ -2,7 +2,6 @@ package pgx_test import ( "context" - "errors" "fmt" "os" "reflect" @@ -815,7 +814,6 @@ func TestCopyFromFunc(t *testing.T) { )`) dataCh := make(chan int, 1) - closeChanErr := errors.New("closed channel") const channelItems = 10 go func() { @@ -829,14 +827,12 @@ func TestCopyFromFunc(t *testing.T) { pgx.CopyFromFunc(func() ([]any, error) { v, ok := <-dataCh if !ok { - return nil, closeChanErr + return nil, nil } return []any{v}, nil })) - fmt.Print(copyCount, err, "\n") - - require.ErrorIs(t, err, closeChanErr) + require.ErrorIs(t, err, nil) require.EqualValues(t, channelItems, copyCount) rows, err := conn.Query(context.Background(), "select * from foo order by a") @@ -845,5 +841,20 @@ func TestCopyFromFunc(t *testing.T) { require.NoError(t, err) require.Equal(t, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, nums) + // simulate a failure + copyCount, err = conn.CopyFrom(context.Background(), pgx.Identifier{"foo"}, []string{"a"}, + pgx.CopyFromFunc(func() func() ([]any, error) { + x := 9 + return func() ([]any, error) { + x++ + if x > 100 { + return nil, fmt.Errorf("simulated error") + } + return []any{x}, nil + } + }())) + require.NotErrorIs(t, err, nil) + require.EqualValues(t, 0, copyCount) // no change, due to error + ensureConnValid(t, conn) } From df5d00eb608ce36f269009dba61607a0507a802e Mon Sep 17 00:00:00 2001 From: Jack Christensen Date: Sat, 11 Nov 2023 10:09:47 -0600 Subject: [PATCH 7/7] Remove PostgreSQL 11 from supported versions --- .github/workflows/ci.yml | 12 +----------- README.md | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93fb9fd72..c1bf91c32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,18 +17,8 @@ jobs: strategy: matrix: go-version: ["1.20", "1.21"] - pg-version: [11, 12, 13, 14, 15, 16, cockroachdb] + pg-version: [12, 13, 14, 15, 16, cockroachdb] include: - - pg-version: 11 - pgx-test-database: "host=127.0.0.1 user=pgx_md5 password=secret dbname=pgx_test" - pgx-test-unix-socket-conn-string: "host=/var/run/postgresql dbname=pgx_test" - pgx-test-tcp-conn-string: "host=127.0.0.1 user=pgx_md5 password=secret dbname=pgx_test" - pgx-test-scram-password-conn-string: "host=127.0.0.1 user=pgx_scram password=secret dbname=pgx_test" - pgx-test-md5-password-conn-string: "host=127.0.0.1 user=pgx_md5 password=secret dbname=pgx_test" - pgx-test-plain-password-conn-string: "host=127.0.0.1 user=pgx_pw password=secret dbname=pgx_test" - pgx-test-tls-conn-string: "host=localhost user=pgx_ssl password=secret sslmode=verify-full sslrootcert=/tmp/ca.pem dbname=pgx_test" - pgx-ssl-password: certpw - pgx-test-tls-client-conn-string: "host=localhost user=pgx_sslcert sslmode=verify-full sslrootcert=/tmp/ca.pem sslcert=/tmp/pgx_sslcert.crt sslkey=/tmp/pgx_sslcert.key dbname=pgx_test" - pg-version: 12 pgx-test-database: "host=127.0.0.1 user=pgx_md5 password=secret dbname=pgx_test" pgx-test-unix-socket-conn-string: "host=/var/run/postgresql dbname=pgx_test" diff --git a/README.md b/README.md index 2a9efc237..8b890836c 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ See the presentation at Golang Estonia, [PGX Top to Bottom](https://www.youtube. ## Supported Go and PostgreSQL Versions -pgx supports the same versions of Go and PostgreSQL that are supported by their respective teams. For [Go](https://golang.org/doc/devel/release.html#policy) that is the two most recent major releases and for [PostgreSQL](https://www.postgresql.org/support/versioning/) the major releases in the last 5 years. This means pgx supports Go 1.20 and higher and PostgreSQL 11 and higher. pgx also is tested against the latest version of [CockroachDB](https://www.cockroachlabs.com/product/). +pgx supports the same versions of Go and PostgreSQL that are supported by their respective teams. For [Go](https://golang.org/doc/devel/release.html#policy) that is the two most recent major releases and for [PostgreSQL](https://www.postgresql.org/support/versioning/) the major releases in the last 5 years. This means pgx supports Go 1.20 and higher and PostgreSQL 12 and higher. pgx also is tested against the latest version of [CockroachDB](https://www.cockroachlabs.com/product/). ## Version Policy