From 82939c6fe65e2f028fe5e5b6e38285519a5efa9f Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Fri, 19 May 2023 23:01:38 +1000 Subject: [PATCH 01/31] Add operations CREATE TABLE and LIST TABLES --- .gitignore | 8 +- LICENSE => LICENSE.md | 0 README.md | 10 +- conn.go | 34 +++++++ driver.go | 94 ++++++++++++++++++ go.mod | 25 +++++ go.sum | 44 +++++++++ godynamo.go | 7 ++ stmt.go | 174 +++++++++++++++++++++++++++++++++ stmt_table.go | 217 ++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 611 insertions(+), 2 deletions(-) rename LICENSE => LICENSE.md (100%) create mode 100644 conn.go create mode 100644 driver.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 godynamo.go create mode 100644 stmt.go create mode 100644 stmt_table.go diff --git a/.gitignore b/.gitignore index 3b735ec..75d3709 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,13 @@ *.out # Dependency directories (remove the comment below to include it) -# vendor/ +vendor/ # Go workspace file go.work + +# IDE +.idea/ + +# Others +Qnd/ diff --git a/LICENSE b/LICENSE.md similarity index 100% rename from LICENSE rename to LICENSE.md diff --git a/README.md b/README.md index cd2b9c8..41d9f6b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ # godynamo -Go driver for AWS DynamoDB + +[![Go Report Card](https://goreportcard.com/badge/github.com/btnguyen2k/godynamo)](https://goreportcard.com/report/github.com/btnguyen2k/godynamo) +[![PkgGoDev](https://pkg.go.dev/badge/github.com/btnguyen2k/godynamo)](https://pkg.go.dev/github.com/btnguyen2k/godynamo) +[![Actions Status](https://github.com/btnguyen2k/godynamo/workflows/godynamo/badge.svg)](https://github.com/btnguyen2k/godynamo/actions) +[![codecov](https://codecov.io/gh/btnguyen2k/godynamo/branch/main/graph/badge.svg?token=pYdHuxbIiI)](https://codecov.io/gh/btnguyen2k/godynamo) +[![Release](https://img.shields.io/github/release/btnguyen2k/godynamo.svg?style=flat-square)](RELEASE-NOTES.md) + +Go driver for [AWS DynamoDB](https://aws.amazon.com/dynamodb/) which can be used with the standard [database/sql](https://golang.org/pkg/database/sql/) package. + diff --git a/conn.go b/conn.go new file mode 100644 index 0000000..33a3e0d --- /dev/null +++ b/conn.go @@ -0,0 +1,34 @@ +package godynamo + +import ( + "database/sql/driver" + "errors" + + "github.com/aws/aws-sdk-go-v2/service/dynamodb" +) + +// Conn is AWS DynamoDB connection handler. +type Conn struct { + client *dynamodb.Client //AWS DynamoDB client +} + +// Close implements driver.Conn.Close. +func (c *Conn) Close() error { + return nil +} + +// Begin implements driver.Conn.Begin. +func (c *Conn) Begin() (driver.Tx, error) { + return nil, errors.New("transaction is not supported") +} + +// CheckNamedValue implements driver.NamedValueChecker.CheckNamedValue. +func (c *Conn) CheckNamedValue(value *driver.NamedValue) error { + // since DynamoDB is document db, it accepts any value types + return nil +} + +// Prepare implements driver.Conn.Prepare. +func (c *Conn) Prepare(query string) (driver.Stmt, error) { + return parseQuery(c, query) +} diff --git a/driver.go b/driver.go new file mode 100644 index 0000000..35de2e1 --- /dev/null +++ b/driver.go @@ -0,0 +1,94 @@ +package godynamo + +import ( + "database/sql" + "database/sql/driver" + "reflect" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/smithy-go" +) + +// init is automatically invoked when the driver is imported +func init() { + sql.Register("godynamo", &Driver{}) +} + +var ( + dataTypes = map[string]types.ScalarAttributeType{ + "BINARY": "B", + "B": "B", + "NUMBER": "N", + "N": "N", + "STRING": "S", + "S": "S", + } + + keyTypes = map[string]types.KeyType{ + "HASH": "HASH", + "RANGE": "RANGE", + } +) + +// IsAwsError returns true if err is an AWS-specific error and it matches awsErrCode. +func IsAwsError(err error, awsErrCode string) bool { + if aerr, ok := err.(*smithy.OperationError); ok { + if herr, ok := aerr.Err.(*http.ResponseError); ok { + return reflect.TypeOf(herr.Err).Elem().Name() == awsErrCode + } + } + return false +} + +// Driver is AWS DynamoDB driver for database/sql. +type Driver struct { +} + +// Open implements driver.Driver.Open. +// +// connStr is expected in the following format: +// +// Region=[Endpoint=] +// +// If not supplied, default value for TimeoutMs is 10 seconds, Version is defaultApiVersion (which is "2018-12-31"), AutoId is true, and InsecureSkipVerify is false +// +// - DefaultDb is added since v0.1.1 +// - AutoId is added since v0.1.2 +// - InsecureSkipVerify is added since v0.1.4 +func (d *Driver) Open(connStr string) (driver.Conn, error) { + params := make(map[string]string) + parts := strings.Split(connStr, ";") + for _, part := range parts { + tokens := strings.SplitN(strings.TrimSpace(part), "=", 2) + key := strings.ToUpper(strings.TrimSpace(tokens[0])) + if len(tokens) == 2 { + params[key] = strings.TrimSpace(tokens[1]) + } else { + params[key] = "" + } + } + + timeoutMs, err := strconv.Atoi(params["TIMEOUTMS"]) + if err != nil || timeoutMs < 0 { + timeoutMs = 10000 + } + opts := dynamodb.Options{ + Credentials: credentials.NewStaticCredentialsProvider(params["AKID"], params["SECRET_KEY"], ""), + HTTPClient: http.NewBuildableClient().WithTimeout(time.Millisecond * time.Duration(timeoutMs)), + Region: params["REGION"], + } + if params["ENDPOINT"] != "" { + opts.EndpointResolver = dynamodb.EndpointResolverFromURL(params["ENDPOINT"]) + if strings.HasPrefix(params["ENDPOINT"], "http://") { + opts.EndpointOptions.DisableHTTPS = true + } + } + client := dynamodb.New(opts) + return &Conn{client: client}, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8ab7679 --- /dev/null +++ b/go.mod @@ -0,0 +1,25 @@ +module github.com/btnguyen2k/godynamo + +go 1.13 + +require ( + github.com/aws/aws-sdk-go-v2/config v1.18.25 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.18.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.27 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect + github.com/aws/smithy-go v1.13.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ff763da --- /dev/null +++ b/go.sum @@ -0,0 +1,44 @@ +github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= +github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.25 h1:JuYyZcnMPBiFqn87L2cRppo+rNwgah6YwD3VuyvaW6Q= +github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= +github.com/aws/aws-sdk-go-v2/credentials v1.13.24 h1:PjiYyls3QdCrzqUN35jMWtUK1vqVZ+zLfdOa/UPFDp0= +github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 h1:jJPgroehGvjrde3XufFIJUZVK5A2L9a3KwSFgKy9n8w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7 h1:yb2o8oh3Y+Gg2g+wlzrWS3pB89+dHrXayT/d9cs8McU= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7/go.mod h1:1MNss6sqoIsFGisX92do/5doiUCBrN7EjhZCS/8DUjI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.27 h1:QmyPCRZNMR1pFbiOi9kBZWZuKrKB9LD4cxltxQk4tNE= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.27/go.mod h1:DfuVY36ixXnsG+uTqnoLWunXAKJ4qjccoFrXUPpj+hs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 h1:UBQjaMTCKwyUYwiVnUt6toEJwGXsLBI6al083tpjJzY= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 h1:PkHIIJs8qvq0e5QybnZoG1K/9QTrLr9OsqCIo59jOBA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 h1:2DQLAKDteoEDI8zpCzqBMaZlJuoE9iTYD0gFmXVax9E= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/godynamo.go b/godynamo.go new file mode 100644 index 0000000..0bd2642 --- /dev/null +++ b/godynamo.go @@ -0,0 +1,7 @@ +// Package godynamo provides database/sql driver for AWS DynamoDB. +package godynamo + +const ( + // Version of package godynamo. + Version = "0.1.0" +) diff --git a/stmt.go b/stmt.go new file mode 100644 index 0000000..ad830dc --- /dev/null +++ b/stmt.go @@ -0,0 +1,174 @@ +package godynamo + +import ( + "database/sql/driver" + "fmt" + "regexp" + "strings" +) + +const ( + field = `([\w\-]+)` + ifNotExists = `(\s+IF\s+NOT\s+EXISTS)?` + with = `(\s+WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+)((\s*,\s*|\s+)WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+))*)?` +) + +var ( + reCreateTable = regexp.MustCompile(`(?is)^CREATE\s+TABLE` + ifNotExists + `\s+` + field + with + `$`) + reListTables = regexp.MustCompile(`(?is)^LIST\s+TABLES?$`) +) + +func parseQuery(c *Conn, query string) (driver.Stmt, error) { + query = strings.TrimSpace(query) + if re := reCreateTable; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtCreateTable{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + ifNotExists: strings.TrimSpace(groups[0][1]) != "", + tableName: strings.TrimSpace(groups[0][2]), + withOptsStr: " " + strings.TrimSpace(groups[0][3]), + } + if err := stmt.parse(); err != nil { + return nil, err + } + return stmt, stmt.validate() + } + if re := reListTables; re.MatchString(query) { + stmt := &StmtListTables{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + } + return stmt, stmt.validate() + } + // if re := reAlterColl; re.MatchString(query) { + // groups := re.FindAllStringSubmatch(query, -1) + // stmt := &StmtAlterCollection{ + // Stmt: &Stmt{query: query, conn: c, numInput: 0}, + // dbName: strings.TrimSpace(groups[0][3]), + // collName: strings.TrimSpace(groups[0][4]), + // withOptsStr: strings.TrimSpace(groups[0][5]), + // } + // if stmt.dbName == "" { + // stmt.dbName = defaultDb + // } + // if err := stmt.parse(); err != nil { + // return nil, err + // } + // return stmt, stmt.validate() + // } + // if re := reDropColl; re.MatchString(query) { + // groups := re.FindAllStringSubmatch(query, -1) + // stmt := &StmtDropCollection{ + // Stmt: &Stmt{query: query, conn: c, numInput: 0}, + // dbName: strings.TrimSpace(groups[0][4]), + // collName: strings.TrimSpace(groups[0][5]), + // ifExists: strings.TrimSpace(groups[0][2]) != "", + // } + // if stmt.dbName == "" { + // stmt.dbName = defaultDb + // } + // return stmt, stmt.validate() + // } + + // if re := reInsert; re.MatchString(query) { + // groups := re.FindAllStringSubmatch(query, -1) + // stmt := &StmtInsert{ + // Stmt: &Stmt{query: query, conn: c, numInput: 0}, + // isUpsert: strings.ToUpper(strings.TrimSpace(groups[0][1])) == "UPSERT", + // dbName: strings.TrimSpace(groups[0][3]), + // collName: strings.TrimSpace(groups[0][4]), + // fieldsStr: strings.TrimSpace(groups[0][5]), + // valuesStr: strings.TrimSpace(groups[0][6]), + // } + // if stmt.dbName == "" { + // stmt.dbName = defaultDb + // } + // if err := stmt.parse(); err != nil { + // return nil, err + // } + // return stmt, stmt.validate() + // } + // if re := reSelect; re.MatchString(query) { + // groups := re.FindAllStringSubmatch(query, -1) + // stmt := &StmtSelect{ + // Stmt: &Stmt{query: query, conn: c, numInput: 0}, + // isCrossPartition: strings.TrimSpace(groups[0][1]) != "", + // collName: strings.TrimSpace(groups[0][2]), + // dbName: defaultDb, + // selectQuery: strings.ReplaceAll(strings.ReplaceAll(query, groups[0][1], ""), groups[0][3], ""), + // } + // if err := stmt.parse(groups[0][3]); err != nil { + // return nil, err + // } + // return stmt, stmt.validate() + // } + // if re := reUpdate; re.MatchString(query) { + // groups := re.FindAllStringSubmatch(query, -1) + // stmt := &StmtUpdate{ + // Stmt: &Stmt{query: query, conn: c, numInput: 0}, + // dbName: strings.TrimSpace(groups[0][2]), + // collName: strings.TrimSpace(groups[0][3]), + // updateStr: strings.TrimSpace(groups[0][4]), + // idStr: strings.TrimSpace(groups[0][5]), + // } + // if stmt.dbName == "" { + // stmt.dbName = defaultDb + // } + // if err := stmt.parse(); err != nil { + // return nil, err + // } + // return stmt, stmt.validate() + // } + // if re := reDelete; re.MatchString(query) { + // groups := re.FindAllStringSubmatch(query, -1) + // stmt := &StmtDelete{ + // Stmt: &Stmt{query: query, conn: c, numInput: 0}, + // dbName: strings.TrimSpace(groups[0][2]), + // collName: strings.TrimSpace(groups[0][3]), + // idStr: strings.TrimSpace(groups[0][4]), + // } + // if stmt.dbName == "" { + // stmt.dbName = defaultDb + // } + // if err := stmt.parse(); err != nil { + // return nil, err + // } + // return stmt, stmt.validate() + // } + + return nil, fmt.Errorf("invalid query: %s", query) +} + +// Stmt is AWS DynamoDB prepared statement handler. +type Stmt struct { + query string // the SQL query + conn *Conn // the connection that this prepared statement is bound to + numInput int // number of placeholder parameters + withOpts map[string]string +} + +var reWithOpts = regexp.MustCompile(`(?i)^(\s*,\s*|\s*)WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+)`) + +// parseWithOpts parses "WITH..." clause and store result in withOpts map. +// This function returns no error. Sub-implementations may override this behavior. +func (s *Stmt) parseWithOpts(withOptsStr string) error { + s.withOpts = make(map[string]string) + for { + matches := reWithOpts.FindStringSubmatch(withOptsStr) + if matches == nil { + break + } + s.withOpts[strings.TrimSpace(strings.ToUpper(matches[2]))] = strings.TrimSpace(matches[3]) + withOptsStr = withOptsStr[len(matches[0]):] + } + return nil +} + +// Close implements driver.Stmt.Close. +func (s *Stmt) Close() error { + return nil +} + +// NumInput implements driver.Stmt.NumInput. +func (s *Stmt) NumInput() int { + return s.numInput +} diff --git a/stmt_table.go b/stmt_table.go new file mode 100644 index 0000000..956cbdd --- /dev/null +++ b/stmt_table.go @@ -0,0 +1,217 @@ +package godynamo + +import ( + "context" + "database/sql/driver" + "errors" + "fmt" + "io" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" +) + +/*----------------------------------------------------------------------*/ + +// StmtCreateTable implements "CREATE TABLE" operation. +// +// Syntax: +// +// CREATE TABLE [IF NOT EXISTS] [, WITH SK=sk-name:sk-type][, WITH RCU=rcu][, WITH WCU=wcu] +// +// - PK: partition key, format name:type (type is one of String, Number, Binary) +// - SK: sort key, format name:type (type is one of String, Number, Binary) +// - rcu: an integer specifying DynamoDB's read capacity, default value is 1. +// - wcu: an integer specifying DynamoDB's write capacity, default value is 1. +// - If "IF NOT EXISTS" is specified, Exec will silently swallow the error "409 Conflict". +type StmtCreateTable struct { + *Stmt + tableName string + ifNotExists bool + pkName, pkType string + skName, skType string + rcu, wcu int64 + withOptsStr string +} + +func (s *StmtCreateTable) parse() error { + if err := s.Stmt.parseWithOpts(s.withOptsStr); err != nil { + return err + } + + // partition key + pkTokens := strings.SplitN(s.withOpts["PK"], ":", 2) + s.pkName = strings.TrimSpace(pkTokens[0]) + if len(pkTokens) > 1 { + s.pkType = strings.TrimSpace(strings.ToUpper(pkTokens[1])) + } + if s.pkName == "" { + return fmt.Errorf("no PartitionKey, specify one using WITH pk=pkname:pktype") + } + if _, ok := dataTypes[s.pkType]; !ok { + return fmt.Errorf("invalid type <%s> for PartitionKey, accepts values are BINARY, NUMBER and STRING", s.pkType) + } + + // sort key + skTokens := strings.SplitN(s.withOpts["SK"], ":", 2) + s.skName = strings.TrimSpace(skTokens[0]) + if len(skTokens) > 1 { + s.skType = strings.TrimSpace(strings.ToUpper(skTokens[1])) + } + if _, ok := dataTypes[s.skType]; !ok && s.skName != "" { + return fmt.Errorf("invalid type SortKey <%s>, accepts values are BINARY, NUMBER and STRING", s.skType) + } + + // RCU + if _, ok := s.withOpts["RCU"]; ok { + rcu, err := strconv.ParseInt(s.withOpts["RCU"], 10, 64) + if err != nil || rcu <= 0 { + return fmt.Errorf("invalid RCU value: %s", s.withOpts["RCU"]) + } + s.rcu = rcu + } + // WCU + if _, ok := s.withOpts["WCU"]; ok { + wcu, err := strconv.ParseInt(s.withOpts["WCU"], 10, 64) + if err != nil || wcu <= 0 { + return fmt.Errorf("invalid WCU value: %s", s.withOpts["WCU"]) + } + s.wcu = wcu + } + if s.rcu < 1 { + s.rcu = 1 + } + if s.wcu < 1 { + s.wcu = 1 + } + + return nil +} + +func (s *StmtCreateTable) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } + return nil +} + +// Query implements driver.Stmt.Query. +// This function is not implemented, use Exec instead. +func (s *StmtCreateTable) Query(_ []driver.Value) (driver.Rows, error) { + return nil, errors.New("this operation is not supported, please use Exec") +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { + attrDefs := make([]types.AttributeDefinition, 0, 2) + attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: &s.pkName, AttributeType: dataTypes[s.pkType]}) + if s.skName != "" { + attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: &s.skName, AttributeType: dataTypes[s.skType]}) + } + + keySchema := make([]types.KeySchemaElement, 0, 2) + keySchema = append(keySchema, types.KeySchemaElement{AttributeName: &s.pkName, KeyType: keyTypes["HASH"]}) + if s.skName != "" { + keySchema = append(keySchema, types.KeySchemaElement{AttributeName: &s.skName, KeyType: keyTypes["RANGE"]}) + } + + input := &dynamodb.CreateTableInput{ + TableName: &s.tableName, + AttributeDefinitions: attrDefs, + KeySchema: keySchema, + ProvisionedThroughput: &types.ProvisionedThroughput{ + ReadCapacityUnits: &s.rcu, + WriteCapacityUnits: &s.rcu, + }, + } + _, err := s.conn.client.CreateTable(context.Background(), input) + result := &ResultCreateTable{Successful: err == nil} + if s.ifNotExists && IsAwsError(err, "ResourceInUseException") { + err = nil + } + return result, err +} + +// ResultCreateTable captures the result from CREATE TABLE operation. +type ResultCreateTable struct { + // Successful flags if the operation was successful or not. + Successful bool +} + +// LastInsertId implements driver.Result.LastInsertId. +func (r *ResultCreateTable) LastInsertId() (int64, error) { + return 0, fmt.Errorf("this operation is not supported.") +} + +// RowsAffected implements driver.Result.RowsAffected. +func (r *ResultCreateTable) RowsAffected() (int64, error) { + if r.Successful { + return 1, nil + } + return 0, nil +} + +/*----------------------------------------------------------------------*/ + +// StmtListTables implements "LIST TABLES" operation. +// +// Syntax: +// +// LIST TABLES|TABLE +type StmtListTables struct { + *Stmt +} + +func (s *StmtListTables) validate() error { + return nil +} + +// Exec implements driver.Stmt.Exec. +// This function is not implemented, use Query instead. +func (s *StmtListTables) Exec(_ []driver.Value) (driver.Result, error) { + return nil, errors.New("this operation is not supported, please use Query") +} + +// Query implements driver.Stmt.Query. +func (s *StmtListTables) Query(_ []driver.Value) (driver.Rows, error) { + output, err := s.conn.client.ListTables(context.Background(), &dynamodb.ListTablesInput{}) + var rows driver.Rows + if err == nil { + rows = &RowsListTables{ + count: len(output.TableNames), + tables: output.TableNames, + cursorCount: 0, + } + } + return rows, err +} + +// RowsListTables captures the result from LIST TABLES operation. +type RowsListTables struct { + count int + tables []string + cursorCount int +} + +// Columns implements driver.Rows.Columns. +func (r *RowsListTables) Columns() []string { + return []string{"$1"} +} + +// Close implements driver.Rows.Close. +func (r *RowsListTables) Close() error { + return nil +} + +// Next implements driver.Rows.Next. +func (r *RowsListTables) Next(dest []driver.Value) error { + if r.cursorCount >= r.count { + return io.EOF + } + rowData := r.tables[r.cursorCount] + r.cursorCount++ + dest[0] = rowData + return nil +} From cbaeb0cbe45f4a440455f92a9315e1bb6e650c65 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 20 May 2023 00:10:58 +1000 Subject: [PATCH 02/31] update docs --- README.md | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++ stmt.go | 6 ++-- stmt_table.go | 2 +- 3 files changed, 98 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 41d9f6b..84cc840 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,97 @@ Go driver for [AWS DynamoDB](https://aws.amazon.com/dynamodb/) which can be used with the standard [database/sql](https://golang.org/pkg/database/sql/) package. +## Usage + +```go +import ( + "database/sql" + _ "github.com/btnguyen2k/gocosmos" +) + +func main() { + driver := "godynamo" + dsn := "Region=;AkId=;SecretKey=" + db, err := sql.Open(driver, dsn) + if err != nil { + panic(err) + } + defer db.Close() + + // db instance is ready to use + dbrows, err := db.Query(`LIST TABLES`) + if err != nil { + panic(err) + } + for dbRows.Next() { + var val interface{} + err := dbRows.Scan(&val) + if err != nil { + panic(err) + } + fmt.Println(val) + } +} +``` + +## Data Source Name (DSN) format for AWS Dynamo DB + +## Supported statements: + +- Tables: + - `CREATE TABLE` + - `LIST TABLES` + +**CREATE TABLE** + +Syntax: +```sql +CREATE TABLE [IF NOT EXIST] +WITH PK=: +[, WITH SK=:] +[, WITH wcu=] +[, WITH rcu=] +``` + +Example: +```go +result, err := db.Exec(`CREATE TABLE...`) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: create a DynamoDB table specified by `table-name`. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- If the specified table already existed: + - If `IF NOT EXISTS` is supplied: `RowsAffected()` returns `0, nil` + - If `IF NOT EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` +- `RCU`: read capacity unit. If not specified or equal to 0, default value of 1 will be used. +- `WCU`: write capacity unit. If not specified or equal to 0, default value of 1 will be used. +- `PK`: partition key, mandatory. +- `SK`: sort key, optional. +- `data-type`: must be one of `BINARY`, `NUMBER` or `STRING` + +Example: +```sql +CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5 +``` + +**LIST TABLES** + +Syntax: +```sql +LIST TABLES +``` + +Example: +```go +result, err := db.Query(`LIST TABLES`) +if err == nil { + ... +} +``` + +Description: return list of all DynamoDB tables. diff --git a/stmt.go b/stmt.go index ad830dc..22275d6 100644 --- a/stmt.go +++ b/stmt.go @@ -14,8 +14,8 @@ const ( ) var ( - reCreateTable = regexp.MustCompile(`(?is)^CREATE\s+TABLE` + ifNotExists + `\s+` + field + with + `$`) - reListTables = regexp.MustCompile(`(?is)^LIST\s+TABLES?$`) + reCreateTable = regexp.MustCompile(`(?im)^CREATE\s+TABLE` + ifNotExists + `\s+` + field + with + `$`) + reListTables = regexp.MustCompile(`(?im)^LIST\s+TABLES?$`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -146,7 +146,7 @@ type Stmt struct { withOpts map[string]string } -var reWithOpts = regexp.MustCompile(`(?i)^(\s*,\s*|\s*)WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+)`) +var reWithOpts = regexp.MustCompile(`(?im)^(\s*,\s*|\s*)WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+)`) // parseWithOpts parses "WITH..." clause and store result in withOpts map. // This function returns no error. Sub-implementations may override this behavior. diff --git a/stmt_table.go b/stmt_table.go index 956cbdd..d498864 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -123,7 +123,7 @@ func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { KeySchema: keySchema, ProvisionedThroughput: &types.ProvisionedThroughput{ ReadCapacityUnits: &s.rcu, - WriteCapacityUnits: &s.rcu, + WriteCapacityUnits: &s.wcu, }, } _, err := s.conn.client.CreateTable(context.Background(), input) From 56ba3c88616b561cc61e429da75067da21d9b090 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 20 May 2023 00:48:08 +1000 Subject: [PATCH 03/31] add operation DROP TABLE --- README.md | 35 +++++++++++++++++++++++++++-- stmt.go | 28 +++++++++++------------ stmt_table.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 107 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 84cc840..c79431a 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,10 @@ func main() { - Tables: - `CREATE TABLE` + - `DROP TABLE` - `LIST TABLES` -**CREATE TABLE** +### CREATE TABLE Syntax: ```sql @@ -86,7 +87,37 @@ Example: CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5 ``` -**LIST TABLES** +### DROP TABLE + +Syntax: +```sql +DROP TABLE [IF EXIST] +``` + +Alias: `DELETE TABLE` + +Example: +```go +result, err := db.Exec(`DROP TABLE...`) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: delete an existing DynamoDB table specified by `table-name`. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- If the specified table does not exist: + - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` + - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` + +Example: +```sql +DROP TABLE IF EXISTS demo +``` + +### LIST TABLES Syntax: ```sql diff --git a/stmt.go b/stmt.go index 22275d6..86ea915 100644 --- a/stmt.go +++ b/stmt.go @@ -10,11 +10,13 @@ import ( const ( field = `([\w\-]+)` ifNotExists = `(\s+IF\s+NOT\s+EXISTS)?` + ifExists = `(\s+IF\s+EXISTS)?` with = `(\s+WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+)((\s*,\s*|\s+)WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+))*)?` ) var ( reCreateTable = regexp.MustCompile(`(?im)^CREATE\s+TABLE` + ifNotExists + `\s+` + field + with + `$`) + reDropTable = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+TABLE` + ifExists + `\s+` + field + `$`) reListTables = regexp.MustCompile(`(?im)^LIST\s+TABLES?$`) ) @@ -24,8 +26,8 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { groups := re.FindAllStringSubmatch(query, -1) stmt := &StmtCreateTable{ Stmt: &Stmt{query: query, conn: c, numInput: 0}, - ifNotExists: strings.TrimSpace(groups[0][1]) != "", - tableName: strings.TrimSpace(groups[0][2]), + ifNotExists: strings.TrimSpace(groups[0][2]) != "", + tableName: strings.TrimSpace(groups[0][3]), withOptsStr: " " + strings.TrimSpace(groups[0][3]), } if err := stmt.parse(); err != nil { @@ -33,6 +35,15 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } return stmt, stmt.validate() } + if re := reDropTable; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtDropTable{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + tableName: strings.TrimSpace(groups[0][3]), + ifExists: strings.TrimSpace(groups[0][2]) != "", + } + return stmt, stmt.validate() + } if re := reListTables; re.MatchString(query) { stmt := &StmtListTables{ Stmt: &Stmt{query: query, conn: c, numInput: 0}, @@ -55,19 +66,6 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { // } // return stmt, stmt.validate() // } - // if re := reDropColl; re.MatchString(query) { - // groups := re.FindAllStringSubmatch(query, -1) - // stmt := &StmtDropCollection{ - // Stmt: &Stmt{query: query, conn: c, numInput: 0}, - // dbName: strings.TrimSpace(groups[0][4]), - // collName: strings.TrimSpace(groups[0][5]), - // ifExists: strings.TrimSpace(groups[0][2]) != "", - // } - // if stmt.dbName == "" { - // stmt.dbName = defaultDb - // } - // return stmt, stmt.validate() - // } // if re := reInsert; re.MatchString(query) { // groups := re.FindAllStringSubmatch(query, -1) diff --git a/stmt_table.go b/stmt_table.go index d498864..3f4e050 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -25,7 +25,7 @@ import ( // - SK: sort key, format name:type (type is one of String, Number, Binary) // - rcu: an integer specifying DynamoDB's read capacity, default value is 1. // - wcu: an integer specifying DynamoDB's write capacity, default value is 1. -// - If "IF NOT EXISTS" is specified, Exec will silently swallow the error "409 Conflict". +// - If "IF NOT EXISTS" is specified, Exec will silently swallow the error "ResourceInUseException". type StmtCreateTable struct { *Stmt tableName string @@ -155,6 +155,66 @@ func (r *ResultCreateTable) RowsAffected() (int64, error) { /*----------------------------------------------------------------------*/ +// StmtDropTable implements "DROP TABLE" operation. +// +// Syntax: +// +// DROP TABLE [IF EXISTS] +// +// If "IF EXISTS" is specified, Exec will silently swallow the error "ResourceNotFoundException". +type StmtDropTable struct { + *Stmt + tableName string + ifExists bool +} + +func (s *StmtDropTable) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } + return nil +} + +// Query implements driver.Stmt.Query. +// This function is not implemented, use Exec instead. +func (s *StmtDropTable) Query(_ []driver.Value) (driver.Rows, error) { + return nil, errors.New("this operation is not supported, please use Exec") +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtDropTable) Exec(_ []driver.Value) (driver.Result, error) { + input := &dynamodb.DeleteTableInput{ + TableName: &s.tableName, + } + _, err := s.conn.client.DeleteTable(context.Background(), input) + result := &ResultDropTable{Successful: err == nil} + if s.ifExists && IsAwsError(err, "ResourceNotFoundException") { + err = nil + } + return result, err +} + +// ResultDropTable captures the result from DROP TABLE operation. +type ResultDropTable struct { + // Successful flags if the operation was successful or not. + Successful bool +} + +// LastInsertId implements driver.Result.LastInsertId. +func (r *ResultDropTable) LastInsertId() (int64, error) { + return 0, fmt.Errorf("this operation is not supported.") +} + +// RowsAffected implements driver.Result.RowsAffected. +func (r *ResultDropTable) RowsAffected() (int64, error) { + if r.Successful { + return 1, nil + } + return 0, nil +} + +/*----------------------------------------------------------------------*/ + // StmtListTables implements "LIST TABLES" operation. // // Syntax: From c738c4f597c5b4de58741967508593afc5b7f2e1 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 20 May 2023 22:02:01 +1000 Subject: [PATCH 04/31] CREATE TABLE support local-secondary-index --- README.md | 14 +++++++-- stmt.go | 21 +++++++++----- stmt_table.go | 78 ++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c79431a..51de929 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,10 @@ WITH PK=: [, WITH SK=:] [, WITH wcu=] [, WITH rcu=] +[, WITH LSI=index-name1:attr-name1:data-type] +[, WITH LSI=index-name2:attr-name2:data-type:*] +[, WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] +[, WITH LSI...] ``` Example: @@ -74,13 +78,17 @@ Description: create a DynamoDB table specified by `table-name`. - If the statement is executed successfully, `RowsAffected()` returns `1, nil`. - If the specified table already existed: - - If `IF NOT EXISTS` is supplied: `RowsAffected()` returns `0, nil` - - If `IF NOT EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` + - If `IF NOT EXISTS` is supplied: `RowsAffected()` returns `0, nil`. + - If `IF NOT EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error`. - `RCU`: read capacity unit. If not specified or equal to 0, default value of 1 will be used. - `WCU`: write capacity unit. If not specified or equal to 0, default value of 1 will be used. - `PK`: partition key, mandatory. - `SK`: sort key, optional. -- `data-type`: must be one of `BINARY`, `NUMBER` or `STRING` +- `LSI`: local secondary index, format `index-name:attr-name:data-type[:projectionAttrs]` + - `projectionAttrs=*`: all attributes from the original table are included in projection (`ProjectionType=ALL`). + - `projectionAttrs=attr1,attr2,...`: specified attributes from the original table are included in projection (`ProjectionType=INCLUDE`). + - _projectionAttrs is not specified_: only key attributes are included in projection (`ProjectionType=KEYS_ONLY`). +- `data-type`: must be one of `BINARY`, `NUMBER` or `STRING`. Example: ```sql diff --git a/stmt.go b/stmt.go index 86ea915..7ad8b90 100644 --- a/stmt.go +++ b/stmt.go @@ -11,7 +11,7 @@ const ( field = `([\w\-]+)` ifNotExists = `(\s+IF\s+NOT\s+EXISTS)?` ifExists = `(\s+IF\s+EXISTS)?` - with = `(\s+WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+)((\s*,\s*|\s+)WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+))*)?` + with = `(\s+WITH\s+` + field + `\s*=\s*([\w/\.\*,;:'"-]+)((\s+|\s*,\s+|\s+,\s*)WITH\s+` + field + `\s*=\s*([\w/\.\*,;:'"-]+))*)?` ) var ( @@ -26,8 +26,8 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { groups := re.FindAllStringSubmatch(query, -1) stmt := &StmtCreateTable{ Stmt: &Stmt{query: query, conn: c, numInput: 0}, - ifNotExists: strings.TrimSpace(groups[0][2]) != "", - tableName: strings.TrimSpace(groups[0][3]), + ifNotExists: strings.TrimSpace(groups[0][1]) != "", + tableName: strings.TrimSpace(groups[0][2]), withOptsStr: " " + strings.TrimSpace(groups[0][3]), } if err := stmt.parse(); err != nil { @@ -136,26 +136,33 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { return nil, fmt.Errorf("invalid query: %s", query) } +type OptStrings []string + +func (s OptStrings) FirstString() string { + return s[0] +} + // Stmt is AWS DynamoDB prepared statement handler. type Stmt struct { query string // the SQL query conn *Conn // the connection that this prepared statement is bound to numInput int // number of placeholder parameters - withOpts map[string]string + withOpts map[string]OptStrings } -var reWithOpts = regexp.MustCompile(`(?im)^(\s*,\s*|\s*)WITH\s+` + field + `\s*=\s*([\w/\.;:'"-]+)`) +var reWithOpts = regexp.MustCompile(`(?im)^(\s+|\s*,\s+|\s+,\s*)WITH\s+` + field + `\s*=\s*([\w/\.\*,;:'"-]+)`) // parseWithOpts parses "WITH..." clause and store result in withOpts map. // This function returns no error. Sub-implementations may override this behavior. func (s *Stmt) parseWithOpts(withOptsStr string) error { - s.withOpts = make(map[string]string) + s.withOpts = make(map[string]OptStrings) for { matches := reWithOpts.FindStringSubmatch(withOptsStr) if matches == nil { break } - s.withOpts[strings.TrimSpace(strings.ToUpper(matches[2]))] = strings.TrimSpace(matches[3]) + k := strings.TrimSpace(strings.ToUpper(matches[2])) + s.withOpts[k] = append(s.withOpts[k], strings.TrimSuffix(strings.TrimSpace(matches[3]), ",")) withOptsStr = withOptsStr[len(matches[0]):] } return nil diff --git a/stmt_table.go b/stmt_table.go index 3f4e050..6d306ac 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -15,17 +15,35 @@ import ( /*----------------------------------------------------------------------*/ +type lsiDef struct { + indexName, fieldName, fieldType string + projectedFields string +} + // StmtCreateTable implements "CREATE TABLE" operation. // // Syntax: // -// CREATE TABLE [IF NOT EXISTS] [, WITH SK=sk-name:sk-type][, WITH RCU=rcu][, WITH WCU=wcu] +// CREATE TABLE [IF NOT EXISTS] +// +// [, WITH SK=sk-attr-name:data-type] +// [, WITH RCU=rcu][, WITH WCU=wcu] +// [, WITH LSI=index-name1:attr-name1:data-type] +// [, WITH LSI=index-name2:attr-name2:data-type:*] +// [, WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] +// [, WITH LSI...] // -// - PK: partition key, format name:type (type is one of String, Number, Binary) -// - SK: sort key, format name:type (type is one of String, Number, Binary) +// - PK: partition key, format name:type (type is one of String, Number, Binary). +// - SK: sort key, format name:type (type is one of String, Number, Binary). +// - LSI: local secondary index, format index-name:attr-name:type[:projectionAttrs], where: +// - type is one of String, Number, Binary. +// - projectionAttrs=*: all attributes from the original table are included in projection (ProjectionType=ALL). +// - projectionAttrs=attr1,attr2,...: specified attributes from the original table are included in projection (ProjectionType=INCLUDE). +// - projectionAttrs is not specified: only key attributes are included in projection (ProjectionType=KEYS_ONLY). // - rcu: an integer specifying DynamoDB's read capacity, default value is 1. // - wcu: an integer specifying DynamoDB's write capacity, default value is 1. // - If "IF NOT EXISTS" is specified, Exec will silently swallow the error "ResourceInUseException". +// - Note: there must be at least one space before the WITH keyword. type StmtCreateTable struct { *Stmt tableName string @@ -33,6 +51,7 @@ type StmtCreateTable struct { pkName, pkType string skName, skType string rcu, wcu int64 + lsi []lsiDef withOptsStr string } @@ -42,7 +61,7 @@ func (s *StmtCreateTable) parse() error { } // partition key - pkTokens := strings.SplitN(s.withOpts["PK"], ":", 2) + pkTokens := strings.SplitN(s.withOpts["PK"].FirstString(), ":", 2) s.pkName = strings.TrimSpace(pkTokens[0]) if len(pkTokens) > 1 { s.pkType = strings.TrimSpace(strings.ToUpper(pkTokens[1])) @@ -55,7 +74,7 @@ func (s *StmtCreateTable) parse() error { } // sort key - skTokens := strings.SplitN(s.withOpts["SK"], ":", 2) + skTokens := strings.SplitN(s.withOpts["SK"].FirstString(), ":", 2) s.skName = strings.TrimSpace(skTokens[0]) if len(skTokens) > 1 { s.skType = strings.TrimSpace(strings.ToUpper(skTokens[1])) @@ -64,9 +83,33 @@ func (s *StmtCreateTable) parse() error { return fmt.Errorf("invalid type SortKey <%s>, accepts values are BINARY, NUMBER and STRING", s.skType) } + // local secondary index + for _, lsiStr := range s.withOpts["LSI"] { + lsiTokens := strings.SplitN(lsiStr, ":", 4) + lsiDef := lsiDef{indexName: strings.TrimSpace(lsiTokens[0])} + if len(lsiTokens) > 1 { + lsiDef.fieldName = strings.TrimSpace(lsiTokens[1]) + } + if len(lsiTokens) > 2 { + lsiDef.fieldType = strings.TrimSpace(strings.ToUpper(lsiTokens[2])) + } + if len(lsiTokens) > 3 { + lsiDef.projectedFields = strings.TrimSpace(lsiTokens[3]) + } + if lsiDef.indexName != "" { + if lsiDef.fieldName == "" { + return fmt.Errorf("invalid LSI definition <%s>: empty field name", lsiDef.indexName) + } + if _, ok := dataTypes[lsiDef.fieldType]; !ok { + return fmt.Errorf("invalid type <%s> of LSI <%s>, accepts values are BINARY, NUMBER and STRING", lsiDef.fieldType, lsiDef.indexName) + } + } + s.lsi = append(s.lsi, lsiDef) + } + // RCU if _, ok := s.withOpts["RCU"]; ok { - rcu, err := strconv.ParseInt(s.withOpts["RCU"], 10, 64) + rcu, err := strconv.ParseInt(s.withOpts["RCU"].FirstString(), 10, 64) if err != nil || rcu <= 0 { return fmt.Errorf("invalid RCU value: %s", s.withOpts["RCU"]) } @@ -74,7 +117,7 @@ func (s *StmtCreateTable) parse() error { } // WCU if _, ok := s.withOpts["WCU"]; ok { - wcu, err := strconv.ParseInt(s.withOpts["WCU"], 10, 64) + wcu, err := strconv.ParseInt(s.withOpts["WCU"].FirstString(), 10, 64) if err != nil || wcu <= 0 { return fmt.Errorf("invalid WCU value: %s", s.withOpts["WCU"]) } @@ -117,6 +160,26 @@ func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { keySchema = append(keySchema, types.KeySchemaElement{AttributeName: &s.skName, KeyType: keyTypes["RANGE"]}) } + lsi := make([]types.LocalSecondaryIndex, len(s.lsi)) + for i := range s.lsi { + attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: &s.lsi[i].fieldName, AttributeType: dataTypes[s.lsi[i].fieldType]}) + lsi[i] = types.LocalSecondaryIndex{ + IndexName: &s.lsi[i].indexName, + KeySchema: []types.KeySchemaElement{ + {AttributeName: &s.pkName, KeyType: keyTypes["HASH"]}, + {AttributeName: &s.lsi[i].fieldName, KeyType: keyTypes["RANGE"]}, + }, + Projection: &types.Projection{ProjectionType: types.ProjectionTypeKeysOnly}, + } + if s.lsi[i].projectedFields == "*" { + lsi[i].Projection.ProjectionType = types.ProjectionTypeAll + } else if s.lsi[i].projectedFields != "" { + lsi[i].Projection.ProjectionType = types.ProjectionTypeInclude + nonKeyAttrs := strings.Split(s.lsi[i].projectedFields, ",") + lsi[i].Projection.NonKeyAttributes = nonKeyAttrs + } + } + input := &dynamodb.CreateTableInput{ TableName: &s.tableName, AttributeDefinitions: attrDefs, @@ -125,6 +188,7 @@ func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { ReadCapacityUnits: &s.rcu, WriteCapacityUnits: &s.wcu, }, + LocalSecondaryIndexes: lsi, } _, err := s.conn.client.CreateTable(context.Background(), input) result := &ResultCreateTable{Successful: err == nil} From c47f368fe0c3890315b9675c8815432d0430f83a Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 20 May 2023 22:27:31 +1000 Subject: [PATCH 05/31] CREATE TABLE support billing modes PAY_PER_REQUEST and PROVISIONED --- README.md | 4 ++++ stmt_table.go | 43 +++++++++++++++++++++---------------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 51de929..dcd7fcd 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ func main() { ## Data Source Name (DSN) format for AWS Dynamo DB +// TODO + ## Supported statements: - Tables: @@ -89,6 +91,8 @@ Description: create a DynamoDB table specified by `table-name`. - `projectionAttrs=attr1,attr2,...`: specified attributes from the original table are included in projection (`ProjectionType=INCLUDE`). - _projectionAttrs is not specified_: only key attributes are included in projection (`ProjectionType=KEYS_ONLY`). - `data-type`: must be one of `BINARY`, `NUMBER` or `STRING`. +- Note: if `RCU` and `WRU` are both `0` or not specified, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. +- Note: there must be _at least one space_ before the `WITH` keyword. Example: ```sql diff --git a/stmt_table.go b/stmt_table.go index 6d306ac..dcf47b0 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -24,14 +24,14 @@ type lsiDef struct { // // Syntax: // -// CREATE TABLE [IF NOT EXISTS] -// -// [, WITH SK=sk-attr-name:data-type] -// [, WITH RCU=rcu][, WITH WCU=wcu] -// [, WITH LSI=index-name1:attr-name1:data-type] -// [, WITH LSI=index-name2:attr-name2:data-type:*] -// [, WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] -// [, WITH LSI...] +// CREATE TABLE [IF NOT EXISTS] +// +// [, WITH SK=sk-attr-name:data-type] +// [, WITH RCU=rcu][, WITH WCU=wcu] +// [, WITH LSI=index-name1:attr-name1:data-type] +// [, WITH LSI=index-name2:attr-name2:data-type:*] +// [, WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] +// [, WITH LSI...] // // - PK: partition key, format name:type (type is one of String, Number, Binary). // - SK: sort key, format name:type (type is one of String, Number, Binary). @@ -40,9 +40,10 @@ type lsiDef struct { // - projectionAttrs=*: all attributes from the original table are included in projection (ProjectionType=ALL). // - projectionAttrs=attr1,attr2,...: specified attributes from the original table are included in projection (ProjectionType=INCLUDE). // - projectionAttrs is not specified: only key attributes are included in projection (ProjectionType=KEYS_ONLY). -// - rcu: an integer specifying DynamoDB's read capacity, default value is 1. -// - wcu: an integer specifying DynamoDB's write capacity, default value is 1. +// - RCU: an integer specifying DynamoDB's read capacity. +// - WCU: an integer specifying DynamoDB's write capacity. // - If "IF NOT EXISTS" is specified, Exec will silently swallow the error "ResourceInUseException". +// - Note: if RCU and WRU are both 0 or not specified, table will be created with PAY_PER_REQUEST billing mode; otherwise table will be creatd with PROVISIONED mode. // - Note: there must be at least one space before the WITH keyword. type StmtCreateTable struct { *Stmt @@ -123,12 +124,6 @@ func (s *StmtCreateTable) parse() error { } s.wcu = wcu } - if s.rcu < 1 { - s.rcu = 1 - } - if s.wcu < 1 { - s.wcu = 1 - } return nil } @@ -181,14 +176,18 @@ func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { } input := &dynamodb.CreateTableInput{ - TableName: &s.tableName, - AttributeDefinitions: attrDefs, - KeySchema: keySchema, - ProvisionedThroughput: &types.ProvisionedThroughput{ + TableName: &s.tableName, + AttributeDefinitions: attrDefs, + KeySchema: keySchema, + LocalSecondaryIndexes: lsi, + } + if s.rcu == 0 && s.wcu == 0 { + input.BillingMode = types.BillingModePayPerRequest + } else { + input.ProvisionedThroughput = &types.ProvisionedThroughput{ ReadCapacityUnits: &s.rcu, WriteCapacityUnits: &s.wcu, - }, - LocalSecondaryIndexes: lsi, + } } _, err := s.conn.client.CreateTable(context.Background(), input) result := &ResultCreateTable{Successful: err == nil} From 1a81d529a858b14de9364ae1dc33707a1ebe0b35 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 20 May 2023 23:24:52 +1000 Subject: [PATCH 06/31] add operation ALTER TABLE --- README.md | 76 +++++++++++----- stmt.go | 48 +++++----- stmt_table.go | 236 +++++++++++++++++++++++++++++++++++--------------- 3 files changed, 245 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index dcd7fcd..e9b970a 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,9 @@ func main() { - Tables: - `CREATE TABLE` - - `DROP TABLE` - `LIST TABLES` + - `ALTER TABLE` + - `DROP TABLE` ### CREATE TABLE @@ -58,13 +59,13 @@ Syntax: ```sql CREATE TABLE [IF NOT EXIST] WITH PK=: -[, WITH SK=:] -[, WITH wcu=] -[, WITH rcu=] -[, WITH LSI=index-name1:attr-name1:data-type] -[, WITH LSI=index-name2:attr-name2:data-type:*] -[, WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] -[, WITH LSI...] +[[,] WITH SK=:] +[[,] WITH wcu=] +[[,] WITH rcu=] +[[,] WITH LSI=index-name1:attr-name1:data-type] +[[,] WITH LSI=index-name2:attr-name2:data-type:*] +[[,] WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] +[[,] WITH LSI...] ``` Example: @@ -82,8 +83,8 @@ Description: create a DynamoDB table specified by `table-name`. - If the specified table already existed: - If `IF NOT EXISTS` is supplied: `RowsAffected()` returns `0, nil`. - If `IF NOT EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error`. -- `RCU`: read capacity unit. If not specified or equal to 0, default value of 1 will be used. -- `WCU`: write capacity unit. If not specified or equal to 0, default value of 1 will be used. +- `RCU`: read capacity unit. +- `WCU`: write capacity unit. - `PK`: partition key, mandatory. - `SK`: sort key, optional. - `LSI`: local secondary index, format `index-name:attr-name:data-type[:projectionAttrs]` @@ -99,49 +100,78 @@ Example: CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5 ``` -### DROP TABLE +### LIST TABLES Syntax: ```sql -DROP TABLE [IF EXIST] +LIST TABLES ``` -Alias: `DELETE TABLE` +Example: +```go +result, err := db.Query(`LIST TABLES`) +if err == nil { + ... +} +``` + +Description: return list of all DynamoDB tables. + +### ALTER TABLE + +Syntax: +```sql +ALTER TABLE WITH wcu= WITH rcu= +``` Example: ```go -result, err := db.Exec(`DROP TABLE...`) +result, err := db.Exec(`ALTER TABLE...`) if err == nil { numAffectedRow, err := result.RowsAffected() ... } ``` -Description: delete an existing DynamoDB table specified by `table-name`. +Description: update WCU and RCU of an existing DynamoDB table specified by `table-name`. - If the statement is executed successfully, `RowsAffected()` returns `1, nil`. -- If the specified table does not exist: - - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` - - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` +- `RCU`: read capacity unit. +- `WCU`: write capacity unit. +- Note: if `RCU` and `WRU` are both `0` or not specified, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. +- Note: there must be _at least one space_ before the `WITH` keyword. Example: ```sql -DROP TABLE IF EXISTS demo +CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5 ``` -### LIST TABLES +### DROP TABLE Syntax: ```sql -LIST TABLES +DROP TABLE [IF EXIST] ``` +Alias: `DELETE TABLE` + Example: ```go -result, err := db.Query(`LIST TABLES`) +result, err := db.Exec(`DROP TABLE...`) if err == nil { + numAffectedRow, err := result.RowsAffected() ... } ``` -Description: return list of all DynamoDB tables. +Description: delete an existing DynamoDB table specified by `table-name`. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- If the specified table does not exist: + - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` + - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` + +Example: +```sql +DROP TABLE IF EXISTS demo +``` diff --git a/stmt.go b/stmt.go index 7ad8b90..c3378c3 100644 --- a/stmt.go +++ b/stmt.go @@ -16,8 +16,9 @@ const ( var ( reCreateTable = regexp.MustCompile(`(?im)^CREATE\s+TABLE` + ifNotExists + `\s+` + field + with + `$`) - reDropTable = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+TABLE` + ifExists + `\s+` + field + `$`) reListTables = regexp.MustCompile(`(?im)^LIST\s+TABLES?$`) + reAlterTable = regexp.MustCompile(`(?im)^ALTER\s+TABLE\s+` + field + with + `$`) + reDropTable = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+TABLE` + ifExists + `\s+` + field + `$`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -35,6 +36,24 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } return stmt, stmt.validate() } + if re := reListTables; re.MatchString(query) { + stmt := &StmtListTables{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + } + return stmt, stmt.validate() + } + if re := reAlterTable; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtAlterTable{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + tableName: strings.TrimSpace(groups[0][1]), + withOptsStr: " " + strings.TrimSpace(groups[0][2]), + } + if err := stmt.parse(); err != nil { + return nil, err + } + return stmt, stmt.validate() + } if re := reDropTable; re.MatchString(query) { groups := re.FindAllStringSubmatch(query, -1) stmt := &StmtDropTable{ @@ -44,28 +63,6 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } return stmt, stmt.validate() } - if re := reListTables; re.MatchString(query) { - stmt := &StmtListTables{ - Stmt: &Stmt{query: query, conn: c, numInput: 0}, - } - return stmt, stmt.validate() - } - // if re := reAlterColl; re.MatchString(query) { - // groups := re.FindAllStringSubmatch(query, -1) - // stmt := &StmtAlterCollection{ - // Stmt: &Stmt{query: query, conn: c, numInput: 0}, - // dbName: strings.TrimSpace(groups[0][3]), - // collName: strings.TrimSpace(groups[0][4]), - // withOptsStr: strings.TrimSpace(groups[0][5]), - // } - // if stmt.dbName == "" { - // stmt.dbName = defaultDb - // } - // if err := stmt.parse(); err != nil { - // return nil, err - // } - // return stmt, stmt.validate() - // } // if re := reInsert; re.MatchString(query) { // groups := re.FindAllStringSubmatch(query, -1) @@ -139,7 +136,10 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { type OptStrings []string func (s OptStrings) FirstString() string { - return s[0] + if len(s) > 0 { + return s[0] + } + return "" } // Stmt is AWS DynamoDB prepared statement handler. diff --git a/stmt_table.go b/stmt_table.go index dcf47b0..54f735e 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -13,25 +13,25 @@ import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" ) -/*----------------------------------------------------------------------*/ - type lsiDef struct { indexName, fieldName, fieldType string projectedFields string } +/*----------------------------------------------------------------------*/ + // StmtCreateTable implements "CREATE TABLE" operation. // // Syntax: // // CREATE TABLE [IF NOT EXISTS] // -// [, WITH SK=sk-attr-name:data-type] -// [, WITH RCU=rcu][, WITH WCU=wcu] -// [, WITH LSI=index-name1:attr-name1:data-type] -// [, WITH LSI=index-name2:attr-name2:data-type:*] -// [, WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] -// [, WITH LSI...] +// [[,] WITH SK=sk-attr-name:data-type] +// [[,] WITH RCU=rcu][, WITH WCU=wcu] +// [[,] WITH LSI=index-name1:attr-name1:data-type] +// [[,] WITH LSI=index-name2:attr-name2:data-type:*] +// [[,] WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] +// [[,] WITH LSI...] // // - PK: partition key, format name:type (type is one of String, Number, Binary). // - SK: sort key, format name:type (type is one of String, Number, Binary). @@ -111,7 +111,7 @@ func (s *StmtCreateTable) parse() error { // RCU if _, ok := s.withOpts["RCU"]; ok { rcu, err := strconv.ParseInt(s.withOpts["RCU"].FirstString(), 10, 64) - if err != nil || rcu <= 0 { + if err != nil || rcu < 0 { return fmt.Errorf("invalid RCU value: %s", s.withOpts["RCU"]) } s.rcu = rcu @@ -119,7 +119,7 @@ func (s *StmtCreateTable) parse() error { // WCU if _, ok := s.withOpts["WCU"]; ok { wcu, err := strconv.ParseInt(s.withOpts["WCU"].FirstString(), 10, 64) - if err != nil || wcu <= 0 { + if err != nil || wcu < 0 { return fmt.Errorf("invalid WCU value: %s", s.withOpts["WCU"]) } s.wcu = wcu @@ -218,20 +218,117 @@ func (r *ResultCreateTable) RowsAffected() (int64, error) { /*----------------------------------------------------------------------*/ -// StmtDropTable implements "DROP TABLE" operation. +// StmtListTables implements "LIST TABLES" operation. // // Syntax: // -// DROP TABLE [IF EXISTS] +// LIST TABLES|TABLE +type StmtListTables struct { + *Stmt +} + +func (s *StmtListTables) validate() error { + return nil +} + +// Exec implements driver.Stmt.Exec. +// This function is not implemented, use Query instead. +func (s *StmtListTables) Exec(_ []driver.Value) (driver.Result, error) { + return nil, errors.New("this operation is not supported, please use Query") +} + +// Query implements driver.Stmt.Query. +func (s *StmtListTables) Query(_ []driver.Value) (driver.Rows, error) { + output, err := s.conn.client.ListTables(context.Background(), &dynamodb.ListTablesInput{}) + var rows driver.Rows + if err == nil { + rows = &RowsListTables{ + count: len(output.TableNames), + tables: output.TableNames, + cursorCount: 0, + } + } + return rows, err +} + +// RowsListTables captures the result from LIST TABLES operation. +type RowsListTables struct { + count int + tables []string + cursorCount int +} + +// Columns implements driver.Rows.Columns. +func (r *RowsListTables) Columns() []string { + return []string{"$1"} +} + +// Close implements driver.Rows.Close. +func (r *RowsListTables) Close() error { + return nil +} + +// Next implements driver.Rows.Next. +func (r *RowsListTables) Next(dest []driver.Value) error { + if r.cursorCount >= r.count { + return io.EOF + } + rowData := r.tables[r.cursorCount] + r.cursorCount++ + dest[0] = rowData + return nil +} + +/*----------------------------------------------------------------------*/ + +// StmtAlterTable implements "ALTER TABLE" operation. // -// If "IF EXISTS" is specified, Exec will silently swallow the error "ResourceNotFoundException". -type StmtDropTable struct { +// Syntax: +// +// ALTER TABLE +// WITH RCU=rcu[,] WITH WCU=wcu +// +// - RCU: an integer specifying DynamoDB's read capacity. +// - WCU: an integer specifying DynamoDB's write capacity. +// - Note: if RCU and WRU are both 0 or not specified, table will be created with PAY_PER_REQUEST billing mode; otherwise table will be creatd with PROVISIONED mode. +// - Note: there must be at least one space before the WITH keyword. +type StmtAlterTable struct { *Stmt - tableName string - ifExists bool + tableName string + rcu, wcu int64 + withOptsStr string } -func (s *StmtDropTable) validate() error { +func (s *StmtAlterTable) parse() error { + if err := s.Stmt.parseWithOpts(s.withOptsStr); err != nil { + return err + } + + // RCU + if _, ok := s.withOpts["RCU"]; ok { + rcu, err := strconv.ParseInt(s.withOpts["RCU"].FirstString(), 10, 64) + if err != nil || rcu < 0 { + return fmt.Errorf("invalid RCU value: %s", s.withOpts["RCU"]) + } + s.rcu = rcu + } else { + return fmt.Errorf("RCU not specified") + } + // WCU + if _, ok := s.withOpts["WCU"]; ok { + wcu, err := strconv.ParseInt(s.withOpts["WCU"].FirstString(), 10, 64) + if err != nil || wcu < 0 { + return fmt.Errorf("invalid WCU value: %s", s.withOpts["WCU"]) + } + s.wcu = wcu + } else { + return fmt.Errorf("WCU not specified") + } + + return nil +} + +func (s *StmtAlterTable) validate() error { if s.tableName == "" { return errors.New("table name is missing") } @@ -240,36 +337,42 @@ func (s *StmtDropTable) validate() error { // Query implements driver.Stmt.Query. // This function is not implemented, use Exec instead. -func (s *StmtDropTable) Query(_ []driver.Value) (driver.Rows, error) { +func (s *StmtAlterTable) Query(_ []driver.Value) (driver.Rows, error) { return nil, errors.New("this operation is not supported, please use Exec") } // Exec implements driver.Stmt.Exec. -func (s *StmtDropTable) Exec(_ []driver.Value) (driver.Result, error) { - input := &dynamodb.DeleteTableInput{ +func (s *StmtAlterTable) Exec(_ []driver.Value) (driver.Result, error) { + input := &dynamodb.UpdateTableInput{ TableName: &s.tableName, } - _, err := s.conn.client.DeleteTable(context.Background(), input) - result := &ResultDropTable{Successful: err == nil} - if s.ifExists && IsAwsError(err, "ResourceNotFoundException") { - err = nil + if s.rcu == 0 && s.wcu == 0 { + input.BillingMode = types.BillingModePayPerRequest + } else { + input.BillingMode = types.BillingModeProvisioned + input.ProvisionedThroughput = &types.ProvisionedThroughput{ + ReadCapacityUnits: &s.rcu, + WriteCapacityUnits: &s.wcu, + } } + _, err := s.conn.client.UpdateTable(context.Background(), input) + result := &ResultAlterTable{Successful: err == nil} return result, err } -// ResultDropTable captures the result from DROP TABLE operation. -type ResultDropTable struct { +// ResultAlterTable captures the result from CREATE TABLE operation. +type ResultAlterTable struct { // Successful flags if the operation was successful or not. Successful bool } // LastInsertId implements driver.Result.LastInsertId. -func (r *ResultDropTable) LastInsertId() (int64, error) { +func (r *ResultAlterTable) LastInsertId() (int64, error) { return 0, fmt.Errorf("this operation is not supported.") } // RowsAffected implements driver.Result.RowsAffected. -func (r *ResultDropTable) RowsAffected() (int64, error) { +func (r *ResultAlterTable) RowsAffected() (int64, error) { if r.Successful { return 1, nil } @@ -278,63 +381,60 @@ func (r *ResultDropTable) RowsAffected() (int64, error) { /*----------------------------------------------------------------------*/ -// StmtListTables implements "LIST TABLES" operation. +// StmtDropTable implements "DROP TABLE" operation. // // Syntax: // -// LIST TABLES|TABLE -type StmtListTables struct { +// DROP TABLE [IF EXISTS] +// +// If "IF EXISTS" is specified, Exec will silently swallow the error "ResourceNotFoundException". +type StmtDropTable struct { *Stmt + tableName string + ifExists bool } -func (s *StmtListTables) validate() error { +func (s *StmtDropTable) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } return nil } -// Exec implements driver.Stmt.Exec. -// This function is not implemented, use Query instead. -func (s *StmtListTables) Exec(_ []driver.Value) (driver.Result, error) { - return nil, errors.New("this operation is not supported, please use Query") -} - // Query implements driver.Stmt.Query. -func (s *StmtListTables) Query(_ []driver.Value) (driver.Rows, error) { - output, err := s.conn.client.ListTables(context.Background(), &dynamodb.ListTablesInput{}) - var rows driver.Rows - if err == nil { - rows = &RowsListTables{ - count: len(output.TableNames), - tables: output.TableNames, - cursorCount: 0, - } - } - return rows, err +// This function is not implemented, use Exec instead. +func (s *StmtDropTable) Query(_ []driver.Value) (driver.Rows, error) { + return nil, errors.New("this operation is not supported, please use Exec") } -// RowsListTables captures the result from LIST TABLES operation. -type RowsListTables struct { - count int - tables []string - cursorCount int +// Exec implements driver.Stmt.Exec. +func (s *StmtDropTable) Exec(_ []driver.Value) (driver.Result, error) { + input := &dynamodb.DeleteTableInput{ + TableName: &s.tableName, + } + _, err := s.conn.client.DeleteTable(context.Background(), input) + result := &ResultDropTable{Successful: err == nil} + if s.ifExists && IsAwsError(err, "ResourceNotFoundException") { + err = nil + } + return result, err } -// Columns implements driver.Rows.Columns. -func (r *RowsListTables) Columns() []string { - return []string{"$1"} +// ResultDropTable captures the result from DROP TABLE operation. +type ResultDropTable struct { + // Successful flags if the operation was successful or not. + Successful bool } -// Close implements driver.Rows.Close. -func (r *RowsListTables) Close() error { - return nil +// LastInsertId implements driver.Result.LastInsertId. +func (r *ResultDropTable) LastInsertId() (int64, error) { + return 0, fmt.Errorf("this operation is not supported.") } -// Next implements driver.Rows.Next. -func (r *RowsListTables) Next(dest []driver.Value) error { - if r.cursorCount >= r.count { - return io.EOF +// RowsAffected implements driver.Result.RowsAffected. +func (r *ResultDropTable) RowsAffected() (int64, error) { + if r.Successful { + return 1, nil } - rowData := r.tables[r.cursorCount] - r.cursorCount++ - dest[0] = rowData - return nil + return 0, nil } From d0dd95a9c02eaad39e7772ab7112c55d3bd58137 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 20 May 2023 23:27:17 +1000 Subject: [PATCH 07/31] update docs --- README.md | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e9b970a..26d36ae 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ WITH PK=: Example: ```go -result, err := db.Exec(`CREATE TABLE...`) +result, err := db.Exec(`CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5`) if err == nil { numAffectedRow, err := result.RowsAffected() ... @@ -95,11 +95,6 @@ Description: create a DynamoDB table specified by `table-name`. - Note: if `RCU` and `WRU` are both `0` or not specified, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. - Note: there must be _at least one space_ before the `WITH` keyword. -Example: -```sql -CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5 -``` - ### LIST TABLES Syntax: @@ -126,7 +121,7 @@ ALTER TABLE WITH wcu= WITH rcu= Example: ```go -result, err := db.Exec(`ALTER TABLE...`) +result, err := db.Exec(`ALTER TABLE demo WITH rcu=0 WITH wcu=0`) if err == nil { numAffectedRow, err := result.RowsAffected() ... @@ -141,11 +136,6 @@ Description: update WCU and RCU of an existing DynamoDB table specified by `tabl - Note: if `RCU` and `WRU` are both `0` or not specified, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. - Note: there must be _at least one space_ before the `WITH` keyword. -Example: -```sql -CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5 -``` - ### DROP TABLE Syntax: @@ -157,7 +147,7 @@ Alias: `DELETE TABLE` Example: ```go -result, err := db.Exec(`DROP TABLE...`) +result, err := db.Exec(`DROP TABLE IF EXISTS demo`) if err == nil { numAffectedRow, err := result.RowsAffected() ... @@ -170,8 +160,3 @@ Description: delete an existing DynamoDB table specified by `table-name`. - If the specified table does not exist: - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` - -Example: -```sql -DROP TABLE IF EXISTS demo -``` From ab55936f87024b9dc96e448244d1a0fa6c752191 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sun, 21 May 2023 23:46:43 +1000 Subject: [PATCH 08/31] support table-class --- README.md | 16 ++-- driver.go | 7 +- go.mod | 23 ++--- go.sum | 12 +-- stm_table_test.go | 127 ++++++++++++++++++++++++++++ stmt.go | 17 +++- stmt_table.go | 209 ++++++++++++++++++++++++++++++++++++++-------- 7 files changed, 337 insertions(+), 74 deletions(-) create mode 100644 stm_table_test.go diff --git a/README.md b/README.md index 26d36ae..2536fc3 100644 --- a/README.md +++ b/README.md @@ -60,12 +60,12 @@ Syntax: CREATE TABLE [IF NOT EXIST] WITH PK=: [[,] WITH SK=:] -[[,] WITH wcu=] -[[,] WITH rcu=] +[[,] WITH wcu=[,] WITH rcu=] [[,] WITH LSI=index-name1:attr-name1:data-type] [[,] WITH LSI=index-name2:attr-name2:data-type:*] [[,] WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] [[,] WITH LSI...] +[[,] WITH CLASS=] ``` Example: @@ -92,6 +92,7 @@ Description: create a DynamoDB table specified by `table-name`. - `projectionAttrs=attr1,attr2,...`: specified attributes from the original table are included in projection (`ProjectionType=INCLUDE`). - _projectionAttrs is not specified_: only key attributes are included in projection (`ProjectionType=KEYS_ONLY`). - `data-type`: must be one of `BINARY`, `NUMBER` or `STRING`. +- `table-class` is either `STANDARD` (default) or `STANDARD_IA`. - Note: if `RCU` and `WRU` are both `0` or not specified, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. - Note: there must be _at least one space_ before the `WITH` keyword. @@ -116,24 +117,27 @@ Description: return list of all DynamoDB tables. Syntax: ```sql -ALTER TABLE WITH wcu= WITH rcu= +ALTER TABLE +[WITH wcu=[,] WITH rcu=] +[[,] WITH CLASS=] ``` Example: ```go -result, err := db.Exec(`ALTER TABLE demo WITH rcu=0 WITH wcu=0`) +result, err := db.Exec(`ALTER TABLE demo WITH rcu=0 WITH wcu=0 WITH CLASS=STANDARD_IA`) if err == nil { numAffectedRow, err := result.RowsAffected() ... } ``` -Description: update WCU and RCU of an existing DynamoDB table specified by `table-name`. +Description: update WCU/RCU or table-class of an existing DynamoDB table specified by `table-name`. - If the statement is executed successfully, `RowsAffected()` returns `1, nil`. - `RCU`: read capacity unit. - `WCU`: write capacity unit. -- Note: if `RCU` and `WRU` are both `0` or not specified, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. +- `table-class` is either `STANDARD` (default) or `STANDARD_IA`. +- Note: if `RCU` and `WRU` are both `0`, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. - Note: there must be _at least one space_ before the `WITH` keyword. ### DROP TABLE diff --git a/driver.go b/driver.go index 35de2e1..65c4d1e 100644 --- a/driver.go +++ b/driver.go @@ -34,6 +34,11 @@ var ( "HASH": "HASH", "RANGE": "RANGE", } + + tableClasses = map[string]types.TableClass{ + "STANDARD": types.TableClassStandard, + "STANDARD_IA": types.TableClassStandardInfrequentAccess, + } ) // IsAwsError returns true if err is an AWS-specific error and it matches awsErrCode. @@ -54,7 +59,7 @@ type Driver struct { // // connStr is expected in the following format: // -// Region=[Endpoint=] +// Region=;AkId=;Secret_Key=[;Endpoint=] // // If not supplied, default value for TimeoutMs is 10 seconds, Version is defaultApiVersion (which is "2018-12-31"), AutoId is true, and InsecureSkipVerify is false // diff --git a/go.mod b/go.mod index 8ab7679..fa33c5f 100644 --- a/go.mod +++ b/go.mod @@ -2,24 +2,11 @@ module github.com/btnguyen2k/godynamo go 1.13 -require ( - github.com/aws/aws-sdk-go-v2/config v1.18.25 - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7 -) +require github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7 require ( - github.com/aws/aws-sdk-go-v2 v1.18.0 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.27 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect - github.com/aws/smithy-go v1.13.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.18.0 + github.com/aws/aws-sdk-go-v2/credentials v1.13.24 + github.com/aws/smithy-go v1.13.5 + github.com/btnguyen2k/consu/reddo v0.1.8 ) diff --git a/go.sum b/go.sum index ff763da..1fb79eb 100644 --- a/go.sum +++ b/go.sum @@ -1,33 +1,26 @@ github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.25 h1:JuYyZcnMPBiFqn87L2cRppo+rNwgah6YwD3VuyvaW6Q= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= github.com/aws/aws-sdk-go-v2/credentials v1.13.24 h1:PjiYyls3QdCrzqUN35jMWtUK1vqVZ+zLfdOa/UPFDp0= github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 h1:jJPgroehGvjrde3XufFIJUZVK5A2L9a3KwSFgKy9n8w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7 h1:yb2o8oh3Y+Gg2g+wlzrWS3pB89+dHrXayT/d9cs8McU= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7/go.mod h1:1MNss6sqoIsFGisX92do/5doiUCBrN7EjhZCS/8DUjI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.27 h1:QmyPCRZNMR1pFbiOi9kBZWZuKrKB9LD4cxltxQk4tNE= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.27/go.mod h1:DfuVY36ixXnsG+uTqnoLWunXAKJ4qjccoFrXUPpj+hs= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 h1:UBQjaMTCKwyUYwiVnUt6toEJwGXsLBI6al083tpjJzY= github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 h1:PkHIIJs8qvq0e5QybnZoG1K/9QTrLr9OsqCIo59jOBA= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 h1:2DQLAKDteoEDI8zpCzqBMaZlJuoE9iTYD0gFmXVax9E= github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/btnguyen2k/consu/reddo v0.1.8 h1:pEAkB6eadp/q+ONy97/JkAAyj058uIgkSu8b862Fwug= +github.com/btnguyen2k/consu/reddo v0.1.8/go.mod h1:pdY5oIVX3noZIaZu3nvoKZ59+seXL/taXNGWh9xJDbg= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= @@ -39,6 +32,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/stm_table_test.go b/stm_table_test.go new file mode 100644 index 0000000..a40b933 --- /dev/null +++ b/stm_table_test.go @@ -0,0 +1,127 @@ +package godynamo + +import ( + "reflect" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" +) + +func TestStmtCreateTable_parse(t *testing.T) { + testName := "TestStmtCreateTable_parse" + testData := []struct { + name string + sql string + expected *StmtCreateTable + }{ + { + name: "basic", + sql: "CREATE TABLE demo WITH pk=id:string", + expected: &StmtCreateTable{tableName: "demo", pkName: "id", pkType: "STRING"}, + }, + { + name: "with_rcu_wcu", + sql: "CREATE TABLE IF NOT EXISTS demo WITH pk=id:number, with WCU=1 WITH rcu=3", + expected: &StmtCreateTable{tableName: "demo", ifNotExists: true, pkName: "id", pkType: "NUMBER", wcu: aws.Int64(1), rcu: aws.Int64(3)}, + }, + { + name: "with_table_class", + sql: "CREATE TABLE demo WITH pk=id:number, with WCU=1 WITH rcu=3, WITH class=STANDARD_ia", + expected: &StmtCreateTable{tableName: "demo", pkName: "id", pkType: "NUMBER", wcu: aws.Int64(1), rcu: aws.Int64(3), tableClass: aws.String("STANDARD_IA")}, + }, + { + name: "with_lsi", + sql: "CREATE TABLE IF NOT EXISTS demo WITH pk=id:number, with LSI=i1:f1:string, with LSI=i2:f2:number:*, , with LSI=i3:f3:binary:a,b,c", + expected: &StmtCreateTable{tableName: "demo", ifNotExists: true, pkName: "id", pkType: "NUMBER", lsi: []lsiDef{ + {indexName: "i1", fieldName: "f1", fieldType: "STRING"}, + {indexName: "i2", fieldName: "f2", fieldType: "NUMBER", projectedFields: "*"}, + {indexName: "i3", fieldName: "f3", fieldType: "BINARY", projectedFields: "a,b,c"}, + }}, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + stmt, err := parseQuery(nil, testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmtCreateTable, ok := stmt.(*StmtCreateTable) + if !ok { + t.Fatalf("%s failed: expected StmtCreateTable but received %T", testName+"/"+testCase.name, stmt) + } + stmtCreateTable.Stmt = nil + stmtCreateTable.withOptsStr = "" + if !reflect.DeepEqual(stmtCreateTable, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmtCreateTable) + } + }) + } +} + +func TestStmtListTables_parse(t *testing.T) { + testName := "TestStmtListTables_parse" + testData := []struct { + name string + sql string + expected *StmtListTables + }{ + { + name: "basic", + sql: "LIST TABLES", + expected: &StmtListTables{}, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + stmt, err := parseQuery(nil, testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmtListTables, ok := stmt.(*StmtListTables) + if !ok { + t.Fatalf("%s failed: expected StmtListTables but received %T", testName+"/"+testCase.name, stmt) + } + stmtListTables.Stmt = nil + if !reflect.DeepEqual(stmtListTables, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmtListTables) + } + }) + } +} + +func TestStmtAlterTable_parse(t *testing.T) { + testName := "TestStmtAlterTable_parse" + testData := []struct { + name string + sql string + expected *StmtAlterTable + }{ + { + name: "with_rcu_wcu", + sql: "ALTER TABLE demo WITH wcu=1 WITH rcu=3", + expected: &StmtAlterTable{tableName: "demo", wcu: aws.Int64(1), rcu: aws.Int64(3)}, + }, + { + name: "with_table_class", + sql: "ALTER TABLE demo WITH CLASS=standard_IA", + expected: &StmtAlterTable{tableName: "demo", tableClass: aws.String("STANDARD_IA")}, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + stmt, err := parseQuery(nil, testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmtAlterTable, ok := stmt.(*StmtAlterTable) + if !ok { + t.Fatalf("%s failed: expected StmtAlterTable but received %T", testName+"/"+testCase.name, stmt) + } + stmtAlterTable.Stmt = nil + stmtAlterTable.withOptsStr = "" + if !reflect.DeepEqual(stmtAlterTable, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmtAlterTable) + } + }) + } +} diff --git a/stmt.go b/stmt.go index c3378c3..a98553f 100644 --- a/stmt.go +++ b/stmt.go @@ -15,10 +15,11 @@ const ( ) var ( - reCreateTable = regexp.MustCompile(`(?im)^CREATE\s+TABLE` + ifNotExists + `\s+` + field + with + `$`) - reListTables = regexp.MustCompile(`(?im)^LIST\s+TABLES?$`) - reAlterTable = regexp.MustCompile(`(?im)^ALTER\s+TABLE\s+` + field + with + `$`) - reDropTable = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+TABLE` + ifExists + `\s+` + field + `$`) + reCreateTable = regexp.MustCompile(`(?im)^CREATE\s+TABLE` + ifNotExists + `\s+` + field + with + `$`) + reListTables = regexp.MustCompile(`(?im)^LIST\s+TABLES?$`) + reAlterTable = regexp.MustCompile(`(?im)^ALTER\s+TABLE\s+` + field + with + `$`) + reDropTable = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+TABLE` + ifExists + `\s+` + field + `$`) + reDescribeTable = regexp.MustCompile(`(?im)^DESCRIBE\s+TABLE\s+` + field + `$`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -63,6 +64,14 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } return stmt, stmt.validate() } + if re := reDescribeTable; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtDescribeTable{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + tableName: strings.TrimSpace(groups[0][1]), + } + return stmt, stmt.validate() + } // if re := reInsert; re.MatchString(query) { // groups := re.FindAllStringSubmatch(query, -1) diff --git a/stmt_table.go b/stmt_table.go index 54f735e..23ad317 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -3,14 +3,18 @@ package godynamo import ( "context" "database/sql/driver" + "encoding/json" "errors" "fmt" "io" + "math" + "reflect" "strconv" "strings" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/btnguyen2k/consu/reddo" ) type lsiDef struct { @@ -27,11 +31,12 @@ type lsiDef struct { // CREATE TABLE [IF NOT EXISTS] // // [[,] WITH SK=sk-attr-name:data-type] -// [[,] WITH RCU=rcu][, WITH WCU=wcu] +// [[,] WITH wcu=[,] WITH rcu=] // [[,] WITH LSI=index-name1:attr-name1:data-type] // [[,] WITH LSI=index-name2:attr-name2:data-type:*] // [[,] WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] // [[,] WITH LSI...] +// [[,] WITH CLASS=] // // - PK: partition key, format name:type (type is one of String, Number, Binary). // - SK: sort key, format name:type (type is one of String, Number, Binary). @@ -42,6 +47,7 @@ type lsiDef struct { // - projectionAttrs is not specified: only key attributes are included in projection (ProjectionType=KEYS_ONLY). // - RCU: an integer specifying DynamoDB's read capacity. // - WCU: an integer specifying DynamoDB's write capacity. +// - CLASS: table class, either STANDARD (default) or STANDARD_IA. // - If "IF NOT EXISTS" is specified, Exec will silently swallow the error "ResourceInUseException". // - Note: if RCU and WRU are both 0 or not specified, table will be created with PAY_PER_REQUEST billing mode; otherwise table will be creatd with PROVISIONED mode. // - Note: there must be at least one space before the WITH keyword. @@ -50,8 +56,9 @@ type StmtCreateTable struct { tableName string ifNotExists bool pkName, pkType string - skName, skType string - rcu, wcu int64 + tableClass *string + skName, skType *string + rcu, wcu *int64 lsi []lsiDef withOptsStr string } @@ -76,12 +83,17 @@ func (s *StmtCreateTable) parse() error { // sort key skTokens := strings.SplitN(s.withOpts["SK"].FirstString(), ":", 2) - s.skName = strings.TrimSpace(skTokens[0]) - if len(skTokens) > 1 { - s.skType = strings.TrimSpace(strings.ToUpper(skTokens[1])) - } - if _, ok := dataTypes[s.skType]; !ok && s.skName != "" { - return fmt.Errorf("invalid type SortKey <%s>, accepts values are BINARY, NUMBER and STRING", s.skType) + skName := strings.TrimSpace(skTokens[0]) + if skName != "" { + s.skName = &skName + skType := "" + if len(skTokens) > 1 { + skType = strings.TrimSpace(strings.ToUpper(skTokens[1])) + } + if _, ok := dataTypes[skType]; !ok { + return fmt.Errorf("invalid type SortKey <%s>, accepts values are BINARY, NUMBER and STRING", skType) + } + s.skType = &skType } // local secondary index @@ -108,13 +120,22 @@ func (s *StmtCreateTable) parse() error { s.lsi = append(s.lsi, lsiDef) } + // table class + if _, ok := s.withOpts["CLASS"]; ok { + tableClass := strings.ToUpper(s.withOpts["CLASS"].FirstString()) + if tableClasses[tableClass] == "" { + return fmt.Errorf("invalid table class <%s>, accepts values are STANDARD, STANDARD_IA", s.withOpts["CLASS"].FirstString()) + } + s.tableClass = &tableClass + } + // RCU if _, ok := s.withOpts["RCU"]; ok { rcu, err := strconv.ParseInt(s.withOpts["RCU"].FirstString(), 10, 64) if err != nil || rcu < 0 { return fmt.Errorf("invalid RCU value: %s", s.withOpts["RCU"]) } - s.rcu = rcu + s.rcu = &rcu } // WCU if _, ok := s.withOpts["WCU"]; ok { @@ -122,7 +143,7 @@ func (s *StmtCreateTable) parse() error { if err != nil || wcu < 0 { return fmt.Errorf("invalid WCU value: %s", s.withOpts["WCU"]) } - s.wcu = wcu + s.wcu = &wcu } return nil @@ -145,14 +166,12 @@ func (s *StmtCreateTable) Query(_ []driver.Value) (driver.Rows, error) { func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { attrDefs := make([]types.AttributeDefinition, 0, 2) attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: &s.pkName, AttributeType: dataTypes[s.pkType]}) - if s.skName != "" { - attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: &s.skName, AttributeType: dataTypes[s.skType]}) - } - keySchema := make([]types.KeySchemaElement, 0, 2) keySchema = append(keySchema, types.KeySchemaElement{AttributeName: &s.pkName, KeyType: keyTypes["HASH"]}) - if s.skName != "" { - keySchema = append(keySchema, types.KeySchemaElement{AttributeName: &s.skName, KeyType: keyTypes["RANGE"]}) + + if s.skName != nil { + attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: s.skName, AttributeType: dataTypes[*s.skType]}) + keySchema = append(keySchema, types.KeySchemaElement{AttributeName: s.skName, KeyType: keyTypes["RANGE"]}) } lsi := make([]types.LocalSecondaryIndex, len(s.lsi)) @@ -181,12 +200,16 @@ func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { KeySchema: keySchema, LocalSecondaryIndexes: lsi, } - if s.rcu == 0 && s.wcu == 0 { + if s.tableClass != nil { + input.TableClass = tableClasses[*s.tableClass] + } + if (s.rcu == nil || *s.rcu == 0) && (s.wcu == nil || *s.wcu == 0) { input.BillingMode = types.BillingModePayPerRequest } else { + input.BillingMode = types.BillingModeProvisioned input.ProvisionedThroughput = &types.ProvisionedThroughput{ - ReadCapacityUnits: &s.rcu, - WriteCapacityUnits: &s.wcu, + ReadCapacityUnits: s.rcu, + WriteCapacityUnits: s.wcu, } } _, err := s.conn.client.CreateTable(context.Background(), input) @@ -279,6 +302,31 @@ func (r *RowsListTables) Next(dest []driver.Value) error { return nil } +// ColumnTypeScanType implements driver.RowsColumnTypeScanType.ColumnTypeScanType +func (r *RowsListTables) ColumnTypeScanType(index int) reflect.Type { + return reddo.TypeString +} + +// ColumnTypeDatabaseTypeName implements driver.RowsColumnTypeDatabaseTypeName.ColumnTypeDatabaseTypeName +func (r *RowsListTables) ColumnTypeDatabaseTypeName(index int) string { + return "STRING" +} + +// ColumnTypeLength implements driver.RowsColumnTypeLength.ColumnTypeLength +func (r *RowsListTables) ColumnTypeLength(index int) (length int64, ok bool) { + return math.MaxInt64, true +} + +// ColumnTypeNullable implements driver.RowsColumnTypeNullable.ColumnTypeNullable +func (r *RowsListTables) ColumnTypeNullable(index int) (nullable, ok bool) { + return false, true +} + +// ColumnTypePrecisionScale implements driver.RowsColumnTypePrecisionScale.ColumnTypePrecisionScale +func (r *RowsListTables) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { + return 0, 0, false +} + /*----------------------------------------------------------------------*/ // StmtAlterTable implements "ALTER TABLE" operation. @@ -286,16 +334,19 @@ func (r *RowsListTables) Next(dest []driver.Value) error { // Syntax: // // ALTER TABLE -// WITH RCU=rcu[,] WITH WCU=wcu +// [WITH RCU=rcu[,] WITH WCU=wcu] +// [[,] WITH CLASS=] // // - RCU: an integer specifying DynamoDB's read capacity. // - WCU: an integer specifying DynamoDB's write capacity. -// - Note: if RCU and WRU are both 0 or not specified, table will be created with PAY_PER_REQUEST billing mode; otherwise table will be creatd with PROVISIONED mode. +// - CLASS: table class, either STANDARD (default) or STANDARD_IA. +// - Note: if RCU and WRU are both 0, table will be created with PAY_PER_REQUEST billing mode; otherwise table will be creatd with PROVISIONED mode. // - Note: there must be at least one space before the WITH keyword. type StmtAlterTable struct { *Stmt tableName string - rcu, wcu int64 + rcu, wcu *int64 + tableClass *string withOptsStr string } @@ -304,15 +355,22 @@ func (s *StmtAlterTable) parse() error { return err } + // table class + if _, ok := s.withOpts["CLASS"]; ok { + tableClass := strings.ToUpper(s.withOpts["CLASS"].FirstString()) + if tableClasses[tableClass] == "" { + return fmt.Errorf("invalid table class <%s>, accepts values are STANDARD, STANDARD_IA", s.withOpts["CLASS"].FirstString()) + } + s.tableClass = &tableClass + } + // RCU if _, ok := s.withOpts["RCU"]; ok { rcu, err := strconv.ParseInt(s.withOpts["RCU"].FirstString(), 10, 64) if err != nil || rcu < 0 { return fmt.Errorf("invalid RCU value: %s", s.withOpts["RCU"]) } - s.rcu = rcu - } else { - return fmt.Errorf("RCU not specified") + s.rcu = &rcu } // WCU if _, ok := s.withOpts["WCU"]; ok { @@ -320,9 +378,7 @@ func (s *StmtAlterTable) parse() error { if err != nil || wcu < 0 { return fmt.Errorf("invalid WCU value: %s", s.withOpts["WCU"]) } - s.wcu = wcu - } else { - return fmt.Errorf("WCU not specified") + s.wcu = &wcu } return nil @@ -346,13 +402,18 @@ func (s *StmtAlterTable) Exec(_ []driver.Value) (driver.Result, error) { input := &dynamodb.UpdateTableInput{ TableName: &s.tableName, } - if s.rcu == 0 && s.wcu == 0 { - input.BillingMode = types.BillingModePayPerRequest - } else { - input.BillingMode = types.BillingModeProvisioned - input.ProvisionedThroughput = &types.ProvisionedThroughput{ - ReadCapacityUnits: &s.rcu, - WriteCapacityUnits: &s.wcu, + if s.tableClass != nil { + input.TableClass = tableClasses[*s.tableClass] + } + if s.rcu != nil || s.wcu != nil { + if s.rcu != nil && *s.rcu == 0 && s.wcu != nil && *s.wcu == 0 { + input.BillingMode = types.BillingModePayPerRequest + } else { + input.BillingMode = types.BillingModeProvisioned + input.ProvisionedThroughput = &types.ProvisionedThroughput{ + ReadCapacityUnits: s.rcu, + WriteCapacityUnits: s.wcu, + } } } _, err := s.conn.client.UpdateTable(context.Background(), input) @@ -438,3 +499,79 @@ func (r *ResultDropTable) RowsAffected() (int64, error) { } return 0, nil } + +/*----------------------------------------------------------------------*/ + +// StmtDescribeTable implements "DESCRIBE TABLE" operation. +// +// Syntax: +// +// DESCRIBE TABLE +type StmtDescribeTable struct { + *Stmt + tableName string +} + +func (s *StmtDescribeTable) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } + return nil +} + +// Query implements driver.Stmt.Query. +func (s *StmtDescribeTable) Query(_ []driver.Value) (driver.Rows, error) { + input := &dynamodb.DescribeTableInput{ + TableName: &s.tableName, + } + output, err := s.conn.client.DescribeTable(context.Background(), input) + result := &RowsDescribeTable{} + if err == nil { + js, _ := json.Marshal(output.Table) + json.Unmarshal(js, &result.tableInfo) + for k := range result.tableInfo { + result.columnList = append(result.columnList, k) + } + result.count = 1 + } + if IsAwsError(err, "ResourceNotFoundException") { + err = nil + } + return result, err +} + +// Exec implements driver.Stmt.Exec. +// This function is not implemented, use Query instead. +func (s *StmtDescribeTable) Exec(_ []driver.Value) (driver.Result, error) { + return nil, errors.New("this operation is not supported, please use Query") +} + +// RowsDescribeTable captures the result from DESCRIBE TABLE operation. +type RowsDescribeTable struct { + count int + columnList []string + tableInfo map[string]interface{} + cursorCount int +} + +// Columns implements driver.Rows.Columns. +func (r *RowsDescribeTable) Columns() []string { + return r.columnList +} + +// Close implements driver.Rows.Close. +func (r *RowsDescribeTable) Close() error { + return nil +} + +// Next implements driver.Rows.Next. +func (r *RowsDescribeTable) Next(dest []driver.Value) error { + if r.cursorCount >= r.count { + return io.EOF + } + for i, colName := range r.columnList { + dest[i] = r.tableInfo[colName] + } + r.cursorCount++ + return nil +} From 188c37ea3b14b696f5e428c837987ae1d76b9c91 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sun, 21 May 2023 23:54:46 +1000 Subject: [PATCH 09/31] more test cases --- stm_table_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/stm_table_test.go b/stm_table_test.go index a40b933..6094aa2 100644 --- a/stm_table_test.go +++ b/stm_table_test.go @@ -125,3 +125,70 @@ func TestStmtAlterTable_parse(t *testing.T) { }) } } + +func TestStmtDropTable_parse(t *testing.T) { + testName := "TestStmtDropTable_parse" + testData := []struct { + name string + sql string + expected *StmtDropTable + }{ + { + name: "basic", + sql: "DROP TABLE demo", + expected: &StmtDropTable{tableName: "demo"}, + }, + { + name: "if_exists", + sql: "DROP TABLE IF EXISTS demo", + expected: &StmtDropTable{tableName: "demo", ifExists: true}, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + stmt, err := parseQuery(nil, testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmtDropTable, ok := stmt.(*StmtDropTable) + if !ok { + t.Fatalf("%s failed: expected StmtDropTable but received %T", testName+"/"+testCase.name, stmt) + } + stmtDropTable.Stmt = nil + if !reflect.DeepEqual(stmtDropTable, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmtDropTable) + } + }) + } +} + +func TestStmtDescribeTable_parse(t *testing.T) { + testName := "TestStmtDescribeTable_parse" + testData := []struct { + name string + sql string + expected *StmtDescribeTable + }{ + { + name: "basic", + sql: "DESCRIBE TABLE demo", + expected: &StmtDescribeTable{tableName: "demo"}, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + stmt, err := parseQuery(nil, testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmtDescribeTable, ok := stmt.(*StmtDescribeTable) + if !ok { + t.Fatalf("%s failed: expected StmtDescribeTable but received %T", testName+"/"+testCase.name, stmt) + } + stmtDescribeTable.Stmt = nil + if !reflect.DeepEqual(stmtDescribeTable, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmtDescribeTable) + } + }) + } +} From e48d028d8572db73a261fafcc81baa866d56c30e Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 00:31:48 +1000 Subject: [PATCH 10/31] add GitHub Actions --- .github/workflows/godynamo.yaml | 33 +++++++++++++++++++ .gitignore | 1 + ...able_test.go => stmt_table_parsing_test.go | 5 +++ 3 files changed, 39 insertions(+) create mode 100644 .github/workflows/godynamo.yaml rename stm_table_test.go => stmt_table_parsing_test.go (96%) diff --git a/.github/workflows/godynamo.yaml b/.github/workflows/godynamo.yaml new file mode 100644 index 0000000..da1020a --- /dev/null +++ b/.github/workflows/godynamo.yaml @@ -0,0 +1,33 @@ +name: godynamo + +on: + push: + branches: [ '*' ] + pull_request: + branches: [ main ] + +jobs: + testLocal: + name: Test against AWS DynamoDB local + runs-on: ubuntu-latest + steps: + - name: Set up Go env + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + - name: Start AWS DynamoDB local server + run: docker run -d --name dynamodb -p 8000:8000 amazon/dynamodb-local -jar DynamoDBLocal.jar -inMemory -sharedDb + - name: Test + run: | + export AWS_REGION="us-east-1" + export AWS_ACCESS_KEY_ID="DUMMYID" + export AWS_SECRET_ACCESS_KEY="DUMMYKEY" + export AWS_DYNAMODB_ENDPOINT="http://localhost:8000" + go test -v -timeout 9999s -count 1 -p 1 -cover -coverprofile coverage_local.txt . + - name: Codecov + uses: codecov/codecov-action@v3 + with: + flags: local + name: local diff --git a/.gitignore b/.gitignore index 75d3709..95a6791 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ go.work # Others Qnd/ +/coverage.txt diff --git a/stm_table_test.go b/stmt_table_parsing_test.go similarity index 96% rename from stm_table_test.go rename to stmt_table_parsing_test.go index 6094aa2..096a5ce 100644 --- a/stm_table_test.go +++ b/stmt_table_parsing_test.go @@ -19,6 +19,11 @@ func TestStmtCreateTable_parse(t *testing.T) { sql: "CREATE TABLE demo WITH pk=id:string", expected: &StmtCreateTable{tableName: "demo", pkName: "id", pkType: "STRING"}, }, + { + name: "with_sk", + sql: "CREATE TABLE demo WITH pk=id:string with SK=grade:binary", + expected: &StmtCreateTable{tableName: "demo", pkName: "id", pkType: "STRING", skName: aws.String("grade"), skType: aws.String("BINARY")}, + }, { name: "with_rcu_wcu", sql: "CREATE TABLE IF NOT EXISTS demo WITH pk=id:number, with WCU=1 WITH rcu=3", From fb74a68a84ccafdc3e9ee9425d404cc7a6f3cfa4 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 00:56:23 +1000 Subject: [PATCH 11/31] update readme --- README.md | 30 +++++++++++++++++------------- stmt_table.go | 3 +++ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2536fc3..42b1847 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,17 @@ Go driver for [AWS DynamoDB](https://aws.amazon.com/dynamodb/) which can be used ## Usage ```go +package main + import ( "database/sql" + "fmt" + _ "github.com/btnguyen2k/gocosmos" ) func main() { - driver := "godynamo" + driver := "godynamo" dsn := "Region=;AkId=;SecretKey=" db, err := sql.Open(driver, dsn) if err != nil { @@ -25,11 +29,11 @@ func main() { } defer db.Close() - // db instance is ready to use - dbrows, err := db.Query(`LIST TABLES`) - if err != nil { - panic(err) - } + // db instance is ready to use + dbrows, err := db.Query(`LIST TABLES`) + if err != nil { + panic(err) + } for dbRows.Next() { var val interface{} err := dbRows.Scan(&val) @@ -72,8 +76,8 @@ Example: ```go result, err := db.Exec(`CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5`) if err == nil { - numAffectedRow, err := result.RowsAffected() - ... + numAffectedRow, err := result.RowsAffected() + ... } ``` @@ -107,7 +111,7 @@ Example: ```go result, err := db.Query(`LIST TABLES`) if err == nil { - ... + ... } ``` @@ -126,8 +130,8 @@ Example: ```go result, err := db.Exec(`ALTER TABLE demo WITH rcu=0 WITH wcu=0 WITH CLASS=STANDARD_IA`) if err == nil { - numAffectedRow, err := result.RowsAffected() - ... + numAffectedRow, err := result.RowsAffected() + ... } ``` @@ -153,8 +157,8 @@ Example: ```go result, err := db.Exec(`DROP TABLE IF EXISTS demo`) if err == nil { - numAffectedRow, err := result.RowsAffected() - ... + numAffectedRow, err := result.RowsAffected() + ... } ``` diff --git a/stmt_table.go b/stmt_table.go index 23ad317..294ecda 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -9,6 +9,7 @@ import ( "io" "math" "reflect" + "sort" "strconv" "strings" @@ -270,6 +271,7 @@ func (s *StmtListTables) Query(_ []driver.Value) (driver.Rows, error) { tables: output.TableNames, cursorCount: 0, } + sort.Strings(rows.(*RowsListTables).tables) } return rows, err } @@ -532,6 +534,7 @@ func (s *StmtDescribeTable) Query(_ []driver.Value) (driver.Rows, error) { for k := range result.tableInfo { result.columnList = append(result.columnList, k) } + sort.Strings(result.columnList) result.count = 1 } if IsAwsError(err, "ResourceNotFoundException") { From 94cb3b624842402cc352810b174a177f612b2c74 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 01:17:29 +1000 Subject: [PATCH 12/31] add operation DESCRIBE TABLE --- README.md | 18 +++++++++++++++++ stmt_table.go | 56 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 42b1847..d2d8237 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ func main() { - `LIST TABLES` - `ALTER TABLE` - `DROP TABLE` + - `DESCRIBE TABLE` ### CREATE TABLE @@ -168,3 +169,20 @@ Description: delete an existing DynamoDB table specified by `table-name`. - If the specified table does not exist: - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` + +### DESCRIBE TABLE + +Syntax: +```sql +DESCRIBE TABLE +``` + +Example: +```go +result, err := db.Query(`DESCRIBE TABLE demo`) +if err == nil { + ... +} +``` + +Description: return info of a DynamoDB table specified by `table-name`. diff --git a/stmt_table.go b/stmt_table.go index 294ecda..6bd7931 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -527,7 +527,7 @@ func (s *StmtDescribeTable) Query(_ []driver.Value) (driver.Rows, error) { TableName: &s.tableName, } output, err := s.conn.client.DescribeTable(context.Background(), input) - result := &RowsDescribeTable{} + result := &RowsDescribeTable{count: 1} if err == nil { js, _ := json.Marshal(output.Table) json.Unmarshal(js, &result.tableInfo) @@ -535,7 +535,10 @@ func (s *StmtDescribeTable) Query(_ []driver.Value) (driver.Rows, error) { result.columnList = append(result.columnList, k) } sort.Strings(result.columnList) - result.count = 1 + result.columnTypeList = make([]reflect.Type, len(result.columnList)) + for i, col := range result.columnList { + result.columnTypeList[i] = reflect.TypeOf(result.tableInfo[col]) + } } if IsAwsError(err, "ResourceNotFoundException") { err = nil @@ -551,10 +554,11 @@ func (s *StmtDescribeTable) Exec(_ []driver.Value) (driver.Result, error) { // RowsDescribeTable captures the result from DESCRIBE TABLE operation. type RowsDescribeTable struct { - count int - columnList []string - tableInfo map[string]interface{} - cursorCount int + count int + columnList []string + columnTypeList []reflect.Type + tableInfo map[string]interface{} + cursorCount int } // Columns implements driver.Rows.Columns. @@ -578,3 +582,43 @@ func (r *RowsDescribeTable) Next(dest []driver.Value) error { r.cursorCount++ return nil } + +// ColumnTypeScanType implements driver.RowsColumnTypeScanType.ColumnTypeScanType +func (r *RowsDescribeTable) ColumnTypeScanType(index int) reflect.Type { + return r.columnTypeList[index] +} + +// ColumnTypeDatabaseTypeName implements driver.RowsColumnTypeDatabaseTypeName.ColumnTypeDatabaseTypeName +func (r *RowsDescribeTable) ColumnTypeDatabaseTypeName(index int) string { + if r.columnTypeList[index] == nil { + return "" + } + switch r.columnTypeList[index].Kind() { + case reflect.Bool: + return "BOOLEAN" + case reflect.String: + return "STRING" + case reflect.Float32, reflect.Float64: + return "NUMBER" + case reflect.Array, reflect.Slice: + return "ARRAY" + case reflect.Map: + return "MAP" + } + return "" +} + +// // ColumnTypeLength implements driver.RowsColumnTypeLength.ColumnTypeLength +// func (r *RowsDescribeTable) ColumnTypeLength(index int) (length int64, ok bool) { +// return math.MaxInt64, true +// } + +// // ColumnTypeNullable implements driver.RowsColumnTypeNullable.ColumnTypeNullable +// func (r *RowsDescribeTable) ColumnTypeNullable(index int) (nullable, ok bool) { +// return false, true +// } + +// // ColumnTypePrecisionScale implements driver.RowsColumnTypePrecisionScale.ColumnTypePrecisionScale +// func (r *RowsDescribeTable) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { +// return 0, 0, false +// } From 586e960cee071f853b2858b89f4d2a01d0359234 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 01:34:49 +1000 Subject: [PATCH 13/31] more test cases --- .github/workflows/godynamo.yaml | 1 + driver.go | 27 ++++++-- godynamo_test.go | 105 ++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 godynamo_test.go diff --git a/.github/workflows/godynamo.yaml b/.github/workflows/godynamo.yaml index da1020a..fafdf62 100644 --- a/.github/workflows/godynamo.yaml +++ b/.github/workflows/godynamo.yaml @@ -25,6 +25,7 @@ jobs: export AWS_ACCESS_KEY_ID="DUMMYID" export AWS_SECRET_ACCESS_KEY="DUMMYKEY" export AWS_DYNAMODB_ENDPOINT="http://localhost:8000" + export AWS_DYNAMODB_URL="Endpoint=http://localhost:8000" go test -v -timeout 9999s -count 1 -p 1 -cover -coverprofile coverage_local.txt . - name: Codecov uses: codecov/codecov-action@v3 diff --git a/driver.go b/driver.go index 65c4d1e..e1947dd 100644 --- a/driver.go +++ b/driver.go @@ -3,6 +3,7 @@ package godynamo import ( "database/sql" "database/sql/driver" + "os" "reflect" "strconv" "strings" @@ -83,14 +84,30 @@ func (d *Driver) Open(connStr string) (driver.Conn, error) { if err != nil || timeoutMs < 0 { timeoutMs = 10000 } + akid := params["AKID"] + if akid == "" { + akid = os.Getenv("AWS_ACCESS_KEY_ID") + } + secretKey := params["SECRET_KEY"] + if secretKey == "" { + secretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + } + region := params["REGION"] + if region == "" { + region = os.Getenv("AWS_REGION") + } opts := dynamodb.Options{ - Credentials: credentials.NewStaticCredentialsProvider(params["AKID"], params["SECRET_KEY"], ""), + Credentials: credentials.NewStaticCredentialsProvider(akid, secretKey, ""), HTTPClient: http.NewBuildableClient().WithTimeout(time.Millisecond * time.Duration(timeoutMs)), - Region: params["REGION"], + Region: region, + } + endpoint := params["ENDPOINT"] + if endpoint == "" { + endpoint = os.Getenv("AWS_DYNAMODB_ENDPOINT") } - if params["ENDPOINT"] != "" { - opts.EndpointResolver = dynamodb.EndpointResolverFromURL(params["ENDPOINT"]) - if strings.HasPrefix(params["ENDPOINT"], "http://") { + if endpoint != "" { + opts.EndpointResolver = dynamodb.EndpointResolverFromURL(endpoint) + if strings.HasPrefix(endpoint, "http://") { opts.EndpointOptions.DisableHTTPS = true } } diff --git a/godynamo_test.go b/godynamo_test.go new file mode 100644 index 0000000..80a78aa --- /dev/null +++ b/godynamo_test.go @@ -0,0 +1,105 @@ +package godynamo + +import ( + "context" + "database/sql" + "os" + "strings" + "testing" +) + +func Test_OpenDatabase(t *testing.T) { + testName := "Test_OpenDatabase" + driver := "godynamo" + dsn := "dummy" + db, err := sql.Open(driver, dsn) + if err != nil { + t.Fatalf("%s failed: %s", testName, err) + } + if db == nil { + t.Fatalf("%s failed: nil", testName) + } +} + +// func TestDriver_invalidConnectionString(t *testing.T) { +// testName := "TestDriver_invalidConnectionString" +// driver := "godynamo" +// { +// db, _ := sql.Open(driver, "Endpoint;AkId=demo") +// if err := db.Ping(); err == nil { +// t.Fatalf("%s failed: should have error", testName) +// } +// } +// { +// db, _ := sql.Open(driver, "Endpoint=demo;AkId") +// if err := db.Ping(); err == nil { +// t.Fatalf("%s failed: should have error", testName) +// } +// } +// { +// db, _ := sql.Open(driver, "Endpoint=http://localhost:8000;AkId=demo/invalid_key") +// if err := db.Ping(); err == nil { +// t.Fatalf("%s failed: should have error", testName) +// } +// } +// } + +/*----------------------------------------------------------------------*/ + +func _openDb(t *testing.T, testName string) *sql.DB { + driver := "godynamo" + url := strings.ReplaceAll(os.Getenv("AWS_DYNAMODB_URL"), `"`, "") + if url == "" { + t.Skipf("%s skipped", testName) + } + db, err := sql.Open(driver, url) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/sql.Open", err) + } + return db +} + +/*----------------------------------------------------------------------*/ + +func TestDriver_Conn(t *testing.T) { + testName := "TestDriver_Conn" + db := _openDb(t, testName) + defer db.Close() + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatalf("%s failed: %s", testName, err) + } + defer conn.Close() +} + +func TestDriver_Transaction(t *testing.T) { + testName := "TestDriver_Transaction" + db := _openDb(t, testName) + defer db.Close() + if tx, err := db.BeginTx(context.Background(), nil); tx != nil || err == nil { + t.Fatalf("%s failed: transaction is not supported yet", testName) + } else if strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: transaction is not supported yet / %s", testName, err) + } +} + +func TestDriver_Open(t *testing.T) { + testName := "TestDriver_Open" + db := _openDb(t, testName) + defer db.Close() + if err := db.Ping(); err != nil { + t.Fatalf("%s failed: %s", testName, err) + } +} + +func TestDriver_Close(t *testing.T) { + testName := "TestDriver_Close" + db := _openDb(t, testName) + defer db.Close() + if err := db.Ping(); err != nil { + t.Fatalf("%s failed: %s", testName, err) + } + if err := db.Close(); err != nil { + t.Fatalf("%s failed: %s", testName, err) + } +} From 240189859279fbb4faba1fc2c05ee55dcd38f182 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 01:40:25 +1000 Subject: [PATCH 14/31] more test cases --- stmt_table_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 stmt_table_test.go diff --git a/stmt_table_test.go b/stmt_table_test.go new file mode 100644 index 0000000..7b1cde3 --- /dev/null +++ b/stmt_table_test.go @@ -0,0 +1,30 @@ +package godynamo + +import ( + "strings" + "testing" +) + +func Test_Query_CreateTable(t *testing.T) { + testName := "Test_Query_CreateTable" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Query("CREATE TABLE tbltemp WITH pk=id:string") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Exec_CreateTable(t *testing.T) { + testName := "Test_Exec_CreateTable" + db := _openDb(t, testName) + defer db.Close() + + tableList := []string{"tbltemp1", "tbltemp2", "tbltemp3"} + defer func() { + for _, tbl := range tableList { + db.Exec("DROP TABLE IF EXISTS " + tbl) + } + }() +} From 3b4b481eef8a1e39ccb639ed4d9c5d801e2aac5d Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 01:51:53 +1000 Subject: [PATCH 15/31] more test cases --- conn.go | 10 +++++----- stmt_table.go | 25 ++++++++++++------------- stmt_table_test.go | 22 +++++++++++++++++++++- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/conn.go b/conn.go index 33a3e0d..bcc951f 100644 --- a/conn.go +++ b/conn.go @@ -22,11 +22,11 @@ func (c *Conn) Begin() (driver.Tx, error) { return nil, errors.New("transaction is not supported") } -// CheckNamedValue implements driver.NamedValueChecker.CheckNamedValue. -func (c *Conn) CheckNamedValue(value *driver.NamedValue) error { - // since DynamoDB is document db, it accepts any value types - return nil -} +// // CheckNamedValue implements driver.NamedValueChecker.CheckNamedValue. +// func (c *Conn) CheckNamedValue(value *driver.NamedValue) error { +// // since DynamoDB is document db, it accepts any value types +// return nil +// } // Prepare implements driver.Conn.Prepare. func (c *Conn) Prepare(query string) (driver.Stmt, error) { diff --git a/stmt_table.go b/stmt_table.go index 6bd7931..f2bece7 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "math" "reflect" "sort" "strconv" @@ -314,20 +313,20 @@ func (r *RowsListTables) ColumnTypeDatabaseTypeName(index int) string { return "STRING" } -// ColumnTypeLength implements driver.RowsColumnTypeLength.ColumnTypeLength -func (r *RowsListTables) ColumnTypeLength(index int) (length int64, ok bool) { - return math.MaxInt64, true -} +// // ColumnTypeLength implements driver.RowsColumnTypeLength.ColumnTypeLength +// func (r *RowsListTables) ColumnTypeLength(index int) (length int64, ok bool) { +// return math.MaxInt64, true +// } -// ColumnTypeNullable implements driver.RowsColumnTypeNullable.ColumnTypeNullable -func (r *RowsListTables) ColumnTypeNullable(index int) (nullable, ok bool) { - return false, true -} +// // ColumnTypeNullable implements driver.RowsColumnTypeNullable.ColumnTypeNullable +// func (r *RowsListTables) ColumnTypeNullable(index int) (nullable, ok bool) { +// return false, true +// } -// ColumnTypePrecisionScale implements driver.RowsColumnTypePrecisionScale.ColumnTypePrecisionScale -func (r *RowsListTables) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { - return 0, 0, false -} +// // ColumnTypePrecisionScale implements driver.RowsColumnTypePrecisionScale.ColumnTypePrecisionScale +// func (r *RowsListTables) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { +// return 0, 0, false +// } /*----------------------------------------------------------------------*/ diff --git a/stmt_table_test.go b/stmt_table_test.go index 7b1cde3..a577068 100644 --- a/stmt_table_test.go +++ b/stmt_table_test.go @@ -21,10 +21,30 @@ func Test_Exec_CreateTable(t *testing.T) { db := _openDb(t, testName) defer db.Close() - tableList := []string{"tbltemp1", "tbltemp2", "tbltemp3"} + tableList := []string{"tbltemp1", "tbltemp2", "tbltemp3", "tbltemp4"} defer func() { for _, tbl := range tableList { db.Exec("DROP TABLE IF EXISTS " + tbl) } }() + + testData := []struct { + name string + sql string + }{ + {name: "basic", sql: `CREATE TABLE tbltemp1 WITH PK=id:string`}, + {name: "if_not_exists", sql: `CREATE TABLE IF NOT EXISTS tbltemp1 WITH PK=id:string`}, + {name: "with_sk", sql: `CREATE TABLE tbltemp2 WITH PK=id:string WITH sk=grade:number, WITH class=standard`}, + {name: "with_rcu_wcu", sql: `CREATE TABLE tbltemp3 WITH PK=id:string WITH rcu=1 WICH wcu=2 WITH class=standard_ia`}, + {name: "with_lsi", sql: `CREATE TABLE tbltemp4 WITH PK=id:string WITH LSI=index1:grade:number, WITH LSI=index2:dob:string:*, WITH LSI=index3:yob:binary:a,b,c`}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + _, err := db.Exec(testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + }) + } } From 814b03dbf6c9ee6556f247d3573649ee90e3fcfd Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 01:53:37 +1000 Subject: [PATCH 16/31] more test cases --- stmt_table_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stmt_table_test.go b/stmt_table_test.go index a577068..4f034fc 100644 --- a/stmt_table_test.go +++ b/stmt_table_test.go @@ -35,7 +35,7 @@ func Test_Exec_CreateTable(t *testing.T) { {name: "basic", sql: `CREATE TABLE tbltemp1 WITH PK=id:string`}, {name: "if_not_exists", sql: `CREATE TABLE IF NOT EXISTS tbltemp1 WITH PK=id:string`}, {name: "with_sk", sql: `CREATE TABLE tbltemp2 WITH PK=id:string WITH sk=grade:number, WITH class=standard`}, - {name: "with_rcu_wcu", sql: `CREATE TABLE tbltemp3 WITH PK=id:string WITH rcu=1 WICH wcu=2 WITH class=standard_ia`}, + {name: "with_rcu_wcu", sql: `CREATE TABLE tbltemp3 WITH PK=id:string WITH rcu=1 WITH wcu=2 WITH class=standard_ia`}, {name: "with_lsi", sql: `CREATE TABLE tbltemp4 WITH PK=id:string WITH LSI=index1:grade:number, WITH LSI=index2:dob:string:*, WITH LSI=index3:yob:binary:a,b,c`}, } From c98ba228ca9331b037a050e46d78e5e0dd60c401 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 01:56:39 +1000 Subject: [PATCH 17/31] more test cases --- stmt_table_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stmt_table_test.go b/stmt_table_test.go index 4f034fc..f8452b1 100644 --- a/stmt_table_test.go +++ b/stmt_table_test.go @@ -35,8 +35,8 @@ func Test_Exec_CreateTable(t *testing.T) { {name: "basic", sql: `CREATE TABLE tbltemp1 WITH PK=id:string`}, {name: "if_not_exists", sql: `CREATE TABLE IF NOT EXISTS tbltemp1 WITH PK=id:string`}, {name: "with_sk", sql: `CREATE TABLE tbltemp2 WITH PK=id:string WITH sk=grade:number, WITH class=standard`}, - {name: "with_rcu_wcu", sql: `CREATE TABLE tbltemp3 WITH PK=id:string WITH rcu=1 WITH wcu=2 WITH class=standard_ia`}, - {name: "with_lsi", sql: `CREATE TABLE tbltemp4 WITH PK=id:string WITH LSI=index1:grade:number, WITH LSI=index2:dob:string:*, WITH LSI=index3:yob:binary:a,b,c`}, + {name: "with_rcu_wcu", sql: `CREATE TABLE tbltemp3 WITH PK=id:number WITH rcu=1 WITH wcu=2 WITH class=standard_ia`}, + {name: "with_lsi", sql: `CREATE TABLE tbltemp4 WITH PK=id:binary WITH SK=username:string WITH LSI=index1:grade:number, WITH LSI=index2:dob:string:*, WITH LSI=index3:yob:binary:a,b,c`}, } for _, testCase := range testData { From de3e5ac6c20973cd377dbd9101828466573eaa73 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 13:17:55 +1000 Subject: [PATCH 18/31] add operation DESCRIBE LSI --- README.md | 5 +- SQL_INDEX.md | 25 +++++++++ go.mod | 1 + go.sum | 3 + stmt.go | 34 ++++++++++++ stmt_index.go | 110 +++++++++++++++++++++++++++++++++++++ stmt_index_parsing_test.go | 46 ++++++++++++++++ stmt_index_test.go | 103 ++++++++++++++++++++++++++++++++++ stmt_table.go | 17 +----- stmt_test.go | 32 +++++++++++ 10 files changed, 359 insertions(+), 17 deletions(-) create mode 100644 SQL_INDEX.md create mode 100644 stmt_index.go create mode 100644 stmt_index_parsing_test.go create mode 100644 stmt_index_test.go create mode 100644 stmt_test.go diff --git a/README.md b/README.md index d2d8237..d2ff366 100644 --- a/README.md +++ b/README.md @@ -51,13 +51,16 @@ func main() { ## Supported statements: -- Tables: +- Table: - `CREATE TABLE` - `LIST TABLES` - `ALTER TABLE` - `DROP TABLE` - `DESCRIBE TABLE` +- [Index](SQL_INDEX.md): + - `DESCRIBE LSI` + ### CREATE TABLE Syntax: diff --git a/SQL_INDEX.md b/SQL_INDEX.md new file mode 100644 index 0000000..9957977 --- /dev/null +++ b/SQL_INDEX.md @@ -0,0 +1,25 @@ +# godynamo - Supported statements for index + +- `DESCRIBE LSI` + +## DESCRIBE LSI + +Syntax: +```sql +DESCRIBE LSI ON +``` + +Example: +```go +result, err := db.Query(`DESCRIBE LSI idxos ON session`) +if err == nil { + ... +} +``` + +Description: return info of a Local Secondary Index specified by `index-name` on a DynamoDB table specified by `table-name`. + +Sample result: +|IndexArn|IndexName|IndexSizeBytes|ItemCount|KeySchema|Projection| +|--------|---------|--------------|---------|---------|----------| +|"arn:aws:dynamodb:ddblocal:000000000000:table/session/index/idxos"|"idxos"|0|0|[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"os","KeyType":"RANGE"}]|{"NonKeyAttributes":["os_name","os_version"],"ProjectionType":"INCLUDE"}| diff --git a/go.mod b/go.mod index fa33c5f..743d465 100644 --- a/go.mod +++ b/go.mod @@ -9,4 +9,5 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.24 github.com/aws/smithy-go v1.13.5 github.com/btnguyen2k/consu/reddo v0.1.8 + github.com/btnguyen2k/consu/semita v0.1.5 ) diff --git a/go.sum b/go.sum index 1fb79eb..e552a26 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,11 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5 github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/btnguyen2k/consu/reddo v0.1.7/go.mod h1:pdY5oIVX3noZIaZu3nvoKZ59+seXL/taXNGWh9xJDbg= github.com/btnguyen2k/consu/reddo v0.1.8 h1:pEAkB6eadp/q+ONy97/JkAAyj058uIgkSu8b862Fwug= github.com/btnguyen2k/consu/reddo v0.1.8/go.mod h1:pdY5oIVX3noZIaZu3nvoKZ59+seXL/taXNGWh9xJDbg= +github.com/btnguyen2k/consu/semita v0.1.5 h1:fu71xNJTbCV8T+6QPJdJu3bxtmLWvTjCepkvujF74+I= +github.com/btnguyen2k/consu/semita v0.1.5/go.mod h1:fksCe3L4kxiJVnKKhUXKI8mcFdB9974mtedwUVVFu1M= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= diff --git a/stmt.go b/stmt.go index a98553f..e550bb7 100644 --- a/stmt.go +++ b/stmt.go @@ -3,10 +3,30 @@ package godynamo import ( "database/sql/driver" "fmt" + "reflect" "regexp" "strings" ) +func goTypeToDynamodbType(typ reflect.Type) string { + if typ == nil { + return "" + } + switch typ.Kind() { + case reflect.Bool: + return "BOOLEAN" + case reflect.String: + return "STRING" + case reflect.Float32, reflect.Float64: + return "NUMBER" + case reflect.Array, reflect.Slice: + return "ARRAY" + case reflect.Map: + return "MAP" + } + return "" +} + const ( field = `([\w\-]+)` ifNotExists = `(\s+IF\s+NOT\s+EXISTS)?` @@ -20,6 +40,10 @@ var ( reAlterTable = regexp.MustCompile(`(?im)^ALTER\s+TABLE\s+` + field + with + `$`) reDropTable = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+TABLE` + ifExists + `\s+` + field + `$`) reDescribeTable = regexp.MustCompile(`(?im)^DESCRIBE\s+TABLE\s+` + field + `$`) + + reDescribeLSI = regexp.MustCompile(`(?im)^DESCRIBE\s+LSI\s+` + field + `\sON\s+` + field + `$`) + + reCreateGSI = regexp.MustCompile(`(?im)^CREATE\s+GSI` + ifNotExists + `\s+` + field + with + `$`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -73,6 +97,16 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { return stmt, stmt.validate() } + if re := reDescribeLSI; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtDescribeLSI{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + tableName: strings.TrimSpace(groups[0][2]), + indexName: strings.TrimSpace(groups[0][1]), + } + return stmt, stmt.validate() + } + // if re := reInsert; re.MatchString(query) { // groups := re.FindAllStringSubmatch(query, -1) // stmt := &StmtInsert{ diff --git a/stmt_index.go b/stmt_index.go new file mode 100644 index 0000000..caec22f --- /dev/null +++ b/stmt_index.go @@ -0,0 +1,110 @@ +package godynamo + +import ( + "context" + "database/sql/driver" + "encoding/json" + "errors" + "io" + "reflect" + "sort" + + "github.com/aws/aws-sdk-go-v2/service/dynamodb" +) + +// RowsDescribeIndex captures the result from DESCRIBE LSI or DESCRIBE GSI operation. +type RowsDescribeIndex struct { + count int + columnList []string + columnTypeList []reflect.Type + indexInfo map[string]interface{} + cursorCount int +} + +// Columns implements driver.Rows.Columns. +func (r *RowsDescribeIndex) Columns() []string { + return r.columnList +} + +// Close implements driver.Rows.Close. +func (r *RowsDescribeIndex) Close() error { + return nil +} + +// Next implements driver.Rows.Next. +func (r *RowsDescribeIndex) Next(dest []driver.Value) error { + if r.cursorCount >= r.count { + return io.EOF + } + for i, colName := range r.columnList { + dest[i] = r.indexInfo[colName] + } + r.cursorCount++ + return nil +} + +// ColumnTypeScanType implements driver.RowsColumnTypeScanType.ColumnTypeScanType +func (r *RowsDescribeIndex) ColumnTypeScanType(index int) reflect.Type { + return r.columnTypeList[index] +} + +// ColumnTypeDatabaseTypeName implements driver.RowsColumnTypeDatabaseTypeName.ColumnTypeDatabaseTypeName +func (r *RowsDescribeIndex) ColumnTypeDatabaseTypeName(index int) string { + return goTypeToDynamodbType(r.columnTypeList[index]) +} + +/*----------------------------------------------------------------------*/ + +// StmtDescribeLSI implements "DESCRIBE LSI" operation. +// +// Syntax: +// +// DESCRIBE LSI ON +type StmtDescribeLSI struct { + *Stmt + tableName, indexName string +} + +func (s *StmtDescribeLSI) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } + if s.indexName == "" { + return errors.New("index name is missing") + } + return nil +} + +// Query implements driver.Stmt.Query. +func (s *StmtDescribeLSI) Query(_ []driver.Value) (driver.Rows, error) { + input := &dynamodb.DescribeTableInput{ + TableName: &s.tableName, + } + output, err := s.conn.client.DescribeTable(context.Background(), input) + result := &RowsDescribeIndex{count: 0} + if err == nil { + for _, lsi := range output.Table.LocalSecondaryIndexes { + if lsi.IndexName != nil && *lsi.IndexName == s.indexName { + result.count = 1 + js, _ := json.Marshal(lsi) + json.Unmarshal(js, &result.indexInfo) + for k := range result.indexInfo { + result.columnList = append(result.columnList, k) + } + sort.Strings(result.columnList) + result.columnTypeList = make([]reflect.Type, len(result.columnList)) + for i, col := range result.columnList { + result.columnTypeList[i] = reflect.TypeOf(result.indexInfo[col]) + } + break + } + } + } + return result, err +} + +// Exec implements driver.Stmt.Exec. +// This function is not implemented, use Query instead. +func (s *StmtDescribeLSI) Exec(_ []driver.Value) (driver.Result, error) { + return nil, errors.New("this operation is not supported, please use Query") +} diff --git a/stmt_index_parsing_test.go b/stmt_index_parsing_test.go new file mode 100644 index 0000000..20d904a --- /dev/null +++ b/stmt_index_parsing_test.go @@ -0,0 +1,46 @@ +package godynamo + +import ( + "reflect" + "testing" +) + +func TestStmtDescribeLSI_parse(t *testing.T) { + testName := "TestStmtDescribeLSI_parse" + testData := []struct { + name string + sql string + expected *StmtDescribeLSI + mustError bool + }{ + { + name: "basic", + sql: "DESCRIBE LSI idxname ON tblname", + expected: &StmtDescribeLSI{tableName: "tblname", indexName: "idxname"}, + }, + { + name: "no_table", + sql: "DESCRIBE LSI idxname", + mustError: true, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + s, err := parseQuery(nil, testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: parsing must fail", testName+"/"+testCase.name) + } + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmt, ok := s.(*StmtDescribeLSI) + if !ok { + t.Fatalf("%s failed: expected StmtDescribeLSI but received %T", testName+"/"+testCase.name, stmt) + } + stmt.Stmt = nil + if !reflect.DeepEqual(stmt, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmt) + } + }) + } +} diff --git a/stmt_index_test.go b/stmt_index_test.go new file mode 100644 index 0000000..dace8f1 --- /dev/null +++ b/stmt_index_test.go @@ -0,0 +1,103 @@ +package godynamo + +import ( + "reflect" + "strings" + "testing" + + "github.com/btnguyen2k/consu/reddo" + "github.com/btnguyen2k/consu/semita" +) + +func Test_Exec_DescribeLSI(t *testing.T) { + testName := "Test_Exec_DescribeLSI" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Exec("DESCRIBE LSI idxname ON tblname") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Query_DescribeLSI(t *testing.T) { + testName := "Test_Query_DescribeLSI" + db := _openDb(t, testName) + defer db.Close() + + defer func() { + db.Exec("DROP TABLE IF EXISTS session") + }() + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS session WITH PK=app:string WITH SK=user:string WITH LSI=idxtime:timestamp:number WITH LSI=idxbrowser:browser:string:* WITH LSI=idxos:os:string:os_name,os_version`) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/createTable", err) + } + + testData := []struct { + name string + sql string + mustError bool + numRows int + indexName string + projectionType string + nonKeyAttrs []string + }{ + {name: "no_table", sql: `DESCRIBE LSI idx ON tblnotexist`, mustError: true}, + {name: "no_index", sql: `DESCRIBE LSI idxnotexists ON session`, numRows: 0}, + {name: "proj_key_only", sql: `DESCRIBE LSI idxtime ON session`, numRows: 1, indexName: "idxtime", projectionType: "KEYS_ONLY"}, + {name: "proj_all", sql: `DESCRIBE LSI idxbrowser ON session`, numRows: 1, indexName: "idxbrowser", projectionType: "ALL"}, + {name: "proj_included", sql: `DESCRIBE LSI idxos ON session`, numRows: 1, indexName: "idxos", projectionType: "INCLUDE", nonKeyAttrs: []string{"os_name", "os_version"}}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + dbresult, err := db.Query(testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: query must fail", testName+"/"+testCase.name) + } + if testCase.mustError { + return + } + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + rows, err := _fetchAllRows(dbresult) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + if len(rows) != testCase.numRows { + t.Fatalf("%s failed: expected %d row(s) but recelved %d", testName+"/"+testCase.name, testCase.numRows, len(rows)) + } + if testCase.numRows > 0 { + s := semita.NewSemita(rows[0]) + + key := "IndexName" + indexName, err := s.GetValueOfType(key, reddo.TypeString) + if err != nil { + t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) + } + if indexName != testCase.indexName { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.indexName, indexName) + } + + key = "Projection.ProjectionType" + projectionType, err := s.GetValueOfType(key, reddo.TypeString) + if err != nil { + t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) + } + if projectionType != testCase.projectionType { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.projectionType, projectionType) + } + + key = "Projection.NonKeyAttributes" + nonKeyAttrs, err := s.GetValueOfType(key, reflect.TypeOf(make([]string, 0))) + if err != nil { + t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) + } + if testCase.nonKeyAttrs != nil && !reflect.DeepEqual(nonKeyAttrs, testCase.nonKeyAttrs) { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.nonKeyAttrs, nonKeyAttrs) + } + } + }) + } +} diff --git a/stmt_table.go b/stmt_table.go index f2bece7..e7e1808 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -589,22 +589,7 @@ func (r *RowsDescribeTable) ColumnTypeScanType(index int) reflect.Type { // ColumnTypeDatabaseTypeName implements driver.RowsColumnTypeDatabaseTypeName.ColumnTypeDatabaseTypeName func (r *RowsDescribeTable) ColumnTypeDatabaseTypeName(index int) string { - if r.columnTypeList[index] == nil { - return "" - } - switch r.columnTypeList[index].Kind() { - case reflect.Bool: - return "BOOLEAN" - case reflect.String: - return "STRING" - case reflect.Float32, reflect.Float64: - return "NUMBER" - case reflect.Array, reflect.Slice: - return "ARRAY" - case reflect.Map: - return "MAP" - } - return "" + return goTypeToDynamodbType(r.columnTypeList[index]) } // // ColumnTypeLength implements driver.RowsColumnTypeLength.ColumnTypeLength diff --git a/stmt_test.go b/stmt_test.go new file mode 100644 index 0000000..69b275b --- /dev/null +++ b/stmt_test.go @@ -0,0 +1,32 @@ +package godynamo + +import ( + "database/sql" +) + +func _fetchAllRows(dbRows *sql.Rows) ([]map[string]interface{}, error) { + colTypes, err := dbRows.ColumnTypes() + if err != nil { + return nil, err + } + + numCols := len(colTypes) + rows := make([]map[string]interface{}, 0) + for dbRows.Next() { + vals := make([]interface{}, numCols) + scanVals := make([]interface{}, numCols) + for i := 0; i < numCols; i++ { + scanVals[i] = &vals[i] + } + if err := dbRows.Scan(scanVals...); err == nil { + row := make(map[string]interface{}) + for i := range colTypes { + row[colTypes[i].Name()] = vals[i] + } + rows = append(rows, row) + } else if err != sql.ErrNoRows { + return nil, err + } + } + return rows, nil +} From 9f697dc705d4aef980f5704a07a485888b3a8caa Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Mon, 22 May 2023 13:19:54 +1000 Subject: [PATCH 19/31] minor fix --- stmt_index_parsing_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stmt_index_parsing_test.go b/stmt_index_parsing_test.go index 20d904a..baa8fe8 100644 --- a/stmt_index_parsing_test.go +++ b/stmt_index_parsing_test.go @@ -30,6 +30,9 @@ func TestStmtDescribeLSI_parse(t *testing.T) { if testCase.mustError && err == nil { t.Fatalf("%s failed: parsing must fail", testName+"/"+testCase.name) } + if testCase.mustError { + return + } if err != nil { t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) } From 9e63b328539929980e318ab16fdf91ee054243cf Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Tue, 23 May 2023 01:17:34 +1000 Subject: [PATCH 20/31] update unit tests --- stmt_table.go | 26 +++---- stmt_table_parsing_test.go | 94 +++++++++++++++++++++--- stmt_table_test.go | 136 +++++++++++++++++++++++++++++++---- stmt_test.go | 142 +++++++++++++++++++++++++++++++++++++ 4 files changed, 364 insertions(+), 34 deletions(-) diff --git a/stmt_table.go b/stmt_table.go index e7e1808..6dcae3f 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -18,8 +18,8 @@ import ( ) type lsiDef struct { - indexName, fieldName, fieldType string - projectedFields string + indexName, attrName, attrType string + projectedAttrs string } /*----------------------------------------------------------------------*/ @@ -101,20 +101,20 @@ func (s *StmtCreateTable) parse() error { lsiTokens := strings.SplitN(lsiStr, ":", 4) lsiDef := lsiDef{indexName: strings.TrimSpace(lsiTokens[0])} if len(lsiTokens) > 1 { - lsiDef.fieldName = strings.TrimSpace(lsiTokens[1]) + lsiDef.attrName = strings.TrimSpace(lsiTokens[1]) } if len(lsiTokens) > 2 { - lsiDef.fieldType = strings.TrimSpace(strings.ToUpper(lsiTokens[2])) + lsiDef.attrType = strings.TrimSpace(strings.ToUpper(lsiTokens[2])) } if len(lsiTokens) > 3 { - lsiDef.projectedFields = strings.TrimSpace(lsiTokens[3]) + lsiDef.projectedAttrs = strings.TrimSpace(lsiTokens[3]) } if lsiDef.indexName != "" { - if lsiDef.fieldName == "" { + if lsiDef.attrName == "" { return fmt.Errorf("invalid LSI definition <%s>: empty field name", lsiDef.indexName) } - if _, ok := dataTypes[lsiDef.fieldType]; !ok { - return fmt.Errorf("invalid type <%s> of LSI <%s>, accepts values are BINARY, NUMBER and STRING", lsiDef.fieldType, lsiDef.indexName) + if _, ok := dataTypes[lsiDef.attrType]; !ok { + return fmt.Errorf("invalid type <%s> of LSI <%s>, accepts values are BINARY, NUMBER and STRING", lsiDef.attrType, lsiDef.indexName) } } s.lsi = append(s.lsi, lsiDef) @@ -176,20 +176,20 @@ func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { lsi := make([]types.LocalSecondaryIndex, len(s.lsi)) for i := range s.lsi { - attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: &s.lsi[i].fieldName, AttributeType: dataTypes[s.lsi[i].fieldType]}) + attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: &s.lsi[i].attrName, AttributeType: dataTypes[s.lsi[i].attrType]}) lsi[i] = types.LocalSecondaryIndex{ IndexName: &s.lsi[i].indexName, KeySchema: []types.KeySchemaElement{ {AttributeName: &s.pkName, KeyType: keyTypes["HASH"]}, - {AttributeName: &s.lsi[i].fieldName, KeyType: keyTypes["RANGE"]}, + {AttributeName: &s.lsi[i].attrName, KeyType: keyTypes["RANGE"]}, }, Projection: &types.Projection{ProjectionType: types.ProjectionTypeKeysOnly}, } - if s.lsi[i].projectedFields == "*" { + if s.lsi[i].projectedAttrs == "*" { lsi[i].Projection.ProjectionType = types.ProjectionTypeAll - } else if s.lsi[i].projectedFields != "" { + } else if s.lsi[i].projectedAttrs != "" { lsi[i].Projection.ProjectionType = types.ProjectionTypeInclude - nonKeyAttrs := strings.Split(s.lsi[i].projectedFields, ",") + nonKeyAttrs := strings.Split(s.lsi[i].projectedAttrs, ",") lsi[i].Projection.NonKeyAttributes = nonKeyAttrs } } diff --git a/stmt_table_parsing_test.go b/stmt_table_parsing_test.go index 096a5ce..e2d54df 100644 --- a/stmt_table_parsing_test.go +++ b/stmt_table_parsing_test.go @@ -10,10 +10,52 @@ import ( func TestStmtCreateTable_parse(t *testing.T) { testName := "TestStmtCreateTable_parse" testData := []struct { - name string - sql string - expected *StmtCreateTable + name string + sql string + expected *StmtCreateTable + mustError bool }{ + { + name: "no_table", + sql: "CREATE TABLE WITH pk=id:string", + mustError: true, + }, + { + name: "no_pk", + sql: "CREATE TABLE demo", + mustError: true, + }, + { + name: "invalid_rcu", + sql: "CREATE TABLE demo WITH pk=id:string WITH RCU=-1", + mustError: true, + }, + { + name: "invalid_wcu", + sql: "CREATE TABLE demo WITH pk=id:string WITH wcu=-1", + mustError: true, + }, + { + name: "invalid_pk_type", + sql: "CREATE TABLE demo WITH pk=id:int", + mustError: true, + }, + { + name: "invalid_sk_type", + sql: "CREATE TABLE demo WITH pk=id:string WITH sk=grade:int", + mustError: true, + }, + { + name: "invalid_table_class", + sql: "CREATE TABLE demo WITH pk=id:string WITH class=table_class", + mustError: true, + }, + { + name: "invalid_lsi_type", + sql: "CREATE TABLE demo WITH pk=id:string WITH LSI=idxname:attrname:float", + mustError: true, + }, + { name: "basic", sql: "CREATE TABLE demo WITH pk=id:string", @@ -38,15 +80,21 @@ func TestStmtCreateTable_parse(t *testing.T) { name: "with_lsi", sql: "CREATE TABLE IF NOT EXISTS demo WITH pk=id:number, with LSI=i1:f1:string, with LSI=i2:f2:number:*, , with LSI=i3:f3:binary:a,b,c", expected: &StmtCreateTable{tableName: "demo", ifNotExists: true, pkName: "id", pkType: "NUMBER", lsi: []lsiDef{ - {indexName: "i1", fieldName: "f1", fieldType: "STRING"}, - {indexName: "i2", fieldName: "f2", fieldType: "NUMBER", projectedFields: "*"}, - {indexName: "i3", fieldName: "f3", fieldType: "BINARY", projectedFields: "a,b,c"}, + {indexName: "i1", attrName: "f1", attrType: "STRING"}, + {indexName: "i2", attrName: "f2", attrType: "NUMBER", projectedAttrs: "*"}, + {indexName: "i3", attrName: "f3", attrType: "BINARY", projectedAttrs: "a,b,c"}, }}, }, } for _, testCase := range testData { t.Run(testCase.name, func(t *testing.T) { stmt, err := parseQuery(nil, testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: parsing must fail", testName+"/"+testCase.name) + } + if testCase.mustError { + return + } if err != nil { t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) } @@ -97,10 +145,32 @@ func TestStmtListTables_parse(t *testing.T) { func TestStmtAlterTable_parse(t *testing.T) { testName := "TestStmtAlterTable_parse" testData := []struct { - name string - sql string - expected *StmtAlterTable + name string + sql string + expected *StmtAlterTable + mustError bool }{ + { + name: "no_table", + sql: "ALTER TABLE WITH wcu=1 WITH rcu=3", + mustError: true, + }, + { + name: "invalid_rcu", + sql: "ALTER TABLE demo WITH wcu=1 WITH rcu=-3", + mustError: true, + }, + { + name: "invalid_wcu", + sql: "ALTER TABLE demo WITH wcu=-1 WITH rcu=3", + mustError: true, + }, + { + name: "invalid_table_class", + sql: "ALTER TABLE demo WITH class=invalid", + mustError: true, + }, + { name: "with_rcu_wcu", sql: "ALTER TABLE demo WITH wcu=1 WITH rcu=3", @@ -115,6 +185,12 @@ func TestStmtAlterTable_parse(t *testing.T) { for _, testCase := range testData { t.Run(testCase.name, func(t *testing.T) { stmt, err := parseQuery(nil, testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: parsing must fail", testName+"/"+testCase.name) + } + if testCase.mustError { + return + } if err != nil { t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) } diff --git a/stmt_table_test.go b/stmt_table_test.go index f8452b1..00f02ae 100644 --- a/stmt_table_test.go +++ b/stmt_table_test.go @@ -1,6 +1,7 @@ package godynamo import ( + "strconv" "strings" "testing" ) @@ -20,23 +21,121 @@ func Test_Exec_CreateTable(t *testing.T) { testName := "Test_Exec_CreateTable" db := _openDb(t, testName) defer db.Close() + _initTest(db) - tableList := []string{"tbltemp1", "tbltemp2", "tbltemp3", "tbltemp4"} - defer func() { - for _, tbl := range tableList { - db.Exec("DROP TABLE IF EXISTS " + tbl) + testData := []struct { + name string + sql string + tableInfo *tableInfo + }{ + {name: "basic", sql: `CREATE TABLE tbltest1 WITH PK=id:string`, tableInfo: &tableInfo{tableName: "tbltest1", + billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "S"}}, + {name: "if_not_exists", sql: `CREATE TABLE IF NOT EXISTS tbltest1 WITH PK=id:NUMBER`, tableInfo: &tableInfo{tableName: "tbltest1", + billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "S"}}, + {name: "with_sk", sql: `CREATE TABLE tbltest2 WITH PK=id:binary WITH sk=grade:number, WITH class=standard`, tableInfo: &tableInfo{tableName: "tbltest2", + billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "B", skAttr: "grade", skType: "N"}}, + {name: "with_rcu_wcu", sql: `CREATE TABLE tbltest3 WITH PK=id:number WITH rcu=1 WITH wcu=2 WITH class=standard_ia`, tableInfo: &tableInfo{tableName: "tbltest3", + billingMode: "", wcu: 2, rcu: 1, pkAttr: "id", pkType: "N"}}, + {name: "with_lsi", sql: `CREATE TABLE tbltest4 WITH PK=id:binary WITH SK=username:string WITH LSI=index1:grade:number, WITH LSI=index2:dob:string:*, WITH LSI=index3:yob:binary:a,b,c`, tableInfo: &tableInfo{tableName: "tbltest4", + billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "B", skAttr: "username", skType: "S", + lsi: map[string]lsiInfo{ + "index1": {projType: "KEYS_ONLY", lsiDef: lsiDef{indexName: "index1", attrName: "grade", attrType: "N"}}, + "index2": {projType: "ALL", lsiDef: lsiDef{indexName: "index2", attrName: "dob", attrType: "S"}}, + "index3": {projType: "INCLUDE", lsiDef: lsiDef{indexName: "index3", attrName: "yob", attrType: "B", projectedAttrs: "a,b,c"}}, + }, + }}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + _, err := db.Exec(testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/create_table", err) + } + + if testCase.tableInfo == nil { + return + } + dbresult, err := db.Query(`DESCRIBE TABLE ` + testCase.tableInfo.tableName) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/describe_table", err) + } + rows, err := _fetchAllRows(dbresult) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/fetch_rows", err) + } + _verifyTableInfo(t, testName+"/"+testCase.name, rows[0], testCase.tableInfo) + }) + } +} + +func Test_Exec_ListTables(t *testing.T) { + testName := "Test_Exec_ListTables" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Exec("LIST TABLES") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Query_ListTables(t *testing.T) { + testName := "Test_Query_ListTables" + db := _openDb(t, testName) + _initTest(db) + defer db.Close() + + tableList := []string{"tbltest2", "tbltest1", "tbltest3", "tbltest0"} + for _, tbl := range tableList { + db.Exec("CREATE TABLE " + tbl + " WITH PK=id:string") + } + + dbresult, err := db.Query(`LIST TABLES`) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/query", err) + } + rows, err := _fetchAllRows(dbresult) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetch_rows", err) + } + if len(rows) != 4 { + t.Fatalf("%s failed: expected 4 rows but received %d", testName+"/fetch_rows", len(rows)) + } + for i := 0; i < 4; i++ { + tblname := rows[i]["$1"].(string) + if tblname != "tbltest"+strconv.Itoa(i) { + t.Fatalf("%s failed: expected row #%d to be %#v but received %#v", testName+"/fetch_rows", i, "tbltemp"+strconv.Itoa(i), tblname) } - }() + } +} +func Test_Query_AlterTable(t *testing.T) { + testName := "Test_Query_AlterTable" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Query("ALTER TABLE tbltemp WITH wcu=0 WITH rcu=0") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Exec_AlterTable(t *testing.T) { + testName := "Test_Exec_AlterTable" + db := _openDb(t, testName) + _initTest(db) + defer db.Close() + + db.Exec(`CREATE TABLE tbltest WITH PK=id:string`) testData := []struct { - name string - sql string + name string + sql string + tableInfo *tableInfo }{ - {name: "basic", sql: `CREATE TABLE tbltemp1 WITH PK=id:string`}, - {name: "if_not_exists", sql: `CREATE TABLE IF NOT EXISTS tbltemp1 WITH PK=id:string`}, - {name: "with_sk", sql: `CREATE TABLE tbltemp2 WITH PK=id:string WITH sk=grade:number, WITH class=standard`}, - {name: "with_rcu_wcu", sql: `CREATE TABLE tbltemp3 WITH PK=id:number WITH rcu=1 WITH wcu=2 WITH class=standard_ia`}, - {name: "with_lsi", sql: `CREATE TABLE tbltemp4 WITH PK=id:binary WITH SK=username:string WITH LSI=index1:grade:number, WITH LSI=index2:dob:string:*, WITH LSI=index3:yob:binary:a,b,c`}, + {name: "change_wcu_rcu", sql: `ALTER TABLE tbltest WITH wcu=3 WITH rcu=5`, tableInfo: &tableInfo{tableName: "tbltest", + billingMode: "PROVISIONED", wcu: 3, rcu: 5, pkAttr: "id", pkType: "S"}}, + // DynamoDB Docker version does not support changing table class } for _, testCase := range testData { @@ -45,6 +144,19 @@ func Test_Exec_CreateTable(t *testing.T) { if err != nil { t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) } + + if testCase.tableInfo == nil { + return + } + dbresult, err := db.Query(`DESCRIBE TABLE ` + testCase.tableInfo.tableName) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/describe_table", err) + } + rows, err := _fetchAllRows(dbresult) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/fetch_rows", err) + } + _verifyTableInfo(t, testName+"/"+testCase.name, rows[0], testCase.tableInfo) }) } } diff --git a/stmt_test.go b/stmt_test.go index 69b275b..75911d2 100644 --- a/stmt_test.go +++ b/stmt_test.go @@ -2,8 +2,150 @@ package godynamo import ( "database/sql" + "fmt" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/btnguyen2k/consu/reddo" + "github.com/btnguyen2k/consu/semita" ) +type lsiInfo struct { + lsiDef + projType string +} + +type tableInfo struct { + tableName string + billingMode string + rcu, wcu int64 + pkAttr, pkType string + skAttr, skType string + lsi map[string]lsiInfo +} + +func _initTest(db *sql.DB) { + db.Exec(`DROP TABLE IF EXISTS tbltest`) + for i := 0; i < 10; i++ { + db.Exec(`DROP TABLE IF EXISTS tbltest` + strconv.Itoa(i)) + } +} + +func _verifyTableInfo(t *testing.T, testName string, row map[string]interface{}, tableInfo *tableInfo) { + s := semita.NewSemita(row) + var key string + + if tableInfo.billingMode != "" { + key = "BillingModeSummary.BillingMode" + billingMode, _ := s.GetValueOfType(key, reddo.TypeString) + if billingMode != tableInfo.billingMode { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.billingMode, billingMode) + } + } + + key = "ProvisionedThroughput.ReadCapacityUnits" + rcu, _ := s.GetValueOfType(key, reddo.TypeInt) + if rcu != tableInfo.rcu { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.rcu, rcu) + } + + key = "ProvisionedThroughput.WriteCapacityUnits" + wcu, _ := s.GetValueOfType(key, reddo.TypeInt) + if wcu != tableInfo.wcu { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.wcu, wcu) + } + + key = "KeySchema[0].AttributeName" + pkAttr, _ := s.GetValueOfType(key, reddo.TypeString) + if pkAttr != tableInfo.pkAttr { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.pkAttr, pkAttr) + } + + key = "AttributeDefinitions[0].AttributeName" + pkName, _ := s.GetValueOfType(key, reddo.TypeString) + if pkName != tableInfo.pkAttr { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.pkAttr, pkName) + } + key = "AttributeDefinitions[0].AttributeType" + pkType, _ := s.GetValueOfType(key, reddo.TypeString) + if pkType != tableInfo.pkType { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.pkType, pkType) + } + + if tableInfo.skAttr != "" { + key = "KeySchema[1].AttributeName" + skAttr, _ := s.GetValueOfType(key, reddo.TypeString) + if skAttr != tableInfo.skAttr { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.skAttr, skAttr) + } + + key = "AttributeDefinitions[1].AttributeName" + skName, _ := s.GetValueOfType(key, reddo.TypeString) + if skName != tableInfo.skAttr { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.skAttr, skName) + } + key = "AttributeDefinitions[1].AttributeType" + skType, _ := s.GetValueOfType(key, reddo.TypeString) + if skType != tableInfo.skType { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, tableInfo.skType, skType) + } + } + + for expectedIdxName, expectedLsi := range tableInfo.lsi { + found := false + tableLsi, _ := s.GetValueOfType("LocalSecondaryIndexes", reflect.TypeOf(make([]interface{}, 0))) + for i := 0; i < len(tableLsi.([]interface{})); i++ { + key = fmt.Sprintf("LocalSecondaryIndexes[%d].IndexName", i) + idxName, _ := s.GetValueOfType(key, reddo.TypeString) + if idxName == expectedIdxName { + found = true + + key = fmt.Sprintf("LocalSecondaryIndexes[%d].Projection.ProjectionType", i) + projType, _ := s.GetValueOfType(key, reddo.TypeString) + if projType != expectedLsi.projType { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, expectedLsi.projType, projType) + } + if projType == "INCLUDE" { + key = fmt.Sprintf("LocalSecondaryIndexes[%d].Projection.NonKeyAttributes", i) + nonKeyAttrs, _ := s.GetValueOfType(key, reflect.TypeOf(make([]string, 0))) + if !reflect.DeepEqual(nonKeyAttrs, strings.Split(expectedLsi.projectedAttrs, ",")) { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, expectedLsi.projectedAttrs, nonKeyAttrs) + } + } + + key = fmt.Sprintf("LocalSecondaryIndexes[%d].KeySchema[1].AttributeName", i) + attrName, _ := s.GetValueOfType(key, reddo.TypeString) + if attrName != expectedLsi.attrName { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, expectedLsi.attrName, attrName) + } + + tableAttrs, _ := s.GetValueOfType("AttributeDefinitions", reflect.TypeOf(make([]interface{}, 0))) + foundAttr := false + for j := 0; j < len(tableAttrs.([]interface{})); j++ { + k := fmt.Sprintf("AttributeDefinitions[%d].AttributeName", j) + attrName, _ := s.GetValueOfType(k, reddo.TypeString) + if attrName == expectedLsi.attrName { + foundAttr = true + k = fmt.Sprintf("AttributeDefinitions[%d].AttributeType", j) + attrType, _ := s.GetValueOfType(k, reddo.TypeString) + if attrType != expectedLsi.attrType { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, k, expectedLsi.attrType, attrType) + } + } + } + if !foundAttr { + t.Fatalf("%s failed: no attribute definition found for LSI <%s>", testName, expectedIdxName) + } + } + } + if !found { + t.Fatalf("%s failed: no LSI <%s> found", testName, expectedIdxName) + } + } +} + func _fetchAllRows(dbRows *sql.Rows) ([]map[string]interface{}, error) { colTypes, err := dbRows.ColumnTypes() if err != nil { From a80e26fcf75a5e3cc11197cf9d920e60aedf308f Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Tue, 23 May 2023 12:04:03 +1000 Subject: [PATCH 21/31] update unit tests and docs --- README.md | 131 +-------------------------------------- SQL_TABLE.md | 149 +++++++++++++++++++++++++++++++++++++++++++++ stmt_index_test.go | 52 ++++++++-------- stmt_table.go | 2 +- stmt_test.go | 2 + 5 files changed, 180 insertions(+), 156 deletions(-) create mode 100644 SQL_TABLE.md diff --git a/README.md b/README.md index d2ff366..457bf29 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ func main() { ## Supported statements: -- Table: +- [Table](SQL_TABLE.md): - `CREATE TABLE` - `LIST TABLES` - `ALTER TABLE` @@ -61,131 +61,6 @@ func main() { - [Index](SQL_INDEX.md): - `DESCRIBE LSI` -### CREATE TABLE +## License -Syntax: -```sql -CREATE TABLE [IF NOT EXIST] -WITH PK=: -[[,] WITH SK=:] -[[,] WITH wcu=[,] WITH rcu=] -[[,] WITH LSI=index-name1:attr-name1:data-type] -[[,] WITH LSI=index-name2:attr-name2:data-type:*] -[[,] WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] -[[,] WITH LSI...] -[[,] WITH CLASS=] -``` - -Example: -```go -result, err := db.Exec(`CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5`) -if err == nil { - numAffectedRow, err := result.RowsAffected() - ... -} -``` - -Description: create a DynamoDB table specified by `table-name`. - -- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. -- If the specified table already existed: - - If `IF NOT EXISTS` is supplied: `RowsAffected()` returns `0, nil`. - - If `IF NOT EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error`. -- `RCU`: read capacity unit. -- `WCU`: write capacity unit. -- `PK`: partition key, mandatory. -- `SK`: sort key, optional. -- `LSI`: local secondary index, format `index-name:attr-name:data-type[:projectionAttrs]` - - `projectionAttrs=*`: all attributes from the original table are included in projection (`ProjectionType=ALL`). - - `projectionAttrs=attr1,attr2,...`: specified attributes from the original table are included in projection (`ProjectionType=INCLUDE`). - - _projectionAttrs is not specified_: only key attributes are included in projection (`ProjectionType=KEYS_ONLY`). -- `data-type`: must be one of `BINARY`, `NUMBER` or `STRING`. -- `table-class` is either `STANDARD` (default) or `STANDARD_IA`. -- Note: if `RCU` and `WRU` are both `0` or not specified, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. -- Note: there must be _at least one space_ before the `WITH` keyword. - -### LIST TABLES - -Syntax: -```sql -LIST TABLES -``` - -Example: -```go -result, err := db.Query(`LIST TABLES`) -if err == nil { - ... -} -``` - -Description: return list of all DynamoDB tables. - -### ALTER TABLE - -Syntax: -```sql -ALTER TABLE -[WITH wcu=[,] WITH rcu=] -[[,] WITH CLASS=] -``` - -Example: -```go -result, err := db.Exec(`ALTER TABLE demo WITH rcu=0 WITH wcu=0 WITH CLASS=STANDARD_IA`) -if err == nil { - numAffectedRow, err := result.RowsAffected() - ... -} -``` - -Description: update WCU/RCU or table-class of an existing DynamoDB table specified by `table-name`. - -- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. -- `RCU`: read capacity unit. -- `WCU`: write capacity unit. -- `table-class` is either `STANDARD` (default) or `STANDARD_IA`. -- Note: if `RCU` and `WRU` are both `0`, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. -- Note: there must be _at least one space_ before the `WITH` keyword. - -### DROP TABLE - -Syntax: -```sql -DROP TABLE [IF EXIST] -``` - -Alias: `DELETE TABLE` - -Example: -```go -result, err := db.Exec(`DROP TABLE IF EXISTS demo`) -if err == nil { - numAffectedRow, err := result.RowsAffected() - ... -} -``` - -Description: delete an existing DynamoDB table specified by `table-name`. - -- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. -- If the specified table does not exist: - - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` - - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` - -### DESCRIBE TABLE - -Syntax: -```sql -DESCRIBE TABLE -``` - -Example: -```go -result, err := db.Query(`DESCRIBE TABLE demo`) -if err == nil { - ... -} -``` - -Description: return info of a DynamoDB table specified by `table-name`. +MIT - See [LICENSE.md](LICENSE.md). diff --git a/SQL_TABLE.md b/SQL_TABLE.md new file mode 100644 index 0000000..b4e80a0 --- /dev/null +++ b/SQL_TABLE.md @@ -0,0 +1,149 @@ +# godynamo - Supported statements for table + +- `CREATE TABLE` +- `LIST TABLES` +- `ALTER TABLE` +- `DROP TABLE` +- `DESCRIBE TABLE` + +## CREATE TABLE + +Syntax: +```sql +CREATE TABLE [IF NOT EXIST] +WITH PK=: +[[,] WITH SK=:] +[[,] WITH wcu=[,] WITH rcu=] +[[,] WITH LSI=index-name1:attr-name1:data-type] +[[,] WITH LSI=index-name2:attr-name2:data-type:*] +[[,] WITH LSI=index-name2:attr-name2:data-type:nonKeyAttr1,nonKeyAttr2,nonKeyAttr3,...] +[[,] WITH LSI...] +[[,] WITH CLASS=] +``` + +Example: +```go +result, err := db.Exec(`CREATE TABLE demo WITH PK=id:string WITH rcu=3 WITH wcu=5`) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: create a DynamoDB table specified by `table-name`. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- If the specified table already existed: + - If `IF NOT EXISTS` is supplied: `RowsAffected()` returns `0, nil`. + - If `IF NOT EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error`. +- `RCU`: read capacity unit. +- `WCU`: write capacity unit. +- `PK`: partition key, mandatory. +- `SK`: sort key, optional. +- `LSI`: local secondary index, format `index-name:attr-name:data-type[:projectionAttrs]` + - `projectionAttrs=*`: all attributes from the original table are included in projection (`ProjectionType=ALL`). + - `projectionAttrs=attr1,attr2,...`: specified attributes from the original table are included in projection (`ProjectionType=INCLUDE`). + - _projectionAttrs is not specified_: only key attributes are included in projection (`ProjectionType=KEYS_ONLY`). +- `data-type`: must be one of `BINARY`, `NUMBER` or `STRING`. +- `table-class` is either `STANDARD` (default) or `STANDARD_IA`. +- Note: if `RCU` and `WRU` are both `0` or not specified, table will be created with `PAY_PER_REQUEST` billing mode; otherwise table will be creatd with `PROVISIONED` mode. +- Note: there must be _at least one space_ before the `WITH` keyword. + +## LIST TABLES + +Syntax: +```sql +LIST TABLES +``` + +Example: +```go +result, err := db.Query(`LIST TABLES`) +if err == nil { + ... +} +``` + +Description: return list of all DynamoDB tables. + +Sample result: +|$1| +|--------| +|tbltest0| +|tbltest1| +|tbltest2| +|tbltest3| + +## ALTER TABLE + +Syntax: +```sql +ALTER TABLE +[WITH wcu=[,] WITH rcu=] +[[,] WITH CLASS=] +``` + +Example: +```go +result, err := db.Exec(`ALTER TABLE demo WITH rcu=5 WITH wcu=7`) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: update WCU/RCU or table-class of an existing DynamoDB table specified by `table-name`. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- `RCU`: read capacity unit. +- `WCU`: write capacity unit. +- `table-class` is either `STANDARD` (default) or `STANDARD_IA`. +- Note: if `RCU` and `WRU` are both `0`, table's billing mode will be updated to `PAY_PER_REQUEST`; otherwise billing mode will be updated to `PROVISIONED`. +- Note: there must be _at least one space_ before the `WITH` keyword. + +## DROP TABLE + +Syntax: +```sql +DROP TABLE [IF EXIST] +``` + +Alias: `DELETE TABLE` + +Example: +```go +result, err := db.Exec(`DROP TABLE IF EXISTS demo`) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: delete an existing DynamoDB table specified by `table-name`. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- If the specified table does not exist: + - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` + - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` + +## DESCRIBE TABLE + +Syntax: +```sql +DESCRIBE TABLE +``` + +Example: +```go +result, err := db.Query(`DESCRIBE TABLE demo`) +if err == nil { + ... +} +``` + +Description: return info of a DynamoDB table specified by `table-name`. + +Sample result: +|ArchivalSummary|AttributeDefinitions|BillingModeSummary|CreationDateTime|DeletionProtectionEnabled|GlobalSecondaryIndexes|GlobalTableVersion|ItemCount|KeySchema|LatestStreamArn|LatestStreamLabel|LocalSecondaryIndexes|ProvisionedThroughput|Replicas|RestoreSummary|SSEDescription|StreamSpecification|TableArn|TableClassSummary|TableId|TableName|TableSizeBytes|TableStatus| +|---------------|--------------------|------------------|----------------|-------------------------|----------------------|------------------|---------|---------|---------------|-----------------|---------------------|---------------------|--------|--------------|--------------|-------------------|--------|-----------------|-------|---------|--------------|-----------| +|null|[{"AttributeName":"app","AttributeType":"S"},{"AttributeName":"user","AttributeType":"S"},{"AttributeName":"timestamp","AttributeType":"N"},{"AttributeName":"browser","AttributeType":"S"},{"AttributeName":"os","AttributeType":"S"}]|{"BillingMode":"PAY_PER_REQUEST","LastUpdateToPayPerRequestDateTime":"2023-05-23T01:58:27.352Z"}|"2023-05-23T01:58:27.352Z"|null|null|null|0|[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"user","KeyType":"RANGE"}]|null|null|[{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxos","IndexName":"idxos","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"os","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":["os_name","os_version"],"ProjectionType":"INCLUDE"}},{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxbrowser","IndexName":"idxbrowser","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"browser","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":null,"ProjectionType":"ALL"}},{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxtime","IndexName":"idxtime","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"timestamp","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":null,"ProjectionType":"KEYS_ONLY"}}]|{"LastDecreaseDateTime":"1970-01-01T00:00:00Z","LastIncreaseDateTime":"1970-01-01T00:00:00Z","NumberOfDecreasesToday":0,"ReadCapacityUnits":0,"WriteCapacityUnits":0}|null|null|null|null|"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp"|null|null|"tbltemp"|0|"ACTIVE"| diff --git a/stmt_index_test.go b/stmt_index_test.go index dace8f1..2e3c6da 100644 --- a/stmt_index_test.go +++ b/stmt_index_test.go @@ -23,30 +23,26 @@ func Test_Exec_DescribeLSI(t *testing.T) { func Test_Query_DescribeLSI(t *testing.T) { testName := "Test_Query_DescribeLSI" db := _openDb(t, testName) + _initTest(db) defer db.Close() - defer func() { - db.Exec("DROP TABLE IF EXISTS session") - }() - _, err := db.Exec(`CREATE TABLE IF NOT EXISTS session WITH PK=app:string WITH SK=user:string WITH LSI=idxtime:timestamp:number WITH LSI=idxbrowser:browser:string:* WITH LSI=idxos:os:string:os_name,os_version`) + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS tbltest WITH PK=app:string WITH SK=user:string WITH LSI=idxtime:timestamp:number WITH LSI=idxbrowser:browser:string:* WITH LSI=idxos:os:string:os_name,os_version`) if err != nil { t.Fatalf("%s failed: %s", testName+"/createTable", err) } testData := []struct { - name string - sql string - mustError bool - numRows int - indexName string - projectionType string - nonKeyAttrs []string + name string + sql string + mustError bool + numRows int + lsi lsiInfo }{ {name: "no_table", sql: `DESCRIBE LSI idx ON tblnotexist`, mustError: true}, - {name: "no_index", sql: `DESCRIBE LSI idxnotexists ON session`, numRows: 0}, - {name: "proj_key_only", sql: `DESCRIBE LSI idxtime ON session`, numRows: 1, indexName: "idxtime", projectionType: "KEYS_ONLY"}, - {name: "proj_all", sql: `DESCRIBE LSI idxbrowser ON session`, numRows: 1, indexName: "idxbrowser", projectionType: "ALL"}, - {name: "proj_included", sql: `DESCRIBE LSI idxos ON session`, numRows: 1, indexName: "idxos", projectionType: "INCLUDE", nonKeyAttrs: []string{"os_name", "os_version"}}, + {name: "no_index", sql: `DESCRIBE LSI idxnotexists ON tbltest`, numRows: 0}, + {name: "proj_key_only", sql: `DESCRIBE LSI idxtime ON tbltest`, numRows: 1, lsi: lsiInfo{projType: "KEYS_ONLY", lsiDef: lsiDef{indexName: "idxtime", attrName: "timestamp"}}}, + {name: "proj_all", sql: `DESCRIBE LSI idxbrowser ON tbltest`, numRows: 1, lsi: lsiInfo{projType: "ALL", lsiDef: lsiDef{indexName: "idxbrowser", attrName: "browser"}}}, + {name: "proj_included", sql: `DESCRIBE LSI idxos ON tbltest`, numRows: 1, lsi: lsiInfo{projType: "INCLUDE", lsiDef: lsiDef{indexName: "idxos", attrName: "os", projectedAttrs: "os_name,os_version"}}}, } for _, testCase := range testData { @@ -76,26 +72,28 @@ func Test_Query_DescribeLSI(t *testing.T) { if err != nil { t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) } - if indexName != testCase.indexName { - t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.indexName, indexName) + if indexName != testCase.lsi.indexName { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.lsi.indexName, indexName) } key = "Projection.ProjectionType" - projectionType, err := s.GetValueOfType(key, reddo.TypeString) + projType, err := s.GetValueOfType(key, reddo.TypeString) if err != nil { t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) } - if projectionType != testCase.projectionType { - t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.projectionType, projectionType) + if projType != testCase.lsi.projType { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.lsi.projType, projType) } - key = "Projection.NonKeyAttributes" - nonKeyAttrs, err := s.GetValueOfType(key, reflect.TypeOf(make([]string, 0))) - if err != nil { - t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) - } - if testCase.nonKeyAttrs != nil && !reflect.DeepEqual(nonKeyAttrs, testCase.nonKeyAttrs) { - t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.nonKeyAttrs, nonKeyAttrs) + if projType == "INCLUDE" { + key = "Projection.NonKeyAttributes" + nonKeyAttrs, err := s.GetValueOfType(key, reflect.TypeOf(make([]string, 0))) + if err != nil { + t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) + } + if !reflect.DeepEqual(nonKeyAttrs, strings.Split(testCase.lsi.projectedAttrs, ",")) { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.lsi.projectedAttrs, nonKeyAttrs) + } } } }) diff --git a/stmt_table.go b/stmt_table.go index 6dcae3f..ed292d1 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -341,7 +341,7 @@ func (r *RowsListTables) ColumnTypeDatabaseTypeName(index int) string { // - RCU: an integer specifying DynamoDB's read capacity. // - WCU: an integer specifying DynamoDB's write capacity. // - CLASS: table class, either STANDARD (default) or STANDARD_IA. -// - Note: if RCU and WRU are both 0, table will be created with PAY_PER_REQUEST billing mode; otherwise table will be creatd with PROVISIONED mode. +// - Note: if RCU and WRU are both 0, table's billing mode will be updated to PAY_PER_REQUEST; otherwise billing mode will be updated to PROVISIONED. // - Note: there must be at least one space before the WITH keyword. type StmtAlterTable struct { *Stmt diff --git a/stmt_test.go b/stmt_test.go index 75911d2..d6992e4 100644 --- a/stmt_test.go +++ b/stmt_test.go @@ -27,6 +27,8 @@ type tableInfo struct { } func _initTest(db *sql.DB) { + db.Exec(`DROP TABLE IF EXISTS tblnotexist`) + db.Exec(`DROP TABLE IF EXISTS tblnotexists`) db.Exec(`DROP TABLE IF EXISTS tbltest`) for i := 0; i < 10; i++ { db.Exec(`DROP TABLE IF EXISTS tbltest` + strconv.Itoa(i)) From 9bb29a7f5db5615bb5239208f03b29252fea9290 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Wed, 24 May 2023 10:48:41 +1000 Subject: [PATCH 22/31] add CREATE GSI and DESCRIBE GSI --- README.md | 2 + SQL_INDEX.md | 63 ++++++++++ SQL_TABLE.md | 2 +- stmt.go | 30 ++++- stmt_index.go | 234 +++++++++++++++++++++++++++++++++++++ stmt_index_parsing_test.go | 130 +++++++++++++++++++++ stmt_index_test.go | 65 +++++++++++ stmt_table_test.go | 90 +++++++++++--- stmt_test.go | 58 +++++++++ 9 files changed, 656 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 457bf29..ee53a6a 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ func main() { - [Index](SQL_INDEX.md): - `DESCRIBE LSI` + - `CREATE GSI` + - `DESCRIBE GSI` ## License diff --git a/SQL_INDEX.md b/SQL_INDEX.md index 9957977..30a55ae 100644 --- a/SQL_INDEX.md +++ b/SQL_INDEX.md @@ -1,6 +1,8 @@ # godynamo - Supported statements for index - `DESCRIBE LSI` +- `CREATE LSI` +- `DESCRIBE GSI` ## DESCRIBE LSI @@ -23,3 +25,64 @@ Sample result: |IndexArn|IndexName|IndexSizeBytes|ItemCount|KeySchema|Projection| |--------|---------|--------------|---------|---------|----------| |"arn:aws:dynamodb:ddblocal:000000000000:table/session/index/idxos"|"idxos"|0|0|[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"os","KeyType":"RANGE"}]|{"NonKeyAttributes":["os_name","os_version"],"ProjectionType":"INCLUDE"}| + +## CREATE GSI + +Syntax: +```sql +CREATE GSI [IF NOT EXISTS] ON + +[[,] WITH SK=sk-attr-name:data-type] +[[,] WITH wcu=[,] WITH rcu=] +[[,] WITH projection=*|attr1,attr2,attr3,...] +``` + +Example: +```go +result, err := db.Exec(`CREATE GSI idxname ON tablename WITH pk=grade:number`) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: create a Global Secondary Index on an existing DynamoDB table. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- If the specified table already existed: + - If `IF NOT EXISTS` is supplied: `RowsAffected()` returns `0, nil`. + - If `IF NOT EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error`. +- `RCU`: GSI's read capacity unit. +- `WCU`: GSI's write capacity unit. +- `PK`: GSI's partition key, mandatory. +- `SK`: GSI's sort key, optional. +- `data-type`: must be one of `BINARY`, `NUMBER` or `STRING`. +- `PROJECTION`: + - `*`: all attributes from the original table are included in projection (`ProjectionType=ALL`). + - `attr1,attr2,...`: specified attributes from the original table are included in projection (`ProjectionType=INCLUDE`). + - _not specified_: only key attributes are included in projection (`ProjectionType=KEYS_ONLY`). +- Note: The provisioned throughput settings of a GSI are separate from those of its base table. +- Note: GSI inherit the RCU and WCU mode from the base table. That means if the base table is in on-demand mode, then DynamoDB also creates the GSI in on-demand mode. +- Note: there must be at least one space before the WITH keyword. + +## DESCRIBE GSI + +Syntax: +```sql +DESCRIBE GSI ON +``` + +Example: +```go +result, err := db.Query(`DESCRIBE GSI idxos ON session`) +if err == nil { + ... +} +``` + +Description: return info of a Local Secondary Index specified by `index-name` on a DynamoDB table specified by `table-name`. + +Sample result: +|Backfilling|IndexArn|IndexName|IndexSizeBytes|IndexStatus|ItemCount|KeySchema|Projection|ProvisionedThroughput| +|-----------|--------|---------|--------------|-----------|---------|---------|----------|---------------------| +|null|"arn:aws:dynamodb:ddblocal:000000000000:table/session/index/idxbrowser"|"idxbrowser"|0|"ACTIVE"|0|[{"AttributeName":"browser","KeyType":"HASH"}]|{"NonKeyAttributes":null,"ProjectionType":"ALL"}|{"LastDecreaseDateTime":null,"LastIncreaseDateTime":null,"NumberOfDecreasesToday":null,"ReadCapacityUnits":1,"WriteCapacityUnits":1}| diff --git a/SQL_TABLE.md b/SQL_TABLE.md index b4e80a0..bc713a6 100644 --- a/SQL_TABLE.md +++ b/SQL_TABLE.md @@ -67,7 +67,7 @@ if err == nil { Description: return list of all DynamoDB tables. Sample result: -|$1| +|$1 <-- this is the column name| |--------| |tbltest0| |tbltest1| diff --git a/stmt.go b/stmt.go index e550bb7..e247924 100644 --- a/stmt.go +++ b/stmt.go @@ -41,9 +41,9 @@ var ( reDropTable = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+TABLE` + ifExists + `\s+` + field + `$`) reDescribeTable = regexp.MustCompile(`(?im)^DESCRIBE\s+TABLE\s+` + field + `$`) - reDescribeLSI = regexp.MustCompile(`(?im)^DESCRIBE\s+LSI\s+` + field + `\sON\s+` + field + `$`) - - reCreateGSI = regexp.MustCompile(`(?im)^CREATE\s+GSI` + ifNotExists + `\s+` + field + with + `$`) + reDescribeLSI = regexp.MustCompile(`(?im)^DESCRIBE\s+LSI\s+` + field + `\s+ON\s+` + field + `$`) + reCreateGSI = regexp.MustCompile(`(?im)^CREATE\s+GSI` + ifNotExists + `\s+` + field + `\s+ON\s+` + field + with + `$`) + reDescribeGSI = regexp.MustCompile(`(?im)^DESCRIBE\s+GSI\s+` + field + `\s+ON\s+` + field + `$`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -107,6 +107,30 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { return stmt, stmt.validate() } + if re := reCreateGSI; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtCreateGSI{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + ifNotExists: strings.TrimSpace(groups[0][1]) != "", + indexName: strings.TrimSpace(groups[0][2]), + tableName: strings.TrimSpace(groups[0][3]), + withOptsStr: " " + strings.TrimSpace(groups[0][4]), + } + if err := stmt.parse(); err != nil { + return nil, err + } + return stmt, stmt.validate() + } + if re := reDescribeGSI; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtDescribeGSI{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + tableName: strings.TrimSpace(groups[0][2]), + indexName: strings.TrimSpace(groups[0][1]), + } + return stmt, stmt.validate() + } + // if re := reInsert; re.MatchString(query) { // groups := re.FindAllStringSubmatch(query, -1) // stmt := &StmtInsert{ diff --git a/stmt_index.go b/stmt_index.go index caec22f..2af6ae4 100644 --- a/stmt_index.go +++ b/stmt_index.go @@ -5,11 +5,15 @@ import ( "database/sql/driver" "encoding/json" "errors" + "fmt" "io" "reflect" "sort" + "strconv" + "strings" "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" ) // RowsDescribeIndex captures the result from DESCRIBE LSI or DESCRIBE GSI operation. @@ -108,3 +112,233 @@ func (s *StmtDescribeLSI) Query(_ []driver.Value) (driver.Rows, error) { func (s *StmtDescribeLSI) Exec(_ []driver.Value) (driver.Result, error) { return nil, errors.New("this operation is not supported, please use Query") } + +/*----------------------------------------------------------------------*/ + +// StmtCreateGSI implements "CREATE GSI" operation. +// +// Syntax: +// +// CREATE GSI [IF NOT EXISTS] ON +// +// [[,] WITH SK=sk-attr-name:data-type] +// [[,] WITH wcu=[,] WITH rcu=] +// [[,] WITH projection=*|attr1,attr2,attr3,...] +// +// - PK: GSI's partition key, format name:type (type is one of String, Number, Binary). +// - SK: GSI's sort key, format name:type (type is one of String, Number, Binary). +// - RCU: an integer specifying DynamoDB's read capacity. +// - WCU: an integer specifying DynamoDB's write capacity. +// - PROJECTION: +// - if not supplied, GSI will be created with projection setting KEYS_ONLY. +// - if equal to "*", GSI will be created with projection setting ALL. +// - if supplied with comma-separated attribute list, for example "attr1,attr2,attr3", GSI will be created with projection setting INCLUDE. +// - If "IF NOT EXISTS" is specified, Exec will silently swallow the error "Attempting to create an index which already exists". +// - Note: if RCU and WRU are both 0 or not specified, GSI will be created with PAY_PER_REQUEST billing mode; otherwise table will be creatd with PROVISIONED mode. +// - Note: there must be at least one space before the WITH keyword. +type StmtCreateGSI struct { + *Stmt + indexName, tableName string + ifNotExists bool + pkName, pkType string + skName, skType *string + rcu, wcu *int64 + projectedAttrs string + withOptsStr string +} + +func (s *StmtCreateGSI) parse() error { + if err := s.Stmt.parseWithOpts(s.withOptsStr); err != nil { + return err + } + + // partition key + pkTokens := strings.SplitN(s.withOpts["PK"].FirstString(), ":", 2) + s.pkName = strings.TrimSpace(pkTokens[0]) + if len(pkTokens) > 1 { + s.pkType = strings.TrimSpace(strings.ToUpper(pkTokens[1])) + } + if s.pkName == "" { + return fmt.Errorf("no PartitionKey, specify one using WITH pk=pkname:pktype") + } + if _, ok := dataTypes[s.pkType]; !ok { + return fmt.Errorf("invalid type <%s> for PartitionKey, accepts values are BINARY, NUMBER and STRING", s.pkType) + } + + // sort key + skTokens := strings.SplitN(s.withOpts["SK"].FirstString(), ":", 2) + skName := strings.TrimSpace(skTokens[0]) + if skName != "" { + s.skName = &skName + skType := "" + if len(skTokens) > 1 { + skType = strings.TrimSpace(strings.ToUpper(skTokens[1])) + } + if _, ok := dataTypes[skType]; !ok { + return fmt.Errorf("invalid type SortKey <%s>, accepts values are BINARY, NUMBER and STRING", skType) + } + s.skType = &skType + } + + //projection + s.projectedAttrs = s.withOpts["PROJECTION"].FirstString() + + // RCU + if _, ok := s.withOpts["RCU"]; ok { + rcu, err := strconv.ParseInt(s.withOpts["RCU"].FirstString(), 10, 64) + if err != nil || rcu < 0 { + return fmt.Errorf("invalid RCU value: %s", s.withOpts["RCU"]) + } + s.rcu = &rcu + } + // WCU + if _, ok := s.withOpts["WCU"]; ok { + wcu, err := strconv.ParseInt(s.withOpts["WCU"].FirstString(), 10, 64) + if err != nil || wcu < 0 { + return fmt.Errorf("invalid WCU value: %s", s.withOpts["WCU"]) + } + s.wcu = &wcu + } + + return nil +} + +func (s *StmtCreateGSI) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } + if s.indexName == "" { + return errors.New("index name is missing") + } + return nil +} + +// Query implements driver.Stmt.Query. +// This function is not implemented, use Exec instead. +func (s *StmtCreateGSI) Query(_ []driver.Value) (driver.Rows, error) { + return nil, errors.New("this operation is not supported, please use Exec") +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtCreateGSI) Exec(_ []driver.Value) (driver.Result, error) { + attrDefs := make([]types.AttributeDefinition, 0, 2) + attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: &s.pkName, AttributeType: dataTypes[s.pkType]}) + keySchema := make([]types.KeySchemaElement, 0, 2) + keySchema = append(keySchema, types.KeySchemaElement{AttributeName: &s.pkName, KeyType: keyTypes["HASH"]}) + if s.skName != nil { + attrDefs = append(attrDefs, types.AttributeDefinition{AttributeName: s.skName, AttributeType: dataTypes[*s.skType]}) + keySchema = append(keySchema, types.KeySchemaElement{AttributeName: s.skName, KeyType: keyTypes["RANGE"]}) + } + + gsiInput := &types.CreateGlobalSecondaryIndexAction{ + IndexName: &s.indexName, + KeySchema: keySchema, + Projection: &types.Projection{ + ProjectionType: types.ProjectionTypeKeysOnly, + }, + } + if s.projectedAttrs == "*" { + gsiInput.Projection.ProjectionType = types.ProjectionTypeAll + } else if s.projectedAttrs != "" { + gsiInput.Projection.ProjectionType = types.ProjectionTypeInclude + nonKeyAttrs := strings.Split(s.projectedAttrs, ",") + gsiInput.Projection.NonKeyAttributes = nonKeyAttrs + } + + if s.rcu != nil && s.wcu != nil { + gsiInput.ProvisionedThroughput = &types.ProvisionedThroughput{ + ReadCapacityUnits: s.rcu, + WriteCapacityUnits: s.wcu, + } + } + + input := &dynamodb.UpdateTableInput{ + TableName: &s.tableName, + AttributeDefinitions: attrDefs, + GlobalSecondaryIndexUpdates: []types.GlobalSecondaryIndexUpdate{{Create: gsiInput}}, + } + + _, err := s.conn.client.UpdateTable(context.Background(), input) + result := &ResultCreateGSI{Successful: err == nil} + if s.ifNotExists && err != nil { + if IsAwsError(err, "ResourceInUseException") || strings.Index(err.Error(), "already exist") >= 0 { + err = nil + } + } + return result, err +} + +// ResultCreateGSI captures the result from CREATE GSI operation. +type ResultCreateGSI struct { + // Successful flags if the operation was successful or not. + Successful bool +} + +// LastInsertId implements driver.Result.LastInsertId. +func (r *ResultCreateGSI) LastInsertId() (int64, error) { + return 0, fmt.Errorf("this operation is not supported.") +} + +// RowsAffected implements driver.Result.RowsAffected. +func (r *ResultCreateGSI) RowsAffected() (int64, error) { + if r.Successful { + return 1, nil + } + return 0, nil +} + +/*----------------------------------------------------------------------*/ + +// StmtDescribeGSI implements "DESCRIBE GSI" operation. +// +// Syntax: +// +// DESCRIBE GSI ON +type StmtDescribeGSI struct { + *Stmt + tableName, indexName string +} + +func (s *StmtDescribeGSI) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } + if s.indexName == "" { + return errors.New("index name is missing") + } + return nil +} + +// Query implements driver.Stmt.Query. +func (s *StmtDescribeGSI) Query(_ []driver.Value) (driver.Rows, error) { + input := &dynamodb.DescribeTableInput{ + TableName: &s.tableName, + } + output, err := s.conn.client.DescribeTable(context.Background(), input) + result := &RowsDescribeIndex{count: 0} + if err == nil { + for _, gsi := range output.Table.GlobalSecondaryIndexes { + if gsi.IndexName != nil && *gsi.IndexName == s.indexName { + result.count = 1 + js, _ := json.Marshal(gsi) + json.Unmarshal(js, &result.indexInfo) + for k := range result.indexInfo { + result.columnList = append(result.columnList, k) + } + sort.Strings(result.columnList) + result.columnTypeList = make([]reflect.Type, len(result.columnList)) + for i, col := range result.columnList { + result.columnTypeList[i] = reflect.TypeOf(result.indexInfo[col]) + } + break + } + } + } + return result, err +} + +// Exec implements driver.Stmt.Exec. +// This function is not implemented, use Query instead. +func (s *StmtDescribeGSI) Exec(_ []driver.Value) (driver.Result, error) { + return nil, errors.New("this operation is not supported, please use Query") +} diff --git a/stmt_index_parsing_test.go b/stmt_index_parsing_test.go index baa8fe8..c09de42 100644 --- a/stmt_index_parsing_test.go +++ b/stmt_index_parsing_test.go @@ -3,6 +3,8 @@ package godynamo import ( "reflect" "testing" + + "github.com/aws/aws-sdk-go-v2/aws" ) func TestStmtDescribeLSI_parse(t *testing.T) { @@ -47,3 +49,131 @@ func TestStmtDescribeLSI_parse(t *testing.T) { }) } } + +func TestStmtCreateGSI_parse(t *testing.T) { + testName := "TestStmtCreateGSI_parse" + testData := []struct { + name string + sql string + expected *StmtCreateGSI + mustError bool + }{ + { + name: "no_table", + sql: "CREATE GSI abc ON WITH pk=id:string", + mustError: true, + }, + { + name: "no_index_name", + sql: "CREATE GSI ON table WITH pk=id:string", + mustError: true, + }, + { + name: "no_pk", + sql: "CREATE GSI index ON table", + mustError: true, + }, + { + name: "invalid_rcu", + sql: "CREATE GSI index ON table WITH pk=id:string WITH RCU=-1", + mustError: true, + }, + { + name: "invalid_wcu", + sql: "CREATE GSI index ON table WITH pk=id:string WITH wcu=-1", + mustError: true, + }, + { + name: "invalid_pk_type", + sql: "CREATE GSI index ON table WITH pk=id:int", + mustError: true, + }, + { + name: "invalid_sk_type", + sql: "CREATE GSI index ON table WITH pk=id:string WITH sk=grade:int", + mustError: true, + }, + + { + name: "basic", + sql: "CREATE GSI index ON table WITH pk=id:string", + expected: &StmtCreateGSI{tableName: "table", indexName: "index", pkName: "id", pkType: "STRING"}, + }, + { + name: "with_sk", + sql: "CREATE GSI index ON table WITH pk=id:string with SK=grade:binary WITH projection=*", + expected: &StmtCreateGSI{tableName: "table", indexName: "index", pkName: "id", pkType: "STRING", skName: aws.String("grade"), skType: aws.String("BINARY"), projectedAttrs: "*"}, + }, + { + name: "with_rcu_wcu", + sql: "CREATE GSI IF NOT EXISTS index ON table WITH pk=id:number, with WCU=1 WITH rcu=0 WITH projection=a,b,c", + expected: &StmtCreateGSI{tableName: "table", indexName: "index", ifNotExists: true, pkName: "id", pkType: "NUMBER", wcu: aws.Int64(1), rcu: aws.Int64(0), projectedAttrs: "a,b,c"}, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + s, err := parseQuery(nil, testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: parsing must fail", testName+"/"+testCase.name) + } + if testCase.mustError { + return + } + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmt, ok := s.(*StmtCreateGSI) + if !ok { + t.Fatalf("%s failed: expected StmtCreateGSI but received %T", testName+"/"+testCase.name, s) + } + stmt.Stmt = nil + stmt.withOptsStr = "" + if !reflect.DeepEqual(stmt, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmt) + } + }) + } +} + +func TestStmtDescribeGSI_parse(t *testing.T) { + testName := "TestStmtDescribeGSI_parse" + testData := []struct { + name string + sql string + expected *StmtDescribeGSI + mustError bool + }{ + { + name: "basic", + sql: "DESCRIBE GSI idxname ON tblname", + expected: &StmtDescribeGSI{tableName: "tblname", indexName: "idxname"}, + }, + { + name: "no_table", + sql: "DESCRIBE LSI idxname", + mustError: true, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + s, err := parseQuery(nil, testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: parsing must fail", testName+"/"+testCase.name) + } + if testCase.mustError { + return + } + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmt, ok := s.(*StmtDescribeGSI) + if !ok { + t.Fatalf("%s failed: expected StmtDescribeGSI but received %T", testName+"/"+testCase.name, stmt) + } + stmt.Stmt = nil + if !reflect.DeepEqual(stmt, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmt) + } + }) + } +} diff --git a/stmt_index_test.go b/stmt_index_test.go index 2e3c6da..21bf03e 100644 --- a/stmt_index_test.go +++ b/stmt_index_test.go @@ -99,3 +99,68 @@ func Test_Query_DescribeLSI(t *testing.T) { }) } } + +func Test_Query_CreateGSI(t *testing.T) { + testName := "Test_Query_CreateGSI" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Query("CREATE GSI idx ON tbltemp WITH pk=id:string") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Exec_CreateGSI(t *testing.T) { + testName := "Test_Exec_CreateGSI" + db := _openDb(t, testName) + defer db.Close() + _initTest(db) + + db.Exec(`CREATE TABLE tbltest WITH pk=id:string WITH rcu=1 WITH wcu=1`) + + testData := []struct { + name string + sql string + gsiInfo *gsiInfo + affectedRows int64 + }{ + {name: "basic", sql: `CREATE GSI index1 ON tbltest WITH PK=grade:number WITH wcu=1 WITH rcu=2`, affectedRows: 1, gsiInfo: &gsiInfo{indexName: "index1", + wcu: 1, rcu: 2, pkAttr: "grade", pkType: "N", projectionType: "KEYS_ONLY"}}, + {name: "if_not_exists", sql: `CREATE GSI IF NOT EXISTS index1 ON tbltest WITH PK=id:string WITH wcu=2 WITH rcu=3 WITH projection=*`, affectedRows: 0, gsiInfo: &gsiInfo{indexName: "index1", + wcu: 1, rcu: 2, pkAttr: "grade", pkType: "N", projectionType: "KEYS_ONLY"}}, + {name: "with_sk", sql: `CREATE GSI index2 ON tbltest WITH PK=grade:number WITH SK=class:string WITH wcu=3 WITH rcu=4 WITH projection=a,b,c`, affectedRows: 1, gsiInfo: &gsiInfo{indexName: "index2", + wcu: 3, rcu: 4, pkAttr: "grade", pkType: "N", skAttr: "class", skType: "S", projectionType: "INCLUDE", projectedAttrs: "a,b,c"}}, + {name: "with_projection_all", sql: `CREATE GSI index3 ON tbltest WITH PK=grade:number WITH wcu=5 WITH rcu=6 WITH projection=*`, affectedRows: 1, gsiInfo: &gsiInfo{indexName: "index3", + wcu: 5, rcu: 6, pkAttr: "grade", pkType: "N", projectionType: "ALL"}}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + execResult, err := db.Exec(testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/create_gsi", err) + } + affectedRows, err := execResult.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/rows_affected", err) + } + if affectedRows != testCase.affectedRows { + t.Fatalf("%s failed: expected %#v affected-rows but received %#v", testName+"/"+testCase.name, testCase.affectedRows, affectedRows) + } + + if testCase.gsiInfo == nil { + return + } + dbresult, err := db.Query(`DESCRIBE GSI ` + testCase.gsiInfo.indexName + ` ON tbltest`) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/describe_gsi", err) + } + rows, err := _fetchAllRows(dbresult) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/fetch_rows", err) + } + _verifyGSIInfo(t, testName+"/"+testCase.name, rows[0], testCase.gsiInfo) + }) + } +} diff --git a/stmt_table_test.go b/stmt_table_test.go index 00f02ae..16f4e8d 100644 --- a/stmt_table_test.go +++ b/stmt_table_test.go @@ -24,19 +24,20 @@ func Test_Exec_CreateTable(t *testing.T) { _initTest(db) testData := []struct { - name string - sql string - tableInfo *tableInfo + name string + sql string + tableInfo *tableInfo + affectedRows int64 }{ - {name: "basic", sql: `CREATE TABLE tbltest1 WITH PK=id:string`, tableInfo: &tableInfo{tableName: "tbltest1", + {name: "basic", sql: `CREATE TABLE tbltest1 WITH PK=id:string`, affectedRows: 1, tableInfo: &tableInfo{tableName: "tbltest1", billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "S"}}, - {name: "if_not_exists", sql: `CREATE TABLE IF NOT EXISTS tbltest1 WITH PK=id:NUMBER`, tableInfo: &tableInfo{tableName: "tbltest1", + {name: "if_not_exists", sql: `CREATE TABLE IF NOT EXISTS tbltest1 WITH PK=id:NUMBER`, affectedRows: 0, tableInfo: &tableInfo{tableName: "tbltest1", billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "S"}}, - {name: "with_sk", sql: `CREATE TABLE tbltest2 WITH PK=id:binary WITH sk=grade:number, WITH class=standard`, tableInfo: &tableInfo{tableName: "tbltest2", + {name: "with_sk", sql: `CREATE TABLE tbltest2 WITH PK=id:binary WITH sk=grade:number, WITH class=standard`, affectedRows: 1, tableInfo: &tableInfo{tableName: "tbltest2", billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "B", skAttr: "grade", skType: "N"}}, - {name: "with_rcu_wcu", sql: `CREATE TABLE tbltest3 WITH PK=id:number WITH rcu=1 WITH wcu=2 WITH class=standard_ia`, tableInfo: &tableInfo{tableName: "tbltest3", + {name: "with_rcu_wcu", sql: `CREATE TABLE tbltest3 WITH PK=id:number WITH rcu=1 WITH wcu=2 WITH class=standard_ia`, affectedRows: 1, tableInfo: &tableInfo{tableName: "tbltest3", billingMode: "", wcu: 2, rcu: 1, pkAttr: "id", pkType: "N"}}, - {name: "with_lsi", sql: `CREATE TABLE tbltest4 WITH PK=id:binary WITH SK=username:string WITH LSI=index1:grade:number, WITH LSI=index2:dob:string:*, WITH LSI=index3:yob:binary:a,b,c`, tableInfo: &tableInfo{tableName: "tbltest4", + {name: "with_lsi", sql: `CREATE TABLE tbltest4 WITH PK=id:binary WITH SK=username:string WITH LSI=index1:grade:number, WITH LSI=index2:dob:string:*, WITH LSI=index3:yob:binary:a,b,c`, affectedRows: 1, tableInfo: &tableInfo{tableName: "tbltest4", billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "B", skAttr: "username", skType: "S", lsi: map[string]lsiInfo{ "index1": {projType: "KEYS_ONLY", lsiDef: lsiDef{indexName: "index1", attrName: "grade", attrType: "N"}}, @@ -48,10 +49,17 @@ func Test_Exec_CreateTable(t *testing.T) { for _, testCase := range testData { t.Run(testCase.name, func(t *testing.T) { - _, err := db.Exec(testCase.sql) + execResult, err := db.Exec(testCase.sql) if err != nil { t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/create_table", err) } + affectedRows, err := execResult.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/rows_affected", err) + } + if affectedRows != testCase.affectedRows { + t.Fatalf("%s failed: expected %#v affected-rows but received %#v", testName+"/"+testCase.name, testCase.affectedRows, affectedRows) + } if testCase.tableInfo == nil { return @@ -129,21 +137,31 @@ func Test_Exec_AlterTable(t *testing.T) { db.Exec(`CREATE TABLE tbltest WITH PK=id:string`) testData := []struct { - name string - sql string - tableInfo *tableInfo + name string + sql string + tableInfo *tableInfo + affectedRows int64 }{ - {name: "change_wcu_rcu", sql: `ALTER TABLE tbltest WITH wcu=3 WITH rcu=5`, tableInfo: &tableInfo{tableName: "tbltest", + {name: "change_wcu_rcu_provisioned", sql: `ALTER TABLE tbltest WITH wcu=3 WITH rcu=5`, affectedRows: 1, tableInfo: &tableInfo{tableName: "tbltest", billingMode: "PROVISIONED", wcu: 3, rcu: 5, pkAttr: "id", pkType: "S"}}, + {name: "change_wcu_rcu_on_demand", sql: `ALTER TABLE tbltest WITH wcu=0 WITH rcu=0`, affectedRows: 1, tableInfo: &tableInfo{tableName: "tbltest", + billingMode: "PAY_PER_REQUEST", wcu: 0, rcu: 0, pkAttr: "id", pkType: "S"}}, // DynamoDB Docker version does not support changing table class } for _, testCase := range testData { t.Run(testCase.name, func(t *testing.T) { - _, err := db.Exec(testCase.sql) + execResult, err := db.Exec(testCase.sql) if err != nil { t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) } + affectedRows, err := execResult.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/rows_affected", err) + } + if affectedRows != testCase.affectedRows { + t.Fatalf("%s failed: expected %#v affected-rows but received %#v", testName+"/"+testCase.name, testCase.affectedRows, affectedRows) + } if testCase.tableInfo == nil { return @@ -160,3 +178,47 @@ func Test_Exec_AlterTable(t *testing.T) { }) } } + +func Test_Query_DropTable(t *testing.T) { + testName := "Test_Query_DropTable" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Query("DROP TABLE tbltemp") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Exec_DropTable(t *testing.T) { + testName := "Test_Exec_DropTable" + db := _openDb(t, testName) + _initTest(db) + defer db.Close() + + db.Exec(`CREATE TABLE tbltest WITH PK=id:string`) + testData := []struct { + name string + sql string + affectedRows int64 + }{ + {name: "basic", sql: `DROP TABLE tbltest`, affectedRows: 1}, + {name: "if_exists", sql: `DROP TABLE IF EXISTS tbltest`, affectedRows: 0}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + execResult, err := db.Exec(testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + affectedRows, err := execResult.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/rows_affected", err) + } + if affectedRows != testCase.affectedRows { + t.Fatalf("%s failed: expected %#v affected-rows but received %#v", testName+"/"+testCase.name, testCase.affectedRows, affectedRows) + } + }) + } +} diff --git a/stmt_test.go b/stmt_test.go index d6992e4..52f4df0 100644 --- a/stmt_test.go +++ b/stmt_test.go @@ -17,6 +17,14 @@ type lsiInfo struct { projType string } +type gsiInfo struct { + indexName string + rcu, wcu int64 + pkAttr, pkType string + skAttr, skType string + projectionType, projectedAttrs string +} + type tableInfo struct { tableName string billingMode string @@ -148,6 +156,56 @@ func _verifyTableInfo(t *testing.T, testName string, row map[string]interface{}, } } +func _verifyGSIInfo(t *testing.T, testName string, row map[string]interface{}, gsiInfo *gsiInfo) { + s := semita.NewSemita(row) + var key string + + key = "IndexName" + indexName, _ := s.GetValueOfType(key, reddo.TypeString) + if indexName != gsiInfo.indexName { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, gsiInfo.indexName, indexName) + } + + key = "ProvisionedThroughput.ReadCapacityUnits" + rcu, _ := s.GetValueOfType(key, reddo.TypeInt) + if rcu != gsiInfo.rcu { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, gsiInfo.rcu, rcu) + } + + key = "ProvisionedThroughput.WriteCapacityUnits" + wcu, _ := s.GetValueOfType(key, reddo.TypeInt) + if wcu != gsiInfo.wcu { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, gsiInfo.wcu, wcu) + } + + key = "KeySchema[0].AttributeName" + pkAttr, _ := s.GetValueOfType(key, reddo.TypeString) + if pkAttr != gsiInfo.pkAttr { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, gsiInfo.pkAttr, pkAttr) + } + + if gsiInfo.skAttr != "" { + key = "KeySchema[1].AttributeName" + skAttr, _ := s.GetValueOfType(key, reddo.TypeString) + if skAttr != gsiInfo.skAttr { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, gsiInfo.skAttr, skAttr) + } + } + + key = "Projection.ProjectionType" + projectionType, _ := s.GetValueOfType(key, reddo.TypeString) + if projectionType != gsiInfo.projectionType { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, gsiInfo.projectionType, projectionType) + } + if projectionType == "INCLUDE" { + key = "Projection.NonKeyAttributes" + nonKeyAttrs, _ := s.GetValueOfType(key, reflect.TypeOf(make([]string, 0))) + if !reflect.DeepEqual(nonKeyAttrs, strings.Split(gsiInfo.projectedAttrs, ",")) { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName, key, gsiInfo.projectedAttrs, nonKeyAttrs) + } + } +} + func _fetchAllRows(dbRows *sql.Rows) ([]map[string]interface{}, error) { colTypes, err := dbRows.ColumnTypes() if err != nil { From 4a1eb477280073f2bd36aead35db572735192a39 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Wed, 24 May 2023 13:55:00 +1000 Subject: [PATCH 23/31] add support ALTER GSI --- README.md | 1 + SQL_INDEX.md | 31 ++++++++++- stmt.go | 32 ++++++++---- stmt_index.go | 103 ++++++++++++++++++++++++++++++++++++- stmt_index_parsing_test.go | 60 +++++++++++++++++++++ stmt_index_test.go | 62 ++++++++++++++++++++++ 6 files changed, 277 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ee53a6a..bc42753 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ func main() { - `DESCRIBE LSI` - `CREATE GSI` - `DESCRIBE GSI` + - `ALTER GSI` ## License diff --git a/SQL_INDEX.md b/SQL_INDEX.md index 30a55ae..c8ff6ab 100644 --- a/SQL_INDEX.md +++ b/SQL_INDEX.md @@ -3,6 +3,7 @@ - `DESCRIBE LSI` - `CREATE LSI` - `DESCRIBE GSI` +- `ALTER GSI` ## DESCRIBE LSI @@ -39,7 +40,7 @@ CREATE GSI [IF NOT EXISTS] ON Example: ```go -result, err := db.Exec(`CREATE GSI idxname ON tablename WITH pk=grade:number`) +result, err := db.Exec(`CREATE GSI idxname ON tablename WITH pk=grade:number, WITH rcu=1 WITH wru=2`) if err == nil { numAffectedRow, err := result.RowsAffected() ... @@ -49,7 +50,7 @@ if err == nil { Description: create a Global Secondary Index on an existing DynamoDB table. - If the statement is executed successfully, `RowsAffected()` returns `1, nil`. -- If the specified table already existed: +- If the specified GSI already existed: - If `IF NOT EXISTS` is supplied: `RowsAffected()` returns `0, nil`. - If `IF NOT EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error`. - `RCU`: GSI's read capacity unit. @@ -86,3 +87,29 @@ Sample result: |Backfilling|IndexArn|IndexName|IndexSizeBytes|IndexStatus|ItemCount|KeySchema|Projection|ProvisionedThroughput| |-----------|--------|---------|--------------|-----------|---------|---------|----------|---------------------| |null|"arn:aws:dynamodb:ddblocal:000000000000:table/session/index/idxbrowser"|"idxbrowser"|0|"ACTIVE"|0|[{"AttributeName":"browser","KeyType":"HASH"}]|{"NonKeyAttributes":null,"ProjectionType":"ALL"}|{"LastDecreaseDateTime":null,"LastIncreaseDateTime":null,"NumberOfDecreasesToday":null,"ReadCapacityUnits":1,"WriteCapacityUnits":1}| + +## ALTER GSI + +Syntax: +```sql +ALTER GSI ON +WITH wcu=[,] WITH rcu= +``` + +Example: +```go +result, err := db.Exec(`ALTER GSI idxname ON tablename WITH rcu=1 WITH wru=2`) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: update WRU/RCU of a Global Secondary Index on an existing DynamoDB table. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- `RCU`: GSI's read capacity unit. +- `WCU`: GSI's write capacity unit. +- Note: The provisioned throughput settings of a GSI are separate from those of its base table. +- Note: GSI inherit the RCU and WCU mode from the base table. That means if the base table is in on-demand mode, then DynamoDB also creates the GSI in on-demand mode. +- Note: there must be at least one space before the WITH keyword. diff --git a/stmt.go b/stmt.go index e247924..ba249a3 100644 --- a/stmt.go +++ b/stmt.go @@ -37,13 +37,14 @@ const ( var ( reCreateTable = regexp.MustCompile(`(?im)^CREATE\s+TABLE` + ifNotExists + `\s+` + field + with + `$`) reListTables = regexp.MustCompile(`(?im)^LIST\s+TABLES?$`) + reDescribeTable = regexp.MustCompile(`(?im)^DESCRIBE\s+TABLE\s+` + field + `$`) reAlterTable = regexp.MustCompile(`(?im)^ALTER\s+TABLE\s+` + field + with + `$`) reDropTable = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+TABLE` + ifExists + `\s+` + field + `$`) - reDescribeTable = regexp.MustCompile(`(?im)^DESCRIBE\s+TABLE\s+` + field + `$`) reDescribeLSI = regexp.MustCompile(`(?im)^DESCRIBE\s+LSI\s+` + field + `\s+ON\s+` + field + `$`) reCreateGSI = regexp.MustCompile(`(?im)^CREATE\s+GSI` + ifNotExists + `\s+` + field + `\s+ON\s+` + field + with + `$`) reDescribeGSI = regexp.MustCompile(`(?im)^DESCRIBE\s+GSI\s+` + field + `\s+ON\s+` + field + `$`) + reAlterGSI = regexp.MustCompile(`(?im)^ALTER\s+GSI\s+` + field + `\s+ON\s+` + field + with + `$`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -67,6 +68,14 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } return stmt, stmt.validate() } + if re := reDescribeTable; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtDescribeTable{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + tableName: strings.TrimSpace(groups[0][1]), + } + return stmt, stmt.validate() + } if re := reAlterTable; re.MatchString(query) { groups := re.FindAllStringSubmatch(query, -1) stmt := &StmtAlterTable{ @@ -88,14 +97,6 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } return stmt, stmt.validate() } - if re := reDescribeTable; re.MatchString(query) { - groups := re.FindAllStringSubmatch(query, -1) - stmt := &StmtDescribeTable{ - Stmt: &Stmt{query: query, conn: c, numInput: 0}, - tableName: strings.TrimSpace(groups[0][1]), - } - return stmt, stmt.validate() - } if re := reDescribeLSI; re.MatchString(query) { groups := re.FindAllStringSubmatch(query, -1) @@ -130,6 +131,19 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } return stmt, stmt.validate() } + if re := reAlterGSI; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtAlterGSI{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + indexName: strings.TrimSpace(groups[0][1]), + tableName: strings.TrimSpace(groups[0][2]), + withOptsStr: " " + strings.TrimSpace(groups[0][3]), + } + if err := stmt.parse(); err != nil { + return nil, err + } + return stmt, stmt.validate() + } // if re := reInsert; re.MatchString(query) { // groups := re.FindAllStringSubmatch(query, -1) diff --git a/stmt_index.go b/stmt_index.go index 2af6ae4..bb6aeb5 100644 --- a/stmt_index.go +++ b/stmt_index.go @@ -134,7 +134,8 @@ func (s *StmtDescribeLSI) Exec(_ []driver.Value) (driver.Result, error) { // - if equal to "*", GSI will be created with projection setting ALL. // - if supplied with comma-separated attribute list, for example "attr1,attr2,attr3", GSI will be created with projection setting INCLUDE. // - If "IF NOT EXISTS" is specified, Exec will silently swallow the error "Attempting to create an index which already exists". -// - Note: if RCU and WRU are both 0 or not specified, GSI will be created with PAY_PER_REQUEST billing mode; otherwise table will be creatd with PROVISIONED mode. +// - Note: The provisioned throughput settings of a GSI are separate from those of its base table. +// - Note: GSI inherit the RCU and WCU mode from the base table. That means if the base table is in on-demand mode, then DynamoDB also creates the GSI in on-demand mode. // - Note: there must be at least one space before the WITH keyword. type StmtCreateGSI struct { *Stmt @@ -342,3 +343,103 @@ func (s *StmtDescribeGSI) Query(_ []driver.Value) (driver.Rows, error) { func (s *StmtDescribeGSI) Exec(_ []driver.Value) (driver.Result, error) { return nil, errors.New("this operation is not supported, please use Query") } + +/*----------------------------------------------------------------------*/ + +// StmtAlterGSI implements "ALTER GSI" operation. +// +// Syntax: +// +// ALTER GSI ON +// WITH wcu=[,] WITH rcu= +// +// - RCU: an integer specifying DynamoDB's read capacity. +// - WCU: an integer specifying DynamoDB's write capacity. +// - Note: The provisioned throughput settings of a GSI are separate from those of its base table. +// - Note: GSI inherit the RCU and WCU mode from the base table. That means if the base table is in on-demand mode, then DynamoDB also creates the GSI in on-demand mode. +// - Note: there must be at least one space before the WITH keyword. +type StmtAlterGSI struct { + *Stmt + indexName, tableName string + rcu, wcu *int64 + withOptsStr string +} + +func (s *StmtAlterGSI) parse() error { + if err := s.Stmt.parseWithOpts(s.withOptsStr); err != nil { + return err + } + + // RCU + if _, ok := s.withOpts["RCU"]; ok { + rcu, err := strconv.ParseInt(s.withOpts["RCU"].FirstString(), 10, 64) + if err != nil || rcu < 0 { + return fmt.Errorf("invalid RCU value: %s", s.withOpts["RCU"]) + } + s.rcu = &rcu + } + // WCU + if _, ok := s.withOpts["WCU"]; ok { + wcu, err := strconv.ParseInt(s.withOpts["WCU"].FirstString(), 10, 64) + if err != nil || wcu < 0 { + return fmt.Errorf("invalid WCU value: %s", s.withOpts["WCU"]) + } + s.wcu = &wcu + } + + return nil +} + +func (s *StmtAlterGSI) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } + if s.indexName == "" { + return errors.New("index name is missing") + } + return nil +} + +// Query implements driver.Stmt.Query. +// This function is not implemented, use Exec instead. +func (s *StmtAlterGSI) Query(_ []driver.Value) (driver.Rows, error) { + return nil, errors.New("this operation is not supported, please use Exec") +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtAlterGSI) Exec(_ []driver.Value) (driver.Result, error) { + gsiInput := &types.UpdateGlobalSecondaryIndexAction{ + IndexName: &s.indexName, + ProvisionedThroughput: &types.ProvisionedThroughput{ + ReadCapacityUnits: s.rcu, + WriteCapacityUnits: s.wcu, + }, + } + input := &dynamodb.UpdateTableInput{ + TableName: &s.tableName, + GlobalSecondaryIndexUpdates: []types.GlobalSecondaryIndexUpdate{{Update: gsiInput}}, + } + + _, err := s.conn.client.UpdateTable(context.Background(), input) + result := &ResultAlterGSI{Successful: err == nil} + return result, err +} + +// ResultAlterGSI captures the result from ALTER GSI operation. +type ResultAlterGSI struct { + // Successful flags if the operation was successful or not. + Successful bool +} + +// LastInsertId implements driver.Result.LastInsertId. +func (r *ResultAlterGSI) LastInsertId() (int64, error) { + return 0, fmt.Errorf("this operation is not supported.") +} + +// RowsAffected implements driver.Result.RowsAffected. +func (r *ResultAlterGSI) RowsAffected() (int64, error) { + if r.Successful { + return 1, nil + } + return 0, nil +} diff --git a/stmt_index_parsing_test.go b/stmt_index_parsing_test.go index c09de42..7c6cfae 100644 --- a/stmt_index_parsing_test.go +++ b/stmt_index_parsing_test.go @@ -177,3 +177,63 @@ func TestStmtDescribeGSI_parse(t *testing.T) { }) } } + +func TestStmtAlterGSI_parse(t *testing.T) { + testName := "TestStmtAlterGSI_parse" + testData := []struct { + name string + sql string + expected *StmtAlterGSI + mustError bool + }{ + { + name: "no_table", + sql: "ALTER GSI abc ON WITH wcu=1 WITH rcu=2", + mustError: true, + }, + { + name: "no_index_name", + sql: "ALTER GSI ON table WITH wcu=1 WITH rcu=2", + mustError: true, + }, + { + name: "invalid_rcu", + sql: "ALTER GSI index ON table WITH RCU=-1", + mustError: true, + }, + { + name: "invalid_wcu", + sql: "ALTER GSI index ON table WITH wcu=-1", + mustError: true, + }, + + { + name: "basic", + sql: "ALTER GSI index ON table WITH wcu=1 WITH rcu=2", + expected: &StmtAlterGSI{tableName: "table", indexName: "index", wcu: aws.Int64(1), rcu: aws.Int64(2)}, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + s, err := parseQuery(nil, testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: parsing must fail", testName+"/"+testCase.name) + } + if testCase.mustError { + return + } + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmt, ok := s.(*StmtAlterGSI) + if !ok { + t.Fatalf("%s failed: expected StmtAlterGSI but received %T", testName+"/"+testCase.name, s) + } + stmt.Stmt = nil + stmt.withOptsStr = "" + if !reflect.DeepEqual(stmt, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmt) + } + }) + } +} diff --git a/stmt_index_test.go b/stmt_index_test.go index 21bf03e..c734e41 100644 --- a/stmt_index_test.go +++ b/stmt_index_test.go @@ -4,6 +4,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/btnguyen2k/consu/reddo" "github.com/btnguyen2k/consu/semita" @@ -164,3 +165,64 @@ func Test_Exec_CreateGSI(t *testing.T) { }) } } + +func Test_Query_AlterGSI(t *testing.T) { + testName := "Test_Query_AlterGSI" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Query("ALTER GSI idx ON tbltemp WITH wcu=1 WITH rcu=2") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Exec_AlterGSI(t *testing.T) { + testName := "Test_Exec_AlterGSI" + db := _openDb(t, testName) + defer db.Close() + _initTest(db) + + db.Exec(`CREATE TABLE tbltest WITH pk=id:string WITH rcu=1 WITH wcu=1`) + db.Exec(`CREATE GSI idxtest ON tbltest WITH pk=grade:number WITH rcu=3 WITH wcu=4`) + time.Sleep(5 * time.Second) + + testData := []struct { + name string + sql string + gsiInfo *gsiInfo + affectedRows int64 + }{ + {name: "basic", sql: `ALTER GSI idxtest ON tbltest WITH wcu=5 WITH rcu=6`, affectedRows: 1, gsiInfo: &gsiInfo{indexName: "idxtest", + wcu: 5, rcu: 6, pkAttr: "grade", pkType: "N", projectionType: "KEYS_ONLY"}}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + execResult, err := db.Exec(testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/alter_gsi", err) + } + affectedRows, err := execResult.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/rows_affected", err) + } + if affectedRows != testCase.affectedRows { + t.Fatalf("%s failed: expected %#v affected-rows but received %#v", testName+"/"+testCase.name, testCase.affectedRows, affectedRows) + } + + if testCase.gsiInfo == nil { + return + } + dbresult, err := db.Query(`DESCRIBE GSI ` + testCase.gsiInfo.indexName + ` ON tbltest`) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/describe_gsi", err) + } + rows, err := _fetchAllRows(dbresult) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/fetch_rows", err) + } + _verifyGSIInfo(t, testName+"/"+testCase.name, rows[0], testCase.gsiInfo) + }) + } +} From d80445c13095a529cf2d1483692ddb07cf951d4b Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Wed, 24 May 2023 16:54:20 +1000 Subject: [PATCH 24/31] update test cases --- stmt_index_test.go | 94 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/stmt_index_test.go b/stmt_index_test.go index c734e41..63f577d 100644 --- a/stmt_index_test.go +++ b/stmt_index_test.go @@ -185,7 +185,7 @@ func Test_Exec_AlterGSI(t *testing.T) { db.Exec(`CREATE TABLE tbltest WITH pk=id:string WITH rcu=1 WITH wcu=1`) db.Exec(`CREATE GSI idxtest ON tbltest WITH pk=grade:number WITH rcu=3 WITH wcu=4`) - time.Sleep(5 * time.Second) + time.Sleep(3 * time.Second) testData := []struct { name string @@ -226,3 +226,95 @@ func Test_Exec_AlterGSI(t *testing.T) { }) } } + +func Test_Exec_DescribeGSI(t *testing.T) { + testName := "Test_Exec_DescribeGSI" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Exec("DESCRIBE GSI idxname ON tblname") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Query_DescribeGSI(t *testing.T) { + testName := "Test_Query_DescribeGSI" + db := _openDb(t, testName) + _initTest(db) + defer db.Close() + + db.Exec(`CREATE TABLE tbltest WITH pk=id:string WITH rcu=1 WITH wcu=2`) + db.Exec(`CREATE GSI idxtime ON tbltest WITH pk=time:number WITH rcu=3 WITH wcu=4`) + db.Exec(`CREATE GSI idxbrowser ON tbltest WITH pk=os:binary WITH SK=version:string WITH rcu=5 WITH wcu=6 WITH projection=*`) + db.Exec(`CREATE GSI idxplatform ON tbltest WITH pk=platform:string WITH rcu=7 WITH wcu=8 WITH projection=a,b,c`) + time.Sleep(3 * time.Second) + + testData := []struct { + name string + sql string + mustError bool + numRows int + gsi gsiInfo + }{ + {name: "no_table", sql: `DESCRIBE GSI idxtest ON tblnotexist`, mustError: true}, + {name: "no_index", sql: `DESCRIBE GSI idxnotexists ON tbltest`, numRows: 0}, + {name: "proj_key_only", sql: `DESCRIBE GSI idxtime ON tbltest`, numRows: 1, gsi: gsiInfo{indexName: "idxtime", rcu: 3, wcu: 4, pkAttr: "time", pkType: "N", projectionType: "KEYS_ONLY"}}, + {name: "proj_all", sql: `DESCRIBE GSI idxbrowser ON tbltest`, numRows: 1, gsi: gsiInfo{indexName: "idxbrowser", rcu: 5, wcu: 6, pkAttr: "os", pkType: "B", skAttr: "version", skType: "S", projectionType: "ALL"}}, + {name: "proj_include", sql: `DESCRIBE GSI idxplatform ON tbltest`, numRows: 1, gsi: gsiInfo{indexName: "idxplatform", rcu: 7, wcu: 8, pkAttr: "platform", pkType: "S", projectionType: "INCLUDE", projectedAttrs: "a,b,c"}}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + dbresult, err := db.Query(testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: query must fail", testName+"/"+testCase.name) + } + if testCase.mustError { + return + } + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + rows, err := _fetchAllRows(dbresult) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + if len(rows) != testCase.numRows { + t.Fatalf("%s failed: expected %d row(s) but recelved %d", testName+"/"+testCase.name, testCase.numRows, len(rows)) + } + if testCase.numRows > 0 { + s := semita.NewSemita(rows[0]) + + key := "IndexName" + indexName, err := s.GetValueOfType(key, reddo.TypeString) + if err != nil { + t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) + } + if indexName != testCase.gsi.indexName { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.gsi.indexName, indexName) + } + + key = "Projection.ProjectionType" + projectionType, err := s.GetValueOfType(key, reddo.TypeString) + if err != nil { + t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) + } + if projectionType != testCase.gsi.projectionType { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.gsi.projectionType, projectionType) + } + + if projectionType == "INCLUDE" { + key = "Projection.NonKeyAttributes" + nonKeyAttrs, err := s.GetValueOfType(key, reflect.TypeOf(make([]string, 0))) + if err != nil { + t.Fatalf("%s failed: cannot fetch value at key <%s> / %s", testName+"/"+testCase.name, key, err) + } + if !reflect.DeepEqual(nonKeyAttrs, strings.Split(testCase.gsi.projectedAttrs, ",")) { + t.Fatalf("%s failed: expected value at key <%s> to be %#v but received %#v", testName+"/"+testCase.name, key, testCase.gsi.projectedAttrs, nonKeyAttrs) + } + } + } + }) + } +} From 6eb79291fee9c2910fcd0cbbea945b61d3025a60 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Wed, 24 May 2023 20:43:43 +1000 Subject: [PATCH 25/31] add DROP GSI --- README.md | 1 + SQL_INDEX.md | 26 +++++++++++++++ stmt.go | 11 +++++++ stmt_index.go | 66 ++++++++++++++++++++++++++++++++++++++ stmt_index_parsing_test.go | 36 +++++++++++++++++++++ stmt_index_test.go | 59 ++++++++++++++++++++++++++++++++++ 6 files changed, 199 insertions(+) diff --git a/README.md b/README.md index bc42753..5a46f65 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ func main() { - `CREATE GSI` - `DESCRIBE GSI` - `ALTER GSI` + - `DROP GSI` ## License diff --git a/SQL_INDEX.md b/SQL_INDEX.md index c8ff6ab..df06206 100644 --- a/SQL_INDEX.md +++ b/SQL_INDEX.md @@ -4,6 +4,7 @@ - `CREATE LSI` - `DESCRIBE GSI` - `ALTER GSI` +- `DROP GSI` ## DESCRIBE LSI @@ -113,3 +114,28 @@ Description: update WRU/RCU of a Global Secondary Index on an existing DynamoDB - Note: The provisioned throughput settings of a GSI are separate from those of its base table. - Note: GSI inherit the RCU and WCU mode from the base table. That means if the base table is in on-demand mode, then DynamoDB also creates the GSI in on-demand mode. - Note: there must be at least one space before the WITH keyword. + +## DROP GSI + +Syntax: +```sql +DROP GSI [IF EXIST] ON +``` + +Alias: `DELETE GSI` + +Example: +```go +result, err := db.Exec(`DROP GSI IF EXISTS index ON table`) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: delete an existing GSI from a DynamoDB table. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- If the specified table does not exist: + - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` + - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` diff --git a/stmt.go b/stmt.go index ba249a3..8f4dac8 100644 --- a/stmt.go +++ b/stmt.go @@ -45,6 +45,7 @@ var ( reCreateGSI = regexp.MustCompile(`(?im)^CREATE\s+GSI` + ifNotExists + `\s+` + field + `\s+ON\s+` + field + with + `$`) reDescribeGSI = regexp.MustCompile(`(?im)^DESCRIBE\s+GSI\s+` + field + `\s+ON\s+` + field + `$`) reAlterGSI = regexp.MustCompile(`(?im)^ALTER\s+GSI\s+` + field + `\s+ON\s+` + field + with + `$`) + reDropGSI = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+GSI` + ifExists + `\s+` + field + `\s+ON\s+` + field + `$`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -144,6 +145,16 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } return stmt, stmt.validate() } + if re := reDropGSI; re.MatchString(query) { + groups := re.FindAllStringSubmatch(query, -1) + stmt := &StmtDropGSI{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + tableName: strings.TrimSpace(groups[0][4]), + indexName: strings.TrimSpace(groups[0][3]), + ifExists: strings.TrimSpace(groups[0][2]) != "", + } + return stmt, stmt.validate() + } // if re := reInsert; re.MatchString(query) { // groups := re.FindAllStringSubmatch(query, -1) diff --git a/stmt_index.go b/stmt_index.go index bb6aeb5..54b9415 100644 --- a/stmt_index.go +++ b/stmt_index.go @@ -443,3 +443,69 @@ func (r *ResultAlterGSI) RowsAffected() (int64, error) { } return 0, nil } + +/*----------------------------------------------------------------------*/ + +// StmtDropGSI implements "DROP GSI" operation. +// +// Syntax: +// +// DROP GSI [IF EXISTS] ON +// +// If "IF EXISTS" is specified, Exec will silently swallow the error "ResourceNotFoundException". +type StmtDropGSI struct { + *Stmt + tableName string + indexName string + ifExists bool +} + +func (s *StmtDropGSI) validate() error { + if s.tableName == "" { + return errors.New("table name is missing") + } + if s.indexName == "" { + return errors.New("index name is missing") + } + return nil +} + +// Query implements driver.Stmt.Query. +// This function is not implemented, use Exec instead. +func (s *StmtDropGSI) Query(_ []driver.Value) (driver.Rows, error) { + return nil, errors.New("this operation is not supported, please use Exec") +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtDropGSI) Exec(_ []driver.Value) (driver.Result, error) { + gsiInput := &types.DeleteGlobalSecondaryIndexAction{IndexName: &s.indexName} + input := &dynamodb.UpdateTableInput{ + TableName: &s.tableName, + GlobalSecondaryIndexUpdates: []types.GlobalSecondaryIndexUpdate{types.GlobalSecondaryIndexUpdate{Delete: gsiInput}}, + } + _, err := s.conn.client.UpdateTable(context.Background(), input) + result := &ResultDropGSI{Successful: err == nil} + if s.ifExists && IsAwsError(err, "ResourceNotFoundException") { + err = nil + } + return result, err +} + +// ResultDropTable captures the result from DROP GSI operation. +type ResultDropGSI struct { + // Successful flags if the operation was successful or not. + Successful bool +} + +// LastInsertId implements driver.Result.LastInsertId. +func (r *ResultDropGSI) LastInsertId() (int64, error) { + return 0, fmt.Errorf("this operation is not supported.") +} + +// RowsAffected implements driver.Result.RowsAffected. +func (r *ResultDropGSI) RowsAffected() (int64, error) { + if r.Successful { + return 1, nil + } + return 0, nil +} diff --git a/stmt_index_parsing_test.go b/stmt_index_parsing_test.go index 7c6cfae..101fb74 100644 --- a/stmt_index_parsing_test.go +++ b/stmt_index_parsing_test.go @@ -237,3 +237,39 @@ func TestStmtAlterGSI_parse(t *testing.T) { }) } } + +func TestStmtDropGSI_parse(t *testing.T) { + testName := "TestStmtDropGSI_parse" + testData := []struct { + name string + sql string + expected *StmtDropGSI + }{ + { + name: "basic", + sql: "DROP GSI index ON table", + expected: &StmtDropGSI{tableName: "table", indexName: "index"}, + }, + { + name: "if_exists", + sql: "DROP GSI IF EXISTS index ON table", + expected: &StmtDropGSI{tableName: "table", indexName: "index", ifExists: true}, + }, + } + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + s, err := parseQuery(nil, testCase.sql) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + stmt, ok := s.(*StmtDropGSI) + if !ok { + t.Fatalf("%s failed: expected StmtDropGSI but received %T", testName+"/"+testCase.name, s) + } + stmt.Stmt = nil + if !reflect.DeepEqual(stmt, testCase.expected) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName+"/"+testCase.name, testCase.expected, stmt) + } + }) + } +} diff --git a/stmt_index_test.go b/stmt_index_test.go index 63f577d..138bf0d 100644 --- a/stmt_index_test.go +++ b/stmt_index_test.go @@ -318,3 +318,62 @@ func Test_Query_DescribeGSI(t *testing.T) { }) } } + +func Test_Query_DropGSI(t *testing.T) { + testName := "Test_Query_DropGSI" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Query("DROP GSI idxname ON tblname") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Exec_DropGSI(t *testing.T) { + testName := "Test_Exec_DropGSI" + db := _openDb(t, testName) + _initTest(db) + defer db.Close() + + db.Exec(`CREATE TABLE tbltest WITH pk=id:string WITH rcu=1 WITH wcu=2`) + db.Exec(`CREATE GSI idxtime ON tbltest WITH pk=time:number WITH rcu=3 WITH wcu=4`) + db.Exec(`CREATE GSI idxbrowser ON tbltest WITH pk=os:binary WITH SK=version:string WITH rcu=5 WITH wcu=6 WITH projection=*`) + db.Exec(`CREATE GSI idxplatform ON tbltest WITH pk=platform:string WITH rcu=7 WITH wcu=8 WITH projection=a,b,c`) + time.Sleep(3 * time.Second) + + testData := []struct { + name string + sql string + mustError bool + affectedRows int64 + }{ + {name: "no_table", sql: `DROP GSI idxtime ON tblnotexist`, mustError: true}, + {name: "no_index", sql: `DROP GSI idxnotexists ON tbltest`, mustError: true}, + {name: "basic", sql: `DROP GSI idxtime ON tbltest`, affectedRows: 1}, + {name: "if_exists", sql: `DROP GSI IF EXISTS idxnotexists ON tbltest`, affectedRows: 0}, + {name: "no_table_if_exists", sql: `DROP GSI IF EXISTS idxtime ON tblnotexist`, affectedRows: 0}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + execResult, err := db.Exec(testCase.sql) + if testCase.mustError && err == nil { + t.Fatalf("%s failed: query must fail", testName+"/"+testCase.name) + } + if testCase.mustError { + return + } + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name, err) + } + affectedRows, err := execResult.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/rows_affected", err) + } + if affectedRows != testCase.affectedRows { + t.Fatalf("%s failed: expected %#v affected-rows but received %#v", testName+"/"+testCase.name, testCase.affectedRows, affectedRows) + } + }) + } +} From 6a3ddd849772273e5a9fbce01281c02871ec4362 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Fri, 26 May 2023 00:04:20 +1000 Subject: [PATCH 26/31] add support INSERT and SELECT --- README.md | 6 ++ SQL_DOCUMENT.md | 24 ++++++ conn.go | 10 +-- driver.go | 34 -------- go.mod | 1 + go.sum | 4 + godynamo.go | 98 +++++++++++++++++++++ stmt.go | 106 +++++------------------ stmt_document.go | 192 ++++++++++++++++++++++++++++++++++++++++++ stmt_document_test.go | 103 ++++++++++++++++++++++ stmt_index.go | 28 +++--- stmt_table.go | 28 +++--- 12 files changed, 482 insertions(+), 152 deletions(-) create mode 100644 SQL_DOCUMENT.md create mode 100644 stmt_document.go create mode 100644 stmt_document_test.go diff --git a/README.md b/README.md index 5a46f65..8c83300 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,12 @@ func main() { - `ALTER GSI` - `DROP GSI` +- [Document](SQL_DOCUMENT.md) + - `INSERT` + - `SELECT` + - `UPDATE` + - `DELETE` + ## License MIT - See [LICENSE.md](LICENSE.md). diff --git a/SQL_DOCUMENT.md b/SQL_DOCUMENT.md new file mode 100644 index 0000000..17088a2 --- /dev/null +++ b/SQL_DOCUMENT.md @@ -0,0 +1,24 @@ +# godynamo - Supported statements for document + +- `INSERT` +- `SELECT` +- `UPDATE` +- `DELETE` + +## INSERT + +Syntax: [PartiQL insert statements for DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.insert.html) + +Example: +```go +result, err := db.Exec(`INSERT INTO "session" VALUE {'app': ?, 'user': ?, 'active': ?}`, "frontend", "user1", true) +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +Description: use the `INSERT` statement to add an item to a table. + +- If the statement is executed successfully, `RowsAffected()` returns `1, nil`. +- Note: the `INSERT` must follow [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.insert.html), e.g. attribute names are enclosed by _single_ quotation marks ('attr-name'), table name is enclosed by _double_ quotation marks ("table-name"), etc. diff --git a/conn.go b/conn.go index bcc951f..33a3e0d 100644 --- a/conn.go +++ b/conn.go @@ -22,11 +22,11 @@ func (c *Conn) Begin() (driver.Tx, error) { return nil, errors.New("transaction is not supported") } -// // CheckNamedValue implements driver.NamedValueChecker.CheckNamedValue. -// func (c *Conn) CheckNamedValue(value *driver.NamedValue) error { -// // since DynamoDB is document db, it accepts any value types -// return nil -// } +// CheckNamedValue implements driver.NamedValueChecker.CheckNamedValue. +func (c *Conn) CheckNamedValue(value *driver.NamedValue) error { + // since DynamoDB is document db, it accepts any value types + return nil +} // Prepare implements driver.Conn.Prepare. func (c *Conn) Prepare(query string) (driver.Stmt, error) { diff --git a/driver.go b/driver.go index e1947dd..3017e11 100644 --- a/driver.go +++ b/driver.go @@ -4,7 +4,6 @@ import ( "database/sql" "database/sql/driver" "os" - "reflect" "strconv" "strings" "time" @@ -12,8 +11,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/aws/smithy-go" ) // init is automatically invoked when the driver is imported @@ -21,37 +18,6 @@ func init() { sql.Register("godynamo", &Driver{}) } -var ( - dataTypes = map[string]types.ScalarAttributeType{ - "BINARY": "B", - "B": "B", - "NUMBER": "N", - "N": "N", - "STRING": "S", - "S": "S", - } - - keyTypes = map[string]types.KeyType{ - "HASH": "HASH", - "RANGE": "RANGE", - } - - tableClasses = map[string]types.TableClass{ - "STANDARD": types.TableClassStandard, - "STANDARD_IA": types.TableClassStandardInfrequentAccess, - } -) - -// IsAwsError returns true if err is an AWS-specific error and it matches awsErrCode. -func IsAwsError(err error, awsErrCode string) bool { - if aerr, ok := err.(*smithy.OperationError); ok { - if herr, ok := aerr.Err.(*http.ResponseError); ok { - return reflect.TypeOf(herr.Err).Elem().Name() == awsErrCode - } - } - return false -} - // Driver is AWS DynamoDB driver for database/sql. type Driver struct { } diff --git a/go.mod b/go.mod index 743d465..f7f295b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7 require ( github.com/aws/aws-sdk-go-v2 v1.18.0 github.com/aws/aws-sdk-go-v2/credentials v1.13.24 + github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.25 github.com/aws/smithy-go v1.13.5 github.com/btnguyen2k/consu/reddo v0.1.8 github.com/btnguyen2k/consu/semita v0.1.5 diff --git a/go.sum b/go.sum index e552a26..a7ff783 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/credentials v1.13.24 h1:PjiYyls3QdCrzqUN35jMWtUK1vqVZ+zLfdOa/UPFDp0= github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.25 h1:/+Z/dCO+1QHOlCm7m9G61snvIaDRUTv/HXp+8HdESiY= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.25/go.mod h1:JQ0HJ+3LaAKHx3uwRUAfR/tb/gOlgAGPT6mZfIq55Ec= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= @@ -9,6 +11,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7im github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7 h1:yb2o8oh3Y+Gg2g+wlzrWS3pB89+dHrXayT/d9cs8McU= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.7/go.mod h1:1MNss6sqoIsFGisX92do/5doiUCBrN7EjhZCS/8DUjI= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.14.11 h1:WHi9VKMYGtWt2DzqeYHXzt55aflymO2EZ6axuKla8oU= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.14.11/go.mod h1:pP+91QTpJMvcFTqGky6puHrkBs8oqoB3XOCiGRDaXwI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.27 h1:QmyPCRZNMR1pFbiOi9kBZWZuKrKB9LD4cxltxQk4tNE= diff --git a/godynamo.go b/godynamo.go index 0bd2642..e0a1a1a 100644 --- a/godynamo.go +++ b/godynamo.go @@ -1,7 +1,105 @@ // Package godynamo provides database/sql driver for AWS DynamoDB. package godynamo +import ( + "reflect" + + "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/smithy-go" +) + const ( // Version of package godynamo. Version = "0.1.0" ) + +var ( + dataTypes = map[string]types.ScalarAttributeType{ + "BINARY": "B", + "B": "B", + "NUMBER": "N", + "N": "N", + "STRING": "S", + "S": "S", + } + + keyTypes = map[string]types.KeyType{ + "HASH": "HASH", + "RANGE": "RANGE", + } + + tableClasses = map[string]types.TableClass{ + "STANDARD": types.TableClassStandard, + "STANDARD_IA": types.TableClassStandardInfrequentAccess, + } +) + +// IsAwsError returns true if err is an AWS-specific error and it matches awsErrCode. +func IsAwsError(err error, awsErrCode string) bool { + if aerr, ok := err.(*smithy.OperationError); ok { + if herr, ok := aerr.Err.(*http.ResponseError); ok { + return reflect.TypeOf(herr.Err).Elem().Name() == awsErrCode + } + } + return false +} + +func ToAttributeValue(value interface{}) (types.AttributeValue, error) { + if av, ok := value.(types.AttributeValue); ok { + return av, nil + } + switch value.(type) { + case types.AttributeValueMemberB: + v := value.(types.AttributeValueMemberB) + return &v, nil + case types.AttributeValueMemberBOOL: + v := value.(types.AttributeValueMemberBOOL) + return &v, nil + case types.AttributeValueMemberBS: + v := value.(types.AttributeValueMemberBS) + return &v, nil + case types.AttributeValueMemberL: + v := value.(types.AttributeValueMemberL) + return &v, nil + case types.AttributeValueMemberM: + v := value.(types.AttributeValueMemberM) + return &v, nil + case types.AttributeValueMemberN: + v := value.(types.AttributeValueMemberN) + return &v, nil + case types.AttributeValueMemberNS: + v := value.(types.AttributeValueMemberNS) + return &v, nil + case types.AttributeValueMemberNULL: + v := value.(types.AttributeValueMemberNULL) + return &v, nil + case types.AttributeValueMemberS: + v := value.(types.AttributeValueMemberS) + return &v, nil + case types.AttributeValueMemberSS: + v := value.(types.AttributeValueMemberSS) + return &v, nil + } + return attributevalue.Marshal(value) +} + +func goTypeToDynamodbType(typ reflect.Type) string { + if typ == nil { + return "" + } + switch typ.Kind() { + case reflect.Bool: + return "BOOLEAN" + case reflect.String: + return "STRING" + case reflect.Float32, reflect.Float64: + return "NUMBER" + case reflect.Array, reflect.Slice: + return "ARRAY" + case reflect.Map: + return "MAP" + } + return "" +} diff --git a/stmt.go b/stmt.go index 8f4dac8..861d34a 100644 --- a/stmt.go +++ b/stmt.go @@ -3,30 +3,10 @@ package godynamo import ( "database/sql/driver" "fmt" - "reflect" "regexp" "strings" ) -func goTypeToDynamodbType(typ reflect.Type) string { - if typ == nil { - return "" - } - switch typ.Kind() { - case reflect.Bool: - return "BOOLEAN" - case reflect.String: - return "STRING" - case reflect.Float32, reflect.Float64: - return "NUMBER" - case reflect.Array, reflect.Slice: - return "ARRAY" - case reflect.Map: - return "MAP" - } - return "" -} - const ( field = `([\w\-]+)` ifNotExists = `(\s+IF\s+NOT\s+EXISTS)?` @@ -46,6 +26,9 @@ var ( reDescribeGSI = regexp.MustCompile(`(?im)^DESCRIBE\s+GSI\s+` + field + `\s+ON\s+` + field + `$`) reAlterGSI = regexp.MustCompile(`(?im)^ALTER\s+GSI\s+` + field + `\s+ON\s+` + field + with + `$`) reDropGSI = regexp.MustCompile(`(?im)^(DROP|DELETE)\s+GSI` + ifExists + `\s+` + field + `\s+ON\s+` + field + `$`) + + reInsert = regexp.MustCompile(`(?im)^INSERT\s+INTO\s+`) + reSelect = regexp.MustCompile(`(?im)^SELECT\s+`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -156,71 +139,24 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { return stmt, stmt.validate() } - // if re := reInsert; re.MatchString(query) { - // groups := re.FindAllStringSubmatch(query, -1) - // stmt := &StmtInsert{ - // Stmt: &Stmt{query: query, conn: c, numInput: 0}, - // isUpsert: strings.ToUpper(strings.TrimSpace(groups[0][1])) == "UPSERT", - // dbName: strings.TrimSpace(groups[0][3]), - // collName: strings.TrimSpace(groups[0][4]), - // fieldsStr: strings.TrimSpace(groups[0][5]), - // valuesStr: strings.TrimSpace(groups[0][6]), - // } - // if stmt.dbName == "" { - // stmt.dbName = defaultDb - // } - // if err := stmt.parse(); err != nil { - // return nil, err - // } - // return stmt, stmt.validate() - // } - // if re := reSelect; re.MatchString(query) { - // groups := re.FindAllStringSubmatch(query, -1) - // stmt := &StmtSelect{ - // Stmt: &Stmt{query: query, conn: c, numInput: 0}, - // isCrossPartition: strings.TrimSpace(groups[0][1]) != "", - // collName: strings.TrimSpace(groups[0][2]), - // dbName: defaultDb, - // selectQuery: strings.ReplaceAll(strings.ReplaceAll(query, groups[0][1], ""), groups[0][3], ""), - // } - // if err := stmt.parse(groups[0][3]); err != nil { - // return nil, err - // } - // return stmt, stmt.validate() - // } - // if re := reUpdate; re.MatchString(query) { - // groups := re.FindAllStringSubmatch(query, -1) - // stmt := &StmtUpdate{ - // Stmt: &Stmt{query: query, conn: c, numInput: 0}, - // dbName: strings.TrimSpace(groups[0][2]), - // collName: strings.TrimSpace(groups[0][3]), - // updateStr: strings.TrimSpace(groups[0][4]), - // idStr: strings.TrimSpace(groups[0][5]), - // } - // if stmt.dbName == "" { - // stmt.dbName = defaultDb - // } - // if err := stmt.parse(); err != nil { - // return nil, err - // } - // return stmt, stmt.validate() - // } - // if re := reDelete; re.MatchString(query) { - // groups := re.FindAllStringSubmatch(query, -1) - // stmt := &StmtDelete{ - // Stmt: &Stmt{query: query, conn: c, numInput: 0}, - // dbName: strings.TrimSpace(groups[0][2]), - // collName: strings.TrimSpace(groups[0][3]), - // idStr: strings.TrimSpace(groups[0][4]), - // } - // if stmt.dbName == "" { - // stmt.dbName = defaultDb - // } - // if err := stmt.parse(); err != nil { - // return nil, err - // } - // return stmt, stmt.validate() - // } + if re := reInsert; re.MatchString(query) { + stmt := &StmtInsert{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + } + if err := stmt.parse(); err != nil { + return nil, err + } + return stmt, stmt.validate() + } + if re := reSelect; re.MatchString(query) { + stmt := &StmtSelect{ + Stmt: &Stmt{query: query, conn: c, numInput: 0}, + } + if err := stmt.parse(); err != nil { + return nil, err + } + return stmt, stmt.validate() + } return nil, fmt.Errorf("invalid query: %s", query) } diff --git a/stmt_document.go b/stmt_document.go new file mode 100644 index 0000000..8ebab43 --- /dev/null +++ b/stmt_document.go @@ -0,0 +1,192 @@ +package godynamo + +import ( + "context" + "database/sql/driver" + "errors" + "fmt" + "io" + "reflect" + "regexp" + "sort" + + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" +) + +var rePlaceholder = regexp.MustCompile(`(?m)\?\s*[\,\]\}\s]`) + +/*----------------------------------------------------------------------*/ + +// StmtInsert implements "INSERT" statement. +// +// Syntax: follow "PartiQL insert statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.insert.html +type StmtInsert struct { + *Stmt +} + +func (s *StmtInsert) parse() error { + matches := rePlaceholder.FindAllString(s.query, -1) + s.numInput = len(matches) + return nil +} + +func (s *StmtInsert) validate() error { + return nil +} + +// Query implements driver.Stmt.Query. +// This function is not implemented, use Exec instead. +func (s *StmtInsert) Query(_ []driver.Value) (driver.Rows, error) { + return nil, errors.New("this operation is not supported, please use Exec") +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtInsert) Exec(values []driver.Value) (driver.Result, error) { + params := make([]types.AttributeValue, len(values)) + var err error + for i, v := range values { + params[i], err = ToAttributeValue(v) + if err != nil { + return &ResultInsert{Successful: false}, fmt.Errorf("error marshalling parameter %d-th: %s", i+1, err) + } + } + input := &dynamodb.ExecuteStatementInput{ + Statement: &s.query, + ReturnConsumedCapacity: types.ReturnConsumedCapacityTotal, + Parameters: params, + } + _, err = s.conn.client.ExecuteStatement(context.Background(), input) + result := &ResultInsert{Successful: err == nil} + return result, err +} + +// ResultInsert captures the result from INSERT statement. +type ResultInsert struct { + // Successful flags if the operation was successful or not. + Successful bool +} + +// LastInsertId implements driver.Result.LastInsertId. +func (r *ResultInsert) LastInsertId() (int64, error) { + return 0, fmt.Errorf("this operation is not supported.") +} + +// RowsAffected implements driver.Result.RowsAffected. +func (r *ResultInsert) RowsAffected() (int64, error) { + if r.Successful { + return 1, nil + } + return 0, nil +} + +/*----------------------------------------------------------------------*/ + +// StmtSelect implements "SELECT" statement. +// +// Syntax: follow "PartiQL select statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html +type StmtSelect struct { + *Stmt +} + +func (s *StmtSelect) parse() error { + matches := rePlaceholder.FindAllString(s.query+" ", -1) + s.numInput = len(matches) + return nil +} + +func (s *StmtSelect) validate() error { + return nil +} + +// Query implements driver.Stmt.Query. +func (s *StmtSelect) Query(values []driver.Value) (driver.Rows, error) { + params := make([]types.AttributeValue, len(values)) + var err error + for i, v := range values { + params[i], err = ToAttributeValue(v) + if err != nil { + return &ResultSelect{}, fmt.Errorf("error marshalling parameter %d-th: %s", i+1, err) + } + } + // fmt.Printf("DEBUG: %T - %#v\n", values[0], values[0]) + // fmt.Printf("DEBUG: %T - %#v\n", params[0], params[0]) + input := &dynamodb.ExecuteStatementInput{ + Statement: &s.query, + ReturnConsumedCapacity: types.ReturnConsumedCapacityTotal, + Parameters: params, + } + dbResult, err := s.conn.client.ExecuteStatement(context.Background(), input) + result := &ResultSelect{dbResult: dbResult, columnTypes: make(map[string]reflect.Type)} + if err == nil { + result.count = len(dbResult.Items) + colMap := make(map[string]bool) + for _, item := range dbResult.Items { + for col, av := range item { + colMap[col] = true + if result.columnTypes[col] == nil { + var value interface{} + attributevalue.Unmarshal(av, &value) + result.columnTypes[col] = reflect.TypeOf(value) + } + } + } + result.columnList = make([]string, 0, len(colMap)) + for col := range colMap { + result.columnList = append(result.columnList, col) + } + sort.Strings(result.columnList) + } + return result, err +} + +// Exec implements driver.Stmt.Exec. +// This function is not implemented, use Query instead. +func (s *StmtSelect) Exec(_ []driver.Value) (driver.Result, error) { + return nil, errors.New("this operation is not supported, please use Query") +} + +// ResultSelect captures the result from SELECT statement. +type ResultSelect struct { + count int + dbResult *dynamodb.ExecuteStatementOutput + cursorCount int + columnList []string + columnTypes map[string]reflect.Type +} + +// Columns implements driver.Rows.Columns. +func (r *ResultSelect) Columns() []string { + return r.columnList +} + +// ColumnTypeScanType implements driver.RowsColumnTypeScanType.ColumnTypeScanType +func (r *ResultSelect) ColumnTypeScanType(index int) reflect.Type { + return r.columnTypes[r.columnList[index]] +} + +// ColumnTypeDatabaseTypeName implements driver.RowsColumnTypeDatabaseTypeName.ColumnTypeDatabaseTypeName +func (r *ResultSelect) ColumnTypeDatabaseTypeName(index int) string { + return goTypeToDynamodbType(r.columnTypes[r.columnList[index]]) +} + +// Close implements driver.Rows.Close. +func (r *ResultSelect) Close() error { + return nil +} + +// Next implements driver.Rows.Next. +func (r *ResultSelect) Next(dest []driver.Value) error { + if r.cursorCount >= r.count { + return io.EOF + } + rowData := r.dbResult.Items[r.cursorCount] + r.cursorCount++ + for i, colName := range r.columnList { + var value interface{} + attributevalue.Unmarshal(rowData[colName], &value) + dest[i] = value + } + return nil +} diff --git a/stmt_document_test.go b/stmt_document_test.go new file mode 100644 index 0000000..1663c8b --- /dev/null +++ b/stmt_document_test.go @@ -0,0 +1,103 @@ +package godynamo + +import ( + "reflect" + "strings" + "testing" +) + +func Test_Query_Insert(t *testing.T) { + testName := "Test_Query_Insert" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Query("INSERT INTO table VALUE {}") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Exec_Insert(t *testing.T) { + testName := "Test_Exec_Insert" + db := _openDb(t, testName) + defer db.Close() + _initTest(db) + + db.Exec(`CREATE TABLE tbltest WITH pk=id:string WITH rcu=1 WITH wcu=1`) + + testData := []struct { + name string + sql string + params []interface{} + affectedRows int64 + }{ + {name: "basic", sql: `INSERT INTO "tbltest" VALUE {'id': '1', 'name': 'User 1'}`, affectedRows: 1}, + {name: "parameterized", sql: `INSERT INTO "tbltest" VALUE {'id': ?, 'name': ?, 'active': ?, 'grade': ?, 'list': ?, 'map': ?}`, affectedRows: 1, + params: []interface{}{"2", "User 2", true, 10, []interface{}{1.2, false, "3"}, map[string]interface{}{"N": -3.4, "B": false, "S": "3"}}}, + } + + for _, testCase := range testData { + t.Run(testCase.name, func(t *testing.T) { + execResult, err := db.Exec(testCase.sql, testCase.params...) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/exec", err) + } + affectedRows, err := execResult.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/"+testCase.name+"/rows_affected", err) + } + if affectedRows != testCase.affectedRows { + t.Fatalf("%s failed: expected %#v affected-rows but received %#v", testName+"/"+testCase.name, testCase.affectedRows, affectedRows) + } + }) + } +} + +func Test_Exec_Select(t *testing.T) { + testName := "Test_Exec_Select" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Exec(`SELECT * FROM "table" WHERE id='a'`) + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} + +func Test_Query_Select(t *testing.T) { + testName := "Test_Query_Select" + db := _openDb(t, testName) + defer db.Close() + _initTest(db) + + _, err := db.Exec(`CREATE TABLE tbltest WITH PK=app:string WITH SK=user:string WITH rcu=100 WITH wcu=100`) + if err != nil { + t.Fatalf("%s failed: %s", testName, err) + } + _, err = db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'os': ?, 'active': ?, 'duration': ?}`, "app0", "user1", "Linux", true, 12.34) + if err != nil { + t.Fatalf("%s failed: %s", testName, err) + } + + dbresult, err := db.Query(`SELECT * FROM "tbltest" WHERE app=?`, "app0") + if err != nil { + t.Fatalf("%s failed: %s", testName, err) + } + rows, err := _fetchAllRows(dbresult) + if err != nil { + t.Fatalf("%s failed: %s", testName, err) + } + if len(rows) != 1 { + t.Fatalf("%s failed: expected 1 row but received %#v", testName, len(rows)) + } + expectedRow := map[string]interface{}{ + "app": "app0", + "user": "user1", + "os": "Linux", + "active": true, + "duration": 12.34, + } + if !reflect.DeepEqual(rows[0], expectedRow) { + t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName, expectedRow, rows) + } +} diff --git a/stmt_index.go b/stmt_index.go index 54b9415..481f163 100644 --- a/stmt_index.go +++ b/stmt_index.go @@ -16,7 +16,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" ) -// RowsDescribeIndex captures the result from DESCRIBE LSI or DESCRIBE GSI operation. +// RowsDescribeIndex captures the result from DESCRIBE LSI or DESCRIBE GSI statement. type RowsDescribeIndex struct { count int columnList []string @@ -59,7 +59,7 @@ func (r *RowsDescribeIndex) ColumnTypeDatabaseTypeName(index int) string { /*----------------------------------------------------------------------*/ -// StmtDescribeLSI implements "DESCRIBE LSI" operation. +// StmtDescribeLSI implements "DESCRIBE LSI" statement. // // Syntax: // @@ -115,7 +115,7 @@ func (s *StmtDescribeLSI) Exec(_ []driver.Value) (driver.Result, error) { /*----------------------------------------------------------------------*/ -// StmtCreateGSI implements "CREATE GSI" operation. +// StmtCreateGSI implements "CREATE GSI" statement. // // Syntax: // @@ -181,7 +181,7 @@ func (s *StmtCreateGSI) parse() error { s.skType = &skType } - //projection + // projection s.projectedAttrs = s.withOpts["PROJECTION"].FirstString() // RCU @@ -269,7 +269,7 @@ func (s *StmtCreateGSI) Exec(_ []driver.Value) (driver.Result, error) { return result, err } -// ResultCreateGSI captures the result from CREATE GSI operation. +// ResultCreateGSI captures the result from CREATE GSI statement. type ResultCreateGSI struct { // Successful flags if the operation was successful or not. Successful bool @@ -277,7 +277,7 @@ type ResultCreateGSI struct { // LastInsertId implements driver.Result.LastInsertId. func (r *ResultCreateGSI) LastInsertId() (int64, error) { - return 0, fmt.Errorf("this operation is not supported.") + return 0, fmt.Errorf("this operation is not supported") } // RowsAffected implements driver.Result.RowsAffected. @@ -290,7 +290,7 @@ func (r *ResultCreateGSI) RowsAffected() (int64, error) { /*----------------------------------------------------------------------*/ -// StmtDescribeGSI implements "DESCRIBE GSI" operation. +// StmtDescribeGSI implements "DESCRIBE GSI" statement. // // Syntax: // @@ -346,7 +346,7 @@ func (s *StmtDescribeGSI) Exec(_ []driver.Value) (driver.Result, error) { /*----------------------------------------------------------------------*/ -// StmtAlterGSI implements "ALTER GSI" operation. +// StmtAlterGSI implements "ALTER GSI" statement. // // Syntax: // @@ -425,7 +425,7 @@ func (s *StmtAlterGSI) Exec(_ []driver.Value) (driver.Result, error) { return result, err } -// ResultAlterGSI captures the result from ALTER GSI operation. +// ResultAlterGSI captures the result from ALTER GSI statement. type ResultAlterGSI struct { // Successful flags if the operation was successful or not. Successful bool @@ -433,7 +433,7 @@ type ResultAlterGSI struct { // LastInsertId implements driver.Result.LastInsertId. func (r *ResultAlterGSI) LastInsertId() (int64, error) { - return 0, fmt.Errorf("this operation is not supported.") + return 0, fmt.Errorf("this operation is not supported") } // RowsAffected implements driver.Result.RowsAffected. @@ -446,7 +446,7 @@ func (r *ResultAlterGSI) RowsAffected() (int64, error) { /*----------------------------------------------------------------------*/ -// StmtDropGSI implements "DROP GSI" operation. +// StmtDropGSI implements "DROP GSI" statement. // // Syntax: // @@ -481,7 +481,7 @@ func (s *StmtDropGSI) Exec(_ []driver.Value) (driver.Result, error) { gsiInput := &types.DeleteGlobalSecondaryIndexAction{IndexName: &s.indexName} input := &dynamodb.UpdateTableInput{ TableName: &s.tableName, - GlobalSecondaryIndexUpdates: []types.GlobalSecondaryIndexUpdate{types.GlobalSecondaryIndexUpdate{Delete: gsiInput}}, + GlobalSecondaryIndexUpdates: []types.GlobalSecondaryIndexUpdate{{Delete: gsiInput}}, } _, err := s.conn.client.UpdateTable(context.Background(), input) result := &ResultDropGSI{Successful: err == nil} @@ -491,7 +491,7 @@ func (s *StmtDropGSI) Exec(_ []driver.Value) (driver.Result, error) { return result, err } -// ResultDropTable captures the result from DROP GSI operation. +// ResultDropGSI captures the result from DROP GSI statement. type ResultDropGSI struct { // Successful flags if the operation was successful or not. Successful bool @@ -499,7 +499,7 @@ type ResultDropGSI struct { // LastInsertId implements driver.Result.LastInsertId. func (r *ResultDropGSI) LastInsertId() (int64, error) { - return 0, fmt.Errorf("this operation is not supported.") + return 0, fmt.Errorf("this operation is not supported") } // RowsAffected implements driver.Result.RowsAffected. diff --git a/stmt_table.go b/stmt_table.go index ed292d1..1e8b0b2 100644 --- a/stmt_table.go +++ b/stmt_table.go @@ -24,7 +24,7 @@ type lsiDef struct { /*----------------------------------------------------------------------*/ -// StmtCreateTable implements "CREATE TABLE" operation. +// StmtCreateTable implements "CREATE TABLE" statement. // // Syntax: // @@ -220,7 +220,7 @@ func (s *StmtCreateTable) Exec(_ []driver.Value) (driver.Result, error) { return result, err } -// ResultCreateTable captures the result from CREATE TABLE operation. +// ResultCreateTable captures the result from CREATE TABLE statement. type ResultCreateTable struct { // Successful flags if the operation was successful or not. Successful bool @@ -228,7 +228,7 @@ type ResultCreateTable struct { // LastInsertId implements driver.Result.LastInsertId. func (r *ResultCreateTable) LastInsertId() (int64, error) { - return 0, fmt.Errorf("this operation is not supported.") + return 0, fmt.Errorf("this operation is not supported") } // RowsAffected implements driver.Result.RowsAffected. @@ -241,7 +241,7 @@ func (r *ResultCreateTable) RowsAffected() (int64, error) { /*----------------------------------------------------------------------*/ -// StmtListTables implements "LIST TABLES" operation. +// StmtListTables implements "LIST TABLES" statement. // // Syntax: // @@ -275,7 +275,7 @@ func (s *StmtListTables) Query(_ []driver.Value) (driver.Rows, error) { return rows, err } -// RowsListTables captures the result from LIST TABLES operation. +// RowsListTables captures the result from LIST TABLES statement. type RowsListTables struct { count int tables []string @@ -304,12 +304,12 @@ func (r *RowsListTables) Next(dest []driver.Value) error { } // ColumnTypeScanType implements driver.RowsColumnTypeScanType.ColumnTypeScanType -func (r *RowsListTables) ColumnTypeScanType(index int) reflect.Type { +func (r *RowsListTables) ColumnTypeScanType(_ int) reflect.Type { return reddo.TypeString } // ColumnTypeDatabaseTypeName implements driver.RowsColumnTypeDatabaseTypeName.ColumnTypeDatabaseTypeName -func (r *RowsListTables) ColumnTypeDatabaseTypeName(index int) string { +func (r *RowsListTables) ColumnTypeDatabaseTypeName(_ int) string { return "STRING" } @@ -330,7 +330,7 @@ func (r *RowsListTables) ColumnTypeDatabaseTypeName(index int) string { /*----------------------------------------------------------------------*/ -// StmtAlterTable implements "ALTER TABLE" operation. +// StmtAlterTable implements "ALTER TABLE" statement. // // Syntax: // @@ -422,7 +422,7 @@ func (s *StmtAlterTable) Exec(_ []driver.Value) (driver.Result, error) { return result, err } -// ResultAlterTable captures the result from CREATE TABLE operation. +// ResultAlterTable captures the result from CREATE TABLE statement. type ResultAlterTable struct { // Successful flags if the operation was successful or not. Successful bool @@ -430,7 +430,7 @@ type ResultAlterTable struct { // LastInsertId implements driver.Result.LastInsertId. func (r *ResultAlterTable) LastInsertId() (int64, error) { - return 0, fmt.Errorf("this operation is not supported.") + return 0, fmt.Errorf("this operation is not supported") } // RowsAffected implements driver.Result.RowsAffected. @@ -443,7 +443,7 @@ func (r *ResultAlterTable) RowsAffected() (int64, error) { /*----------------------------------------------------------------------*/ -// StmtDropTable implements "DROP TABLE" operation. +// StmtDropTable implements "DROP TABLE" statement. // // Syntax: // @@ -482,7 +482,7 @@ func (s *StmtDropTable) Exec(_ []driver.Value) (driver.Result, error) { return result, err } -// ResultDropTable captures the result from DROP TABLE operation. +// ResultDropTable captures the result from DROP TABLE statement. type ResultDropTable struct { // Successful flags if the operation was successful or not. Successful bool @@ -490,7 +490,7 @@ type ResultDropTable struct { // LastInsertId implements driver.Result.LastInsertId. func (r *ResultDropTable) LastInsertId() (int64, error) { - return 0, fmt.Errorf("this operation is not supported.") + return 0, fmt.Errorf("this operation is not supported") } // RowsAffected implements driver.Result.RowsAffected. @@ -551,7 +551,7 @@ func (s *StmtDescribeTable) Exec(_ []driver.Value) (driver.Result, error) { return nil, errors.New("this operation is not supported, please use Query") } -// RowsDescribeTable captures the result from DESCRIBE TABLE operation. +// RowsDescribeTable captures the result from DESCRIBE TABLE statement. type RowsDescribeTable struct { count int columnList []string From ea17410f6e135e2e451cccecc7bc9399d0f596c1 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Fri, 26 May 2023 00:10:13 +1000 Subject: [PATCH 27/31] update docs --- SQL_DOCUMENT.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/SQL_DOCUMENT.md b/SQL_DOCUMENT.md index 17088a2..fddccb6 100644 --- a/SQL_DOCUMENT.md +++ b/SQL_DOCUMENT.md @@ -22,3 +22,24 @@ Description: use the `INSERT` statement to add an item to a table. - If the statement is executed successfully, `RowsAffected()` returns `1, nil`. - Note: the `INSERT` must follow [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.insert.html), e.g. attribute names are enclosed by _single_ quotation marks ('attr-name'), table name is enclosed by _double_ quotation marks ("table-name"), etc. + +## SELECT + +Syntax: [PartiQL select statements for DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html) + +Example: +```go +result, err := db.Query(`SELECT * FROM "session" WHERE app='frontend'`) +if err == nil { + ... +} +``` + +Description: use the `SELECT` statement to retrieve data from a table. + +- Note: the `SELECT` must follow [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html). + +Sample result: +|active|app|user| +|------|---|----| +|true|"frontend"|"user1"| From ba0d0d90ab4af630eab7d08dc54ca43ae0c42e18 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Fri, 26 May 2023 00:16:53 +1000 Subject: [PATCH 28/31] update test --- stmt_table_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/stmt_table_test.go b/stmt_table_test.go index 16f4e8d..d48331e 100644 --- a/stmt_table_test.go +++ b/stmt_table_test.go @@ -222,3 +222,14 @@ func Test_Exec_DropTable(t *testing.T) { }) } } + +func Test_Exec_DescribeTable(t *testing.T) { + testName := "Test_Exec_DescribeTable" + db := _openDb(t, testName) + defer db.Close() + + _, err := db.Exec("DESCRIBE TABLE tbltemp") + if err == nil || strings.Index(err.Error(), "not supported") < 0 { + t.Fatalf("%s failed: expected 'not support' error, but received %#v", testName, err) + } +} From 3e1d61db34fbc28f2fa53437c26f0f7ac5450054 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 27 May 2023 01:21:27 +1000 Subject: [PATCH 29/31] add support DELETE statement --- stmt.go | 24 +++- stmt_document.go | 267 ++++++++++++++++++++++++++---------------- stmt_document_test.go | 114 ++++++++++++++++++ 3 files changed, 305 insertions(+), 100 deletions(-) diff --git a/stmt.go b/stmt.go index 861d34a..81b2762 100644 --- a/stmt.go +++ b/stmt.go @@ -29,6 +29,8 @@ var ( reInsert = regexp.MustCompile(`(?im)^INSERT\s+INTO\s+`) reSelect = regexp.MustCompile(`(?im)^SELECT\s+`) + reUpdate = regexp.MustCompile(`(?im)^UPDATE\s+`) + reDelete = regexp.MustCompile(`(?im)^DELETE\s+FROM\s+`) ) func parseQuery(c *Conn, query string) (driver.Stmt, error) { @@ -141,7 +143,7 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { if re := reInsert; re.MatchString(query) { stmt := &StmtInsert{ - Stmt: &Stmt{query: query, conn: c, numInput: 0}, + StmtExecutable: &StmtExecutable{Stmt: &Stmt{query: query, conn: c, numInput: 0}}, } if err := stmt.parse(); err != nil { return nil, err @@ -150,7 +152,25 @@ func parseQuery(c *Conn, query string) (driver.Stmt, error) { } if re := reSelect; re.MatchString(query) { stmt := &StmtSelect{ - Stmt: &Stmt{query: query, conn: c, numInput: 0}, + StmtExecutable: &StmtExecutable{Stmt: &Stmt{query: query, conn: c, numInput: 0}}, + } + if err := stmt.parse(); err != nil { + return nil, err + } + return stmt, stmt.validate() + } + if re := reUpdate; re.MatchString(query) { + stmt := &StmtUpdate{ + StmtExecutable: &StmtExecutable{Stmt: &Stmt{query: query, conn: c, numInput: 0}}, + } + if err := stmt.parse(); err != nil { + return nil, err + } + return stmt, stmt.validate() + } + if re := reDelete; re.MatchString(query) { + stmt := &StmtDelete{ + StmtExecutable: &StmtExecutable{Stmt: &Stmt{query: query, conn: c, numInput: 0}}, } if err := stmt.parse(); err != nil { return nil, err diff --git a/stmt_document.go b/stmt_document.go index 8ebab43..ddeb608 100644 --- a/stmt_document.go +++ b/stmt_document.go @@ -19,37 +19,28 @@ var rePlaceholder = regexp.MustCompile(`(?m)\?\s*[\,\]\}\s]`) /*----------------------------------------------------------------------*/ -// StmtInsert implements "INSERT" statement. -// -// Syntax: follow "PartiQL insert statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.insert.html -type StmtInsert struct { +// StmtExecutable is the base implementation for INSERT, SELECT, UPDATE and DELETE statements. +type StmtExecutable struct { *Stmt } -func (s *StmtInsert) parse() error { - matches := rePlaceholder.FindAllString(s.query, -1) +func (s *StmtExecutable) parse() error { + matches := rePlaceholder.FindAllString(s.query+" ", -1) s.numInput = len(matches) return nil } -func (s *StmtInsert) validate() error { +func (s *StmtExecutable) validate() error { return nil } -// Query implements driver.Stmt.Query. -// This function is not implemented, use Exec instead. -func (s *StmtInsert) Query(_ []driver.Value) (driver.Rows, error) { - return nil, errors.New("this operation is not supported, please use Exec") -} - -// Exec implements driver.Stmt.Exec. -func (s *StmtInsert) Exec(values []driver.Value) (driver.Result, error) { +func (s *StmtExecutable) Execute(values []driver.Value) (*dynamodb.ExecuteStatementOutput, error) { params := make([]types.AttributeValue, len(values)) var err error for i, v := range values { params[i], err = ToAttributeValue(v) if err != nil { - return &ResultInsert{Successful: false}, fmt.Errorf("error marshalling parameter %d-th: %s", i+1, err) + return nil, fmt.Errorf("error marshalling parameter %d-th: %s", i+1, err) } } input := &dynamodb.ExecuteStatementInput{ @@ -57,127 +48,85 @@ func (s *StmtInsert) Exec(values []driver.Value) (driver.Result, error) { ReturnConsumedCapacity: types.ReturnConsumedCapacityTotal, Parameters: params, } - _, err = s.conn.client.ExecuteStatement(context.Background(), input) - result := &ResultInsert{Successful: err == nil} - return result, err + return s.conn.client.ExecuteStatement(context.Background(), input) } -// ResultInsert captures the result from INSERT statement. -type ResultInsert struct { +// ResultNoResultSet captures the result from statements that do not expect a ResultSet to be returned. +type ResultNoResultSet struct { // Successful flags if the operation was successful or not. - Successful bool + Successful bool + AffectedRows int64 } // LastInsertId implements driver.Result.LastInsertId. -func (r *ResultInsert) LastInsertId() (int64, error) { +func (r *ResultNoResultSet) LastInsertId() (int64, error) { return 0, fmt.Errorf("this operation is not supported.") } // RowsAffected implements driver.Result.RowsAffected. -func (r *ResultInsert) RowsAffected() (int64, error) { - if r.Successful { - return 1, nil - } - return 0, nil -} - -/*----------------------------------------------------------------------*/ - -// StmtSelect implements "SELECT" statement. -// -// Syntax: follow "PartiQL select statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html -type StmtSelect struct { - *Stmt -} - -func (s *StmtSelect) parse() error { - matches := rePlaceholder.FindAllString(s.query+" ", -1) - s.numInput = len(matches) - return nil +func (r *ResultNoResultSet) RowsAffected() (int64, error) { + return r.AffectedRows, nil } -func (s *StmtSelect) validate() error { - return nil +// ResultResultSet captures the result from statements that expect a ResultSet to be returned. +type ResultResultSet struct { + count int + dbResult *dynamodb.ExecuteStatementOutput + cursorCount int + columnList []string + columnTypes map[string]reflect.Type } -// Query implements driver.Stmt.Query. -func (s *StmtSelect) Query(values []driver.Value) (driver.Rows, error) { - params := make([]types.AttributeValue, len(values)) - var err error - for i, v := range values { - params[i], err = ToAttributeValue(v) - if err != nil { - return &ResultSelect{}, fmt.Errorf("error marshalling parameter %d-th: %s", i+1, err) - } +func (r *ResultResultSet) init() *ResultResultSet { + if r.dbResult == nil { + return r } - // fmt.Printf("DEBUG: %T - %#v\n", values[0], values[0]) - // fmt.Printf("DEBUG: %T - %#v\n", params[0], params[0]) - input := &dynamodb.ExecuteStatementInput{ - Statement: &s.query, - ReturnConsumedCapacity: types.ReturnConsumedCapacityTotal, - Parameters: params, + if r.columnTypes == nil { + r.columnTypes = make(map[string]reflect.Type) } - dbResult, err := s.conn.client.ExecuteStatement(context.Background(), input) - result := &ResultSelect{dbResult: dbResult, columnTypes: make(map[string]reflect.Type)} - if err == nil { - result.count = len(dbResult.Items) - colMap := make(map[string]bool) - for _, item := range dbResult.Items { - for col, av := range item { - colMap[col] = true - if result.columnTypes[col] == nil { - var value interface{} - attributevalue.Unmarshal(av, &value) - result.columnTypes[col] = reflect.TypeOf(value) - } + r.count = len(r.dbResult.Items) + colMap := make(map[string]bool) + for _, item := range r.dbResult.Items { + for col, av := range item { + colMap[col] = true + if r.columnTypes[col] == nil { + var value interface{} + attributevalue.Unmarshal(av, &value) + r.columnTypes[col] = reflect.TypeOf(value) } } - result.columnList = make([]string, 0, len(colMap)) - for col := range colMap { - result.columnList = append(result.columnList, col) - } - sort.Strings(result.columnList) } - return result, err -} - -// Exec implements driver.Stmt.Exec. -// This function is not implemented, use Query instead. -func (s *StmtSelect) Exec(_ []driver.Value) (driver.Result, error) { - return nil, errors.New("this operation is not supported, please use Query") -} + r.columnList = make([]string, 0, len(colMap)) + for col := range colMap { + r.columnList = append(r.columnList, col) + } + sort.Strings(r.columnList) -// ResultSelect captures the result from SELECT statement. -type ResultSelect struct { - count int - dbResult *dynamodb.ExecuteStatementOutput - cursorCount int - columnList []string - columnTypes map[string]reflect.Type + return r } // Columns implements driver.Rows.Columns. -func (r *ResultSelect) Columns() []string { +func (r *ResultResultSet) Columns() []string { return r.columnList } // ColumnTypeScanType implements driver.RowsColumnTypeScanType.ColumnTypeScanType -func (r *ResultSelect) ColumnTypeScanType(index int) reflect.Type { +func (r *ResultResultSet) ColumnTypeScanType(index int) reflect.Type { return r.columnTypes[r.columnList[index]] } // ColumnTypeDatabaseTypeName implements driver.RowsColumnTypeDatabaseTypeName.ColumnTypeDatabaseTypeName -func (r *ResultSelect) ColumnTypeDatabaseTypeName(index int) string { +func (r *ResultResultSet) ColumnTypeDatabaseTypeName(index int) string { return goTypeToDynamodbType(r.columnTypes[r.columnList[index]]) } // Close implements driver.Rows.Close. -func (r *ResultSelect) Close() error { +func (r *ResultResultSet) Close() error { return nil } // Next implements driver.Rows.Next. -func (r *ResultSelect) Next(dest []driver.Value) error { +func (r *ResultResultSet) Next(dest []driver.Value) error { if r.cursorCount >= r.count { return io.EOF } @@ -190,3 +139,125 @@ func (r *ResultSelect) Next(dest []driver.Value) error { } return nil } + +/*----------------------------------------------------------------------*/ + +// StmtInsert implements "INSERT" statement. +// +// Syntax: follow "PartiQL insert statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.insert.html +type StmtInsert struct { + *StmtExecutable +} + +// Query implements driver.Stmt.Query. +// This function is not implemented, use Exec instead. +func (s *StmtInsert) Query(_ []driver.Value) (driver.Rows, error) { + return nil, errors.New("this operation is not supported, please use Exec") +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtInsert) Exec(values []driver.Value) (driver.Result, error) { + _, err := s.Execute(values) + result := &ResultNoResultSet{Successful: err == nil} + return result, err +} + +/*----------------------------------------------------------------------*/ + +// StmtSelect implements "SELECT" statement. +// +// Syntax: follow "PartiQL select statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html +type StmtSelect struct { + *StmtExecutable +} + +// Query implements driver.Stmt.Query. +func (s *StmtSelect) Query(values []driver.Value) (driver.Rows, error) { + output, err := s.Execute(values) + result := &ResultResultSet{dbResult: output} + if err == nil { + result.init() + } + return result, err +} + +// Exec implements driver.Stmt.Exec. +// This function is not implemented, use Query instead. +func (s *StmtSelect) Exec(_ []driver.Value) (driver.Result, error) { + return nil, errors.New("this operation is not supported, please use Query") +} + +/*----------------------------------------------------------------------*/ + +// StmtUpdate implements "UPDATE" statement. +// +// Syntax: follow "PartiQL update statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.update.html +type StmtUpdate struct { + *StmtExecutable +} + +// Query implements driver.Stmt.Query. +func (s *StmtUpdate) Query(values []driver.Value) (driver.Rows, error) { + output, err := s.Execute(values) + result := &ResultResultSet{dbResult: output, columnTypes: make(map[string]reflect.Type)} + if err == nil { + result.init() + } + return result, err +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtUpdate) Exec(values []driver.Value) (driver.Result, error) { + _, err := s.Execute(values) + result := &ResultNoResultSet{Successful: err == nil} + if err != nil { + result.AffectedRows = 0 + } else { + result.AffectedRows = 1 + } + return result, err +} + +/*----------------------------------------------------------------------*/ + +// StmtDelete implements "DELETE" statement. +// +// Syntax: follow "PartiQL delete statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.delete.html +// +// Note: StmtDelete returns the deleted item by appending "RETURNING RETURNING ALL OLD *" to the statement. +type StmtDelete struct { + *StmtExecutable +} + +var ( + reReturning = regexp.MustCompile(`(?im)\s+RETURNING\s+((ALL\s+OLD)|(MODIFIED\s+OLD)|(ALL\s+NEW)|(MODIFIED\s+NEW))\s+\*\s*$`) +) + +func (s *StmtDelete) parse() error { + if !reReturning.MatchString(s.query) { + s.query += " RETURNING ALL OLD *" + } + return s.StmtExecutable.parse() +} + +// Query implements driver.Stmt.Query. +func (s *StmtDelete) Query(values []driver.Value) (driver.Rows, error) { + output, err := s.Execute(values) + result := &ResultResultSet{dbResult: output, columnTypes: make(map[string]reflect.Type)} + if err == nil { + result.init() + } + return result, err +} + +// Exec implements driver.Stmt.Exec. +func (s *StmtDelete) Exec(values []driver.Value) (driver.Result, error) { + output, err := s.Execute(values) + if IsAwsError(err, "ConditionalCheckFailedException") { + return &ResultNoResultSet{Successful: true, AffectedRows: 0}, nil + } + if err != nil { + return &ResultNoResultSet{Successful: false, AffectedRows: 0}, err + } + return &ResultNoResultSet{Successful: true, AffectedRows: int64(len(output.Items))}, nil +} diff --git a/stmt_document_test.go b/stmt_document_test.go index 1663c8b..e214eea 100644 --- a/stmt_document_test.go +++ b/stmt_document_test.go @@ -101,3 +101,117 @@ func Test_Query_Select(t *testing.T) { t.Fatalf("%s failed:\nexpected %#v\nreceived %#v", testName, expectedRow, rows) } } + +func Test_Exec_Delete(t *testing.T) { + testName := "Test_Exec_Delete" + db := _openDb(t, testName) + defer db.Close() + _initTest(db) + + db.Exec(`DROP TABLE IF EXISTS tbltest`) + db.Exec(`CREATE TABLE tbltest WITH PK=app:string WITH SK=user:string WITH rcu=100 WITH wcu=100`) + _, err := db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'os': ?, 'active': ?, 'duration': ?}`, "app0", "user1", "Ubuntu", true, 12.34) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + _, err = db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'platform': ?, 'location': ?}`, "app0", "user2", "Windows", "AU") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + + dbRows1, _ := db.Query(`SELECT * FROM "tbltest"`) + rows1, _ := _fetchAllRows(dbRows1) + if len(rows1) != 2 { + t.Fatalf("%s failed: expected 2 rows in table, but there is %#v", testName, len(rows1)) + } + + sql := `DELETE FROM "tbltest" WHERE "app"=? AND "user"=?` + result1, err := db.Exec(sql, "app0", "user1") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/delete", err) + } + rowsAffected1, err := result1.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/rows_affected", err) + } + if rowsAffected1 != 1 { + t.Fatalf("%s failed: expected 1 row affected but received %#v", testName+"/rows_affected", rowsAffected1) + } + + dbRows2, _ := db.Query(`SELECT * FROM "tbltest"`) + rows2, _ := _fetchAllRows(dbRows2) + if len(rows2) != 1 { + t.Fatalf("%s failed: expected 1 rows in table, but there is %#v", testName, len(rows1)) + } + + result0, err := db.Exec(sql, "app0", "user0") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/delete", err) + } + rowsAffected0, err := result0.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/rows_affected", err) + } + if rowsAffected0 != 0 { + t.Fatalf("%s failed: expected 0 row affected but received %#v", testName+"/rows_affected", rowsAffected0) + } +} + +func Test_Query_Delete(t *testing.T) { + testName := "Test_Query_Delete" + db := _openDb(t, testName) + defer db.Close() + _initTest(db) + + db.Exec(`DROP TABLE IF EXISTS tbltest`) + db.Exec(`CREATE TABLE tbltest WITH PK=app:string WITH SK=user:string WITH rcu=100 WITH wcu=100`) + _, err := db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'os': ?, 'active': ?, 'duration': ?}`, "app0", "user1", "Ubuntu", true, 12.34) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + _, err = db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'platform': ?, 'location': ?}`, "app0", "user2", "Windows", "AU") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + + dbRows1, _ := db.Query(`SELECT * FROM "tbltest"`) + rows1, _ := _fetchAllRows(dbRows1) + if len(rows1) != 2 { + t.Fatalf("%s failed: expected 2 rows in table, but there is %#v", testName, len(rows1)) + } + + sql := `DELETE FROM "tbltest" WHERE "app"=? AND "user"=? RETURNING ALL OLD *` + result1, err := db.Query(sql, "app0", "user1") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/delete", err) + } + rows, err := _fetchAllRows(result1) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/delete", err) + } + if len(rows) != 1 { + t.Fatalf("%s failed: expected 1 row returned, but received %#v", testName+"/delete", len(rows)) + } + expected := map[string]interface{}{"app": "app0", "user": "user1", "os": "Ubuntu", "active": true, "duration": float64(12.34)} + if !reflect.DeepEqual(rows[0], expected) { + t.Fatalf("%s failed:\nexpected %#v\nbut received %#v", testName+"/delete", expected, rows[0]) + } + + dbRows2, _ := db.Query(`SELECT * FROM "tbltest"`) + rows2, _ := _fetchAllRows(dbRows2) + if len(rows2) != 1 { + t.Fatalf("%s failed: expected 1 rows in table, but there is %#v", testName, len(rows1)) + } + + result0, err := db.Query(sql, "app0", "user0") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/delete", err) + } + rows, err = _fetchAllRows(result0) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/delete", err) + } + if len(rows) != 0 { + t.Fatalf("%s failed: expected 0 row returned, but received %#v", testName+"/delete", len(rows)) + } +} From 1bf77caa75473a8224d49a459519ddbb3f70ec6a Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 27 May 2023 01:37:15 +1000 Subject: [PATCH 30/31] update docs --- SQL_DOCUMENT.md | 29 +++++++++++++++++++++++++++++ stmt_document.go | 3 +++ 2 files changed, 32 insertions(+) diff --git a/SQL_DOCUMENT.md b/SQL_DOCUMENT.md index fddccb6..d217687 100644 --- a/SQL_DOCUMENT.md +++ b/SQL_DOCUMENT.md @@ -43,3 +43,32 @@ Sample result: |active|app|user| |------|---|----| |true|"frontend"|"user1"| + +## DELETE + +Syntax: [PartiQL delete statements for DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.delete.html) + +Example: +```go +result, err := db.Exec(`DELETE FROM "tbltest" WHERE "app"=? AND "user"=?`, "app0", "user1") +if err == nil { + numAffectedRow, err := result.RowsAffected() + ... +} +``` + +`Query` can also be used to have the content of the old item returned. +```go +if err == nil { + ... +} +``` + +Description: use the `DELETE` statement to delete an existing item from a table. + +- Note: the `DELETE` must follow [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.delete.html). + +Sample result: +|app|location|platform|user| +|---|--------|--------|----| +|"app0"|"AU"|"Windows"|"user2"| diff --git a/stmt_document.go b/stmt_document.go index ddeb608..422c920 100644 --- a/stmt_document.go +++ b/stmt_document.go @@ -159,6 +159,9 @@ func (s *StmtInsert) Query(_ []driver.Value) (driver.Rows, error) { func (s *StmtInsert) Exec(values []driver.Value) (driver.Result, error) { _, err := s.Execute(values) result := &ResultNoResultSet{Successful: err == nil} + if err == nil { + result.AffectedRows = 1 + } return result, err } From 23ff7b2965e6fe75bddfeecccdc7e365b3e87496 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Sat, 27 May 2023 22:45:58 +1000 Subject: [PATCH 31/31] prepare to release v0.1.0 --- README.md | 12 +++-- RELEASE-NOTES.md | 8 ++++ SQL_DOCUMENT.md | 45 +++++++++++++++--- SQL_TABLE.md | 46 +++++++++---------- driver.go | 13 ++++-- godynamo.go | 2 +- release.sh | 16 +++++++ stmt_document.go | 38 +++++++++------ stmt_document_test.go | 104 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 231 insertions(+), 53 deletions(-) create mode 100644 RELEASE-NOTES.md create mode 100644 release.sh diff --git a/README.md b/README.md index 8c83300..0837317 100644 --- a/README.md +++ b/README.md @@ -47,16 +47,22 @@ func main() { ## Data Source Name (DSN) format for AWS Dynamo DB -// TODO +`Region=;AkId=;Secret_Key=[;Endpoint=][TimeoutMs=]` + +- `Region`: AWS region, for example `us-east-1`. If not supplied, the value of the environment `AWS_REGION` is used. +- `AkId`: AWS Access Key ID, for example `AKIA1234567890ABCDEF`. If not supplied, the value of the environment `AWS_ACCESS_KEY_ID` is used. +- `Secret_Key`: AWS Secret Key, for example `0***F`. If not supplied, the value of the environment `AWS_SECRET_ACCESS_KEY` is used. +- `Endpoint`: (optional) AWS DynamoDB endpoint, for example `http://localhost:8000`; useful when AWS DynamoDB is running on local machine. +- `TimeoutMs`: (optional) timeout in milliseconds. If not specified, default value is `10000`. ## Supported statements: - [Table](SQL_TABLE.md): - `CREATE TABLE` - `LIST TABLES` + - `DESCRIBE TABLE` - `ALTER TABLE` - `DROP TABLE` - - `DESCRIBE TABLE` - [Index](SQL_INDEX.md): - `DESCRIBE LSI` @@ -65,7 +71,7 @@ func main() { - `ALTER GSI` - `DROP GSI` -- [Document](SQL_DOCUMENT.md) +- [Document](SQL_DOCUMENT.md): - `INSERT` - `SELECT` - `UPDATE` diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md new file mode 100644 index 0000000..1b96d5c --- /dev/null +++ b/RELEASE-NOTES.md @@ -0,0 +1,8 @@ +# godynamo release notes + +## 2023-05-27 - v0.1.0 + +- Driver for `database/sql`, supported statements: + - Table: `CREATE TABLE`, `LIST TABLES`, `DESCRIBE TABLE`, `ALTER TABLE`, `DROP TABLE`. + - Index: `DESCRIBE LSI`, `CREATE GSI`, `DESCRIBE GSI`, `ALTER GSI`, `DROP GSI`. + - Document: `INSERT`, `SELECT`, `UPDATE`, `DELETE`. diff --git a/SQL_DOCUMENT.md b/SQL_DOCUMENT.md index d217687..b013457 100644 --- a/SQL_DOCUMENT.md +++ b/SQL_DOCUMENT.md @@ -29,9 +29,9 @@ Syntax: [PartiQL select statements for DynamoDB](https://docs.aws.amazon.com/ama Example: ```go -result, err := db.Query(`SELECT * FROM "session" WHERE app='frontend'`) +dbrows, err := db.Query(`SELECT * FROM "session" WHERE app='frontend'`) if err == nil { - ... + fetchAndPrintAllRows(dbrows) } ``` @@ -44,22 +44,45 @@ Sample result: |------|---|----| |true|"frontend"|"user1"| -## DELETE +## UPDATE -Syntax: [PartiQL delete statements for DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.delete.html) +Syntax: [PartiQL update statements for DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.update.html) Example: ```go -result, err := db.Exec(`DELETE FROM "tbltest" WHERE "app"=? AND "user"=?`, "app0", "user1") +result, err := db.Exec(`UPDATE "tbltest" SET location=? SET os=? WHERE "app"=? AND "user"=?`, "VN", "Ubuntu", "app0", "user1") if err == nil { numAffectedRow, err := result.RowsAffected() ... } ``` -`Query` can also be used to have the content of the old item returned. +Description: use the `UPDATE` statement to modify the value of one or more attributes within an item in a table. + +- Note: the `UPDATE` must follow [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.update.html). + +`Query` can also be used to fetch returned values. ```go +dbrows, err := db.Query(`UPDATE "tbltest" SET location=? SET os=? WHERE "app"=? AND "user"=? RETURNING MODIFIED OLD *`, "VN", "Ubuntu", "app0", "user0") if err == nil { + fetchAndPrintAllRows(dbrows) +} +``` + +Sample result: +|location| +|--------| +|"AU" | + +## DELETE + +Syntax: [PartiQL delete statements for DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.delete.html) + +Example: +```go +result, err := db.Exec(`DELETE FROM "tbltest" WHERE "app"=? AND "user"=?`, "app0", "user1") +if err == nil { + numAffectedRow, err := result.RowsAffected() ... } ``` @@ -68,7 +91,15 @@ Description: use the `DELETE` statement to delete an existing item from a table. - Note: the `DELETE` must follow [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.delete.html). +`Query` can also be used to have the content of the old item returned. +```go +dbrows, err := db.Query(`DELETE FROM "tbltest" WHERE "app"=? AND "user"=?`, "app0", "user1") +if err == nil { + fetchAndPrintAllRows(dbrows) +} +``` + Sample result: |app|location|platform|user| |---|--------|--------|----| -|"app0"|"AU"|"Windows"|"user2"| +|"app0"|"AU"|"Windows"|"user1"| diff --git a/SQL_TABLE.md b/SQL_TABLE.md index bc713a6..75f19ed 100644 --- a/SQL_TABLE.md +++ b/SQL_TABLE.md @@ -2,9 +2,9 @@ - `CREATE TABLE` - `LIST TABLES` +- `DESCRIBE TABLE` - `ALTER TABLE` - `DROP TABLE` -- `DESCRIBE TABLE` ## CREATE TABLE @@ -74,6 +74,28 @@ Sample result: |tbltest2| |tbltest3| +## DESCRIBE TABLE + +Syntax: +```sql +DESCRIBE TABLE +``` + +Example: +```go +result, err := db.Query(`DESCRIBE TABLE demo`) +if err == nil { + ... +} +``` + +Description: return info of a DynamoDB table specified by `table-name`. + +Sample result: +|ArchivalSummary|AttributeDefinitions|BillingModeSummary|CreationDateTime|DeletionProtectionEnabled|GlobalSecondaryIndexes|GlobalTableVersion|ItemCount|KeySchema|LatestStreamArn|LatestStreamLabel|LocalSecondaryIndexes|ProvisionedThroughput|Replicas|RestoreSummary|SSEDescription|StreamSpecification|TableArn|TableClassSummary|TableId|TableName|TableSizeBytes|TableStatus| +|---------------|--------------------|------------------|----------------|-------------------------|----------------------|------------------|---------|---------|---------------|-----------------|---------------------|---------------------|--------|--------------|--------------|-------------------|--------|-----------------|-------|---------|--------------|-----------| +|null|[{"AttributeName":"app","AttributeType":"S"},{"AttributeName":"user","AttributeType":"S"},{"AttributeName":"timestamp","AttributeType":"N"},{"AttributeName":"browser","AttributeType":"S"},{"AttributeName":"os","AttributeType":"S"}]|{"BillingMode":"PAY_PER_REQUEST","LastUpdateToPayPerRequestDateTime":"2023-05-23T01:58:27.352Z"}|"2023-05-23T01:58:27.352Z"|null|null|null|0|[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"user","KeyType":"RANGE"}]|null|null|[{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxos","IndexName":"idxos","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"os","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":["os_name","os_version"],"ProjectionType":"INCLUDE"}},{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxbrowser","IndexName":"idxbrowser","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"browser","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":null,"ProjectionType":"ALL"}},{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxtime","IndexName":"idxtime","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"timestamp","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":null,"ProjectionType":"KEYS_ONLY"}}]|{"LastDecreaseDateTime":"1970-01-01T00:00:00Z","LastIncreaseDateTime":"1970-01-01T00:00:00Z","NumberOfDecreasesToday":0,"ReadCapacityUnits":0,"WriteCapacityUnits":0}|null|null|null|null|"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp"|null|null|"tbltemp"|0|"ACTIVE"| + ## ALTER TABLE Syntax: @@ -125,25 +147,3 @@ Description: delete an existing DynamoDB table specified by `table-name`. - If the specified table does not exist: - If `IF EXISTS` is supplied: `RowsAffected()` returns `0, nil` - If `IF EXISTS` is _not_ supplied: `RowsAffected()` returns `_, error` - -## DESCRIBE TABLE - -Syntax: -```sql -DESCRIBE TABLE -``` - -Example: -```go -result, err := db.Query(`DESCRIBE TABLE demo`) -if err == nil { - ... -} -``` - -Description: return info of a DynamoDB table specified by `table-name`. - -Sample result: -|ArchivalSummary|AttributeDefinitions|BillingModeSummary|CreationDateTime|DeletionProtectionEnabled|GlobalSecondaryIndexes|GlobalTableVersion|ItemCount|KeySchema|LatestStreamArn|LatestStreamLabel|LocalSecondaryIndexes|ProvisionedThroughput|Replicas|RestoreSummary|SSEDescription|StreamSpecification|TableArn|TableClassSummary|TableId|TableName|TableSizeBytes|TableStatus| -|---------------|--------------------|------------------|----------------|-------------------------|----------------------|------------------|---------|---------|---------------|-----------------|---------------------|---------------------|--------|--------------|--------------|-------------------|--------|-----------------|-------|---------|--------------|-----------| -|null|[{"AttributeName":"app","AttributeType":"S"},{"AttributeName":"user","AttributeType":"S"},{"AttributeName":"timestamp","AttributeType":"N"},{"AttributeName":"browser","AttributeType":"S"},{"AttributeName":"os","AttributeType":"S"}]|{"BillingMode":"PAY_PER_REQUEST","LastUpdateToPayPerRequestDateTime":"2023-05-23T01:58:27.352Z"}|"2023-05-23T01:58:27.352Z"|null|null|null|0|[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"user","KeyType":"RANGE"}]|null|null|[{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxos","IndexName":"idxos","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"os","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":["os_name","os_version"],"ProjectionType":"INCLUDE"}},{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxbrowser","IndexName":"idxbrowser","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"browser","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":null,"ProjectionType":"ALL"}},{"IndexArn":"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp/index/idxtime","IndexName":"idxtime","IndexSizeBytes":0,"ItemCount":0,"KeySchema":[{"AttributeName":"app","KeyType":"HASH"},{"AttributeName":"timestamp","KeyType":"RANGE"}],"Projection":{"NonKeyAttributes":null,"ProjectionType":"KEYS_ONLY"}}]|{"LastDecreaseDateTime":"1970-01-01T00:00:00Z","LastIncreaseDateTime":"1970-01-01T00:00:00Z","NumberOfDecreasesToday":0,"ReadCapacityUnits":0,"WriteCapacityUnits":0}|null|null|null|null|"arn:aws:dynamodb:ddblocal:000000000000:table/tbltemp"|null|null|"tbltemp"|0|"ACTIVE"| diff --git a/driver.go b/driver.go index 3017e11..b42cd3e 100644 --- a/driver.go +++ b/driver.go @@ -50,17 +50,20 @@ func (d *Driver) Open(connStr string) (driver.Conn, error) { if err != nil || timeoutMs < 0 { timeoutMs = 10000 } + region := params["REGION"] + if region == "" { + region = os.Getenv("AWS_REGION") + } akid := params["AKID"] if akid == "" { akid = os.Getenv("AWS_ACCESS_KEY_ID") } secretKey := params["SECRET_KEY"] if secretKey == "" { - secretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") - } - region := params["REGION"] - if region == "" { - region = os.Getenv("AWS_REGION") + secretKey = params["SECRETKEY"] + if secretKey == "" { + secretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + } } opts := dynamodb.Options{ Credentials: credentials.NewStaticCredentialsProvider(akid, secretKey, ""), diff --git a/godynamo.go b/godynamo.go index e0a1a1a..cb65423 100644 --- a/godynamo.go +++ b/godynamo.go @@ -36,7 +36,7 @@ var ( } ) -// IsAwsError returns true if err is an AWS-specific error and it matches awsErrCode. +// IsAwsError returns true if err is an AWS-specific error, and it matches awsErrCode. func IsAwsError(err error, awsErrCode string) bool { if aerr, ok := err.(*smithy.OperationError); ok { if herr, ok := aerr.Err.(*http.ResponseError); ok { diff --git a/release.sh b/release.sh new file mode 100644 index 0000000..02f1e4f --- /dev/null +++ b/release.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +## Utility script to release project with a tag +## Usage: +## ./release.sh + +if [ "$1" == "" ]; then + echo "Usage: $0 tag-name" + exit -1 +fi + +echo "$1" +git commit -m "$1" +git tag -f -a "$1" -m "$1" +git push origin "$1" -f +git push diff --git a/stmt_document.go b/stmt_document.go index 422c920..77765fd 100644 --- a/stmt_document.go +++ b/stmt_document.go @@ -15,7 +15,10 @@ import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" ) -var rePlaceholder = regexp.MustCompile(`(?m)\?\s*[\,\]\}\s]`) +var ( + rePlaceholder = regexp.MustCompile(`(?m)\?\s*[\,\]\}\s]`) + reReturning = regexp.MustCompile(`(?im)\s+RETURNING\s+((ALL\s+OLD)|(MODIFIED\s+OLD)|(ALL\s+NEW)|(MODIFIED\s+NEW))\s+\*\s*$`) +) /*----------------------------------------------------------------------*/ @@ -195,30 +198,40 @@ func (s *StmtSelect) Exec(_ []driver.Value) (driver.Result, error) { // StmtUpdate implements "UPDATE" statement. // // Syntax: follow "PartiQL update statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.update.html +// +// Note: StmtUpdate returns the updated item by appending "RETURNING ALL OLD *" to the statement. type StmtUpdate struct { *StmtExecutable } +func (s *StmtUpdate) parse() error { + if !reReturning.MatchString(s.query) { + s.query += " RETURNING ALL OLD *" + } + return s.StmtExecutable.parse() +} + // Query implements driver.Stmt.Query. func (s *StmtUpdate) Query(values []driver.Value) (driver.Rows, error) { output, err := s.Execute(values) result := &ResultResultSet{dbResult: output, columnTypes: make(map[string]reflect.Type)} - if err == nil { + if err == nil || IsAwsError(err, "ConditionalCheckFailedException") { result.init() + err = nil } return result, err } // Exec implements driver.Stmt.Exec. func (s *StmtUpdate) Exec(values []driver.Value) (driver.Result, error) { - _, err := s.Execute(values) - result := &ResultNoResultSet{Successful: err == nil} + output, err := s.Execute(values) + if IsAwsError(err, "ConditionalCheckFailedException") { + return &ResultNoResultSet{Successful: true, AffectedRows: 0}, nil + } if err != nil { - result.AffectedRows = 0 - } else { - result.AffectedRows = 1 + return &ResultNoResultSet{Successful: false, AffectedRows: 0}, err } - return result, err + return &ResultNoResultSet{Successful: true, AffectedRows: int64(len(output.Items))}, nil } /*----------------------------------------------------------------------*/ @@ -227,15 +240,11 @@ func (s *StmtUpdate) Exec(values []driver.Value) (driver.Result, error) { // // Syntax: follow "PartiQL delete statements for DynamoDB" https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.delete.html // -// Note: StmtDelete returns the deleted item by appending "RETURNING RETURNING ALL OLD *" to the statement. +// Note: StmtDelete returns the deleted item by appending "RETURNING ALL OLD *" to the statement. type StmtDelete struct { *StmtExecutable } -var ( - reReturning = regexp.MustCompile(`(?im)\s+RETURNING\s+((ALL\s+OLD)|(MODIFIED\s+OLD)|(ALL\s+NEW)|(MODIFIED\s+NEW))\s+\*\s*$`) -) - func (s *StmtDelete) parse() error { if !reReturning.MatchString(s.query) { s.query += " RETURNING ALL OLD *" @@ -247,8 +256,9 @@ func (s *StmtDelete) parse() error { func (s *StmtDelete) Query(values []driver.Value) (driver.Rows, error) { output, err := s.Execute(values) result := &ResultResultSet{dbResult: output, columnTypes: make(map[string]reflect.Type)} - if err == nil { + if err == nil || IsAwsError(err, "ConditionalCheckFailedException") { result.init() + err = nil } return result, err } diff --git a/stmt_document_test.go b/stmt_document_test.go index e214eea..c976b28 100644 --- a/stmt_document_test.go +++ b/stmt_document_test.go @@ -1,9 +1,13 @@ package godynamo import ( + "fmt" "reflect" "strings" "testing" + + "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/smithy-go" ) func Test_Query_Insert(t *testing.T) { @@ -108,6 +112,7 @@ func Test_Exec_Delete(t *testing.T) { defer db.Close() _initTest(db) + // setup table db.Exec(`DROP TABLE IF EXISTS tbltest`) db.Exec(`CREATE TABLE tbltest WITH PK=app:string WITH SK=user:string WITH rcu=100 WITH wcu=100`) _, err := db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'os': ?, 'active': ?, 'duration': ?}`, "app0", "user1", "Ubuntu", true, 12.34) @@ -119,12 +124,14 @@ func Test_Exec_Delete(t *testing.T) { t.Fatalf("%s failed: %s", testName+"/insert", err) } + // make sure table has 2 rows dbRows1, _ := db.Query(`SELECT * FROM "tbltest"`) rows1, _ := _fetchAllRows(dbRows1) if len(rows1) != 2 { t.Fatalf("%s failed: expected 2 rows in table, but there is %#v", testName, len(rows1)) } + // delete 1 row sql := `DELETE FROM "tbltest" WHERE "app"=? AND "user"=?` result1, err := db.Exec(sql, "app0", "user1") if err != nil { @@ -138,12 +145,14 @@ func Test_Exec_Delete(t *testing.T) { t.Fatalf("%s failed: expected 1 row affected but received %#v", testName+"/rows_affected", rowsAffected1) } + // make sure table has 1 row dbRows2, _ := db.Query(`SELECT * FROM "tbltest"`) rows2, _ := _fetchAllRows(dbRows2) if len(rows2) != 1 { t.Fatalf("%s failed: expected 1 rows in table, but there is %#v", testName, len(rows1)) } + // delete 0 row result0, err := db.Exec(sql, "app0", "user0") if err != nil { t.Fatalf("%s failed: %s", testName+"/delete", err) @@ -163,6 +172,7 @@ func Test_Query_Delete(t *testing.T) { defer db.Close() _initTest(db) + // setup table db.Exec(`DROP TABLE IF EXISTS tbltest`) db.Exec(`CREATE TABLE tbltest WITH PK=app:string WITH SK=user:string WITH rcu=100 WITH wcu=100`) _, err := db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'os': ?, 'active': ?, 'duration': ?}`, "app0", "user1", "Ubuntu", true, 12.34) @@ -174,17 +184,20 @@ func Test_Query_Delete(t *testing.T) { t.Fatalf("%s failed: %s", testName+"/insert", err) } + // make sure table has 2 rows dbRows1, _ := db.Query(`SELECT * FROM "tbltest"`) rows1, _ := _fetchAllRows(dbRows1) if len(rows1) != 2 { t.Fatalf("%s failed: expected 2 rows in table, but there is %#v", testName, len(rows1)) } + // delete 1 row sql := `DELETE FROM "tbltest" WHERE "app"=? AND "user"=? RETURNING ALL OLD *` result1, err := db.Query(sql, "app0", "user1") if err != nil { t.Fatalf("%s failed: %s", testName+"/delete", err) } + // the old row should be returned rows, err := _fetchAllRows(result1) if err != nil { t.Fatalf("%s failed: %s", testName+"/delete", err) @@ -197,12 +210,14 @@ func Test_Query_Delete(t *testing.T) { t.Fatalf("%s failed:\nexpected %#v\nbut received %#v", testName+"/delete", expected, rows[0]) } + // make sure table has 1 row dbRows2, _ := db.Query(`SELECT * FROM "tbltest"`) rows2, _ := _fetchAllRows(dbRows2) if len(rows2) != 1 { t.Fatalf("%s failed: expected 1 rows in table, but there is %#v", testName, len(rows1)) } + // delete 0 row result0, err := db.Query(sql, "app0", "user0") if err != nil { t.Fatalf("%s failed: %s", testName+"/delete", err) @@ -215,3 +230,92 @@ func Test_Query_Delete(t *testing.T) { t.Fatalf("%s failed: expected 0 row returned, but received %#v", testName+"/delete", len(rows)) } } + +func Test_Exec_Update(t *testing.T) { + testName := "Test_Exec_Update" + db := _openDb(t, testName) + defer db.Close() + _initTest(db) + + // setup table + db.Exec(`DROP TABLE IF EXISTS tbltest`) + db.Exec(`CREATE TABLE tbltest WITH PK=app:string WITH SK=user:string WITH rcu=100 WITH wcu=100`) + _, err := db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'platform': ?, 'location': ?}`, "app0", "user0", "Linux", "AU") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + + // update 1 row + sql := `UPDATE "tbltest" SET location=? SET os=? WHERE "app"=? AND "user"=?` + result1, err := db.Exec(sql, "VN", "Ubuntu", "app0", "user0") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/update", err) + } + rowsAffected1, err := result1.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/rows_affected", err) + } + if rowsAffected1 != 1 { + t.Fatalf("%s failed: expected 1 row affected but received %#v", testName+"/rows_affected", rowsAffected1) + } + + // update 0 row + result2, err := db.Exec(sql, "VN", "Ubuntu", "app0", "user2") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/update", err) + } + rowsAffected2, err := result2.RowsAffected() + if err != nil { + t.Fatalf("%s failed: %s", testName+"/rows_affected", err) + } + if rowsAffected2 != 0 { + t.Fatalf("%s failed: expected 0 row affected but received %#v", testName+"/rows_affected", rowsAffected2) + } +} + +func Test_Query_Update(t *testing.T) { + testName := "Test_Query_Update" + db := _openDb(t, testName) + defer db.Close() + _initTest(db) + + // setup table + db.Exec(`DROP TABLE IF EXISTS tbltest`) + db.Exec(`CREATE TABLE tbltest WITH PK=app:string WITH SK=user:string WITH rcu=100 WITH wcu=100`) + _, err := db.Exec(`INSERT INTO "tbltest" VALUE {'app': ?, 'user': ?, 'platform': ?, 'location': ?}`, "app0", "user0", "Linux", "AU") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + + // update 1 row + sql := `UPDATE "tbltest" SET location=? SET os=? WHERE "app"=? AND "user"=?` + dbrows1, err := db.Query(sql+" RETURNING MODIFIED OLD *", "VN", "Ubuntu", "app0", "user0") + if err != nil { + t.Fatalf("%s failed: %s", testName+"/update", err) + } + // values of modified attributes should be returned + rows1, _ := _fetchAllRows(dbrows1) + if len(rows1) != 1 { + t.Fatalf("%s failed: expected 1 affected row, but received %#v", testName, len(rows1)) + } + expected := map[string]interface{}{"location": "AU"} + if !reflect.DeepEqual(rows1[0], expected) { + t.Fatalf("%s failed:\nexpected %#v\nbut received %#v", testName+"/delete", expected, rows1[0]) + } + + dbrows2, err := db.Query(sql+" RETURNING ALL OLD *", "US", "Fedora", "app0", "user2") + if err != nil { + if aerr, ok := err.(*smithy.OperationError); ok { + if herr, ok := aerr.Err.(*http.ResponseError); ok { + fmt.Printf("DEBUG: %#v\n", herr.Err) + fmt.Printf("DEBUG: %#v\n", reflect.TypeOf(herr.Err).Name()) + fmt.Printf("DEBUG: %#v\n", reflect.TypeOf(herr.Err).Elem().Name()) + } + } + t.Fatalf("%s failed: %s", testName+"/update", err) + } + rows2, _ := _fetchAllRows(dbrows2) + if len(rows2) != 0 { + t.Fatalf("%s failed: expected 0 affected row, but received %#v", testName, len(rows2)) + } +}