From 631f6dd7d6936490470397a764732edf372366dd Mon Sep 17 00:00:00 2001 From: Joshua Humphries <2035234+jhump@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:02:03 -0400 Subject: [PATCH] feat: preserve unrecognized logical types and properties (#469) --- gen/testdata/golden.avsc | 3 - ocf/ocf_test.go | 1 + ocf/testdata/full-schema.json | 3 +- schema.go | 73 +++- schema_json_test.go | 4 + schema_parse.go | 97 +++-- schema_test.go | 687 ++++++++++++++++++++++++++++++++++ 7 files changed, 823 insertions(+), 45 deletions(-) diff --git a/gen/testdata/golden.avsc b/gen/testdata/golden.avsc index 438d842..592d91b 100644 --- a/gen/testdata/golden.avsc +++ b/gen/testdata/golden.avsc @@ -106,7 +106,6 @@ { "name": "mapOfStrings", "type": { - "name": "aMapOfStrings", "type": "map", "values": "string" } @@ -114,7 +113,6 @@ { "name": "mapOfRecords", "type": { - "name": "aMapOfRecords", "type": "map", "values": { "name": "RecordInMap", @@ -175,7 +173,6 @@ { "name": "aRecordArray", "type": { - "name": "someRecordArray", "type": "array", "items": { "name": "recordInArray", diff --git a/ocf/ocf_test.go b/ocf/ocf_test.go index 38a462b..200a779 100644 --- a/ocf/ocf_test.go +++ b/ocf/ocf_test.go @@ -1074,6 +1074,7 @@ func TestWithSchemaMarshaler(t *testing.T) { "name": "meta", "type": { "type": "array", + "logicalType": "map", "items": { "type": "record", "name": "FooMetadataEntry", diff --git a/ocf/testdata/full-schema.json b/ocf/testdata/full-schema.json index a1b5999..003d072 100644 --- a/ocf/testdata/full-schema.json +++ b/ocf/testdata/full-schema.json @@ -35,7 +35,8 @@ "field-id": 5 } ] - } + }, + "logicalType": "map" }, "field-id": 3 } diff --git a/schema.go b/schema.go index 0d82f7a..b3b97d6 100644 --- a/schema.go +++ b/schema.go @@ -25,11 +25,21 @@ func (nullDefaultType) MarshalJSON() ([]byte, error) { var nullDefault nullDefaultType = struct{}{} var ( - schemaReserved = []string{ - "doc", "fields", "items", "name", "namespace", "size", "symbols", - "values", "type", "aliases", "logicalType", "precision", "scale", - } - fieldReserved = []string{"default", "doc", "name", "order", "type", "aliases"} + // Note: order matches the order of properties as they are named in the spec. + // https://avro.apache.org/docs/1.12.0/specification + recordReserved = []string{"type", "name", "namespace", "doc", "aliases", "fields"} + fieldReserved = []string{"name", "doc", "type", "order", "aliases", "default"} + enumReserved = []string{"type", "name", "namespace", "aliases", "doc", "symbols", "default"} + arrayReserved = []string{"type", "items"} + mapReserved = []string{"type", "values"} + fixedReserved = []string{"type", "name", "namespace", "aliases", "size"} + fixedWithLogicalTypeReserved = []string{"type", "name", "namespace", "aliases", "size", "logicalType"} + fixedWithDecimalTypeReserved = []string{ + "type", "name", "namespace", "aliases", "size", "logicalType", "precision", "scale", + } + primitiveReserved = []string{"type"} + primitiveWithLogicalTypeReserved = []string{"type", "logicalType"} + primitiveWithDecimalTypeReserved = []string{"type", "logicalType", "precision", "scale"} ) // Type is a schema type. @@ -482,9 +492,16 @@ func NewPrimitiveSchema(t Type, l LogicalSchema, opts ...SchemaOption) *Primitiv for _, opt := range opts { opt(&cfg) } - + reservedProps := primitiveReserved + if l != nil { + if l.Type() == Decimal { + reservedProps = primitiveWithDecimalTypeReserved + } else { + reservedProps = primitiveWithLogicalTypeReserved + } + } return &PrimitiveSchema{ - properties: newProperties(cfg.props, schemaReserved), + properties: newProperties(cfg.props, reservedProps), cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp}, typ: t, logical: l, @@ -574,7 +591,7 @@ func NewRecordSchema(name, namespace string, fields []*Field, opts ...SchemaOpti return &RecordSchema{ name: n, - properties: newProperties(cfg.props, schemaReserved), + properties: newProperties(cfg.props, recordReserved), cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp}, fields: fields, doc: cfg.doc, @@ -919,7 +936,7 @@ func NewEnumSchema(name, namespace string, symbols []string, opts ...SchemaOptio return &EnumSchema{ name: n, - properties: newProperties(cfg.props, schemaReserved), + properties: newProperties(cfg.props, enumReserved), cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp}, symbols: symbols, def: def, @@ -1072,7 +1089,7 @@ func NewArraySchema(items Schema, opts ...SchemaOption) *ArraySchema { } return &ArraySchema{ - properties: newProperties(cfg.props, schemaReserved), + properties: newProperties(cfg.props, arrayReserved), cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp}, items: items, } @@ -1142,7 +1159,7 @@ func NewMapSchema(values Schema, opts ...SchemaOption) *MapSchema { } return &MapSchema{ - properties: newProperties(cfg.props, schemaReserved), + properties: newProperties(cfg.props, mapReserved), cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp}, values: values, } @@ -1323,9 +1340,17 @@ func NewFixedSchema( return nil, err } + reservedProps := fixedReserved + if logical != nil { + if logical.Type() == Decimal { + reservedProps = fixedWithDecimalTypeReserved + } else { + reservedProps = fixedWithLogicalTypeReserved + } + } return &FixedSchema{ name: n, - properties: newProperties(cfg.props, schemaReserved), + properties: newProperties(cfg.props, reservedProps), cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp}, size: size, logical: logical, @@ -1406,9 +1431,22 @@ func (s *FixedSchema) CacheFingerprint() [32]byte { // NullSchema is an Avro null type schema. type NullSchema struct { + properties fingerprinter } +// NewNullSchema creates a new NullSchema. +func NewNullSchema(opts ...SchemaOption) *NullSchema { + var cfg schemaConfig + for _, opt := range opts { + opt(&cfg) + } + + return &NullSchema{ + properties: newProperties(cfg.props, primitiveReserved), + } +} + // Type returns the type of the schema. func (s *NullSchema) Type() Type { return Null @@ -1421,7 +1459,16 @@ func (s *NullSchema) String() string { // MarshalJSON marshals the schema to json. func (s *NullSchema) MarshalJSON() ([]byte, error) { - return []byte(`"null"`), nil + if len(s.props) == 0 { + return []byte(`"null"`), nil + } + buf := new(bytes.Buffer) + buf.WriteString(`{"type":"null"`) + if err := s.marshalPropertiesToJSON(buf); err != nil { + return nil, err + } + buf.WriteString("}") + return buf.Bytes(), nil } // Fingerprint returns the SHA256 fingerprint of the schema. diff --git a/schema_json_test.go b/schema_json_test.go index 2db4ac0..40f04ad 100644 --- a/schema_json_test.go +++ b/schema_json_test.go @@ -23,6 +23,10 @@ func TestSchema_JSON(t *testing.T) { input: `{"type":"null"}`, json: `"null"`, }, + { + input: `{"type":"null","other":"foo"}`, + json: `{"type":"null","other":"foo"}`, + }, { input: `"boolean"`, json: `"boolean"`, diff --git a/schema_parse.go b/schema_parse.go index 889ada0..6903c02 100644 --- a/schema_parse.go +++ b/schema_parse.go @@ -121,6 +121,12 @@ func parsePrimitiveType(namespace, s string, cache *SchemaCache) (Schema, error) func parseComplexType(namespace string, m map[string]any, seen seenCache, cache *SchemaCache) (Schema, error) { if val, ok := m["type"].([]any); ok { + // Note: According to the spec, this is not allowed: + // https://avro.apache.org/docs/1.12.0/specification/#schema-declaration + // The "type" property in an object must be a string. A union type will be a slice, + // but NOT an object with a "type" property that is a slice. + // Might be advisable to remove this call (tradeoff between better conformance + // with the spec vs. possible backwards-compatibility issue). return parseUnion(namespace, val, seen, cache) } @@ -131,10 +137,7 @@ func parseComplexType(namespace string, m map[string]any, seen seenCache, cache typ := Type(str) switch typ { - case Null: - return &NullSchema{}, nil - - case String, Bytes, Int, Long, Float, Double, Boolean: + case String, Bytes, Int, Long, Float, Double, Boolean, Null: return parsePrimitive(typ, m) case Record, Error: @@ -158,14 +161,15 @@ func parseComplexType(namespace string, m map[string]any, seen seenCache, cache } type primitiveSchema struct { - LogicalType string `mapstructure:"logicalType"` - Precision int `mapstructure:"precision"` - Scale int `mapstructure:"scale"` - Props map[string]any `mapstructure:",remain"` + Type string `mapstructure:"type"` + Props map[string]any `mapstructure:",remain"` } func parsePrimitive(typ Type, m map[string]any) (Schema, error) { - if m == nil { + if len(m) == 0 { + if typ == Null { + return &NullSchema{}, nil + } return NewPrimitiveSchema(typ, nil), nil } @@ -178,14 +182,20 @@ func parsePrimitive(typ Type, m map[string]any) (Schema, error) { } var logical LogicalSchema - if p.LogicalType != "" { - logical = parsePrimitiveLogicalType(typ, p.LogicalType, p.Precision, p.Scale) + if logicalType := logicalTypeProperty(p.Props); logicalType != "" { + logical = parsePrimitiveLogicalType(typ, logicalType, p.Props) + if logical != nil { + delete(p.Props, "logicalType") + } } + if typ == Null { + return NewNullSchema(WithProps(p.Props)), nil + } return NewPrimitiveSchema(typ, logical, WithProps(p.Props)), nil } -func parsePrimitiveLogicalType(typ Type, lt string, prec, scale int) LogicalSchema { +func parsePrimitiveLogicalType(typ Type, lt string, props map[string]any) LogicalSchema { ltyp := LogicalType(lt) if (typ == String && ltyp == UUID) || (typ == Int && ltyp == Date) || @@ -199,10 +209,10 @@ func parsePrimitiveLogicalType(typ Type, lt string, prec, scale int) LogicalSche } if typ == Bytes && ltyp == Decimal { - return parseDecimalLogicalType(-1, prec, scale) + return parseDecimalLogicalType(-1, props) } - return nil + return nil // otherwise, not a recognized logical type } type recordSchema struct { @@ -368,6 +378,7 @@ func parseEnum(namespace string, m map[string]any, seen seenCache, cache *Schema } type arraySchema struct { + Type string `mapstructure:"type"` Items any `mapstructure:"items"` Props map[string]any `mapstructure:",remain"` } @@ -393,6 +404,7 @@ func parseArray(namespace string, m map[string]any, seen seenCache, cache *Schem } type mapSchema struct { + Type string `mapstructure:"type"` Values any `mapstructure:"values"` Props map[string]any `mapstructure:",remain"` } @@ -431,15 +443,12 @@ func parseUnion(namespace string, v []any, seen seenCache, cache *SchemaCache) ( } type fixedSchema struct { - Name string `mapstructure:"name"` - Namespace string `mapstructure:"namespace"` - Aliases []string `mapstructure:"aliases"` - Type string `mapstructure:"type"` - Size int `mapstructure:"size"` - LogicalType string `mapstructure:"logicalType"` - Precision int `mapstructure:"precision"` - Scale int `mapstructure:"scale"` - Props map[string]any `mapstructure:",remain"` + Name string `mapstructure:"name"` + Namespace string `mapstructure:"namespace"` + Aliases []string `mapstructure:"aliases"` + Type string `mapstructure:"type"` + Size int `mapstructure:"size"` + Props map[string]any `mapstructure:",remain"` } func parseFixed(namespace string, m map[string]any, seen seenCache, cache *SchemaCache) (Schema, error) { @@ -463,8 +472,11 @@ func parseFixed(namespace string, m map[string]any, seen seenCache, cache *Schem } var logical LogicalSchema - if f.LogicalType != "" { - logical = parseFixedLogicalType(f.Size, f.LogicalType, f.Precision, f.Scale) + if logicalType := logicalTypeProperty(f.Props); logicalType != "" { + logical = parseFixedLogicalType(f.Size, logicalType, f.Props) + if logical != nil { + delete(f.Props, "logicalType") + } } fixed, err := NewFixedSchema(f.Name, f.Namespace, f.Size, logical, WithAliases(f.Aliases), WithProps(f.Props)) @@ -485,19 +497,41 @@ func parseFixed(namespace string, m map[string]any, seen seenCache, cache *Schem return fixed, nil } -func parseFixedLogicalType(size int, lt string, prec, scale int) LogicalSchema { +func parseFixedLogicalType(size int, lt string, props map[string]any) LogicalSchema { ltyp := LogicalType(lt) switch { case ltyp == Duration && size == 12: return NewPrimitiveLogicalSchema(Duration) case ltyp == Decimal: - return parseDecimalLogicalType(size, prec, scale) + return parseDecimalLogicalType(size, props) } return nil } -func parseDecimalLogicalType(size, prec, scale int) LogicalSchema { +type decimalSchema struct { + Precision int `mapstructure:"precision"` + Scale int `mapstructure:"scale"` +} + +func parseDecimalLogicalType(size int, props map[string]any) LogicalSchema { + var ( + d decimalSchema + meta mapstructure.Metadata + ) + if err := decodeMap(props, &d, &meta); err != nil { + return nil + } + decType := newDecimalLogicalType(size, d.Precision, d.Scale) + if decType != nil { + // Remove the properties that we consumed + delete(props, "precision") + delete(props, "scale") + } + return decType +} + +func newDecimalLogicalType(size, prec, scale int) LogicalSchema { if prec <= 0 { return nil } @@ -594,3 +628,10 @@ func (c seenCache) Add(name string) error { c[name] = struct{}{} return nil } + +func logicalTypeProperty(props map[string]any) string { + if lt, ok := props["logicalType"].(string); ok { + return lt + } + return "" +} diff --git a/schema_test.go b/schema_test.go index 469a0e0..cc8ccea 100644 --- a/schema_test.go +++ b/schema_test.go @@ -58,6 +58,7 @@ func TestNullSchema(t *testing.T) { schemas := []string{ `null`, `{"type":"null"}`, + `{"type":"null", "other-property": 123, "another-property": ["a","b","c"]}`, } for _, schm := range schemas { @@ -1551,3 +1552,689 @@ func TestSchema_DereferencingRectifiesAlreadySeenSchema(t *testing.T) { n := strings.Count(strSchema, `"name":"org.hamba.avro.test1"`) assert.Equal(t, 1, n) } + +func TestParse_PreservesAllProperties(t *testing.T) { + testCases := []struct { + name string + schema string + check func(t *testing.T, schema avro.Schema) + }{ + { + name: "record", + schema: `{ + "type": "record", + "name": "SomeRecord", + "logicalType": "complex-number", + "precision": "abc", + "scale": "def", + "other": [1,2,3], + "fields": [ + { + "name": "r", + "type": "double" + }, + { + "name": "i", + "type": "double" + } + ] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.RecordSchema) + assert.Equal(t, map[string]any{ + "logicalType": "complex-number", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "array", + schema: `{ + "type": "array", + "name": "SomeArray", + "logicalType": "complex-number", + "precision": "abc", + "scale": "def", + "other": [1,2,3], + "items": "double" + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.ArraySchema) + assert.Equal(t, map[string]any{ + "name": "SomeArray", + "logicalType": "complex-number", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "map", + schema: `{ + "type": "map", + "name": "SomeMap", + "logicalType": "weights", + "precision": "abc", + "scale": "def", + "other": [1,2,3], + "values": "double" + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.MapSchema) + assert.Equal(t, map[string]any{ + "name": "SomeMap", + "logicalType": "weights", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "enum", + schema: `{ + "type": "enum", + "name": "SomeEnum", + "logicalType": "status", + "precision": "abc", + "scale": "def", + "other": [1,2,3], + "symbols": ["A", "B", "C"], + "default": "A" + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.EnumSchema) + assert.Equal(t, map[string]any{ + "logicalType": "status", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "fixed-no-logical-type", + schema: `{ + "type": "fixed", + "name": "SomeFixed", + "size": 16, + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.FixedSchema) + assert.Equal(t, map[string]any{ + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "fixed-unknown-logical-type", + schema: `{ + "type": "fixed", + "name": "SomeFixed", + "size": 16, + "logicalType": "uuid", + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.FixedSchema) + assert.Equal(t, map[string]any{ + "logicalType": "uuid", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "fixed-invalid-logical-type", + schema: `{ + "type": "fixed", + "name": "SomeFixed", + "size": 16, + "logicalType": {"foo": "bar", "baz": ["x","y","z"]}, + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.FixedSchema) + assert.Equal(t, map[string]any{ + "logicalType": map[string]any{ + "foo": "bar", + "baz": []any{"x", "y", "z"}, + }, + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "fixed-decimal-logical-type", + schema: `{ + "type": "fixed", + "name": "SomeFixed", + "size": 16, + "logicalType": "decimal", + "precision": 10, + "scale": 3, + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.FixedSchema) + assert.Equal(t, map[string]any{ + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + assert.Equal(t, avro.Decimal, rec.Logical().Type()) + }, + }, + { + name: "fixed-invalid-decimal-logical-type-bad-values", // scale is too high + schema: `{ + "type": "fixed", + "name": "SomeFixed", + "size": 16, + "logicalType": "decimal", + "precision": 10, + "scale": 20, + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.FixedSchema) + assert.Equal(t, map[string]any{ + "logicalType": "decimal", + "precision": 10.0, + "scale": 20.0, + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "fixed-invalid-decimal-logical-type-bad-type", // precision and scale aren't ints + schema: `{ + "type": "fixed", + "name": "SomeFixed", + "size": 16, + "logicalType": "decimal", + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.FixedSchema) + assert.Equal(t, map[string]any{ + "logicalType": "decimal", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "fixed-duration-logical", + schema: `{ + "type": "fixed", + "name": "SomeFixed", + "size": 12, + "logicalType": "duration", + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.FixedSchema) + assert.Equal(t, map[string]any{ + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + assert.Equal(t, avro.Duration, rec.Logical().Type()) + }, + }, + { + name: "primitive-no-logical-type", + schema: `{ + "type": "long", + "name": "SomeLong", + "size": 16, + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.PrimitiveSchema) + assert.Equal(t, map[string]any{ + "name": "SomeLong", + "size": 16.0, + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "primitive-unknown-logical-type", + schema: `{ + "type": "string", + "name": "SomeString", + "logicalType": "enum-name", + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.PrimitiveSchema) + assert.Equal(t, map[string]any{ + "name": "SomeString", + "logicalType": "enum-name", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "primitive-invalid-logical-type", + schema: `{ + "type": "string", + "name": "SomeString", + "logicalType": {"foo": "bar", "baz": ["x","y","z"]}, + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.PrimitiveSchema) + assert.Equal(t, map[string]any{ + "name": "SomeString", + "logicalType": map[string]any{ + "foo": "bar", + "baz": []any{"x", "y", "z"}, + }, + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "primitive-date-logical-type", + schema: `{ + "type": "int", + "logicalType": "date", + "precision": 10, + "scale": 3, + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.PrimitiveSchema) + assert.Equal(t, map[string]any{ + "precision": 10.0, + "scale": 3.0, + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + assert.Equal(t, avro.Date, rec.Logical().Type()) + }, + }, + { + name: "primitive-decimal-logical-type", + schema: `{ + "type": "bytes", + "logicalType": "decimal", + "precision": 10, + "scale": 3, + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.PrimitiveSchema) + assert.Equal(t, map[string]any{ + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + assert.Equal(t, avro.Decimal, rec.Logical().Type()) + }, + }, + { + name: "primitive-invalid-decimal-logical-type-bad-values", // scale is too high + schema: `{ + "type": "bytes", + "logicalType": "decimal", + "precision": 10, + "scale": 20, + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.PrimitiveSchema) + assert.Equal(t, map[string]any{ + "logicalType": "decimal", + "precision": 10.0, + "scale": 20.0, + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "primitive-invalid-decimal-logical-type-bad-type", // precision and scale aren't ints + schema: `{ + "type": "bytes", + "logicalType": "decimal", + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.PrimitiveSchema) + assert.Equal(t, map[string]any{ + "logicalType": "decimal", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + { + name: "null", + schema: `{ + "type": "null", + "name": "SomeMap", + "logicalType": "weights", + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.NullSchema) + assert.Equal(t, map[string]any{ + "name": "SomeMap", + "logicalType": "weights", + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + schema, err := avro.Parse(testCase.schema) + require.NoError(t, err) + testCase.check(t, schema) + }) + } +} + +func TestNewSchema_IgnoresInvalidProperties(t *testing.T) { + t.Run("record", func(t *testing.T) { + rec, err := avro.NewRecordSchema("abc.def.Xyz", "", nil, + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "name": 123, + "namespace": "abc", + "doc": "blah", + "fields": []any{1, 2, 3}, + "aliases": "foo", + "type": false, + // valid + "other": true, + "logicalType": "baz", + })) + require.NoError(t, err) + assert.Equal(t, map[string]any{ + "other": true, + "logicalType": "baz", + }, rec.Props()) + }) + t.Run("enum", func(t *testing.T) { + rec, err := avro.NewEnumSchema("abc.def.Xyz", "", []string{"ABC"}, + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "name": 123, + "namespace": "abc", + "doc": "blah", + "symbols": []any{1, 2, 3}, + "aliases": "foo", + "type": false, + "default": 123.456, + // valid + "other": true, + "logicalType": "baz", + })) + require.NoError(t, err) + assert.Equal(t, map[string]any{ + "other": true, + "logicalType": "baz", + }, rec.Props()) + }) + t.Run("fixed-no-logical-type", func(t *testing.T) { + rec, err := avro.NewFixedSchema("abc.def.Xyz", "", 10, nil, + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "type": false, + // valid + "doc": "blah", + "other": true, + "logicalType": "baz", + "precision": "abc", + "scale": "def", + })) + require.NoError(t, err) + assert.Equal(t, map[string]any{ + "doc": "blah", + "other": true, + "logicalType": "baz", + "precision": "abc", + "scale": "def", + }, rec.Props()) + }) + t.Run("fixed-logical-type", func(t *testing.T) { + rec, err := avro.NewFixedSchema("abc.def.Xyz", "", 10, avro.NewPrimitiveLogicalSchema(avro.Duration), + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "type": false, + "logicalType": "baz", + // valid + "doc": "blah", + "other": true, + "precision": "abc", + "scale": "def", + })) + require.NoError(t, err) + assert.Equal(t, map[string]any{ + "doc": "blah", + "other": true, + "precision": "abc", + "scale": "def", + }, rec.Props()) + }) + t.Run("fixed-decimal-logical-type", func(t *testing.T) { + rec, err := avro.NewFixedSchema("abc.def.Xyz", "", 10, avro.NewDecimalLogicalSchema(10, 0), + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "type": false, + "logicalType": "baz", + "precision": "abc", + "scale": "def", + // valid + "doc": "blah", + "other": true, + })) + require.NoError(t, err) + assert.Equal(t, map[string]any{ + "doc": "blah", + "other": true, + }, rec.Props()) + }) + t.Run("array", func(t *testing.T) { + rec := avro.NewArraySchema(avro.NewPrimitiveSchema(avro.String, nil), + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "items": []any{1, 2, 3}, + "type": false, + // valid + "name": 123, + "namespace": "abc", + "doc": "blah", + "aliases": "foo", + "other": true, + "logicalType": "baz", + })) + assert.Equal(t, map[string]any{ + "name": 123, + "namespace": "abc", + "doc": "blah", + "aliases": "foo", + "other": true, + "logicalType": "baz", + }, rec.Props()) + }) + t.Run("map", func(t *testing.T) { + rec := avro.NewMapSchema(avro.NewPrimitiveSchema(avro.String, nil), + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "values": []any{1, 2, 3}, + "type": false, + // valid + "name": 123, + "namespace": "abc", + "doc": "blah", + "aliases": "foo", + "other": true, + "logicalType": "baz", + })) + assert.Equal(t, map[string]any{ + "name": 123, + "namespace": "abc", + "doc": "blah", + "aliases": "foo", + "other": true, + "logicalType": "baz", + }, rec.Props()) + }) + t.Run("primitive-no-logical-type", func(t *testing.T) { + rec := avro.NewPrimitiveSchema(avro.Long, nil, + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "type": false, + // valid + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "doc": "blah", + "other": true, + "logicalType": "baz", + "precision": "abc", + "scale": "def", + })) + assert.Equal(t, map[string]any{ + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "doc": "blah", + "other": true, + "logicalType": "baz", + "precision": "abc", + "scale": "def", + }, rec.Props()) + }) + t.Run("primitive-logical-type", func(t *testing.T) { + rec := avro.NewPrimitiveSchema(avro.Long, avro.NewPrimitiveLogicalSchema(avro.TimestampMicros), + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "type": false, + "logicalType": "baz", + // valid + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "doc": "blah", + "other": true, + "precision": "abc", + "scale": "def", + })) + assert.Equal(t, map[string]any{ + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "doc": "blah", + "other": true, + "precision": "abc", + "scale": "def", + }, rec.Props()) + }) + t.Run("primitive-decimal-logical-type", func(t *testing.T) { + rec := avro.NewPrimitiveSchema(avro.Bytes, avro.NewDecimalLogicalSchema(10, 0), + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "type": false, + "logicalType": "baz", + "precision": "abc", + "scale": "def", + // valid + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "doc": "blah", + "other": true, + })) + assert.Equal(t, map[string]any{ + "name": 123, + "namespace": "abc", + "size": []any{1, 2, 3}, + "aliases": "foo", + "doc": "blah", + "other": true, + }, rec.Props()) + }) + t.Run("null", func(t *testing.T) { + rec := avro.NewNullSchema( + avro.WithProps(map[string]any{ + // invalid (conflict with other type properties) + "type": false, + // valid + "name": 123, + "namespace": "abc", + "doc": "blah", + "aliases": "foo", + "other": true, + "logicalType": "baz", + "precision": "abc", + "scale": "def", + })) + assert.Equal(t, map[string]any{ + "name": 123, + "namespace": "abc", + "doc": "blah", + "aliases": "foo", + "other": true, + "logicalType": "baz", + "precision": "abc", + "scale": "def", + }, rec.Props()) + }) +}