diff --git a/.dockerignore b/.dockerignore index 6244185b0..2e8e40bc6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ # Include source directories and files required for building. !karapace +!go !requirements/*.txt !setup.py !version.py diff --git a/.github/workflows/schema.yml b/.github/workflows/schema.yml index 1ee9a8049..6665ab235 100644 --- a/.github/workflows/schema.yml +++ b/.github/workflows/schema.yml @@ -21,7 +21,7 @@ jobs: requirements.txt - name: Install libsnappy-dev run: sudo apt install libsnappy-dev - - run: pip install -r requirements/requirements.txt + - run: make install # Compare with latest release when running on main. - run: make schema against=$(git describe --abbrev=0 --tags) if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 029a3a011..1df7f9ca8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,11 @@ jobs: cache: pip python-version: ${{ matrix.python-version }} + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.21.0' + - run: make install version - run: make unit-tests env: diff --git a/MANIFEST.in b/MANIFEST.in index fa15133f4..0b55e5636 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,5 +9,6 @@ include setup.py include setup.cfg include LICENSE include MANIFEST.in +include *.so recursive-exclude examples *~ *.pyc \.* diff --git a/README.rst b/README.rst index 15f9dd52f..650b17165 100644 --- a/README.rst +++ b/README.rst @@ -475,6 +475,9 @@ Keys to take special care are the ones needed to configure Kafka and advertised_ - ``true`` - If enabled, kafka errors which can be retried or custom errors specififed for the service will not be raised, instead, a warning log is emitted. This will denoise issue tracking systems, i.e. sentry + * - ``use_protobuf_formatter`` + - ``false`` + - If protobuf formatter should be used on protobuf schemas in order to normalize schemas. The formatter is used on top and independent of regular normalization and schemas will be persisted in a formatted state. Authentication and authorization of Karapace Schema Registry REST API diff --git a/container/Dockerfile b/container/Dockerfile index 61ad142b8..4fe2a46a5 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -8,6 +8,13 @@ RUN python3 -m venv /venv ENV PATH="/venv/bin:$PATH" ENV PIP_REQUIRE_VIRTUALENV=true +# Install golang needed by extensions +ENV GO_VERSION=1.21.0 +ENV PATH="/usr/local/go/bin:${PATH}" +RUN wget --progress=dot:giga "https://go.dev/dl/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \ + && tar -C /usr/local -xzf "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \ + && rm "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" + # Copy the requirements.txt and install dependencies in venv. Using a separate # command to use layer caching. # diff --git a/go/protopace/.gitignore b/go/protopace/.gitignore new file mode 100644 index 000000000..6f72f8926 --- /dev/null +++ b/go/protopace/.gitignore @@ -0,0 +1,25 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env diff --git a/go/protopace/Makefile b/go/protopace/Makefile new file mode 100644 index 000000000..f6acf75db --- /dev/null +++ b/go/protopace/Makefile @@ -0,0 +1,69 @@ +# Change these variables as necessary. +MAIN_PACKAGE_PATH := . +BINARY_NAME := protopace +BUILD_DIR := ../../karapace/protobuf/protopace/bin + +# ==================================================================================== # +# HELPERS +# ==================================================================================== # + +## help: print this help message +.PHONY: help +help: + @echo 'Usage:' + @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /' + +.PHONY: confirm +confirm: + @echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ] + +.PHONY: no-dirty +no-dirty: + git diff --exit-code + + +# ==================================================================================== # +# QUALITY CONTROL +# ==================================================================================== # + +## format: format code +.PHONY: format +format: + go fmt ./... + +## tidy: format code and tidy modfile +.PHONY: tidy +tidy: + go fmt ./... + go mod tidy -v + +## audit: run quality control checks +.PHONY: audit +audit: + go mod verify + go vet ./... + go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./... + go run golang.org/x/vuln/cmd/govulncheck@latest ./... + go test -race -buildvcs -vet=off ./... + + +# ==================================================================================== # +# DEVELOPMENT +# ==================================================================================== # + +## test: run all tests +.PHONY: test +test: + go test -v -race -buildvcs ./... + +## test/cover: run all tests and display coverage +.PHONY: test/cover +test/cover: + go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./... + go tool cover -html=/tmp/coverage.out + +## build: build the application +.PHONY: build +build: + # Include additional build steps, like TypeScript, SCSS or Tailwind compilation here... + go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH} diff --git a/go/protopace/compatibility.go b/go/protopace/compatibility.go new file mode 100644 index 000000000..ee90df999 --- /dev/null +++ b/go/protopace/compatibility.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + + "github.com/Aiven-Open/karapace/go/protopace/schema" + + "github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/pkg/tracing" + "go.uber.org/zap" +) + +func Check(schema schema.Schema, previousSchema schema.Schema) error { + handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer) + ctx := context.Background() + image, err := schema.CompileBufImage() + if err != nil { + return err + } + previousImage, err := previousSchema.CompileBufImage() + if err != nil { + return err + } + checkConfig, _ := bufconfig.NewEnabledCheckConfig( + bufconfig.FileVersionV2, + nil, + []string{ + "FIELD_NO_DELETE", + "FILE_SAME_PACKAGE", + "FIELD_SAME_NAME", + "FIELD_SAME_JSON_NAME", + "FILE_NO_DELETE", + "ENUM_NO_DELETE", + }, + nil, + nil, + ) + config := bufconfig.NewBreakingConfig(checkConfig, false) + return handler.Check(ctx, config, previousImage, image) +} diff --git a/go/protopace/compatibility_test.go b/go/protopace/compatibility_test.go new file mode 100644 index 000000000..a9a5dd16e --- /dev/null +++ b/go/protopace/compatibility_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "os" + "testing" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/stretchr/testify/assert" +) + +func TestCompatibility(t *testing.T) { + assert := assert.New(t) + + data, _ := os.ReadFile("./fixtures/dependency.proto") + dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil) + assert.NoError(err) + assert.NotNil(dependencySchema) + + data, _ = os.ReadFile("./fixtures/test.proto") + testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(testSchema) + + data, _ = os.ReadFile("./fixtures/test_previous.proto") + previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(previousSchema) + + err = Check(*testSchema, *testSchema) + assert.NoError(err) + + err = Check(*testSchema, *previousSchema) + assert.ErrorContains(err, "Field \"5\" with name \"foo\" on message \"EventValue\" changed type from \"string\" to \"int32\".") +} diff --git a/go/protopace/fixtures/dependency.proto b/go/protopace/fixtures/dependency.proto new file mode 100644 index 000000000..45b94a1de --- /dev/null +++ b/go/protopace/fixtures/dependency.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} diff --git a/go/protopace/fixtures/test.proto b/go/protopace/fixtures/test.proto new file mode 100644 index 000000000..e973b579d --- /dev/null +++ b/go/protopace/fixtures/test.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; +option php_namespace = "My\\Awesome\\Customer\\V1"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; + int32 foo = 5; +} diff --git a/go/protopace/fixtures/test_previous.proto b/go/protopace/fixtures/test_previous.proto new file mode 100644 index 000000000..6ca7b7c06 --- /dev/null +++ b/go/protopace/fixtures/test_previous.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; +option php_namespace = "My\\Awesome\\Customer\\V1"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; + string foo = 5; +} diff --git a/go/protopace/formatter.go b/go/protopace/formatter.go new file mode 100644 index 000000000..749fa16a6 --- /dev/null +++ b/go/protopace/formatter.go @@ -0,0 +1,2469 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Modifications made by Aiven Ltd, 2024: +// - Use fully qualified name for field types +// - Always sort options by name + +package main + +import ( + "errors" + "fmt" + "io" + "reflect" + "sort" + "strings" + "unicode" + "unicode/utf8" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/bufbuild/protocompile/ast" + "github.com/bufbuild/protocompile/walk" + "go.uber.org/multierr" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +func Format(schema s.Schema) (s.Schema, error) { + all, err := schema.Compile() + if err != nil { + return schema, err + } + res := all[0] + + astNodeMapping := map[ast.Node]protoreflect.FullName{} + + walk.DescriptorProtos(res.FileDescriptorProto(), func(fn protoreflect.FullName, m proto.Message) error { + astNode := res.Node(m) + astNodeMapping[astNode] = fn + return nil + }) + + fieldTypeMapping := map[protoreflect.FullName]string{} + + walk.Descriptors(res, func(d protoreflect.Descriptor) error { + fd, ok := d.(protoreflect.FieldDescriptor) + if ok { + message := fd.Message() + if message != nil { + fieldTypeMapping[fd.FullName()] = string(message.FullName()) + return nil + } + enum := fd.Enum() + if enum != nil { + fieldTypeMapping[fd.FullName()] = string(enum.FullName()) + return nil + } + } + return nil + }) + writer := strings.Builder{} + fileNode := res.FileNode().(*ast.FileNode) + f := newFormatter(&writer, fileNode, astNodeMapping, fieldTypeMapping) + f.Run() + newSchema := schema + newSchema.Schema = writer.String() + return newSchema, nil +} + +type formatter struct { + writer io.Writer + fileNode *ast.FileNode + astNodeMapping map[ast.Node]protoreflect.FullName + fieldTypeMapping map[protoreflect.FullName]string + + // Used to adjust comments when we remove superfluous + // separators tp canonicalize message literals + overrideTrailingComments map[ast.Node]ast.Comments + + // Current level of indentation. + indent int + // The last character written to writer. + lastWritten rune + + // The last node written. This must be updated from all functions + // that write comments with a node. This flag informs how the next + // node's leading comments and whitespace should be written. + previousNode ast.Node + + // If true, a space will be written to the output unless the next character + // written is a newline (don't wait errant trailing spaces). + pendingSpace bool + // If true, the formatter is in the middle of printing compact options. + inCompactOptions bool + + // Track runes that open blocks/scopes and are expected to increase indention + // level. For example, when runes "{" "[" "(" ")" are written, the pending + // value is 2 (increment three times for "{" "[" "("; decrement once for ")"). + // If it's greater than zero at the end of a line, we call In() so that + // subsequent lines are indented. If it's less than zero at the end of a line, + // we call Out(). This minimizes the amount of explicit indent/unindent code + // that is needed and makes it less error-prone. + pendingIndent int + // If true, an inline node/sequence is being written. We treat whitespace a + // little differently for when blocks are printed inline vs. across multiple + // lines. So this flag informs the logic that makes those whitespace decisions. + inline bool + + // Records all errors that occur during the formatting process. Nearly any + // non-nil error represents a bug in the implementation. + err error +} + +// newFormatter returns a new formatter for the given file. +func newFormatter( + writer io.Writer, + fileNode *ast.FileNode, + astNodeMapping map[ast.Node]protoreflect.FullName, + fieldTypeMapping map[protoreflect.FullName]string, +) *formatter { + return &formatter{ + writer: writer, + fileNode: fileNode, + astNodeMapping: astNodeMapping, + fieldTypeMapping: fieldTypeMapping, + overrideTrailingComments: map[ast.Node]ast.Comments{}, + } +} + +// Run runs the formatter and writes the file's content to the formatter's writer. +func (f *formatter) Run() error { + f.writeFile() + return f.err +} + +// P prints a line to the generated output. +// +// This will emit a newline and proper indentation. If you do not +// want to emit a newline and want to write a raw string, use +// WriteString (which P calls). +// +// If strings.TrimSpace(elem) is empty, no indentation is produced. +func (f *formatter) P(elem string) { + if len(strings.TrimSpace(elem)) > 0 { + // We only want to write an indent if we're + // writing a non-empty string (not just a newline). + f.Indent(nil) + f.WriteString(elem) + } + f.WriteString("\n") + + if f.pendingIndent > 0 { + f.In() + } else if f.pendingIndent < 0 { + f.Out() + } + f.pendingIndent = 0 +} + +// Space adds a space to the generated output. +func (f *formatter) Space() { + f.pendingSpace = true +} + +// In increases the current level of indentation. +func (f *formatter) In() { + f.indent++ +} + +// Out reduces the current level of indentation. +func (f *formatter) Out() { + if f.indent <= 0 { + // Unreachable. + f.err = multierr.Append( + f.err, + errors.New("internal error: attempted to decrement indentation at zero"), + ) + return + } + f.indent-- +} + +// Indent writes the number of spaces associated +// with the current level of indentation. +func (f *formatter) Indent(nextNode ast.Node) { + // only indent at beginning of line + if f.lastWritten != '\n' { + return + } + indent := f.indent + if rn, ok := nextNode.(*ast.RuneNode); ok && indent > 0 { + if strings.ContainsRune("}])>", rn.Rune) { + indent-- + } + } + f.WriteString(strings.Repeat(" ", indent)) +} + +// WriteString writes the given element to the generated output. +// +// This will not write indentation or newlines. Use P if you +// want to emit identation or newlines. +func (f *formatter) WriteString(elem string) { + if f.pendingSpace { + f.pendingSpace = false + first, _ := utf8.DecodeRuneInString(elem) + + // We don't want "dangling spaces" before certain characters: + // newlines, commas, and semicolons. Also, when writing + // elements inline, we don't want spaces before close parens + // and braces. Similarly, we don't want extra/doubled spaces + // or dangling spaces after certain characters when printing + // inline, like open parens/braces. So only print the space + // if the previous and next character don't match above + // conditions. + + prevBlockList := "\x00 \t\n" + nextBlockList := "\n;," + if f.inline { + prevBlockList = "\x00 \t\n<[{(" + nextBlockList = "\n;,)]}>" + } + + if !strings.ContainsRune(prevBlockList, f.lastWritten) && + !strings.ContainsRune(nextBlockList, first) { + if _, err := f.writer.Write([]byte{' '}); err != nil { + f.err = multierr.Append(f.err, err) + return + } + } + } + if len(elem) == 0 { + return + } + f.lastWritten, _ = utf8.DecodeLastRuneInString(elem) + if _, err := f.writer.Write([]byte(elem)); err != nil { + f.err = multierr.Append(f.err, err) + } +} + +// SetPreviousNode sets the previously written node. This should +// be called in all of the comment writing functions. +func (f *formatter) SetPreviousNode(node ast.Node) { + f.previousNode = node +} + +// writeFile writes the file node. +func (f *formatter) writeFile() { + f.writeFileHeader() + f.writeFileTypes() + if f.fileNode.EOF != nil { + info := f.nodeInfo(f.fileNode.EOF) + f.writeMultilineComments(info.LeadingComments()) + } + if f.lastWritten != 0 && f.lastWritten != '\n' { + // If anything was written, we always conclude with + // a newline. + f.P("") + } +} + +// writeFileHeader writes the header of a .proto file. This includes the syntax, +// package, imports, and options (in that order). The imports and options are +// sorted. All other file elements are handled by f.writeFileTypes. +// +// For example, +// +// syntax = "proto3"; +// +// package acme.v1.weather; +// +// import "acme/payment/v1/payment.proto"; +// import "google/type/datetime.proto"; +// +// option cc_enable_arenas = true; +// option optimize_for = SPEED; +func (f *formatter) writeFileHeader() { + var ( + packageNode *ast.PackageNode + importNodes []*ast.ImportNode + optionNodes []*ast.OptionNode + ) + for _, fileElement := range f.fileNode.Decls { + switch node := fileElement.(type) { + case *ast.PackageNode: + packageNode = node + case *ast.ImportNode: + importNodes = append(importNodes, node) + case *ast.OptionNode: + optionNodes = append(optionNodes, node) + default: + continue + } + } + if f.fileNode.Syntax == nil && f.fileNode.Edition == nil && + packageNode == nil && importNodes == nil && optionNodes == nil { + // There aren't any header values, so we can return early. + return + } + editionNode := f.fileNode.Edition + if editionNode != nil { + f.writeEdition(editionNode) + } + if syntaxNode := f.fileNode.Syntax; syntaxNode != nil && editionNode == nil { + f.writeSyntax(syntaxNode) + } + if packageNode != nil { + f.writePackage(packageNode) + } + sort.Slice(importNodes, func(i, j int) bool { + iName := importNodes[i].Name.AsString() + jName := importNodes[j].Name.AsString() + // sort by public > None > weak + iOrder := importSortOrder(importNodes[i]) + jOrder := importSortOrder(importNodes[j]) + + if iName < jName { + return true + } + if iName > jName { + return false + } + if iOrder > jOrder { + return true + } + if iOrder < jOrder { + return false + } + + // put commented import first + return !f.importHasComment(importNodes[j]) + }) + for i, importNode := range importNodes { + if i == 0 && f.previousNode != nil && !f.leadingCommentsContainBlankLine(importNode) { + f.P("") + } + + // since the imports are sorted, this will skip write imports + // if they have appear before and dont have comment + if i > 0 && importNode.Name.AsString() == importNodes[i-1].Name.AsString() && + !f.importHasComment(importNode) { + continue + } + + f.writeImport(importNode, i > 0) + } + sort.Sort(ByOptionName(optionNodes)) + for i, optionNode := range optionNodes { + if i == 0 && f.previousNode != nil && !f.leadingCommentsContainBlankLine(optionNode) { + f.P("") + } + f.writeFileOption(optionNode, i > 0) + } +} + +// writeFileTypes writes the types defined in a .proto file. This includes the messages, enums, +// services, etc. All other elements are ignored since they are handled by f.writeFileHeader. +func (f *formatter) writeFileTypes() { + for i, fileElement := range f.fileNode.Decls { + switch node := fileElement.(type) { + case *ast.PackageNode, *ast.OptionNode, *ast.ImportNode, *ast.EmptyDeclNode: + // These elements have already been written by f.writeFileHeader. + continue + default: + info := f.nodeInfo(node) + wantNewline := f.previousNode != nil && (i == 0 || info.LeadingComments().Len() > 0) + if wantNewline && !f.leadingCommentsContainBlankLine(node) { + f.P("") + } + f.writeNode(node) + } + } +} + +// writeSyntax writes the syntax. +// +// For example, +// +// syntax = "proto3"; +func (f *formatter) writeSyntax(syntaxNode *ast.SyntaxNode) { + f.writeStart(syntaxNode.Keyword) + f.Space() + f.writeInline(syntaxNode.Equals) + f.Space() + f.writeInline(syntaxNode.Syntax) + f.writeLineEnd(syntaxNode.Semicolon) +} + +// writeEdition writes the edition. +// +// For example, +// +// edition = "2023"; +func (f *formatter) writeEdition(editionNode *ast.EditionNode) { + f.writeStart(editionNode.Keyword) + f.Space() + f.writeInline(editionNode.Equals) + f.Space() + f.writeInline(editionNode.Edition) + f.writeLineEnd(editionNode.Semicolon) +} + +// writePackage writes the package. +// +// For example, +// +// package acme.weather.v1; +func (f *formatter) writePackage(packageNode *ast.PackageNode) { + f.writeStart(packageNode.Keyword) + f.Space() + f.writeInline(packageNode.Name) + f.writeLineEnd(packageNode.Semicolon) +} + +// writeImport writes an import statement. +// +// For example, +// +// import "google/protobuf/descriptor.proto"; +func (f *formatter) writeImport(importNode *ast.ImportNode, forceCompact bool) { + f.writeStartMaybeCompact(importNode.Keyword, forceCompact) + f.Space() + // We don't want to write the "public" and "weak" nodes + // if they aren't defined. One could be set, but never both. + switch { + case importNode.Public != nil: + f.writeInline(importNode.Public) + f.Space() + case importNode.Weak != nil: + f.writeInline(importNode.Weak) + f.Space() + } + f.writeInline(importNode.Name) + f.writeLineEnd(importNode.Semicolon) +} + +// writeFileOption writes a file option. This function is slightly +// different than f.writeOption because file options are sorted at +// the top of the file, and leading comments are adjusted accordingly. +func (f *formatter) writeFileOption(optionNode *ast.OptionNode, forceCompact bool) { + f.writeStartMaybeCompact(optionNode.Keyword, forceCompact) + f.Space() + f.writeNode(optionNode.Name) + f.Space() + f.writeInline(optionNode.Equals) + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // Compound string literals are written across multiple lines + // immediately after the '=', so we don't need a trailing + // space in the option prefix. + f.writeCompoundStringLiteralIndentEndInline(node) + f.writeLineEnd(optionNode.Semicolon) + return + } + f.Space() + f.writeInline(optionNode.Val) + f.writeLineEnd(optionNode.Semicolon) +} + +// writeOption writes an option. +// +// For example, +// +// option go_package = "github.com/foo/bar"; +func (f *formatter) writeOption(optionNode *ast.OptionNode) { + f.writeOptionPrefix(optionNode) + if optionNode.Semicolon != nil { + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // Compound string literals are written across multiple lines + // immediately after the '=', so we don't need a trailing + // space in the option prefix. + f.writeCompoundStringLiteralIndentEndInline(node) + f.writeLineEnd(optionNode.Semicolon) + return + } + f.writeInline(optionNode.Val) + f.writeLineEnd(optionNode.Semicolon) + return + } + + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + f.writeCompoundStringLiteralIndent(node) + return + } + f.writeInline(optionNode.Val) +} + +// writeLastCompactOption writes a compact option but preserves its the +// trailing end comments. This is only used for the last compact option +// since it's the only time a trailing ',' will be omitted. +// +// For example, +// +// [ +// deprecated = true, +// json_name = "something" // Trailing comment on the last element. +// ] +func (f *formatter) writeLastCompactOption(optionNode *ast.OptionNode) { + f.writeOptionPrefix(optionNode) + f.writeLineEnd(optionNode.Val) +} + +// writeOptionValue writes the option prefix, which makes up all of the +// option's definition, excluding the final token(s). +// +// For example, +// +// deprecated = +func (f *formatter) writeOptionPrefix(optionNode *ast.OptionNode) { + if optionNode.Keyword != nil { + // Compact options don't have the keyword. + f.writeStart(optionNode.Keyword) + f.Space() + f.writeNode(optionNode.Name) + } else { + f.writeStart(optionNode.Name) + } + f.Space() + f.writeInline(optionNode.Equals) + f.Space() +} + +// writeOptionName writes an option name. +// +// For example, +// +// go_package +// (custom.thing) +// (custom.thing).bridge.(another.thing) +func (f *formatter) writeOptionName(optionNameNode *ast.OptionNameNode) { + for i := 0; i < len(optionNameNode.Parts); i++ { + if f.inCompactOptions && i == 0 { + // The leading comments of the first token (either open rune or the + // name) will have already been written, so we need to handle this + // case specially. + fieldReferenceNode := optionNameNode.Parts[0] + if fieldReferenceNode.Open != nil { + f.writeNode(fieldReferenceNode.Open) + if info := f.nodeInfo(fieldReferenceNode.Open); info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } + f.writeInline(fieldReferenceNode.Name) + } else { + f.writeNode(fieldReferenceNode.Name) + if info := f.nodeInfo(fieldReferenceNode.Name); info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } + } + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } + continue + } + if i > 0 { + // The length of this slice must be exactly len(Parts)-1. + f.writeInline(optionNameNode.Dots[i-1]) + } + f.writeNode(optionNameNode.Parts[i]) + } +} + +// writeMessage writes the message node. +// +// For example, +// +// message Foo { +// option deprecated = true; +// reserved 50 to 100; +// extensions 150 to 200; +// +// message Bar { +// string name = 1; +// } +// enum Baz { +// BAZ_UNSPECIFIED = 0; +// } +// extend Bar { +// string value = 2; +// } +// +// Bar bar = 1; +// Baz baz = 2; +// } +func (f *formatter) writeMessage(messageNode *ast.MessageNode) { + var elementWriterFunc func() + if len(messageNode.Decls) != 0 { + elementWriterFunc = func() { + writeNodes(messageNode.Decls, f) + } + } + f.writeStart(messageNode.Keyword) + f.Space() + f.writeInline(messageNode.Name) + f.Space() + f.writeCompositeTypeBody( + messageNode.OpenBrace, + messageNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeMessageLiteral writes a message literal. +// +// For example, +// +// { +// foo: 1 +// foo: 2 +// foo: 3 +// bar: < +// name:"abc" +// id:123 +// > +// } +func (f *formatter) writeMessageLiteral(messageLiteralNode *ast.MessageLiteralNode) { + if f.maybeWriteCompactMessageLiteral(messageLiteralNode, false) { + return + } + var elementWriterFunc func() + if len(messageLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + f.writeMessageLiteralElements(messageLiteralNode) + } + } + f.writeCompositeValueBody( + messageLiteralOpen(messageLiteralNode), + messageLiteralClose(messageLiteralNode), + elementWriterFunc, + ) +} + +// writeMessageLiteral writes a message literal suitable for +// an element in an array literal. +func (f *formatter) writeMessageLiteralForArray( + messageLiteralNode *ast.MessageLiteralNode, + lastElement bool, +) { + if f.maybeWriteCompactMessageLiteral(messageLiteralNode, true) { + return + } + var elementWriterFunc func() + if len(messageLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + f.writeMessageLiteralElements(messageLiteralNode) + } + } + closeWriter := f.writeBodyEndInline + if lastElement { + closeWriter = f.writeBodyEnd + } + f.writeBody( + messageLiteralOpen(messageLiteralNode), + messageLiteralClose(messageLiteralNode), + elementWriterFunc, + f.writeOpenBracePrefixForArray, + closeWriter, + ) +} + +func (f *formatter) maybeWriteCompactMessageLiteral( + messageLiteralNode *ast.MessageLiteralNode, + inArrayLiteral bool, +) bool { + if len(messageLiteralNode.Elements) == 0 || len(messageLiteralNode.Elements) > 1 || + f.hasInteriorComments(messageLiteralNode.Children()...) || + messageLiteralHasNestedMessageOrArray(messageLiteralNode) { + return false + } + // messages with a single scalar field and no comments can be + // printed all on one line + openNode := messageLiteralOpen(messageLiteralNode) + closeNode := messageLiteralClose(messageLiteralNode) + if inArrayLiteral { + f.Indent(openNode) + } + f.writeInline(openNode) + fieldNode := messageLiteralNode.Elements[0] + f.writeInline(fieldNode.Name) + if fieldNode.Sep != nil { + f.writeInline(fieldNode.Sep) + } else { + f.WriteString(":") + } + f.Space() + if messageLiteralNode.Seps[0] != nil { + // We are dropping the optional trailing separator. If it had + // trailing comments and the value does not, move the separator's + // trailing comment to the value. + sepTrailingComments := f.nodeInfo(messageLiteralNode.Seps[0]).TrailingComments() + if sepTrailingComments.Len() > 0 && + f.nodeInfo(fieldNode.Val).TrailingComments().Len() == 0 { + f.setTrailingComments(fieldNode.Val, sepTrailingComments) + } + } + f.writeInline(fieldNode.Val) + f.writeInline(closeNode) + return true +} + +func messageLiteralHasNestedMessageOrArray(messageLiteralNode *ast.MessageLiteralNode) bool { + for _, elem := range messageLiteralNode.Elements { + switch elem.Val.(type) { + case *ast.ArrayLiteralNode, *ast.MessageLiteralNode: + return true + } + } + return false +} + +func arrayLiteralHasNestedMessageOrArray(arrayLiteralNode *ast.ArrayLiteralNode) bool { + for _, elem := range arrayLiteralNode.Elements { + switch elem.(type) { + case *ast.ArrayLiteralNode, *ast.MessageLiteralNode: + return true + } + } + return false +} + +// writeMessageLiteralElements writes the message literal's elements. +// +// For example, +// +// foo: 1 +// foo: 2 +func (f *formatter) writeMessageLiteralElements(messageLiteralNode *ast.MessageLiteralNode) { + for i := 0; i < len(messageLiteralNode.Elements); i++ { + // Separators ("," or ";") are optional. To avoid inconsistent formatted output, + // we suppress them, since they aren't needed. So we just write the element and + // ignore any optional separator in the AST. + if messageLiteralNode.Seps[i] != nil { + // Since we are dropping the optional trailing separator, we should + // possibly move its trailing comment to the element value so we don't + // lose it. Skip this step if the value already has a trailing comment. + sepTrailingComments := f.nodeInfo(messageLiteralNode.Seps[i]).TrailingComments() + if sepTrailingComments.Len() > 0 && + f.nodeInfo(messageLiteralNode.Elements[i].Val).TrailingComments().Len() == 0 { + f.setTrailingComments(messageLiteralNode.Elements[i].Val, sepTrailingComments) + } + } + f.writeNode(messageLiteralNode.Elements[i]) + } +} + +// writeMessageField writes the message field node, and concludes the +// line without leaving room for a trailing separator in the parent +// message literal. +func (f *formatter) writeMessageField(messageFieldNode *ast.MessageFieldNode) { + f.writeMessageFieldPrefix(messageFieldNode) + if compoundStringLiteral, ok := messageFieldNode.Val.(*ast.CompoundStringLiteralNode); ok { + f.writeCompoundStringLiteralIndent(compoundStringLiteral) + return + } + f.writeLineEnd(messageFieldNode.Val) +} + +// writeMessageFieldPrefix writes the message field node as a single line. +// +// For example, +// +// foo:"bar" +func (f *formatter) writeMessageFieldPrefix(messageFieldNode *ast.MessageFieldNode) { + // The comments need to be written as a multiline comment above + // the message field name. + // + // Note that this is different than how field reference nodes are + // normally formatted in-line (i.e. as option name components). + fieldReferenceNode := messageFieldNode.Name + if fieldReferenceNode.Open != nil { + f.writeStart(fieldReferenceNode.Open) + if fieldReferenceNode.URLPrefix != nil { + f.writeInline(fieldReferenceNode.URLPrefix) + f.writeInline(fieldReferenceNode.Slash) + } + f.writeInline(fieldReferenceNode.Name) + } else { + f.writeStart(fieldReferenceNode.Name) + } + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } + // The colon separator is optional sometimes, but we don't have enough + // information here to know whether it's necessary. For more consistent + // output, just always include it. + if messageFieldNode.Sep != nil { + f.writeInline(messageFieldNode.Sep) + } else { + f.WriteString(":") + } + f.Space() +} + +// writeEnum writes the enum node. +// +// For example, +// +// enum Foo { +// option deprecated = true; +// reserved 1 to 5; +// +// FOO_UNSPECIFIED = 0; +// } +func (f *formatter) writeEnum(enumNode *ast.EnumNode) { + var elementWriterFunc func() + if len(enumNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(enumNode.Decls, f) + } + } + f.writeStart(enumNode.Keyword) + f.Space() + f.writeInline(enumNode.Name) + f.Space() + f.writeCompositeTypeBody( + enumNode.OpenBrace, + enumNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeEnumValue writes the enum value as a single line. If the enum has +// compact options, it will be written across multiple lines. +// +// For example, +// +// FOO_UNSPECIFIED = 1 [ +// deprecated = true +// ]; +func (f *formatter) writeEnumValue(enumValueNode *ast.EnumValueNode) { + f.writeStart(enumValueNode.Name) + f.Space() + f.writeInline(enumValueNode.Equals) + f.Space() + f.writeInline(enumValueNode.Number) + if enumValueNode.Options != nil { + f.Space() + f.writeNode(enumValueNode.Options) + } + f.writeLineEnd(enumValueNode.Semicolon) +} + +// writeField writes the field node as a single line. If the field has +// compact options, it will be written across multiple lines. +// +// For example, +// +// repeated string name = 1 [ +// deprecated = true, +// json_name = "name" +// ]; +func (f *formatter) writeField(fieldNode *ast.FieldNode) { + // We need to handle the comments for the field label specially since + // a label might not be defined, but it has the leading comments attached + // to it. + n := f.astNodeMapping[fieldNode] + fullType := f.fieldTypeMapping[n] + var t ast.IdentValueNode + if fullType == "" { + t = fieldNode.FldType + } else { + t = ast.NewIdentNode(fullType, fieldNode.FldType.Start()) + } + + if fieldNode.Label.KeywordNode != nil { + f.writeStart(fieldNode.Label) + f.Space() + //f.writeInline(fieldNode.FldType) + f.writeInline(t) + } else { + // If a label was not written, the multiline comments will be + // attached to the type. + //if compoundIdentNode, ok := fieldNode.FldType.(*ast.CompoundIdentNode); ok { + // f.writeCompountIdentForFieldName(compoundIdentNode) + //} else { + //f.writeStart(fieldNode.FldType) + //} + f.writeStart(t) + } + f.Space() + f.writeInline(fieldNode.Name) + f.Space() + f.writeInline(fieldNode.Equals) + f.Space() + f.writeInline(fieldNode.Tag) + if fieldNode.Options != nil { + f.Space() + f.writeNode(fieldNode.Options) + } + f.writeLineEnd(fieldNode.Semicolon) +} + +// writeMapField writes a map field (e.g. 'map pairs = 1;'). +func (f *formatter) writeMapField(mapFieldNode *ast.MapFieldNode) { + f.writeNode(mapFieldNode.MapType) + f.Space() + f.writeInline(mapFieldNode.Name) + f.Space() + f.writeInline(mapFieldNode.Equals) + f.Space() + f.writeInline(mapFieldNode.Tag) + if mapFieldNode.Options != nil { + f.Space() + f.writeNode(mapFieldNode.Options) + } + f.writeLineEnd(mapFieldNode.Semicolon) +} + +// writeMapType writes a map type (e.g. 'map'). +func (f *formatter) writeMapType(mapTypeNode *ast.MapTypeNode) { + f.writeStart(mapTypeNode.Keyword) + f.writeInline(mapTypeNode.OpenAngle) + f.writeInline(mapTypeNode.KeyType) + f.writeInline(mapTypeNode.Comma) + f.Space() + f.writeInline(mapTypeNode.ValueType) + f.writeInline(mapTypeNode.CloseAngle) +} + +// writeFieldReference writes a field reference (e.g. '(foo.bar)'). +func (f *formatter) writeFieldReference(fieldReferenceNode *ast.FieldReferenceNode) { + if fieldReferenceNode.Open != nil { + f.writeInline(fieldReferenceNode.Open) + } + f.writeInline(fieldReferenceNode.Name) + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } +} + +// writeExtend writes the extend node. +// +// For example, +// +// extend google.protobuf.FieldOptions { +// bool redacted = 33333; +// } +func (f *formatter) writeExtend(extendNode *ast.ExtendNode) { + var elementWriterFunc func() + if len(extendNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(extendNode.Decls, f) + } + } + f.writeStart(extendNode.Keyword) + f.Space() + f.writeInline(extendNode.Extendee) + f.Space() + f.writeCompositeTypeBody( + extendNode.OpenBrace, + extendNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeService writes the service node. +// +// For example, +// +// service FooService { +// option deprecated = true; +// +// rpc Foo(FooRequest) returns (FooResponse) {}; +func (f *formatter) writeService(serviceNode *ast.ServiceNode) { + var elementWriterFunc func() + if len(serviceNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(serviceNode.Decls, f) + } + } + f.writeStart(serviceNode.Keyword) + f.Space() + f.writeInline(serviceNode.Name) + f.Space() + f.writeCompositeTypeBody( + serviceNode.OpenBrace, + serviceNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeRPC writes the RPC node. RPCs are formatted in +// the following order: +// +// For example, +// +// rpc Foo(FooRequest) returns (FooResponse) { +// option deprecated = true; +// }; +func (f *formatter) writeRPC(rpcNode *ast.RPCNode) { + var elementWriterFunc func() + if len(rpcNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(rpcNode.Decls, f) + } + } + f.writeStart(rpcNode.Keyword) + f.Space() + f.writeInline(rpcNode.Name) + f.writeInline(rpcNode.Input) + f.Space() + f.writeInline(rpcNode.Returns) + f.Space() + f.writeInline(rpcNode.Output) + if rpcNode.OpenBrace == nil { + // This RPC doesn't have any elements, so we prefer the + // ';' form. + // + // rpc Ping(PingRequest) returns (PingResponse); + // + f.writeLineEnd(rpcNode.Semicolon) + return + } + f.Space() + f.writeCompositeTypeBody( + rpcNode.OpenBrace, + rpcNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeRPCType writes the RPC type node (e.g. (stream foo.Bar)). +func (f *formatter) writeRPCType(rpcTypeNode *ast.RPCTypeNode) { + f.writeInline(rpcTypeNode.OpenParen) + if rpcTypeNode.Stream != nil { + f.writeInline(rpcTypeNode.Stream) + f.Space() + } + f.writeInline(rpcTypeNode.MessageType) + f.writeInline(rpcTypeNode.CloseParen) +} + +// writeOneOf writes the oneof node. +// +// For example, +// +// oneof foo { +// option deprecated = true; +// +// string name = 1; +// int number = 2; +// } +func (f *formatter) writeOneOf(oneOfNode *ast.OneofNode) { + var elementWriterFunc func() + if len(oneOfNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(oneOfNode.Decls, f) + } + } + f.writeStart(oneOfNode.Keyword) + f.Space() + f.writeInline(oneOfNode.Name) + f.Space() + f.writeCompositeTypeBody( + oneOfNode.OpenBrace, + oneOfNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeGroup writes the group node. +// +// For example, +// +// optional group Key = 4 [ +// deprecated = true, +// json_name = "key" +// ] { +// optional uint64 id = 1; +// optional string name = 2; +// } +func (f *formatter) writeGroup(groupNode *ast.GroupNode) { + var elementWriterFunc func() + if len(groupNode.Decls) > 0 { + elementWriterFunc = func() { + writeNodes(groupNode.Decls, f) + } + } + // We need to handle the comments for the group label specially since + // a label might not be defined, but it has the leading comments attached + // to it. + if groupNode.Label.KeywordNode != nil { + f.writeStart(groupNode.Label) + f.Space() + f.writeInline(groupNode.Keyword) + } else { + // If a label was not written, the multiline comments will be + // attached to the keyword. + f.writeStart(groupNode.Keyword) + } + f.Space() + f.writeInline(groupNode.Name) + f.Space() + f.writeInline(groupNode.Equals) + f.Space() + f.writeInline(groupNode.Tag) + if groupNode.Options != nil { + f.Space() + f.writeNode(groupNode.Options) + } + f.Space() + f.writeCompositeTypeBody( + groupNode.OpenBrace, + groupNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeExtensionRange writes the extension range node. +// +// For example, +// +// extensions 5-10, 100 to max [ +// deprecated = true +// ]; +func (f *formatter) writeExtensionRange(extensionRangeNode *ast.ExtensionRangeNode) { + f.writeStart(extensionRangeNode.Keyword) + f.Space() + for i := 0; i < len(extensionRangeNode.Ranges); i++ { + if i > 0 { + // The length of this slice must be exactly len(Ranges)-1. + f.writeInline(extensionRangeNode.Commas[i-1]) + f.Space() + } + f.writeNode(extensionRangeNode.Ranges[i]) + } + if extensionRangeNode.Options != nil { + f.Space() + f.writeNode(extensionRangeNode.Options) + } + f.writeLineEnd(extensionRangeNode.Semicolon) +} + +// writeReserved writes a reserved node. +// +// For example, +// +// reserved 5-10, 100 to max; +func (f *formatter) writeReserved(reservedNode *ast.ReservedNode) { + f.writeStart(reservedNode.Keyword) + // Either names or ranges will be set, but never both. + elements := make([]ast.Node, 0, len(reservedNode.Names)+len(reservedNode.Ranges)) + switch { + case reservedNode.Names != nil: + for _, nameNode := range reservedNode.Names { + elements = append(elements, nameNode) + } + case reservedNode.Identifiers != nil: + for _, identNode := range reservedNode.Identifiers { + elements = append(elements, identNode) + } + case reservedNode.Ranges != nil: + for _, rangeNode := range reservedNode.Ranges { + elements = append(elements, rangeNode) + } + } + f.Space() + for i := 0; i < len(elements); i++ { + if i > 0 { + // The length of this slice must be exactly len({Names,Ranges})-1. + f.writeInline(reservedNode.Commas[i-1]) + f.Space() + } + f.writeInline(elements[i]) + } + f.writeLineEnd(reservedNode.Semicolon) +} + +// writeRange writes the given range node (e.g. '1 to max'). +func (f *formatter) writeRange(rangeNode *ast.RangeNode) { + f.writeInline(rangeNode.StartVal) + if rangeNode.To != nil { + f.Space() + f.writeInline(rangeNode.To) + } + // Either EndVal or Max will be set, but never both. + switch { + case rangeNode.EndVal != nil: + f.Space() + f.writeInline(rangeNode.EndVal) + case rangeNode.Max != nil: + f.Space() + f.writeInline(rangeNode.Max) + } +} + +// writeCompactOptions writes a compact options node. +// +// For example, +// +// [ +// deprecated = true, +// json_name = "something" +// ] +func (f *formatter) writeCompactOptions(compactOptionsNode *ast.CompactOptionsNode) { + f.inCompactOptions = true + defer func() { + f.inCompactOptions = false + }() + if len(compactOptionsNode.Options) == 1 && + !f.hasInteriorComments(compactOptionsNode.OpenBracket, compactOptionsNode.Options[0].Name) { + // If there's only a single compact scalar option without comments, we can write it + // in-line. For example: + // + // [deprecated = true] + // + // However, this does not include the case when the '[' has trailing comments, + // or the option name has leading comments. In those cases, we write the option + // across multiple lines. For example: + // + // [ + // // This type is deprecated. + // deprecated = true + // ] + // + optionNode := compactOptionsNode.Options[0] + f.writeInline(compactOptionsNode.OpenBracket) + f.writeInline(optionNode.Name) + f.Space() + f.writeInline(optionNode.Equals) + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // If there's only a single compact option, the value needs to + // write its comments (if any) in a way that preserves the closing ']'. + f.writeCompoundStringLiteralNoIndentEndInline(node) + f.writeInline(compactOptionsNode.CloseBracket) + return + } + f.Space() + f.writeInline(optionNode.Val) + f.writeInline(compactOptionsNode.CloseBracket) + return + } + var elementWriterFunc func() + if len(compactOptionsNode.Options) > 0 { + elementWriterFunc = func() { + sort.Sort(ByOptionName(compactOptionsNode.Options)) + for i, opt := range compactOptionsNode.Options { + if i == len(compactOptionsNode.Options)-1 { + // The last element won't have a trailing comma. + f.writeLastCompactOption(opt) + return + } + f.writeNode(opt) + f.writeLineEnd(compactOptionsNode.Commas[i]) + } + } + } + f.writeCompositeValueBody( + compactOptionsNode.OpenBracket, + compactOptionsNode.CloseBracket, + elementWriterFunc, + ) +} + +func (f *formatter) hasInteriorComments(nodes ...ast.Node) bool { + for i, n := range nodes { + // interior comments mean we ignore leading comments on first + // token and trailing comments on the last one + info := f.nodeInfo(n) + if i > 0 && info.LeadingComments().Len() > 0 { + return true + } + if i < len(nodes)-1 && info.TrailingComments().Len() > 0 { + return true + } + } + return false +} + +// writeArrayLiteral writes an array literal across multiple lines. +// +// For example, +// +// [ +// "foo", +// "bar" +// ] +func (f *formatter) writeArrayLiteral(arrayLiteralNode *ast.ArrayLiteralNode) { + if len(arrayLiteralNode.Elements) == 1 && + !f.hasInteriorComments(arrayLiteralNode.Children()...) && + !arrayLiteralHasNestedMessageOrArray(arrayLiteralNode) { + // arrays with a single scalar value and no comments can be + // printed all on one line + valueNode := arrayLiteralNode.Elements[0] + f.writeInline(arrayLiteralNode.OpenBracket) + f.writeInline(valueNode) + f.writeInline(arrayLiteralNode.CloseBracket) + return + } + + var elementWriterFunc func() + if len(arrayLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + for i := 0; i < len(arrayLiteralNode.Elements); i++ { + lastElement := i == len(arrayLiteralNode.Elements)-1 + if compositeNode, ok := arrayLiteralNode.Elements[i].(ast.CompositeNode); ok { + f.writeCompositeValueForArrayLiteral(compositeNode, lastElement) + if !lastElement { + f.writeLineEnd(arrayLiteralNode.Commas[i]) + } + continue + } + if lastElement { + // The last element won't have a trailing comma. + f.writeLineElement(arrayLiteralNode.Elements[i]) + return + } + f.writeStart(arrayLiteralNode.Elements[i]) + f.writeLineEnd(arrayLiteralNode.Commas[i]) + } + } + } + f.writeCompositeValueBody( + arrayLiteralNode.OpenBracket, + arrayLiteralNode.CloseBracket, + elementWriterFunc, + ) +} + +// writeCompositeForArrayLiteral writes the composite node in a way that's suitable +// for array literals. In general, signed integers and compound strings should have their +// comments written in-line because they are one of many components in a single line. +// +// However, each of these composite types occupy a single line in an array literal, +// so they need their comments to be formatted like a standalone node. +// +// For example, +// +// option (value) = /* In-line comment for '-42' */ -42; +// +// option (thing) = { +// values: [ +// // Leading comment on -42. +// -42, // Trailing comment on -42. +// ] +// } +// +// The lastElement boolean is used to signal whether or not the composite value +// should be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeCompositeValueForArrayLiteral( + compositeNode ast.CompositeNode, + lastElement bool, +) { + switch node := compositeNode.(type) { + case *ast.CompoundStringLiteralNode: + f.writeCompoundStringLiteralForArray(node, lastElement) + case *ast.NegativeIntLiteralNode: + f.writeNegativeIntLiteralForArray(node, lastElement) + case *ast.SignedFloatLiteralNode: + f.writeSignedFloatLiteralForArray(node, lastElement) + case *ast.MessageLiteralNode: + f.writeMessageLiteralForArray(node, lastElement) + default: + f.err = multierr.Append(f.err, fmt.Errorf("unexpected array value node %T", node)) + } +} + +// writeCompositeTypeBody writes the body of a composite type, e.g. message, enum, extend, oneof, etc. +func (f *formatter) writeCompositeTypeBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), +) { + f.writeBody( + openBrace, + closeBrace, + elementWriterFunc, + f.writeOpenBracePrefix, + f.writeBodyEnd, + ) +} + +// writeCompositeValueBody writes the body of a composite value, e.g. compact options, +// array literal, etc. We need to handle the ']' different than composite types because +// there could be more tokens following the final ']'. +func (f *formatter) writeCompositeValueBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), +) { + f.writeBody( + openBrace, + closeBrace, + elementWriterFunc, + f.writeOpenBracePrefix, + f.writeBodyEndInline, + ) +} + +// writeBody writes the body of a type or value, e.g. message, enum, compact options, etc. +// The elementWriterFunc is used to write the declarations within the composite type (e.g. +// fields in a message). The openBraceWriterFunc and closeBraceWriterFunc functions are used +// to customize how the '{' and '} nodes are written, respectively. +func (f *formatter) writeBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), + openBraceWriterFunc func(ast.Node), + closeBraceWriterFunc func(ast.Node, bool), +) { + if elementWriterFunc == nil && !f.hasInteriorComments(openBrace, closeBrace) { + // completely empty body + f.writeInline(openBrace) + closeBraceWriterFunc(closeBrace, true) + return + } + + openBraceWriterFunc(openBrace) + if elementWriterFunc != nil { + elementWriterFunc() + } + closeBraceWriterFunc(closeBrace, false) +} + +// writeOpenBracePrefix writes the open brace with its leading comments in-line. +// This is used for nearly every use case of f.writeBody, excluding the instances +// in array literals. +func (f *formatter) writeOpenBracePrefix(openBrace ast.Node) { + defer f.SetPreviousNode(openBrace) + info := f.nodeInfo(openBrace) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(openBrace) + if info.TrailingComments().Len() > 0 { + f.writeTrailingEndComments(info.TrailingComments()) + } else { + f.P("") + } +} + +// writeOpenBracePrefixForArray writes the open brace with its leading comments +// on multiple lines. This is only used for message literals in arrays. +func (f *formatter) writeOpenBracePrefixForArray(openBrace ast.Node) { + defer f.SetPreviousNode(openBrace) + info := f.nodeInfo(openBrace) + if info.LeadingComments().Len() > 0 { + f.writeMultilineComments(info.LeadingComments()) + } + f.Indent(openBrace) + f.writeNode(openBrace) + if info.TrailingComments().Len() > 0 { + f.writeTrailingEndComments(info.TrailingComments()) + } else { + f.P("") + } +} + +// writeCompoundIdent writes a compound identifier (e.g. '.com.foo.Bar'). +func (f *formatter) writeCompoundIdent(compoundIdentNode *ast.CompoundIdentNode) { + if compoundIdentNode.LeadingDot != nil { + f.writeInline(compoundIdentNode.LeadingDot) + } + for i := 0; i < len(compoundIdentNode.Components); i++ { + if i > 0 { + // The length of this slice must be exactly len(Components)-1. + f.writeInline(compoundIdentNode.Dots[i-1]) + } + f.writeInline(compoundIdentNode.Components[i]) + } +} + +// writeCompountIdentForFieldName writes a compound identifier, but handles comments +// specially for field names. +// +// For example, +// +// message Foo { +// // These are comments attached to bar. +// bar.v1.Bar bar = 1; +// } +func (f *formatter) writeCompountIdentForFieldName(compoundIdentNode *ast.CompoundIdentNode) { + if compoundIdentNode.LeadingDot != nil { + f.writeStart(compoundIdentNode.LeadingDot) + } + for i := 0; i < len(compoundIdentNode.Components); i++ { + if i == 0 && compoundIdentNode.LeadingDot == nil { + f.writeStart(compoundIdentNode.Components[i]) + continue + } + if i > 0 { + // The length of this slice must be exactly len(Components)-1. + f.writeInline(compoundIdentNode.Dots[i-1]) + } + f.writeInline(compoundIdentNode.Components[i]) + } +} + +// writeFieldLabel writes the field label node. +// +// For example, +// +// optional +// repeated +// required +func (f *formatter) writeFieldLabel(fieldLabel ast.FieldLabel) { + f.WriteString(fieldLabel.Val) +} + +// writeCompoundStringLiteral writes a compound string literal value. +// +// For example, +// +// "one," +// "two," +// "three" +func (f *formatter) writeCompoundStringLiteral( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, + needsIndent bool, + hasTrailingPunctuation bool, +) { + f.P("") + if needsIndent { + f.In() + } + for i, child := range compoundStringLiteralNode.Children() { + if hasTrailingPunctuation && i == len(compoundStringLiteralNode.Children())-1 { + // inline because there may be a subsequent comma or punctuation from enclosing element + f.writeStart(child) + break + } + f.writeLineElement(child) + } + if needsIndent { + f.Out() + } +} + +func (f *formatter) writeCompoundStringLiteralIndent( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, true, false) +} + +func (f *formatter) writeCompoundStringLiteralIndentEndInline( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, true, true) +} + +func (f *formatter) writeCompoundStringLiteralNoIndentEndInline( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, false, true) +} + +// writeCompoundStringLiteralForArray writes a compound string literal value, +// but writes its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeCompoundStringLiteralForArray( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, + lastElement bool, +) { + for i, child := range compoundStringLiteralNode.Children() { + if !lastElement && i == len(compoundStringLiteralNode.Children())-1 { + f.writeStart(child) + return + } + f.writeLineElement(child) + } +} + +// writeFloatLiteral writes a float literal value (e.g. '42.2'). +func (f *formatter) writeFloatLiteral(floatLiteralNode *ast.FloatLiteralNode) { + f.writeRaw(floatLiteralNode) +} + +// writeSignedFloatLiteral writes a signed float literal value (e.g. '-42.2'). +func (f *formatter) writeSignedFloatLiteral(signedFloatLiteralNode *ast.SignedFloatLiteralNode) { + f.writeInline(signedFloatLiteralNode.Sign) + f.writeInline(signedFloatLiteralNode.Float) +} + +// writeSignedFloatLiteralForArray writes a signed float literal value, but writes +// its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeSignedFloatLiteralForArray( + signedFloatLiteralNode *ast.SignedFloatLiteralNode, + lastElement bool, +) { + f.writeStart(signedFloatLiteralNode.Sign) + if lastElement { + f.writeLineEnd(signedFloatLiteralNode.Float) + return + } + f.writeInline(signedFloatLiteralNode.Float) +} + +// writeSpecialFloatLiteral writes a special float literal value (e.g. "nan" or "inf"). +func (f *formatter) writeSpecialFloatLiteral(specialFloatLiteralNode *ast.SpecialFloatLiteralNode) { + f.WriteString(specialFloatLiteralNode.KeywordNode.Val) +} + +// writeStringLiteral writes a string literal value (e.g. "foo"). +// Note that the raw string is written as-is so that it preserves +// the quote style used in the original source. +func (f *formatter) writeStringLiteral(stringLiteralNode *ast.StringLiteralNode) { + f.writeRaw(stringLiteralNode) +} + +// writeUintLiteral writes a uint literal (e.g. '42'). +func (f *formatter) writeUintLiteral(uintLiteralNode *ast.UintLiteralNode) { + f.writeRaw(uintLiteralNode) +} + +// writeNegativeIntLiteral writes a negative int literal (e.g. '-42'). +func (f *formatter) writeNegativeIntLiteral(negativeIntLiteralNode *ast.NegativeIntLiteralNode) { + f.writeInline(negativeIntLiteralNode.Minus) + f.writeInline(negativeIntLiteralNode.Uint) +} + +func (f *formatter) writeRaw(n ast.Node) { + info := f.nodeInfo(n) + f.WriteString(info.RawText()) +} + +// writeNegativeIntLiteralForArray writes a negative int literal value, but writes +// its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeNegativeIntLiteralForArray( + negativeIntLiteralNode *ast.NegativeIntLiteralNode, + lastElement bool, +) { + f.writeStart(negativeIntLiteralNode.Minus) + if lastElement { + f.writeLineEnd(negativeIntLiteralNode.Uint) + return + } + f.writeInline(negativeIntLiteralNode.Uint) +} + +// writeIdent writes an identifier (e.g. 'foo'). +func (f *formatter) writeIdent(identNode *ast.IdentNode) { + f.WriteString(identNode.Val) +} + +// writeKeyword writes a keyword (e.g. 'syntax'). +func (f *formatter) writeKeyword(keywordNode *ast.KeywordNode) { + f.WriteString(keywordNode.Val) +} + +// writeRune writes a rune (e.g. '='). +func (f *formatter) writeRune(runeNode *ast.RuneNode) { + if strings.ContainsRune("{[(<", runeNode.Rune) { + f.pendingIndent++ + } else if strings.ContainsRune("}])>", runeNode.Rune) { + f.pendingIndent-- + } + f.WriteString(string(runeNode.Rune)) +} + +// writeNode writes the node by dispatching to a function tailored to its concrete type. +// +// Comments are handled in each respective write function so that it can determine whether +// to write the comments in-line or not. +func (f *formatter) writeNode(node ast.Node) { + switch element := node.(type) { + case *ast.ArrayLiteralNode: + f.writeArrayLiteral(element) + case *ast.CompactOptionsNode: + f.writeCompactOptions(element) + case *ast.CompoundIdentNode: + f.writeCompoundIdent(element) + case *ast.CompoundStringLiteralNode: + f.writeCompoundStringLiteralIndent(element) + case *ast.EnumNode: + f.writeEnum(element) + case *ast.EnumValueNode: + f.writeEnumValue(element) + case *ast.ExtendNode: + f.writeExtend(element) + case *ast.ExtensionRangeNode: + f.writeExtensionRange(element) + case ast.FieldLabel: + f.writeFieldLabel(element) + case *ast.FieldNode: + f.writeField(element) + case *ast.FieldReferenceNode: + f.writeFieldReference(element) + case *ast.FloatLiteralNode: + f.writeFloatLiteral(element) + case *ast.GroupNode: + f.writeGroup(element) + case *ast.IdentNode: + f.writeIdent(element) + case *ast.ImportNode: + f.writeImport(element, false) + case *ast.KeywordNode: + f.writeKeyword(element) + case *ast.MapFieldNode: + f.writeMapField(element) + case *ast.MapTypeNode: + f.writeMapType(element) + case *ast.MessageNode: + f.writeMessage(element) + case *ast.MessageFieldNode: + f.writeMessageField(element) + case *ast.MessageLiteralNode: + f.writeMessageLiteral(element) + case *ast.NegativeIntLiteralNode: + f.writeNegativeIntLiteral(element) + case *ast.OneofNode: + f.writeOneOf(element) + case *ast.OptionNode: + f.writeOption(element) + case *ast.OptionNameNode: + f.writeOptionName(element) + case *ast.PackageNode: + f.writePackage(element) + case *ast.RangeNode: + f.writeRange(element) + case *ast.ReservedNode: + f.writeReserved(element) + case *ast.RPCNode: + f.writeRPC(element) + case *ast.RPCTypeNode: + f.writeRPCType(element) + case *ast.RuneNode: + f.writeRune(element) + case *ast.ServiceNode: + f.writeService(element) + case *ast.SignedFloatLiteralNode: + f.writeSignedFloatLiteral(element) + case *ast.SpecialFloatLiteralNode: + f.writeSpecialFloatLiteral(element) + case *ast.StringLiteralNode: + f.writeStringLiteral(element) + case *ast.SyntaxNode: + f.writeSyntax(element) + case *ast.UintLiteralNode: + f.writeUintLiteral(element) + case *ast.EmptyDeclNode: + // Nothing to do here. + default: + f.err = multierr.Append(f.err, fmt.Errorf("unexpected node: %T", node)) + } +} + +// writeStart writes the node across as the start of a line. +// Start nodes have their leading comments written across +// multiple lines, but their trailing comments must be written +// in-line to preserve the line structure. +// +// For example, +// +// // Leading comment on 'message'. +// // Spread across multiple lines. +// message /* This is a trailing comment on 'message' */ Foo {} +// +// Newlines are preserved, so that any logical grouping of elements +// is maintained in the formatted result. +// +// For example, +// +// // Type represents a set of different types. +// enum Type { +// // Unspecified is the naming convention for default enum values. +// TYPE_UNSPECIFIED = 0; +// +// // The following elements are the real values. +// TYPE_ONE = 1; +// TYPE_TWO = 2; +// } +// +// Start nodes are always indented according to the formatter's +// current level of indentation (e.g. nested messages, fields, etc). +// +// Note that this is one of the most complex component of the formatter - it +// controls how each node should be separated from one another and preserves +// newlines in the original source. +func (f *formatter) writeStart(node ast.Node) { + f.writeStartMaybeCompact(node, true) +} + +func (f *formatter) writeStartMaybeCompact(node ast.Node, forceCompact bool) { + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + var ( + nodeNewlineCount = newlineCount(info.LeadingWhitespace()) + compact = forceCompact || isOpenBrace(f.previousNode) + ) + if length := info.LeadingComments().Len(); length > 0 { + // If leading comments are defined, the whitespace we care about + // is attached to the first comment. + f.writeMultilineCommentsMaybeCompact(info.LeadingComments(), forceCompact) + if !forceCompact && nodeNewlineCount > 1 { + // At this point, we're looking at the lines between + // a comment and the node its attached to. + // + // If the last comment is a standard comment, a single newline + // character is sufficient to warrant a separation of the + // two. + // + // If the last comment is a C-style comment, multiple newline + // characters are required because C-style comments don't consume + // a newline. + f.P("") + } + } else if !compact && nodeNewlineCount > 1 { + // If the previous node is an open brace, this is the first element + // in the body of a composite type, so we don't want to write a + // newline. This makes it so that trailing newlines are removed. + // + // For example, + // + // message Foo { + // + // string bar = 1; + // } + // + // Is formatted into the following: + // + // message Foo { + // string bar = 1; + // } + f.P("") + } + f.Indent(node) + f.writeNode(node) + if info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } +} + +// writeInline writes the node and its surrounding comments in-line. +// +// This is useful for writing individual nodes like keywords, runes, +// string literals, etc. +// +// For example, +// +// // This is a leading comment on the syntax keyword. +// syntax = /* This is a leading comment on 'proto3' */" proto3"; +func (f *formatter) writeInline(node ast.Node) { + f.inline = true + defer func() { + f.inline = false + }() + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(node) + f.writeInlineComments(info.TrailingComments()) +} + +// writeBodyEnd writes the node as the end of a body. +// Leading comments are written above the token across +// multiple lines, whereas the trailing comments are +// written in-line and preserve their format. +// +// Body end nodes are always indented according to the +// formatter's current level of indentation (e.g. nested +// messages). +// +// This is useful for writing a node that concludes a +// composite node: ']', '}', '>', etc. +// +// For example, +// +// message Foo { +// string bar = 1; +// // Leading comment on '}'. +// } // Trailing comment on '}. +func (f *formatter) writeBodyEnd(node ast.Node, leadingEndline bool) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + if f.lastWritten != '\n' { + f.P("") + } + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if leadingEndline { + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + } else { + f.writeMultilineComments(info.LeadingComments()) + f.Indent(node) + } + f.writeNode(node) + f.writeTrailingEndComments(info.TrailingComments()) +} + +func (f *formatter) writeLineElement(node ast.Node) { + f.writeBodyEnd(node, false) +} + +// writeBodyEndInline writes the node as the end of a body. +// Leading comments are written above the token across +// multiple lines, whereas the trailing comments are +// written in-line and adapt their comment style if they +// exist. +// +// Body end nodes are always indented according to the +// formatter's current level of indentation (e.g. nested +// messages). +// +// This is useful for writing a node that concludes either +// compact options or an array literal. +// +// This is behaviorally similar to f.writeStart, but it ignores +// the preceding newline logic because these body ends should +// always be compact. +// +// For example, +// +// message Foo { +// string bar = 1 [ +// deprecated = true +// +// // Leading comment on ']'. +// ] /* Trailing comment on ']' */ ; +// } +func (f *formatter) writeBodyEndInline(node ast.Node, leadingInline bool) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if leadingInline { + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + } else { + f.writeMultilineComments(info.LeadingComments()) + f.Indent(node) + } + f.writeNode(node) + if info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } +} + +// writeLineEnd writes the node so that it ends a line. +// +// This is useful for writing individual nodes like ';' and other +// tokens that conclude the end of a single line. In this case, we +// don't want to transform the trailing comment's from '//' to C-style +// because it's not necessary. +// +// For example, +// +// // This is a leading comment on the syntax keyword. +// syntax = " proto3" /* This is a leading comment on the ';'; // This is a trailing comment on the ';'. +func (f *formatter) writeLineEnd(node ast.Node) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + if f.lastWritten != '\n' { + f.P("") + } + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(node) + f.Space() + f.writeTrailingEndComments(info.TrailingComments()) +} + +// writeMultilineComments writes the given comments as a newline-delimited block. +// This is useful for both the beginning of a type (e.g. message, field, etc), as +// well as the trailing comments attached to the beginning of a body block (e.g. +// '{', '[', '<', etc). +// +// For example, +// +// // This is a comment spread across +// // multiple lines. +// message Foo {} +func (f *formatter) writeMultilineComments(comments ast.Comments) { + f.writeMultilineCommentsMaybeCompact(comments, false) +} + +func (f *formatter) writeMultilineCommentsMaybeCompact(comments ast.Comments, forceCompact bool) { + compact := forceCompact || isOpenBrace(f.previousNode) + for i := 0; i < comments.Len(); i++ { + comment := comments.Index(i) + if !compact && newlineCount(comment.LeadingWhitespace()) > 1 { + // Newlines between blocks of comments should be preserved. + // + // For example, + // + // // This is a license header + // // spread across multiple lines. + // + // // Package pet.v1 defines a PetStore API. + // package pet.v1; + // + f.P("") + } + compact = false + f.writeComment(comment.RawText()) + f.WriteString("\n") + } +} + +// writeInlineComments writes the given comments in-line. Standard comments are +// transformed to C-style comments so that we can safely write the comment in-line. +// +// Nearly all of these comments will already be C-style comments. The only cases we're +// preventing are when the type is defined across multiple lines. +// +// For example, given the following: +// +// extend . google. // in-line comment +// protobuf . +// ExtensionRangeOptions { +// optional string label = 20000; +// } +// +// The formatted result is shown below: +// +// extend .google.protobuf./* in-line comment */ExtensionRangeOptions { +// optional string label = 20000; +// } +func (f *formatter) writeInlineComments(comments ast.Comments) { + for i := 0; i < comments.Len(); i++ { + if i > 0 || comments.Index(i).LeadingWhitespace() != "" || f.lastWritten == ';' || f.lastWritten == '}' { + f.Space() + } + text := comments.Index(i).RawText() + if strings.HasPrefix(text, "//") { + text = strings.TrimSpace(strings.TrimPrefix(text, "//")) + text = "/* " + text + " */" + } else { + // no multi-line comments + lines := strings.Split(text, "\n") + for i := range lines { + lines[i] = strings.TrimSpace(lines[i]) + } + text = strings.Join(lines, " ") + } + f.WriteString(text) + } +} + +// writeTrailingEndComments writes the given comments at the end of a line and +// preserves the comment style. This is useful or writing comments attached to +// things like ';' and other tokens that conclude a type definition on a single +// line. +// +// If there is a newline between this trailing comment and the previous node, the +// comments are written immediately underneath the node on a newline. +// +// For example, +// +// enum Type { +// TYPE_UNSPECIFIED = 0; +// } +// // This comment is attached to the '}' +// // So is this one. +func (f *formatter) writeTrailingEndComments(comments ast.Comments) { + for i := 0; i < comments.Len(); i++ { + comment := comments.Index(i) + if i > 0 || comment.LeadingWhitespace() != "" { + f.Space() + } + f.writeComment(comment.RawText()) + } + f.P("") +} + +func (f *formatter) writeComment(comment string) { + if strings.HasPrefix(comment, "/*") && newlineCount(comment) > 0 { + lines := strings.Split(comment, "\n") + // find minimum indent, so we can make all other lines relative to that + minIndent := -1 // sentinel that means unset + // start at 1 because line at index zero starts with "/*", not whitespace + var prefix string + for i := 1; i < len(lines); i++ { + indent, ok := computeIndent(lines[i]) + if ok && (minIndent == -1 || indent < minIndent) { + minIndent = indent + } + if i > 1 && len(prefix) == 0 { + // no shared prefix + continue + } + line := strings.TrimSpace(lines[i]) + if line == "*/" { + continue + } + var linePrefix string + if len(line) > 0 && isCommentPrefix(line[0]) { + linePrefix = line[:1] + } + if i == 1 { + prefix = linePrefix + } else if linePrefix != prefix { + // they do not share prefix + prefix = "" + } + } + if minIndent < 0 { + // This shouldn't be necessary. + // But we do it just in case, to avoid possible panic + minIndent = 0 + } + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine == "" || trimmedLine == "*/" || len(prefix) > 0 { + line = trimmedLine + } else { + // we only trim space from the right; for the left, + // we unindent based on indentation found above. + line = unindent(line, minIndent) + line = strings.TrimRightFunc(line, unicode.IsSpace) + } + // If we have a block comment with no prefix, we'll format + // like so: + + /* + This is a multi-line comment example. + It has no comment prefix on each line. + */ + + // But if there IS a prefix, "|" for example, we'll left-align + // the prefix symbol under the asterisk of the comment start + // like this: + + /* + | This comment has a prefix before each line. + | Usually the prefix is asterisk, but it's a + | pipe in this example. + */ + + // Finally, if the comment prefix is an asterisk, we'll left-align + // the comment end so its asterisk also aligns, like so: + + /* + * This comment has a prefix before each line. + * Usually the prefix is asterisk, which is the + * case in this example. + */ + + if i > 0 && line != "*/" { + if len(prefix) == 0 { + line = " " + line + } else { + line = " " + line + } + } + if line == "*/" && prefix == "*" { + // align the comment end with the other asterisks + line = " " + line + } + + if i != len(lines)-1 { + f.P(line) + } else { + // for last line, we don't use P because we don't + // want to print a trailing newline + f.Indent(nil) + f.WriteString(line) + } + } + } else { + f.Indent(nil) + f.WriteString(strings.TrimSpace(comment)) + } +} + +func isCommentPrefix(ch byte) bool { + r := rune(ch) + // A multi-line comment prefix is *usually* an asterisk, like in the following + /* + * Foo + * Bar + * Baz + */ + // But we'll allow other prefixes. But if it's a letter or number, it's not a prefix. + return !unicode.IsLetter(r) && !unicode.IsNumber(r) +} + +func unindent(s string, unindent int) string { + pos := 0 + for i, r := range s { + if pos == unindent { + return s[i:] + } + if pos > unindent { + // removing tab-stop unindented too far, so we + // add back some spaces to compensate + return strings.Repeat(" ", pos-unindent) + s[i:] + } + + switch r { + case ' ': + pos++ + case '\t': + // jump to next tab stop + pos += 8 - (pos % 8) + default: + return s[i:] + } + } + // nothing but whitespace... + return "" +} + +func computeIndent(s string) (int, bool) { + if strings.TrimSpace(s) == "*/" { + return 0, false + } + indent := 0 + for _, r := range s { + switch r { + case ' ': + indent++ + case '\t': + // jump to next tab stop + indent += 8 - (indent % 8) + default: + return indent, true + } + } + // if we get here, line is nothing but whitespace + return 0, false +} + +func (f *formatter) leadingCommentsContainBlankLine(n ast.Node) bool { + info := f.nodeInfo(n) + comments := info.LeadingComments() + for i := 0; i < comments.Len(); i++ { + if newlineCount(comments.Index(i).LeadingWhitespace()) > 1 { + return true + } + } + return newlineCount(info.LeadingWhitespace()) > 1 +} + +func (f *formatter) importHasComment(importNode *ast.ImportNode) bool { + if f.nodeHasComment(importNode) { + return true + } + if importNode == nil { + return false + } + + return f.nodeHasComment(importNode.Keyword) || + f.nodeHasComment(importNode.Name) || + f.nodeHasComment(importNode.Semicolon) || + f.nodeHasComment(importNode.Public) || + f.nodeHasComment(importNode.Weak) +} + +func (f *formatter) nodeHasComment(node ast.Node) bool { + // when node != nil, node's value could be nil, see: https://go.dev/doc/faq#nil_error + if node == nil || reflect.ValueOf(node).IsNil() { + return false + } + + nodeinfo := f.nodeInfo(node) + return nodeinfo.LeadingComments().Len() > 0 || + nodeinfo.TrailingComments().Len() > 0 +} + +func (f *formatter) setTrailingComments(node ast.Node, comments ast.Comments) { + f.overrideTrailingComments[node] = comments +} + +func (f *formatter) nodeInfo(node ast.Node) nodeInfo { + info := f.fileNode.NodeInfo(node) + if trailingComments, ok := f.overrideTrailingComments[node]; ok { + return infoWithTrailingComments{info, trailingComments} + } + return info +} + +type nodeInfo interface { + Start() ast.SourcePos + End() ast.SourcePos + LeadingComments() ast.Comments + TrailingComments() ast.Comments + LeadingWhitespace() string + RawText() string +} + +type infoWithTrailingComments struct { + ast.NodeInfo + trailing ast.Comments +} + +func (n infoWithTrailingComments) TrailingComments() ast.Comments { + return n.trailing +} + +// importSortOrder maps import types to a sort order number, so it can be compared and sorted. +// `import`=3, `import public`=2, `import weak`=1 +func importSortOrder(node *ast.ImportNode) int { + switch { + case node.Public != nil: + return 2 + case node.Weak != nil: + return 1 + default: + return 3 + } +} + +// stringForOptionName returns the string representation of the given option name node. +// This is used for sorting file-level options. +func stringForOptionName(optionNameNode *ast.OptionNameNode) string { + var result string + for j, part := range optionNameNode.Parts { + if j > 0 { + // Add a dot between each of the parts. + result += "." + } + result += stringForFieldReference(part) + } + return result +} + +// stringForFieldReference returns the string representation of the given field reference. +// This is used for sorting file-level options. +func stringForFieldReference(fieldReference *ast.FieldReferenceNode) string { + var result string + if fieldReference.Open != nil { + result += "(" + } + result += string(fieldReference.Name.AsIdentifier()) + if fieldReference.Close != nil { + result += ")" + } + return result +} + +// isOpenBrace returns true if the given node represents one of the +// possible open brace tokens, namely '{', '[', or '<'. +func isOpenBrace(node ast.Node) bool { + if node == nil { + return false + } + runeNode, ok := node.(*ast.RuneNode) + if !ok { + return false + } + return runeNode.Rune == '{' || runeNode.Rune == '[' || runeNode.Rune == '<' +} + +// newlineCount returns the number of newlines in the given value. +// This is useful for determining whether or not we should preserve +// the newline between nodes. +// +// The newlines don't need to be adjacent to each other - all of the +// tokens between them are other whitespace characters, so we can +// safely ignore them. +func newlineCount(value string) int { + return strings.Count(value, "\n") +} + +func messageLiteralOpen(msg *ast.MessageLiteralNode) *ast.RuneNode { + node := msg.Open + if node.Rune == '{' { + return node + } + // If it's not "{" then this message literal used "<" and ">" to enclose it. + // For consistent formatted output, change it to "{". + return ast.NewRuneNode('{', node.Token()) +} + +func messageLiteralClose(msg *ast.MessageLiteralNode) *ast.RuneNode { + node := msg.Close + if node.Rune == '}' { + return node + } + // If it's not "}" then this message literal used "<" and ">" to enclose it. + // For consistent formatted output, change it to "}". + return ast.NewRuneNode('}', node.Token()) +} + +type ByOptionName []*ast.OptionNode + +func (a ByOptionName) Len() int { return len(a) } +func (a ByOptionName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByOptionName) Less(i, j int) bool { + // The default options (e.g. cc_enable_arenas) should always + // be sorted above custom options (which are identified by a + // leading '('). + left := stringForOptionName(a[i].Name) + right := stringForOptionName(a[j].Name) + if strings.HasPrefix(left, "(") && !strings.HasPrefix(right, "(") { + // Prefer the default option on the right. + return false + } + if !strings.HasPrefix(left, "(") && strings.HasPrefix(right, "(") { + // Prefer the default option on the left. + return true + } + // Both options are custom, so we defer to the standard sorting. + return left < right +} + +// writeNodes writes nodes with sorted options. +func writeNodes[T ast.Node](nodes []T, f *formatter) { + optionNodes := []*ast.OptionNode{} + for _, node := range nodes { + if option, ok := ast.Node(node).(*ast.OptionNode); ok { + optionNodes = append(optionNodes, option) + } + } + + sort.Sort(ByOptionName(optionNodes)) + + for _, node := range optionNodes { + f.writeNode(node) + } + + for _, node := range nodes { + if _, ok := ast.Node(node).(*ast.OptionNode); !ok { + f.writeNode(node) + } + } +} diff --git a/go/protopace/formatter_test.go b/go/protopace/formatter_test.go new file mode 100644 index 000000000..bdc960fd7 --- /dev/null +++ b/go/protopace/formatter_test.go @@ -0,0 +1,169 @@ +package main + +import ( + "testing" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/stretchr/testify/assert" +) + +func TestFormatEnum(t *testing.T) { + + testProto := + `syntax = "proto3"; + +package pkg; + +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +enum MyEnum { + option (my_option3) = "my_value3"; + option (my_option) = "my_value"; + option (my_option2) = "my_value2"; + + ACTIVE = 0; +}` + + expected := + `syntax = "proto3"; +package pkg; + +import "google/protobuf/descriptor.proto"; +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} +enum MyEnum { + option (my_option) = "my_value"; + option (my_option2) = "my_value2"; + option (my_option3) = "my_value3"; + ACTIVE = 0; +} +` + + assert := assert.New(t) + + testSchema, err := s.FromString("test.proto", testProto, []s.Schema{}) + assert.NoError(err) + assert.NotNil(testSchema) + + result, err := Format(*testSchema) + assert.NoError(err) + assert.Equal(result.Schema, expected) +} + +func TestFormatFullyQualifiedNames(t *testing.T) { + + dependency := + ` +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} +` + + dependencyV2 := + ` +syntax = "proto3"; +package my.awesome.customer.v2; + +enum Status { + ACTIVE = 0; + INACTIVE = 1; +} + +message NestedValue { + string value = 1; + Status status = 2; +} +` + + testProtoA := + `syntax = "proto3"; +package my.awesome.customer.v2; + +import "my/awesome/customer/v1/nested_value.proto"; +import "my/awesome/customer/v2/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option php_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option ruby_package = "My::Awesome::Customer::V1"; + +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; + + +message EventValue { + .my.awesome.customer.v2.NestedValue nested_value = 1; + .google.protobuf.Timestamp created_at = 2; + + + .my.awesome.customer.v2.Status status = 3; +} +` + + testProtoB := + `syntax = "proto3"; +package my.awesome.customer.v2; + +import "my/awesome/customer/v1/nested_value.proto"; +import "my/awesome/customer/v2/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option php_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option ruby_package = "My::Awesome::Customer::V1"; + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; +} +` + + assert := assert.New(t) + + testDependency, err := s.FromString("my/awesome/customer/v1/nested_value.proto", dependency, []s.Schema{}) + assert.NoError(err) + assert.NotNil(testDependency) + + testDependencyV2, err := s.FromString("my/awesome/customer/v2/nested_value.proto", dependencyV2, []s.Schema{}) + assert.NoError(err) + assert.NotNil(testDependencyV2) + + testSchemaA, err := s.FromString("test.proto", testProtoA, []s.Schema{*testDependency, *testDependencyV2}) + assert.NoError(err) + assert.NotNil(testSchemaA) + a, err := Format(*testSchemaA) + assert.NoError(err) + + testSchemaB, err := s.FromString("test.proto", testProtoB, []s.Schema{*testDependency, *testDependencyV2}) + assert.NoError(err) + assert.NotNil(testSchemaB) + b, err := Format(*testSchemaB) + assert.NoError(err) + + assert.Equal(a.Schema, b.Schema) + err = Check(b, a) + assert.NoError(err) +} diff --git a/go/protopace/go.mod b/go/protopace/go.mod new file mode 100644 index 000000000..ad7dccca3 --- /dev/null +++ b/go/protopace/go.mod @@ -0,0 +1,42 @@ +module github.com/Aiven-Open/karapace/go/protopace + +go 1.21.0 + +toolchain go1.21.10 + +replace github.com/bufbuild/buf => github.com/Aiven-Open/buf v0.0.0-20240829124012-099cfb8b4b31 + +require ( + github.com/bufbuild/buf v1.34.0 + github.com/bufbuild/protocompile v0.14.0 + github.com/gofrs/uuid/v5 v5.3.0 + github.com/stretchr/testify v1.9.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.27.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee // indirect + github.com/bufbuild/protovalidate-go v0.6.4 // indirect + github.com/bufbuild/protoyaml-go v0.1.11 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/cel-go v0.21.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect + golang.org/x/mod v0.20.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240823204242-4ba0660f739c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240823204242-4ba0660f739c // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go/protopace/go.sum b/go/protopace/go.sum new file mode 100644 index 000000000..5a2301354 --- /dev/null +++ b/go/protopace/go.sum @@ -0,0 +1,83 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2 h1:SZRVx928rbYZ6hEKUIN+vtGDkl7uotABRWGY4OAg5gM= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw= +github.com/Aiven-Open/buf v0.0.0-20240829124012-099cfb8b4b31 h1:sTDz5KNYsvDl4mjQ3o/grLdesCnu/YJilBmNVTNZs04= +github.com/Aiven-Open/buf v0.0.0-20240829124012-099cfb8b4b31/go.mod h1:1P0U+x/ky1KhpK7o7mGraDAYjQUG7710wk5lEZFWsTA= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/bufbuild/protocompile v0.14.0 h1:z3DW4IvXE5G/uTOnSQn+qwQQxvhckkTWLS/0No/o7KU= +github.com/bufbuild/protocompile v0.14.0/go.mod h1:N6J1NYzkspJo3ZwyL4Xjvli86XOj1xq4qAasUFxGups= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee h1:E6ET8YUcYJ1lAe6ctR3as7yqzW2BNItDFnaB5zQq/8M= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee/go.mod h1:HjGFxsck9RObrTJp2hXQZfWhPgZqnR6sR1U5fCA/Kus= +github.com/bufbuild/protovalidate-go v0.6.4 h1:QtNIz4LGclM3UArQv/R1AKNF7MO8wriT9v7b8Gnmqak= +github.com/bufbuild/protovalidate-go v0.6.4/go.mod h1:HlkVnkE/zVYZvHIG/a7QZuzqC9bSqHaOOTeRomYF0Q8= +github.com/bufbuild/protoyaml-go v0.1.11 h1:Iyixd6Y5dx6ws6Uh8APgC1lMyvXt710NayoY8cY0Vj8= +github.com/bufbuild/protoyaml-go v0.1.11/go.mod h1:KCBItkvZOK/zwGueLdH1Wx1RLyFn5rCH7YjQrdty2Wc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk= +github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= +github.com/google/cel-go v0.21.0 h1:cl6uW/gxN+Hy50tNYvI691+sXxioCnstFzLp2WO4GCI= +github.com/google/cel-go v0.21.0/go.mod h1:rHUlWCcBKgyEk+eV03RPdZUekPp6YcJwV0FxuUksYxc= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/api v0.0.0-20240823204242-4ba0660f739c h1:e0zB268kOca6FbuJkYUGxfwG4DKFZG/8DLyv9Zv66cE= +google.golang.org/genproto/googleapis/api v0.0.0-20240823204242-4ba0660f739c/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240823204242-4ba0660f739c h1:Kqjm4WpoWvwhMPcrAczoTyMySQmYa9Wy2iL6Con4zn8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240823204242-4ba0660f739c/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/protopace/main.go b/go/protopace/main.go new file mode 100644 index 000000000..84eb82d55 --- /dev/null +++ b/go/protopace/main.go @@ -0,0 +1,86 @@ +package main + +/* +#include + +struct result { + char* res; + char* err; +}; +*/ +import "C" +import ( + "unsafe" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" +) + +func result(schema string, err error) *C.struct_result { + res := (*C.struct_result)(C.malloc(C.size_t(unsafe.Sizeof(C.struct_result{})))) + res.res = C.CString(schema) + res.err = nil + if err != nil { + res.err = C.CString(err.Error()) + } + return res +} + +//export FreeResult +func FreeResult(result *C.struct_result) { + C.free(unsafe.Pointer(result.res)) + C.free(unsafe.Pointer(result.err)) + C.free(unsafe.Pointer(result)) +} + +func createSchema(cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int) (*s.Schema, error) { + depArray := unsafe.Slice(cDependencies, depsLenght) + depNamesArray := unsafe.Slice(cDependencyNames, depsLenght) + dependencies := []s.Schema{} + for i, dep := range depArray { + dependency, err := s.FromString(C.GoString(depNamesArray[i]), C.GoString(dep), []s.Schema{}) + if err != nil { + return nil, err + } + dependencies = append(dependencies, *dependency) + } + + schema, err := s.FromString(C.GoString(cSchemaName), C.GoString(cSchema), dependencies) + return schema, err +} + +//export FormatSchema +func FormatSchema(cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int) *C.struct_result { + schema, err := createSchema(cSchemaName, cSchema, cDependencyNames, cDependencies, depsLenght) + if err != nil { + return result("", err) + } + + res, err := Format(*schema) + if err != nil { + return result("", err) + } + return result(res.Schema, err) +} + +//export CheckCompatibility +func CheckCompatibility( + cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int, + cSchemaNamePrev *C.char, cSchemaPrev *C.char, cDependencyNamesPrev **C.char, cDependenciesPrev **C.char, depsLenghtPrev C.int) *C.char { + + schema, err := createSchema(cSchemaName, cSchema, cDependencyNames, cDependencies, depsLenght) + if err != nil { + return C.CString(err.Error()) + } + prevSchema, err := createSchema(cSchemaNamePrev, cSchemaPrev, cDependencyNamesPrev, cDependenciesPrev, depsLenghtPrev) + if err != nil { + return C.CString(err.Error()) + } + + err = Check(*schema, *prevSchema) + if err != nil { + return C.CString(err.Error()) + } + return nil +} + +func main() {} diff --git a/go/protopace/schema/compiler.go b/go/protopace/schema/compiler.go new file mode 100644 index 000000000..5311d022c --- /dev/null +++ b/go/protopace/schema/compiler.go @@ -0,0 +1,10 @@ +package schema + +import ( + "github.com/bufbuild/protocompile" +) + +func NewCompiler(resolver protocompile.Resolver) *protocompile.Compiler { + compiler := protocompile.Compiler{Resolver: resolver, RetainASTs: true} + return &compiler +} diff --git a/go/protopace/schema/google/type/README.md b/go/protopace/schema/google/type/README.md new file mode 100644 index 000000000..adf1563a8 --- /dev/null +++ b/go/protopace/schema/google/type/README.md @@ -0,0 +1,7 @@ +## Google Common Types + +This package contains definitions of common types for Google APIs. +All types defined in this package are suitable for different APIs to +exchange data, and will never break binary compatibility. They should +have design quality comparable to major programming languages like +Java and C#. diff --git a/go/protopace/schema/google/type/calendar_period.proto b/go/protopace/schema/google/type/calendar_period.proto new file mode 100644 index 000000000..25a8f6441 --- /dev/null +++ b/go/protopace/schema/google/type/calendar_period.proto @@ -0,0 +1,56 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/go/protopace/schema/google/type/color.proto b/go/protopace/schema/google/type/color.proto new file mode 100644 index 000000000..3e57c1fb2 --- /dev/null +++ b/go/protopace/schema/google/type/color.proto @@ -0,0 +1,174 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness. For example, the fields of this representation +// can be trivially provided to the constructor of `java.awt.Color` in Java; it +// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS `rgba()` string in JavaScript. +// +// This reference page doesn't carry information about the absolute color +// space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color +// space. +// +// When color equality needs to be decided, implementations, unless +// documented otherwise, treat two colors as equal if all their red, +// green, blue, and alpha values each differ by at most 1e-5. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is rendered as a solid color + // (as if the alpha value had been explicitly given a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/go/protopace/schema/google/type/date.proto b/go/protopace/schema/google/type/date.proto new file mode 100644 index 000000000..6370cd869 --- /dev/null +++ b/go/protopace/schema/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/go/protopace/schema/google/type/datetime.proto b/go/protopace/schema/google/type/datetime.proto new file mode 100644 index 000000000..a363a41ef --- /dev/null +++ b/go/protopace/schema/google/type/datetime.proto @@ -0,0 +1,104 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time (or occasionally physical time). +// +// This type can represent a civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type may also be used to represent a physical time if all the date and +// time fields are set and either case of the `time_offset` oneof is set. +// Consider using `Timestamp` message for physical time instead. If your use +// case also would like to store the user's timezone, that can be done in +// another field. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/go/protopace/schema/google/type/dayofweek.proto b/go/protopace/schema/google/type/dayofweek.proto new file mode 100644 index 000000000..e16c19469 --- /dev/null +++ b/go/protopace/schema/google/type/dayofweek.proto @@ -0,0 +1,50 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of the week. +enum DayOfWeek { + // The day of the week is unspecified. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // Monday + MONDAY = 1; + + // Tuesday + TUESDAY = 2; + + // Wednesday + WEDNESDAY = 3; + + // Thursday + THURSDAY = 4; + + // Friday + FRIDAY = 5; + + // Saturday + SATURDAY = 6; + + // Sunday + SUNDAY = 7; +} diff --git a/go/protopace/schema/google/type/decimal.proto b/go/protopace/schema/google/type/decimal.proto new file mode 100644 index 000000000..293d08273 --- /dev/null +++ b/go/protopace/schema/google/type/decimal.proto @@ -0,0 +1,95 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; +option java_multiple_files = true; +option java_outer_classname = "DecimalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A representation of a decimal value, such as 2.5. Clients may convert values +// into language-native decimal formats, such as Java's [BigDecimal][] or +// Python's [decimal.Decimal][]. +// +// [BigDecimal]: +// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html +// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html +message Decimal { + // The decimal value, as a string. + // + // The string representation consists of an optional sign, `+` (`U+002B`) + // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits + // ("the integer"), optionally followed by a fraction, optionally followed + // by an exponent. + // + // The fraction consists of a decimal point followed by zero or more decimal + // digits. The string must contain at least one digit in either the integer + // or the fraction. The number formed by the sign, the integer and the + // fraction is referred to as the significand. + // + // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) + // followed by one or more decimal digits. + // + // Services **should** normalize decimal values before storing them by: + // + // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). + // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). + // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). + // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). + // + // Services **may** perform additional normalization based on its own needs + // and the internal decimal implementation selected, such as shifting the + // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). + // Additionally, services **may** preserve trailing zeroes in the fraction + // to indicate increased precision, but are not required to do so. + // + // Note that only the `.` character is supported to divide the integer + // and the fraction; `,` **should not** be supported regardless of locale. + // Additionally, thousand separators **should not** be supported. If a + // service does support them, values **must** be normalized. + // + // The ENBF grammar is: + // + // DecimalString = + // [Sign] Significand [Exponent]; + // + // Sign = '+' | '-'; + // + // Significand = + // Digits ['.'] [Digits] | [Digits] '.' Digits; + // + // Exponent = ('e' | 'E') [Sign] Digits; + // + // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; + // + // Services **should** clearly document the range of supported values, the + // maximum supported precision (total number of digits), and, if applicable, + // the scale (number of digits after the decimal point), as well as how it + // behaves when receiving out-of-bounds values. + // + // Services **may** choose to accept values passed as input even when the + // value has a higher precision or scale than the service supports, and + // **should** round the value to fit the supported scale. Alternatively, the + // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) + // if precision would be lost. + // + // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in + // gRPC) if the service receives a value outside of the supported range. + string value = 1; +} diff --git a/go/protopace/schema/google/type/expr.proto b/go/protopace/schema/google/type/expr.proto new file mode 100644 index 000000000..544e66874 --- /dev/null +++ b/go/protopace/schema/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/go/protopace/schema/google/type/fraction.proto b/go/protopace/schema/google/type/fraction.proto new file mode 100644 index 000000000..06f072322 --- /dev/null +++ b/go/protopace/schema/google/type/fraction.proto @@ -0,0 +1,33 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The numerator in the fraction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/go/protopace/schema/google/type/interval.proto b/go/protopace/schema/google/type/interval.proto new file mode 100644 index 000000000..fcf94c866 --- /dev/null +++ b/go/protopace/schema/google/type/interval.proto @@ -0,0 +1,46 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/timestamp.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; +option java_multiple_files = true; +option java_outer_classname = "IntervalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time interval, encoded as a Timestamp start (inclusive) and a +// Timestamp end (exclusive). +// +// The start must be less than or equal to the end. +// When the start equals the end, the interval is empty (matches no time). +// When both start and end are unspecified, the interval matches any time. +message Interval { + // Optional. Inclusive start of the interval. + // + // If specified, a Timestamp matching this interval will have to be the same + // or after the start. + google.protobuf.Timestamp start_time = 1; + + // Optional. Exclusive end of the interval. + // + // If specified, a Timestamp matching this interval will have to be before the + // end. + google.protobuf.Timestamp end_time = 2; +} diff --git a/go/protopace/schema/google/type/latlng.proto b/go/protopace/schema/google/type/latlng.proto new file mode 100644 index 000000000..daeba48b4 --- /dev/null +++ b/go/protopace/schema/google/type/latlng.proto @@ -0,0 +1,37 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object that represents a latitude/longitude pair. This is expressed as a +// pair of doubles to represent degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/go/protopace/schema/google/type/localized_text.proto b/go/protopace/schema/google/type/localized_text.proto new file mode 100644 index 000000000..82d083c43 --- /dev/null +++ b/go/protopace/schema/google/type/localized_text.proto @@ -0,0 +1,36 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; +option java_multiple_files = true; +option java_outer_classname = "LocalizedTextProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Localized variant of a text in a particular language. +message LocalizedText { + // Localized string in the language corresponding to `language_code' below. + string text = 1; + + // The text's BCP-47 language code, such as "en-US" or "sr-Latn". + // + // For more information, see + // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. + string language_code = 2; +} diff --git a/go/protopace/schema/google/type/money.proto b/go/protopace/schema/google/type/money.proto new file mode 100644 index 000000000..c61094336 --- /dev/null +++ b/go/protopace/schema/google/type/money.proto @@ -0,0 +1,42 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The three-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/go/protopace/schema/google/type/month.proto b/go/protopace/schema/google/type/month.proto new file mode 100644 index 000000000..19982cb51 --- /dev/null +++ b/go/protopace/schema/google/type/month.proto @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecified month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/go/protopace/schema/google/type/phone_number.proto b/go/protopace/schema/google/type/phone_number.proto new file mode 100644 index 000000000..370d1623d --- /dev/null +++ b/go/protopace/schema/google/type/phone_number.proto @@ -0,0 +1,113 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; +option java_multiple_files = true; +option java_outer_classname = "PhoneNumberProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a phone number, suitable as an API wire format. +// +// This representation: +// +// - should not be used for locale-specific formatting of a phone number, such +// as "+1 (650) 253-0000 ext. 123" +// +// - is not designed for efficient storage +// - may not be suitable for dialing - specialized libraries (see references) +// should be used to parse the number for that purpose +// +// To do something meaningful with this number, such as format it for various +// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. +// +// For instance, in Java this would be: +// +// com.google.type.PhoneNumber wireProto = +// com.google.type.PhoneNumber.newBuilder().build(); +// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = +// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); +// if (!wireProto.getExtension().isEmpty()) { +// phoneNumber.setExtension(wireProto.getExtension()); +// } +// +// Reference(s): +// - https://github.com/google/libphonenumber +message PhoneNumber { + // An object representing a short code, which is a phone number that is + // typically much shorter than regular phone numbers and can be used to + // address messages in MMS and SMS systems, as well as for abbreviated dialing + // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). + // + // Short codes are restricted to a region and are not internationally + // dialable, which means the same short code can exist in different regions, + // with different usage and pricing, even if those regions share the same + // country calling code (e.g. US and CA). + message ShortCode { + // Required. The BCP-47 region code of the location where calls to this + // short code can be made, such as "US" and "BB". + // + // Reference(s): + // - http://www.unicode.org/reports/tr35/#unicode_region_subtag + string region_code = 1; + + // Required. The short code digits, without a leading plus ('+') or country + // calling code, e.g. "611". + string number = 2; + } + + // Required. Either a regular number, or a short code. New fields may be + // added to the oneof below in the future, so clients should ignore phone + // numbers for which none of the fields they coded against are set. + oneof kind { + // The phone number, represented as a leading plus sign ('+'), followed by a + // phone number that uses a relaxed ITU E.164 format consisting of the + // country calling code (1 to 3 digits) and the subscriber number, with no + // additional spaces or formatting, e.g.: + // - correct: "+15552220123" + // - incorrect: "+1 (555) 222-01234 x123". + // + // The ITU E.164 format limits the latter to 12 digits, but in practice not + // all countries respect that, so we relax that restriction here. + // National-only numbers are not allowed. + // + // References: + // - https://www.itu.int/rec/T-REC-E.164-201011-I + // - https://en.wikipedia.org/wiki/E.164. + // - https://en.wikipedia.org/wiki/List_of_country_calling_codes + string e164_number = 1; + + // A short code. + // + // Reference(s): + // - https://en.wikipedia.org/wiki/Short_code + ShortCode short_code = 2; + } + + // The phone number's extension. The extension is not standardized in ITU + // recommendations, except for being defined as a series of numbers with a + // maximum length of 40 digits. Other than digits, some other dialing + // characters such as ',' (indicating a wait) or '#' may be stored here. + // + // Note that no regions currently use extensions with short codes, so this + // field is normally only set in conjunction with an E.164 number. It is held + // separately from the E.164 number to allow for short code extensions in the + // future. + string extension = 3; +} diff --git a/go/protopace/schema/google/type/postal_address.proto b/go/protopace/schema/google/type/postal_address.proto new file mode 100644 index 000000000..7023a9b3e --- /dev/null +++ b/go/protopace/schema/google/type/postal_address.proto @@ -0,0 +1,134 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/go/protopace/schema/google/type/quaternion.proto b/go/protopace/schema/google/type/quaternion.proto new file mode 100644 index 000000000..416de30cf --- /dev/null +++ b/go/protopace/schema/google/type/quaternion.proto @@ -0,0 +1,94 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/go/protopace/schema/google/type/timeofday.proto b/go/protopace/schema/google/type/timeofday.proto new file mode 100644 index 000000000..3735745a4 --- /dev/null +++ b/go/protopace/schema/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and +// `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} diff --git a/go/protopace/schema/google/type/type.yaml b/go/protopace/schema/google/type/type.yaml new file mode 100644 index 000000000..d5c71364d --- /dev/null +++ b/go/protopace/schema/google/type/type.yaml @@ -0,0 +1,40 @@ +type: google.api.Service +config_version: 3 +name: type.googleapis.com +title: Common Types + +types: +- name: google.type.Color +- name: google.type.Date +- name: google.type.DateTime +- name: google.type.Decimal +- name: google.type.Expr +- name: google.type.Fraction +- name: google.type.Interval +- name: google.type.LatLng +- name: google.type.LocalizedText +- name: google.type.Money +- name: google.type.PhoneNumber +- name: google.type.PostalAddress +- name: google.type.Quaternion +- name: google.type.TimeOfDay + +enums: +- name: google.type.CalendarPeriod +- name: google.type.DayOfWeek +- name: google.type.Month + +documentation: + summary: Defines common types for Google APIs. + overview: |- + # Google Common Types + + This package contains definitions of common types for Google APIs. + All types defined in this package are suitable for different APIs to + exchange data, and will never break binary compatibility. They should + have design quality comparable to major programming languages like + Java and C#. + + NOTE: Some common types are defined in the package `google.protobuf` + as they are directly supported by Protocol Buffers compiler and + runtime. Those types are called Well-Known Types. diff --git a/go/protopace/schema/google_types.go b/go/protopace/schema/google_types.go new file mode 100644 index 000000000..e81f5a533 --- /dev/null +++ b/go/protopace/schema/google_types.go @@ -0,0 +1,23 @@ +package schema + +import ( + "embed" + "io" + + "github.com/bufbuild/protocompile" +) + +//go:embed google/type/*.proto +var files embed.FS + +// WithGoogleTypeImports returns a new resolver that can provide the source code for google/types protos +func WithGoogleTypeImports(resolver protocompile.Resolver) protocompile.Resolver { + return protocompile.CompositeResolver{ + resolver, + &protocompile.SourceResolver{ + Accessor: func(path string) (io.ReadCloser, error) { + return files.Open(path) + }, + }, + } +} diff --git a/go/protopace/schema/resolver.go b/go/protopace/schema/resolver.go new file mode 100644 index 000000000..9ace3e55d --- /dev/null +++ b/go/protopace/schema/resolver.go @@ -0,0 +1,39 @@ +package schema + +import ( + "fmt" + "strings" + + "github.com/bufbuild/protocompile" +) + +type SchemaResolver struct { + schemas map[string]Schema +} + +func NewSchemaResolver(schemas []Schema) protocompile.Resolver { + schemasIndex := map[string]Schema{} + for _, schema := range schemas { + schemasIndex[schema.Name] = schema + } + resolver := SchemaResolver{schemas: schemasIndex} + return WithGoogleTypeImports(protocompile.WithStandardImports(&resolver)) +} + +func (s *SchemaResolver) AddSchema(schema Schema) { + s.schemas[schema.Name] = schema +} + +// FindFileByPath implements protocompile.Resolver. +func (s *SchemaResolver) FindFileByPath(path string) (protocompile.SearchResult, error) { + searchResult := protocompile.SearchResult{} + schema, ok := s.schemas[path] + if !ok { + return searchResult, fmt.Errorf("schema not found: %s", path) + } + searchResult.Source = strings.NewReader(schema.Schema) + searchResult.ParseResult = schema.ParserResult + return searchResult, nil +} + +var _ protocompile.Resolver = (*SchemaResolver)(nil) diff --git a/go/protopace/schema/schema.go b/go/protopace/schema/schema.go new file mode 100644 index 000000000..1a0bb822e --- /dev/null +++ b/go/protopace/schema/schema.go @@ -0,0 +1,81 @@ +package schema + +import ( + "context" + "strings" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/protocompile/linker" + "github.com/bufbuild/protocompile/parser" + "github.com/bufbuild/protocompile/reporter" + "github.com/gofrs/uuid/v5" +) + +var ( + handler = reporter.NewHandler(nil) +) + +type Schema struct { + Schema string + Name string + ParserResult parser.Result + Dependencies []Schema +} + +func FromString(name string, proto string, dependencies []Schema) (*Schema, error) { + fileNode, err := parser.Parse(name, strings.NewReader(proto), handler) + if err != nil { + return nil, err + } + result, err := parser.ResultFromAST(fileNode, true, handler) + if err != nil { + return nil, err + } + return &Schema{Schema: proto, Name: name, ParserResult: result, Dependencies: dependencies}, nil +} + +func (s Schema) Compile() ([]linker.Result, error) { + resolver := NewSchemaResolver(append(s.Dependencies, s)) + compiler := NewCompiler(resolver) + ctx := context.Background() + schemas := []string{s.Name} + for _, dep := range s.Dependencies { + schemas = append(schemas, dep.Name) + } + files, err := compiler.Compile(ctx, schemas...) + if err != nil { + return nil, err + } + res := make([]linker.Result, len(files)) + for i, f := range files { + res[i] = f.(linker.Result) + } + return res, nil +} + +func (s Schema) CompileBufImage() (bufimage.Image, error) { + res, err := s.Compile() + if err != nil { + return nil, err + } + files := make([]bufimage.ImageFile, len(res)) + for i, r := range res { + file, err := bufimage.NewImageFile( + r.FileDescriptorProto(), + nil, + uuid.Nil, + "", + "", + false, + false, + nil, + ) + if err != nil { + return nil, err + } + files[i] = file + } + + image, err := bufimage.NewImage(files) + return image, err +} diff --git a/karapace/config.py b/karapace/config.py index bbae62701..2618158a2 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -82,6 +82,7 @@ class Config(TypedDict): statsd_port: int kafka_schema_reader_strict_mode: bool kafka_retriable_errors_silenced: bool + use_protobuf_formatter: bool sentry: NotRequired[Mapping[str, object]] tags: NotRequired[Mapping[str, object]] @@ -158,6 +159,7 @@ class ConfigDefaults(Config, total=False): "statsd_port": 8125, "kafka_schema_reader_strict_mode": False, "kafka_retriable_errors_silenced": True, + "use_protobuf_formatter": False, } SECRET_CONFIG_OPTIONS = [SASL_PLAIN_PASSWORD] diff --git a/karapace/dependency.py b/karapace/dependency.py index a289e1f25..11c84781b 100644 --- a/karapace/dependency.py +++ b/karapace/dependency.py @@ -7,7 +7,10 @@ from __future__ import annotations +from karapace.errors import InvalidSchema +from karapace.protobuf.protopace.protopace import Proto from karapace.schema_references import Reference +from karapace.schema_type import SchemaType from karapace.typing import JsonData, Subject, Version from typing import TYPE_CHECKING @@ -41,6 +44,16 @@ def get_schema(self) -> ValidatedTypedSchema: def of(reference: Reference, target_schema: ValidatedTypedSchema) -> Dependency: return Dependency(reference.name, reference.subject, reference.version, target_schema) + def to_proto(self) -> Proto: + if self.schema.schema_type is not SchemaType.PROTOBUF: + raise InvalidSchema("Only supported for Protobuf") + + dependencies = [] + if self.schema.dependencies: + for _, dep in self.schema.dependencies.items(): + dependencies.append(dep.to_proto()) + return Proto(self.name, self.schema.schema_str, dependencies) + def to_dict(self) -> JsonData: return { "name": self.name, diff --git a/karapace/protobuf/protopace/__init__.py b/karapace/protobuf/protopace/__init__.py new file mode 100644 index 000000000..ec6c72f0b --- /dev/null +++ b/karapace/protobuf/protopace/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from .protopace import check_compatibility, format_proto, IncompatibleError, Proto # noqa: F401 diff --git a/karapace/protobuf/protopace/protopace.py b/karapace/protobuf/protopace/protopace.py new file mode 100644 index 000000000..a65f90582 --- /dev/null +++ b/karapace/protobuf/protopace/protopace.py @@ -0,0 +1,189 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from dataclasses import dataclass, field +from functools import cached_property +from karapace.errors import InvalidSchema +from typing import Dict, List + +import ctypes +import importlib.util +import timeit + +spec = importlib.util.find_spec("protopacelib") +if not spec: + raise FileNotFoundError("Unable to find protopace shared library") + +lib_file = spec.origin +lib = ctypes.CDLL(lib_file) + +lib.FormatSchema.argtypes = [ + ctypes.c_char_p, # schema name + ctypes.c_char_p, # schema string + ctypes.Array, # dependency names + ctypes.Array, # dependency schema strings + ctypes.c_int, # number of dependencies +] +lib.FormatSchema.restype = ctypes.c_void_p +lib.CheckCompatibility.restype = ctypes.c_char_p + + +class FormatResult(ctypes.Structure): + _fields_ = [ + ("res", ctypes.c_char_p), + ("err", ctypes.c_char_p), + ] + + +@dataclass +class Proto: + name: str + schema: str + dependencies: List["Proto"] = field(default_factory=list) + + @cached_property + def all_dependencies(self) -> List["Proto"]: + dependencies: Dict[str, "Proto"] = {} + for dep in self.dependencies: + if dep.dependencies: + dependencies.update([(d.name, d) for d in dep.all_dependencies]) + dependencies[dep.name] = dep + return list(dependencies.values()) + + +class IncompatibleError(Exception): + pass + + +def format_proto(proto: Proto) -> str: + length = len(proto.all_dependencies) + c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.all_dependencies]) + c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.all_dependencies]) + c_name = ctypes.c_char_p(proto.name.encode()) + c_schema = ctypes.c_char_p(proto.schema.encode()) + res_ptr = lib.FormatSchema(c_name, c_schema, c_dependency_names, c_dependencies, length) + res = FormatResult.from_address(res_ptr) + + if res.err: + err = res.err + msg = err.decode() + lib.FreeResult(ctypes.c_void_p(res_ptr)) + raise InvalidSchema(msg) + + result = res.res.decode() + lib.FreeResult(ctypes.c_void_p(res_ptr)) + return result + + +def check_compatibility(proto: Proto, prev_proto: Proto) -> None: + length = len(proto.all_dependencies) + c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.all_dependencies]) + c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.all_dependencies]) + + prev_length = len(prev_proto.all_dependencies) + prev_c_dependencies = (ctypes.c_char_p * prev_length)(*[d.schema.encode() for d in prev_proto.all_dependencies]) + prev_c_dependency_names = (ctypes.c_char_p * prev_length)(*[d.name.encode() for d in prev_proto.all_dependencies]) + + err = lib.CheckCompatibility( + proto.name.encode(), + proto.schema.encode(), + c_dependency_names, + c_dependencies, + length, + prev_proto.name.encode(), + prev_proto.schema.encode(), + prev_c_dependency_names, + prev_c_dependencies, + prev_length, + ) + + if err is not None: + msg = err.decode() + raise IncompatibleError(msg) + + +SCHEMA = """ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; +} +""" + +DEPENDENCY = """ +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} +""" + + +def _format_time(seconds: float) -> str: + units = [("s", 1), ("ms", 1e-3), ("µs", 1e-6), ("ns", 1e-9)] + for unit, factor in units: + if seconds >= factor: + return f"{seconds / factor:.3f} {unit}" + return f"{seconds:.3f} s" + + +def _time_format() -> None: + def test() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + format_proto(proto) + + number = 10000 + seconds = timeit.timeit(test, number=number) + + print("----- Format -----") + print(f"Total time: {_format_time(seconds)}") + print(f"Execution time per loop: {_format_time(seconds / number)}") + + +def _time_check_compatibility() -> None: + def test() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + check_compatibility(proto, proto) + + number = 10000 + seconds = timeit.timeit(test, number=number) + + print("----- Compatibility Check -----") + print(f"Total time: {_format_time(seconds)}") + print(f"Execution time per loop: {_format_time(seconds / number)}") + + +if __name__ == "__main__": + _time_format() + _time_check_compatibility() diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 896e73f26..eab1a5c9f 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -11,6 +11,7 @@ from jsonschema.exceptions import SchemaError from karapace.dependency import Dependency from karapace.errors import InvalidSchema, InvalidVersion, VersionNotFoundException +from karapace.protobuf import protopace from karapace.protobuf.exception import ( Error as ProtobufError, IllegalArgumentException, @@ -25,7 +26,7 @@ from karapace.schema_type import SchemaType from karapace.typing import JsonObject, SchemaId, Subject, Version, VersionTag from karapace.utils import assert_never, json_decode, json_encode, JSONDecodeError -from typing import Any, cast, Dict, Final, final, Mapping, Sequence +from typing import Any, cast, Collection, Dict, Final, final, Mapping, Sequence import hashlib import logging @@ -58,12 +59,19 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: return Draft7Validator(schema) # type: ignore[arg-type] +def _format_protobuf(schema: str, dependencies: Collection[Dependency], name: str = "schema.proto") -> str: + deps = [dep.to_proto() for dep in dependencies] + proto = protopace.Proto(name, schema, deps) + return protopace.format_proto(proto) + + def parse_protobuf_schema_definition( schema_definition: str, references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, validate_references: bool = True, normalize: bool = False, + use_protobuf_formatter: bool = False, ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -71,6 +79,12 @@ def parse_protobuf_schema_definition( ProtobufUnresolvedDependencyException if Protobuf dependency cannot be resolved. """ + if use_protobuf_formatter: + try: + schema_definition = _format_protobuf(schema_definition, dependencies.values() if dependencies else []) + except InvalidSchema as err: + LOG.error("Error formatting schema %s", err) + protobuf_schema = ( ProtobufSchema(schema_definition, references, dependencies) if not normalize @@ -188,6 +202,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protobuf_formatter: bool = False, ) -> ParsedTypedSchema: if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -213,7 +228,12 @@ def parse( elif schema_type is SchemaType.PROTOBUF: try: parsed_schema = parse_protobuf_schema_definition( - schema_str, references, dependencies, validate_references=True, normalize=normalize + schema_str, + references, + dependencies, + validate_references=True, + normalize=normalize, + use_protobuf_formatter=use_protobuf_formatter, ) except ( TypeError, @@ -282,6 +302,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protobuf_formatter: bool = False, ) -> ParsedTypedSchema: return parse( schema_type=schema_type, @@ -291,6 +312,7 @@ def parse( references=references, dependencies=dependencies, normalize=normalize, + use_protobuf_formatter=use_protobuf_formatter, ) def __str__(self) -> str: @@ -366,6 +388,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protobuf_formatter: bool = False, ) -> ValidatedTypedSchema: parsed_schema = parse( schema_type=schema_type, @@ -375,6 +398,7 @@ def parse( references=references, dependencies=dependencies, normalize=normalize, + use_protobuf_formatter=use_protobuf_formatter, ) return cast(ValidatedTypedSchema, parsed_schema) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index d3b90dac6..5a9196087 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -391,6 +391,7 @@ async def compatibility_check( schema_str=body["schema"], references=references, dependencies=new_schema_dependencies, + use_protobuf_formatter=self.config["use_protobuf_formatter"], ) except InvalidSchema: self.r( @@ -1121,6 +1122,7 @@ async def subjects_schema_post( references=references, dependencies=new_schema_dependencies, normalize=normalize, + use_protobuf_formatter=self.config["use_protobuf_formatter"], ) except InvalidSchema: self.log.warning("Invalid schema: %r", schema_str) @@ -1222,6 +1224,7 @@ async def subject_post( references=references, dependencies=resolved_dependencies, normalize=normalize, + use_protobuf_formatter=self.config["use_protobuf_formatter"], ) except (InvalidReferences, InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in index 73dfa6bad..b11b729cb 100644 --- a/requirements/requirements-dev.in +++ b/requirements/requirements-dev.in @@ -20,3 +20,6 @@ locust # Sentry SDK sentry-sdk + +# Golang +setuptools-golang diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index e9024431f..8a7f8ac58 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -6,7 +6,11 @@ # accept-types==0.4.1 # via -r requirements.txt -aiohttp==3.9.5 +aiohappyeyeballs==2.4.0 + # via + # -r requirements.txt + # aiohttp +aiohttp==3.10.5 # via -r requirements.txt aiokafka==0.10.0 # via -r requirements.txt @@ -23,7 +27,7 @@ async-timeout==4.0.3 # -r requirements.txt # aiohttp # aiokafka -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements.txt # aiohttp @@ -52,13 +56,13 @@ configargparse==1.7 # via locust confluent-kafka==2.4.0 # via -r requirements.txt -coverage[toml]==7.5.3 +coverage[toml]==7.6.1 # via pytest-cov cramjam==2.8.3 # via # -r requirements.txt # python-snappy -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via # -r requirements.txt # anyio @@ -68,7 +72,7 @@ execnet==2.1.1 # via pytest-xdist fancycompleter==0.9.1 # via pdbpp -filelock==3.14.0 +filelock==3.15.4 # via -r requirements-dev.in flask==3.0.3 # via @@ -92,17 +96,17 @@ geventhttpclient==2.0.12 # via locust greenlet==3.0.3 # via gevent -hypothesis==6.103.1 +hypothesis==6.111.2 # via -r requirements-dev.in -idna==3.7 +idna==3.8 # via # -r requirements.txt # anyio # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.4.0 # via flask -importlib-resources==6.4.0 +importlib-resources==6.4.4 # via # -r requirements.txt # jsonschema @@ -115,7 +119,7 @@ itsdangerous==2.2.0 # via flask jinja2==3.1.4 # via flask -jsonschema==4.22.0 +jsonschema==4.23.0 # via -r requirements.txt jsonschema-specifications==2023.12.1 # via @@ -146,7 +150,7 @@ multidict==6.0.5 # yarl networkx==3.1 # via -r requirements.txt -packaging==24.0 +packaging==24.1 # via # -r requirements.txt # aiokafka @@ -163,7 +167,7 @@ prometheus-client==0.20.0 # via -r requirements.txt protobuf==3.20.3 # via -r requirements.txt -psutil==5.9.8 +psutil==6.0.0 # via # -r requirements-dev.in # locust @@ -173,11 +177,11 @@ pygments==2.18.0 # -r requirements.txt # pdbpp # rich -pyjwt==2.8.0 +pyjwt==2.9.0 # via -r requirements.txt pyrepl==0.9.0 # via fancycompleter -pytest==8.2.2 +pytest==8.3.2 # via # -r requirements-dev.in # pytest-cov @@ -194,9 +198,9 @@ pytest-xdist[psutil]==3.6.1 # via -r requirements-dev.in python-dateutil==2.9.0.post0 # via -r requirements.txt -python-snappy==0.7.1 +python-snappy==0.7.2 # via -r requirements.txt -pyzmq==26.0.3 +pyzmq==26.2.0 # via locust referencing==0.35.1 # via @@ -211,12 +215,14 @@ rich==13.7.1 # via -r requirements.txt roundrobin==0.0.4 # via locust -rpds-py==0.18.1 +rpds-py==0.20.0 # via # -r requirements.txt # jsonschema # referencing -sentry-sdk==2.5.0 +sentry-sdk==2.13.0 + # via -r requirements-dev.in +setuptools-golang==2.9.0 # via -r requirements-dev.in six==1.16.0 # via @@ -237,7 +243,7 @@ tomli==2.0.1 # coverage # locust # pytest -typing-extensions==4.12.1 +typing-extensions==4.12.2 # via # -r requirements.txt # anyio @@ -248,31 +254,31 @@ urllib3==2.2.2 # via # requests # sentry-sdk -watchfiles==0.22.0 +watchfiles==0.23.0 # via -r requirements.txt -werkzeug==3.0.3 +werkzeug==3.0.4 # via # flask # flask-login # locust wmctrl==0.5 # via pdbpp -xxhash==3.4.1 +xxhash==3.5.0 # via -r requirements.txt yarl==1.9.4 # via # -r requirements.txt # aiohttp -zipp==3.19.2 +zipp==3.20.1 # via # -r requirements.txt # importlib-metadata # importlib-resources zope-event==5.0 # via gevent -zope-interface==6.4.post2 +zope-interface==7.0.2 # via gevent -zstandard==0.22.0 +zstandard==0.23.0 # via -r requirements.txt # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index 0e109ac7c..7b064f425 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -4,7 +4,7 @@ # # 'make requirements' # -attrs==23.2.0 +attrs==24.2.0 # via # -c requirements-dev.txt # -c requirements.txt @@ -13,7 +13,7 @@ certifi==2024.7.4 # via # -c requirements-dev.txt # sentry-sdk -mypy==1.11.1 +mypy==1.11.2 # via -r requirements-typing.in mypy-extensions==1.0.0 # via mypy @@ -22,12 +22,12 @@ referencing==0.35.1 # -c requirements-dev.txt # -c requirements.txt # types-jsonschema -rpds-py==0.18.1 +rpds-py==0.20.0 # via # -c requirements-dev.txt # -c requirements.txt # referencing -sentry-sdk==2.5.0 +sentry-sdk==2.13.0 # via # -c requirements-dev.txt # -r requirements-typing.in @@ -35,13 +35,13 @@ tomli==2.0.1 # via # -c requirements-dev.txt # mypy -types-cachetools==5.3.0.7 +types-cachetools==5.5.0.20240820 # via -r requirements-typing.in -types-jsonschema==4.23.0.20240712 +types-jsonschema==4.23.0.20240813 # via -r requirements-typing.in types-protobuf==3.20.4.6 # via -r requirements-typing.in -typing-extensions==4.12.1 +typing-extensions==4.12.2 # via # -c requirements-dev.txt # -c requirements.txt diff --git a/requirements/requirements.in b/requirements/requirements.in index a9401892e..fa61d5d54 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -7,9 +7,9 @@ confluent-kafka==2.4.0 isodate<1 jsonschema<5 lz4 -networkx<4 +networkx==3.1 protobuf<4 -pyjwt>=2.4.0<3 +pyjwt>=2.4.0,<3 python-dateutil<3 python-snappy rich~=13.7.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6f1a771f9..368ffb097 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -6,7 +6,9 @@ # accept-types==0.4.1 # via -r requirements.in -aiohttp==3.9.5 +aiohappyeyeballs==2.4.0 + # via aiohttp +aiohttp==3.10.5 # via -r requirements.in aiokafka==0.10.0 # via -r requirements.in @@ -18,7 +20,7 @@ async-timeout==4.0.3 # via # aiohttp # aiokafka -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema @@ -31,23 +33,23 @@ confluent-kafka==2.4.0 # via -r requirements.in cramjam==2.8.3 # via python-snappy -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via anyio frozenlist==1.4.1 # via # aiohttp # aiosignal -idna==3.7 +idna==3.8 # via # anyio # yarl -importlib-resources==6.4.0 +importlib-resources==6.4.4 # via # jsonschema # jsonschema-specifications isodate==0.6.1 # via -r requirements.in -jsonschema==4.22.0 +jsonschema==4.23.0 # via -r requirements.in jsonschema-specifications==2023.12.1 # via jsonschema @@ -63,7 +65,7 @@ multidict==6.0.5 # yarl networkx==3.1 # via -r requirements.in -packaging==24.0 +packaging==24.1 # via aiokafka pkgutil-resolve-name==1.3.10 # via jsonschema @@ -73,11 +75,11 @@ protobuf==3.20.3 # via -r requirements.in pygments==2.18.0 # via rich -pyjwt==2.8.0 +pyjwt==2.9.0 # via -r requirements.in python-dateutil==2.9.0.post0 # via -r requirements.in -python-snappy==0.7.1 +python-snappy==0.7.2 # via -r requirements.in referencing==0.35.1 # via @@ -85,7 +87,7 @@ referencing==0.35.1 # jsonschema-specifications rich==13.7.1 # via -r requirements.in -rpds-py==0.18.1 +rpds-py==0.20.0 # via # jsonschema # referencing @@ -97,20 +99,20 @@ sniffio==1.3.1 # via anyio tenacity==9.0.0 # via -r requirements.in -typing-extensions==4.12.1 +typing-extensions==4.12.2 # via # -r requirements.in # anyio # rich ujson==5.10.0 # via -r requirements.in -watchfiles==0.22.0 +watchfiles==0.23.0 # via -r requirements.in -xxhash==3.4.1 +xxhash==3.5.0 # via -r requirements.in yarl==1.9.4 # via aiohttp -zipp==3.19.2 +zipp==3.20.1 # via importlib-resources -zstandard==0.22.0 +zstandard==0.23.0 # via -r requirements.in diff --git a/setup.py b/setup.py index 3b50270d1..cd8bac10e 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ Copyright (c) 2023 Aiven Ltd See LICENSE for details """ -from setuptools import find_packages, setup +from setuptools import Extension, find_packages, setup import os import version @@ -21,6 +21,7 @@ version=version_for_setup_py, zip_safe=False, packages=find_packages(exclude=["test"]), + setup_requires=["setuptools-golang"], install_requires=[ "accept-types", "aiohttp", @@ -70,4 +71,12 @@ "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries", ], + include_package_data=True, + ext_modules=[ + Extension( + "protopacelib", + ["go/protopace/main.go"], + ), + ], + build_golang={"root": "go/protopace"}, ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 04fbd7aa1..ecc52470a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -510,8 +510,9 @@ async def fixture_registry_cluster( endpoint = RegistryEndpoint(registry.scheme, registry.hostname, registry.port) yield RegistryDescription(endpoint, "_schemas") return - + user_config = request.param.get("config", {}) if hasattr(request, "param") else {} config = {"bootstrap_uri": kafka_servers.bootstrap_servers} + config.update(user_config) async with start_schema_registry_cluster( config_templates=[config], data_dir=session_logdir / _clear_test_name(request.node.name), diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 716a03e12..9eae2b994 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -43,6 +43,7 @@ def add_slashes(text: str) -> str: # This test ProtoBuf schemas in subject registeration, compatibility of evolved version and querying the schema # w.r.t. normalization of whitespace and other minor differences to verify equality and inequality comparison of such schemas @pytest.mark.parametrize("trail", ["", "/"]) +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_schema_normalization(registry_async_client: Client, trail: str) -> None: subject = create_subject_name_factory(f"test_protobuf_schema_compatibility-{trail}")() @@ -175,6 +176,7 @@ async def test_protobuf_schema_normalization(registry_async_client: Client, trai assert evolved_id == res.json()["id"], "Check returns evolved id" +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_schema_references(registry_async_client: Client) -> None: customer_schema = """ |syntax = "proto3"; @@ -310,6 +312,7 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None assert res.status_code == 200 +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> None: no_ref = """ |syntax = "proto3"; @@ -384,6 +387,7 @@ async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> No assert "id" in res.json() +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: customer_schema = """ |syntax = "proto3"; @@ -963,6 +967,7 @@ class ReferenceTestCase(BaseTestCase): ], ids=str, ) +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) @pytest.mark.parametrize("metadata", [None, {}]) @pytest.mark.parametrize("rule_set", [None, {}]) async def test_references( @@ -1019,6 +1024,7 @@ async def test_references( assert fetch_schema_res.status_code == 200 +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_reference_update_creates_new_schema_version(registry_async_client: Client): test_schemas = [ TestCaseSchema( @@ -1079,6 +1085,7 @@ async def test_reference_update_creates_new_schema_version(registry_async_client assert res.json_result.get("id") == expected_schema_id +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_error(registry_async_client: Client) -> None: testdata = TestCaseSchema( schema_type=SchemaType.PROTOBUF, @@ -1129,6 +1136,7 @@ async def test_protobuf_error(registry_async_client: Client) -> None: assert res.status_code == 200 +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_missing_google_import(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_missing_google_import")() @@ -1149,6 +1157,7 @@ async def test_protobuf_missing_google_import(registry_async_client: Client) -> assert myjson["error_code"] == 42201 and '"google.type.PostalAddress" is not defined' in myjson["message"] +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_customer_update(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_customer_update")() @@ -1185,6 +1194,7 @@ async def test_protobuf_customer_update(registry_async_client: Client) -> None: assert res.status_code == 200 +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_binary_serialized(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_binary_serialized")() @@ -1236,6 +1246,7 @@ async def test_protobuf_binary_serialized(registry_async_client: Client) -> None assert schema_id == res.json()["id"] +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_protobuf_update_ordering(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_update_ordering")() @@ -1315,7 +1326,12 @@ async def test_protobuf_update_ordering(registry_async_client: Client) -> None: """ -async def test_registering_normalized_schema(registry_async_client: Client) -> None: +@pytest.mark.parametrize( + "registry_cluster, status", + [({"config": {}}, 404), ({"config": {"use_protobuf_formatter": True}}, 200)], + indirect=["registry_cluster"], +) +async def test_registering_normalized_schema(registry_async_client: Client, status: int) -> None: subject = create_subject_name_factory("test_protobuf_normalization")() body = {"schemaType": "PROTOBUF", "schema": SCHEMA_WITH_OPTION_ORDERED} @@ -1327,7 +1343,7 @@ async def test_registering_normalized_schema(registry_async_client: Client) -> N body = {"schemaType": "PROTOBUF", "schema": SCHEMA_WITH_OPTION_UNORDERDERED} res = await registry_async_client.post(f"subjects/{subject}", json=body) - assert res.status_code == 404 + assert res.status_code == status res = await registry_async_client.post(f"subjects/{subject}?normalize=true", json=body) @@ -1336,6 +1352,7 @@ async def test_registering_normalized_schema(registry_async_client: Client) -> N assert original_schema_id == res.json()["id"] +@pytest.mark.parametrize("registry_cluster", [{"config": {}}, {"config": {"use_protobuf_formatter": True}}], indirect=True) async def test_normalized_schema_idempotence_produce_and_fetch(registry_async_client: Client) -> None: subject = create_subject_name_factory("test_protobuf_normalization")() diff --git a/tests/unit/protobuf/test_protobuf_normalization.py b/tests/unit/protobuf/test_protobuf_normalization.py index 0f752d563..0cc6b8184 100644 --- a/tests/unit/protobuf/test_protobuf_normalization.py +++ b/tests/unit/protobuf/test_protobuf_normalization.py @@ -68,10 +68,20 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + ACTIVE = 0; } """ @@ -80,10 +90,20 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + ACTIVE = 0; } """ @@ -92,6 +112,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.ServiceOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + service MyService { option (my_option) = "my_value"; option (my_option2) = "my_value2"; @@ -104,6 +132,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.ServiceOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + service MyService { option (my_option3) = "my_value3"; option (my_option) = "my_value"; @@ -116,6 +152,18 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +message Foo { + string res = 1; +} + service MyService { rpc MyRpc (Foo) returns (Foo) { option (my_option) = "my_value"; @@ -130,6 +178,18 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +message Foo { + string res = 1; +} + service MyService { rpc MyRpc (Foo) returns (Foo) { option (my_option3) = "my_value3"; @@ -139,48 +199,26 @@ } """ -PROTO_WITH_OPTIONS_IN_EXTEND_ORDERED = """\ -syntax = "proto3"; - -package pkg; - -message Foo { - string fieldA = 1; -} - -extend Foo { - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; -} -""" - -PROTO_WITH_OPTIONS_IN_EXTEND_UNORDERED = """\ +PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED = """\ syntax = "proto3"; package pkg; -message Foo { - string fieldA = 1; -} +import "google/protobuf/descriptor.proto"; -extend Foo { - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; +extend google.protobuf.OneofOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; } -""" - -PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED = """\ -syntax = "proto3"; - -package pkg; message Foo { oneof my_oneof { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + string test = 1; } } """ @@ -190,11 +228,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.OneofOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { oneof my_oneof { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + string test = 1; } } """ @@ -204,6 +252,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -214,6 +270,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -224,6 +288,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -234,6 +306,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -244,11 +324,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { enum MyEnum { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + ACTIVE = 0; } } """ @@ -258,11 +348,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { enum MyEnum { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + ACTIVE = 0; } } """ @@ -272,6 +372,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { message Bar { string fieldA = 1 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; @@ -284,6 +392,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { message Bar { string fieldA = 1 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; @@ -297,6 +413,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0; } @@ -311,6 +435,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0; } @@ -325,6 +457,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -339,6 +479,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -353,6 +501,50 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option_EnumValue = 50002; + string my_option2_EnumValue = 50003; + string my_option3_EnumValue = 50004; +} + +extend google.protobuf.EnumOptions { + string my_option_Enum = 50002; + string my_option2_Enum = 50003; + string my_option3_Enum = 50004; +} + +extend google.protobuf.FieldOptions { + string my_option_Field = 50002; + string my_option2_Field = 50003; + string my_option3_Field = 50004; +} + +extend google.protobuf.OneofOptions { + string my_option_Oneof = 50002; + string my_option2_Oneof = 50003; + string my_option3_Oneof = 50004; +} + +extend google.protobuf.MessageOptions { + string my_option_Message = 50002; + string my_option2_Message = 50003; + string my_option3_Message = 50004; +} + +extend google.protobuf.MethodOptions { + string my_option_Method = 50002; + string my_option2_Method = 50003; + string my_option3_Method = 50004; +} + +extend google.protobuf.ServiceOptions { + string my_option_Service = 50002; + string my_option2_Service = 50003; + string my_option3_Service = 50004; +} + option cc_generic_services = true; option java_generate_equals_and_hash = true; option java_generic_services = true; @@ -373,47 +565,45 @@ message NestedFoo { string fieldA = 1; - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; + option (my_option_Message) = "my_value"; + option (my_option2_Message) = "my_value2"; } - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Message) = "my_value3"; + option (my_option2_Message) = "my_value2"; + option (my_option_Message) = "my_value"; oneof my_oneof { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Oneof) = "my_value3"; + option (my_option2_Oneof) = "my_value2"; + option (my_option_Oneof) = "my_value"; + + string test = 5; } enum MyEnum { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Enum) = "my_value3"; + option (my_option2_Enum) = "my_value2"; + option (my_option_Enum) = "my_value"; + + ACTIVE = 0; } } -extend Foo { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; -} - service MyService { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Service) = "my_value3"; + option (my_option2_Service) = "my_value2"; + option (my_option_Service) = "my_value"; rpc MyRpc (Foo) returns (Foo) { - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; + option (my_option_Method) = "my_value"; + option (my_option2_Method) = "my_value2"; + option (my_option3_Method) = "my_value3"; } } @@ -425,6 +615,50 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option_EnumValue = 50002; + string my_option2_EnumValue = 50003; + string my_option3_EnumValue = 50004; +} + +extend google.protobuf.EnumOptions { + string my_option_Enum = 50002; + string my_option2_Enum = 50003; + string my_option3_Enum = 50004; +} + +extend google.protobuf.FieldOptions { + string my_option_Field = 50002; + string my_option2_Field = 50003; + string my_option3_Field = 50004; +} + +extend google.protobuf.OneofOptions { + string my_option_Oneof = 50002; + string my_option2_Oneof = 50003; + string my_option3_Oneof = 50004; +} + +extend google.protobuf.MessageOptions { + string my_option_Message = 50002; + string my_option2_Message = 50003; + string my_option3_Message = 50004; +} + +extend google.protobuf.MethodOptions { + string my_option_Method = 50002; + string my_option2_Method = 50003; + string my_option3_Method = 50004; +} + +extend google.protobuf.ServiceOptions { + string my_option_Service = 50002; + string my_option2_Service = 50003; + string my_option3_Service = 50004; +} + option cc_generic_services = true; option java_outer_classname = "FooProto"; option optimize_for = SPEED; @@ -445,45 +679,42 @@ message NestedFoo { string fieldA = 1; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option2_Message) = "my_value2"; + option (my_option_Message) = "my_value"; } - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + option (my_option2_Message) = "my_value2"; + option (my_option3_Message) = "my_value3"; + option (my_option_Message) = "my_value"; oneof my_oneof { - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; + option (my_option_Oneof) = "my_value"; + option (my_option3_Oneof) = "my_value3"; + option (my_option2_Oneof) = "my_value2"; + + string test = 5; } enum MyEnum { - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - } -} - + option (my_option_Enum) = "my_value"; + option (my_option3_Enum) = "my_value3"; + option (my_option2_Enum) = "my_value2"; -extend Foo { - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + ACTIVE = 0; + } } service MyService { - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; + option (my_option2_Service) = "my_value2"; + option (my_option_Service) = "my_value"; + option (my_option3_Service) = "my_value3"; rpc MyRpc (Foo) returns (Foo) { - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + option (my_option2_Method) = "my_value2"; + option (my_option3_Method) = "my_value3"; + option (my_option_Method) = "my_value"; } } @@ -498,7 +729,6 @@ (PROTO_WITH_OPTIONS_IN_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_UNORDERED), (PROTO_WITH_OPTIONS_IN_SERVICE_ORDERED, PROTO_WITH_OPTIONS_IN_SERVICE_UNORDERED), (PROTO_WITH_OPTIONS_IN_RPC_ORDERED, PROTO_WITH_OPTIONS_IN_RPC_UNORDERED), - (PROTO_WITH_OPTIONS_IN_EXTEND_ORDERED, PROTO_WITH_OPTIONS_IN_EXTEND_UNORDERED), (PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED, PROTO_WITH_OPTIONS_IN_ONEOF_UNORDERED), (PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_UNORDERED), (PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_UNORDERED), @@ -537,6 +767,53 @@ def test_differently_ordered_options_normalizes_equally(ordered_schema: str, uno assert normalize(ordered_proto).to_schema() == normalize(unordered_proto).to_schema() +@pytest.mark.parametrize( + ("ordered_schema", "unordered_schema"), + ( + (PROTO_WITH_OPTIONS_ORDERED, PROTO_WITH_OPTIONS_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_UNORDERED), + (PROTO_WITH_OPTIONS_IN_SERVICE_ORDERED, PROTO_WITH_OPTIONS_IN_SERVICE_UNORDERED), + (PROTO_WITH_OPTIONS_IN_RPC_ORDERED, PROTO_WITH_OPTIONS_IN_RPC_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED, PROTO_WITH_OPTIONS_IN_ONEOF_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_UNORDERED), + (PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_UNORDERED), + (PROTO_WITH_NEASTED_ENUM_IN_MESSAGE_WITH_OPTIONS_ORDERED, PROTO_WITH_NEASTED_ENUM_IN_MESSAGE_WITH_OPTIONS_UNORDERED), + ( + PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_WITH_OPTIONS_ORDERED, + PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_WITH_OPTIONS_UNORDERED, + ), + (PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_UNORDERED), + ( + PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_WITH_OPTIONS_ORDERED, + PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_WITH_OPTIONS_UNORDERED, + ), + (PROTO_WITH_COMPLEX_SCHEMA_ORDERED, PROTO_WITH_COMPLEX_SCHEMA_UNORDERED), + ), +) +def test_differently_ordered_options_normalizes_equally_with_formatter(ordered_schema: str, unordered_schema: str) -> None: + ordered_proto = parse_protobuf_schema_definition( + schema_definition=ordered_schema, + references=None, + dependencies=None, + validate_references=True, + normalize=True, + use_protobuf_formatter=True, + ) + unordered_proto = parse_protobuf_schema_definition( + schema_definition=unordered_schema, + references=None, + dependencies=None, + validate_references=True, + normalize=True, + use_protobuf_formatter=True, + ) + + result = CompareResult() + ordered_proto.compare(unordered_proto, result) + assert result.is_compatible() + assert ordered_proto.schema == unordered_proto.schema + + DEPENDENCY = """\ syntax = "proto3"; package my.awesome.customer.v1; @@ -560,8 +837,8 @@ def test_differently_ordered_options_normalizes_equally(ordered_schema: str, uno option java_outer_classname = "EventValueProto"; option java_package = "com.my.awesome.customer.v1"; option objc_class_prefix = "TDD"; -option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; -option php_namespace = "My\\Awesome\\Customer\\V1"; +option php_metadata_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option php_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; option ruby_package = "My::Awesome::Customer::V1"; message EventValue { @@ -584,8 +861,8 @@ def test_differently_ordered_options_normalizes_equally(ordered_schema: str, uno option java_outer_classname = "EventValueProto"; option java_package = "com.my.awesome.customer.v1"; option objc_class_prefix = "TDD"; -option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; -option php_namespace = "My\\Awesome\\Customer\\V1"; +option php_metadata_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; +option php_namespace = "My\\\\Awesome\\\\Customer\\\\V1"; option ruby_package = "My::Awesome::Customer::V1"; message EventValue { @@ -667,6 +944,34 @@ def test_full_path_and_simple_names_are_equal() -> None: ), "also the string rendering shouldn't change a simple name notation protofile" +def test_full_path_and_simple_names_are_equal_with_formatter() -> None: + no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, DEPENDENCY, normalize=True) + dep = Dependency("my/awesome/customer/v1/nested_value.proto", Subject("nested_value"), Version(1), no_ref_schema) + dependencies = {"my/awesome/customer/v1/nested_value.proto": dep} + fully_qualitifed_simple_name_notation = parse_protobuf_schema_definition( + schema_definition=PROTO_WITH_SIMPLE_NAMES, + references=None, + dependencies=dependencies, + validate_references=True, + normalize=True, + use_protobuf_formatter=True, + ) + fully_qualitifed_dot_notation = parse_protobuf_schema_definition( + schema_definition=PROTO_WITH_FULLY_QUALIFIED_PATHS, + references=None, + dependencies=dependencies, + validate_references=True, + normalize=True, + use_protobuf_formatter=True, + ) + result = CompareResult() + fully_qualitifed_simple_name_notation.compare(fully_qualitifed_dot_notation, result) + assert result.is_compatible(), "normalized schemas are not compatible" + assert ( + fully_qualitifed_dot_notation.schema == fully_qualitifed_simple_name_notation.schema + ), "normalized schemas should match" + + TRICKY_DEPENDENCY = """\ syntax = "proto3"; package org.my.awesome.customer.v1; @@ -725,3 +1030,36 @@ def test_full_path_and_simple_names_are_not_equal_if_simple_name_is_not_unique() ).to_schema() assert normalized_schema == schema, "Since the simple name is not unique identifying the type isn't replacing the source" + + +def test_full_path_and_simple_names_are_not_equal_if_simple_name_is_not_unique_with_formatter() -> None: + no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, DEPENDENCY, normalize=True) + tricky_no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, TRICKY_DEPENDENCY, normalize=True) + dep = Dependency("my/awesome/customer/v1/nested_value.proto", Subject("nested_value"), Version(1), no_ref_schema) + tricky_dep = Dependency( + "org/my/awesome/customer/v1/nested_value.proto", Subject("tricky_nested_value"), Version(1), tricky_no_ref_schema + ) + dependencies = { + "my/awesome/customer/v1/nested_value.proto": dep, + "org/my/awesome/customer/v1/nested_value.proto": tricky_dep, + } + schema = parse_protobuf_schema_definition( + schema_definition=PROTO_WITH_FULLY_QUALIFIED_PATHS_AND_TRICKY_DEPENDENCY, + references=None, + dependencies=dependencies, + validate_references=True, + normalize=False, + use_protobuf_formatter=True, + ) + normalized_schema = parse_protobuf_schema_definition( + schema_definition=PROTO_WITH_FULLY_QUALIFIED_PATHS_AND_TRICKY_DEPENDENCY, + references=None, + dependencies=dependencies, + validate_references=True, + normalize=False, + use_protobuf_formatter=True, + ) + + assert ( + normalized_schema.schema == schema.schema + ), "Since the simple name is not unique identifying the type isn't replacing the source"