Merge pull request #933 from Aiven-Open/keejon/protobuf-formatter-nor…

…malization

Introduce protobuf formatter

.dockerignore

@@ -4,6 +4,7 @@
 
 # Include source directories and files required for building.
 !karapace
+!go
 !requirements/*.txt
 !setup.py
 !version.py

.github/workflows/schema.yml

@@ -21,7 +21,7 @@ jobs:
             requirements.txt
       - name: Install libsnappy-dev
         run: sudo apt install libsnappy-dev
-      - run: pip install -r requirements/requirements.txt
+      - run: make install
       # Compare with latest release when running on main.
       - run: make schema against=$(git describe --abbrev=0 --tags)
         if: github.ref == 'refs/heads/main'

.github/workflows/tests.yml

@@ -35,6 +35,11 @@ jobs:
         cache: pip
         python-version: ${{ matrix.python-version }}
 
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.21.0'
+
     - run: make install version
     - run: make unit-tests
       env:

MANIFEST.in

@@ -9,5 +9,6 @@ include setup.py
 include setup.cfg
 include LICENSE
 include MANIFEST.in
+include *.so
 
 recursive-exclude examples *~ *.pyc \.*

README.rst

@@ -475,6 +475,9 @@ Keys to take special care are the ones needed to configure Kafka and advertised_
      - ``true``
      - If enabled, kafka errors which can be retried or custom errors specififed for the service will not be raised,
        instead, a warning log is emitted. This will denoise issue tracking systems, i.e. sentry
+   * - ``use_protobuf_formatter``
+     - ``false``
+     - If protobuf formatter should be used on protobuf schemas in order to normalize schemas. The formatter is used on top and independent of regular normalization and schemas will be persisted in a formatted state.
 
 
 Authentication and authorization of Karapace Schema Registry REST API

container/Dockerfile

@@ -8,6 +8,13 @@ RUN python3 -m venv /venv
 ENV PATH="/venv/bin:$PATH"
 ENV PIP_REQUIRE_VIRTUALENV=true
 
+# Install golang needed by extensions
+ENV GO_VERSION=1.21.0
+ENV PATH="/usr/local/go/bin:${PATH}"
+RUN wget --progress=dot:giga "https://go.dev/dl/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \
+ && tar -C /usr/local -xzf "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \
+ && rm "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz"
+
 # Copy the requirements.txt and install dependencies in venv. Using a separate
 # command to use layer caching.
 #

go/protopace/.gitignore

@@ -0,0 +1,25 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env

go/protopace/Makefile

@@ -0,0 +1,69 @@
+# Change these variables as necessary.
+MAIN_PACKAGE_PATH := .
+BINARY_NAME := protopace
+BUILD_DIR := ../../karapace/protobuf/protopace/bin
+
+# ==================================================================================== #
+# HELPERS
+# ==================================================================================== #
+
+## help: print this help message
+.PHONY: help
+help:
+	@echo 'Usage:'
+	@sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' |  sed -e 's/^/ /'
+
+.PHONY: confirm
+confirm:
+	@echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ]
+
+.PHONY: no-dirty
+no-dirty:
+	git diff --exit-code
+
+
+# ==================================================================================== #
+# QUALITY CONTROL
+# ==================================================================================== #
+
+## format: format code
+.PHONY: format
+format:
+	go fmt ./...
+
+## tidy: format code and tidy modfile
+.PHONY: tidy
+tidy:
+	go fmt ./...
+	go mod tidy -v
+
+## audit: run quality control checks
+.PHONY: audit
+audit:
+	go mod verify
+	go vet ./...
+	go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./...
+	go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+	go test -race -buildvcs -vet=off ./...
+
+
+# ==================================================================================== #
+# DEVELOPMENT
+# ==================================================================================== #
+
+## test: run all tests
+.PHONY: test
+test:
+	go test -v -race -buildvcs ./...
+
+## test/cover: run all tests and display coverage
+.PHONY: test/cover
+test/cover:
+	go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./...
+	go tool cover -html=/tmp/coverage.out
+
+## build: build the application
+.PHONY: build
+build:
+	# Include additional build steps, like TypeScript, SCSS or Tailwind compilation here...
+	go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH}

go/protopace/compatibility.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"context"
+
+	"github.com/Aiven-Open/karapace/go/protopace/schema"
+
+	"github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking"
+	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
+	"github.com/bufbuild/buf/private/pkg/tracing"
+	"go.uber.org/zap"
+)
+
+func Check(schema schema.Schema, previousSchema schema.Schema) error {
+	handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer)
+	ctx := context.Background()
+	image, err := schema.CompileBufImage()
+	if err != nil {
+		return err
+	}
+	previousImage, err := previousSchema.CompileBufImage()
+	if err != nil {
+		return err
+	}
+	checkConfig, _ := bufconfig.NewEnabledCheckConfig(
+		bufconfig.FileVersionV2,
+		nil,
+		[]string{
+			"FIELD_NO_DELETE",
+			"FILE_SAME_PACKAGE",
+			"FIELD_SAME_NAME",
+			"FIELD_SAME_JSON_NAME",
+			"FILE_NO_DELETE",
+			"ENUM_NO_DELETE",
+		},
+		nil,
+		nil,
+	)
+	config := bufconfig.NewBreakingConfig(checkConfig, false)
+	return handler.Check(ctx, config, previousImage, image)
+}

go/protopace/compatibility_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	s "github.com/Aiven-Open/karapace/go/protopace/schema"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCompatibility(t *testing.T) {
+	assert := assert.New(t)
+
+	data, _ := os.ReadFile("./fixtures/dependency.proto")
+	dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil)
+	assert.NoError(err)
+	assert.NotNil(dependencySchema)
+
+	data, _ = os.ReadFile("./fixtures/test.proto")
+	testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
+	assert.NoError(err)
+	assert.NotNil(testSchema)
+
+	data, _ = os.ReadFile("./fixtures/test_previous.proto")
+	previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
+	assert.NoError(err)
+	assert.NotNil(previousSchema)
+
+	err = Check(*testSchema, *testSchema)
+	assert.NoError(err)
+
+	err = Check(*testSchema, *previousSchema)
+	assert.ErrorContains(err, "Field \"5\" with name \"foo\" on message \"EventValue\" changed type from \"string\" to \"int32\".")
+}

go/protopace/fixtures/dependency.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+package my.awesome.customer.v1;
+
+message NestedValue {
+    string value = 1;
+}
+
+enum Status {
+    UNKNOWN = 0;
+    ACTIVE = 1;
+    INACTIVE = 2;
+}

go/protopace/fixtures/test.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package my.awesome.customer.v1;
+
+import "my/awesome/customer/v1/nested_value.proto";
+import "google/protobuf/timestamp.proto";
+
+option ruby_package = "My::Awesome::Customer::V1";
+option csharp_namespace = "my.awesome.customer.V1";
+option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
+option java_multiple_files = true;
+option java_outer_classname = "EventValueProto";
+option java_package = "com.my.awesome.customer.v1";
+option objc_class_prefix = "TDD";
+option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
+option php_namespace = "My\\Awesome\\Customer\\V1";
+
+message Local {
+    message NestedValue {
+        string foo = 1;
+    }
+}
+
+message EventValue {
+  NestedValue nested_value = 1;
+  google.protobuf.Timestamp created_at = 2;
+  Status status = 3;
+  Local.NestedValue local_nested_value = 4;
+  int32 foo = 5;
+}

go/protopace/fixtures/test_previous.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package my.awesome.customer.v1;
+
+import "my/awesome/customer/v1/nested_value.proto";
+import "google/protobuf/timestamp.proto";
+
+option ruby_package = "My::Awesome::Customer::V1";
+option csharp_namespace = "my.awesome.customer.V1";
+option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
+option java_multiple_files = true;
+option java_outer_classname = "EventValueProto";
+option java_package = "com.my.awesome.customer.v1";
+option objc_class_prefix = "TDD";
+option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
+option php_namespace = "My\\Awesome\\Customer\\V1";
+
+message Local {
+    message NestedValue {
+        string foo = 1;
+    }
+}
+
+message EventValue {
+  NestedValue nested_value = 1;
+  google.protobuf.Timestamp created_at = 2;
+  Status status = 3;
+  Local.NestedValue local_nested_value = 4;
+  string foo = 5;
+}