From b6e7093d3b243530ccba3b3fe4023ff9455eb827 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Wed, 12 Jun 2024 19:07:47 +0200 Subject: [PATCH 01/21] [Schemas] Add name, version to metadata, remove ID --- opencdc/metadata.go | 40 +++++++++--- opencdc/metadata_test.go | 3 +- proto/opencdc/v1/opencdc.pb.go | 112 ++++++++++++++++++++------------- proto/opencdc/v1/opencdc.proto | 12 ++-- 4 files changed, 109 insertions(+), 58 deletions(-) diff --git a/opencdc/metadata.go b/opencdc/metadata.go index 0086bbc..398d23b 100644 --- a/opencdc/metadata.go +++ b/opencdc/metadata.go @@ -43,9 +43,12 @@ const ( // MetadataCollection is a Record.Metadata key for the name of the collection // where the record originated from and/or where it should be stored. MetadataCollection = "opencdc.collection" - // MetadataSchemaID is a Record.Metadata key for the ID of the schema of + // MetadataSchemaName is a Record.Metadata key for the name of the schema of // the record's .Payload.After field. - MetadataSchemaID = "opencdc.schema.id" + MetadataSchemaName = "opencdc.schema.name" + // MetadataSchemaVersion is a Record.Metadata key for the version of the schema of + // the record's .Payload.After field. + MetadataSchemaVersion = "opencdc.schema.version" // MetadataSchemaType is a Record.Metadata key for the type of the schema of // the record's .Payload.After field. MetadataSchemaType = "opencdc.schema.type" @@ -241,15 +244,36 @@ func (m Metadata) SetConduitDLQNackNodeID(id string) { m[MetadataConduitDLQNackNodeID] = id } -// GetSchemaID returns the value for key MetadataSchemaID. +// GetSchemaName returns the value for key MetadataSchemaName. +// If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. +func (m Metadata) GetSchemaName() (string, error) { + return m.getValue(MetadataSchemaName) +} + +// SetSchemaName sets the metadata value for key MetadataSchemaName. +func (m Metadata) SetSchemaName(name string) { + m[MetadataSchemaName] = name +} + +// GetSchemaVersion returns the value for key MetadataSchemaVersion. // If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetSchemaID() (string, error) { - return m.getValue(MetadataSchemaID) +func (m Metadata) GetSchemaVersion() (int, error) { + vs, err := m.getValue(MetadataSchemaVersion) + if err != nil { + return 0, err + } + + v, err := strconv.Atoi(vs) + if err != nil { + return 0, fmt.Errorf("invalid version %q: %w", vs, err) + } + + return v, nil } -// SetSchemaID sets the metadata value for key MetadataSchemaID. -func (m Metadata) SetSchemaID(id string) { - m[MetadataSchemaID] = id +// SetSchemaVersion sets the metadata value for key MetadataSchemaVersion. +func (m Metadata) SetSchemaVersion(version int) { + m[MetadataSchemaVersion] = strconv.Itoa(version) } // GetSchemaType returns the value for key MetadataSchemaType. diff --git a/opencdc/metadata_test.go b/opencdc/metadata_test.go index ababe79..ceef5e9 100644 --- a/opencdc/metadata_test.go +++ b/opencdc/metadata_test.go @@ -30,7 +30,8 @@ func TestMetadataConstants(t *testing.T) { MetadataCreatedAt: opencdcv1.E_MetadataCreatedAt, MetadataReadAt: opencdcv1.E_MetadataReadAt, MetadataCollection: opencdcv1.E_MetadataCollection, - MetadataSchemaID: opencdcv1.E_MetadataSchemaId, + MetadataSchemaName: opencdcv1.E_MetadataSchemaName, + MetadataSchemaVersion: opencdcv1.E_MetadataSchemaVersion, MetadataSchemaType: opencdcv1.E_MetadataSchemaType, MetadataConduitSourcePluginName: metadatav1.E_MetadataConduitSourcePluginName, diff --git a/proto/opencdc/v1/opencdc.pb.go b/proto/opencdc/v1/opencdc.pb.go index 601126b..dd104f2 100644 --- a/proto/opencdc/v1/opencdc.pb.go +++ b/proto/opencdc/v1/opencdc.pb.go @@ -367,16 +367,24 @@ var file_opencdc_v1_opencdc_proto_extTypes = []protoimpl.ExtensionInfo{ ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), Field: 10004, - Name: "opencdc.v1.metadata_schema_id", - Tag: "bytes,10004,opt,name=metadata_schema_id", + Name: "opencdc.v1.metadata_schema_name", + Tag: "bytes,10004,opt,name=metadata_schema_name", Filename: "opencdc/v1/opencdc.proto", }, { ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), Field: 10005, + Name: "opencdc.v1.metadata_schema_version", + Tag: "bytes,10005,opt,name=metadata_schema_version", + Filename: "opencdc/v1/opencdc.proto", + }, + { + ExtendedType: (*descriptorpb.FileOptions)(nil), + ExtensionType: (*string)(nil), + Field: 10006, Name: "opencdc.v1.metadata_schema_type", - Tag: "bytes,10005,opt,name=metadata_schema_type", + Tag: "bytes,10006,opt,name=metadata_schema_type", Filename: "opencdc/v1/opencdc.proto", }, } @@ -414,16 +422,21 @@ var ( // // optional string metadata_collection = 10003; E_MetadataCollection = &file_opencdc_v1_opencdc_proto_extTypes[4] - // Metadata field "opencdc.schema.id" contains the ID of the schema of + // Metadata field "opencdc.schema.name" contains the name of the schema of + // the record's .Payload.After field. + // + // optional string metadata_schema_name = 10004; + E_MetadataSchemaName = &file_opencdc_v1_opencdc_proto_extTypes[5] + // Metadata field "opencdc.schema.version" contains the version of the schema of // the record's .Payload.After field. // - // optional string metadata_schema_id = 10004; - E_MetadataSchemaId = &file_opencdc_v1_opencdc_proto_extTypes[5] + // optional string metadata_schema_version = 10005; + E_MetadataSchemaVersion = &file_opencdc_v1_opencdc_proto_extTypes[6] // Metadata field "opencdc.schema.type" contains the type of the schema of // the record's .Payload.After field (currently, the only supported value is "avro") // - // optional string metadata_schema_type = 10005; - E_MetadataSchemaType = &file_opencdc_v1_opencdc_proto_extTypes[6] + // optional string metadata_schema_type = 10006; + E_MetadataSchemaType = &file_opencdc_v1_opencdc_proto_extTypes[7] ) var File_opencdc_v1_opencdc_proto protoreflect.FileDescriptor @@ -498,36 +511,44 @@ var file_opencdc_v1_opencdc_proto_rawDesc = []byte{ 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x93, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x43, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x4b, 0x0a, 0x12, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x69, 0x64, 0x12, - 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x94, 0x4e, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x49, 0x64, 0x3a, 0x4f, 0x0a, 0x14, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x95, 0x4e, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0xaa, 0x02, 0xfa, 0xf0, 0x04, 0x02, 0x76, 0x31, - 0x82, 0xf1, 0x04, 0x0f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x8a, 0xf1, 0x04, 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x92, 0xf1, 0x04, 0x0e, 0x6f, 0x70, 0x65, 0x6e, - 0x63, 0x64, 0x63, 0x2e, 0x72, 0x65, 0x61, 0x64, 0x41, 0x74, 0x9a, 0xf1, 0x04, 0x12, 0x6f, 0x70, - 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0xa2, 0xf1, 0x04, 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x2e, 0x69, 0x64, 0xaa, 0xf1, 0x04, 0x13, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x0a, 0x0e, 0x63, 0x6f, - 0x6d, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, - 0x65, 0x6e, 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, - 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, - 0x2f, 0x76, 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, - 0x4f, 0x58, 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, - 0xca, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, - 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, - 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x4f, 0x0a, 0x14, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x94, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x4e, 0x61, 0x6d, 0x65, 0x3a, 0x55, 0x0a, 0x17, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x95, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x3a, 0x4f, 0x0a, 0x14, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x96, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x42, 0xc6, 0x02, 0xfa, 0xf0, 0x04, 0x02, 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, + 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, + 0x04, 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x64, 0x41, 0x74, 0x92, 0xf1, 0x04, 0x0e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x72, + 0x65, 0x61, 0x64, 0x41, 0x74, 0x9a, 0xf1, 0x04, 0x12, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, + 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x13, 0x6f, + 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, + 0x6d, 0x65, 0xaa, 0xf1, 0x04, 0x16, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xb2, 0xf1, 0x04, 0x13, + 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, + 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, + 0x74, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, + 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, + 0x64, 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, + 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, + 0x63, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, + 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, + 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -566,12 +587,13 @@ var file_opencdc_v1_opencdc_proto_depIdxs = []int32{ 6, // 9: opencdc.v1.metadata_created_at:extendee -> google.protobuf.FileOptions 6, // 10: opencdc.v1.metadata_read_at:extendee -> google.protobuf.FileOptions 6, // 11: opencdc.v1.metadata_collection:extendee -> google.protobuf.FileOptions - 6, // 12: opencdc.v1.metadata_schema_id:extendee -> google.protobuf.FileOptions - 6, // 13: opencdc.v1.metadata_schema_type:extendee -> google.protobuf.FileOptions - 14, // [14:14] is the sub-list for method output_type - 14, // [14:14] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 7, // [7:14] is the sub-list for extension extendee + 6, // 12: opencdc.v1.metadata_schema_name:extendee -> google.protobuf.FileOptions + 6, // 13: opencdc.v1.metadata_schema_version:extendee -> google.protobuf.FileOptions + 6, // 14: opencdc.v1.metadata_schema_type:extendee -> google.protobuf.FileOptions + 15, // [15:15] is the sub-list for method output_type + 15, // [15:15] is the sub-list for method input_type + 15, // [15:15] is the sub-list for extension type_name + 7, // [7:15] is the sub-list for extension extendee 0, // [0:7] is the sub-list for field type_name } @@ -629,7 +651,7 @@ func file_opencdc_v1_opencdc_proto_init() { RawDescriptor: file_opencdc_v1_opencdc_proto_rawDesc, NumEnums: 1, NumMessages: 4, - NumExtensions: 7, + NumExtensions: 8, NumServices: 0, }, GoTypes: file_opencdc_v1_opencdc_proto_goTypes, diff --git a/proto/opencdc/v1/opencdc.proto b/proto/opencdc/v1/opencdc.proto index a5e234d..2b19cc1 100644 --- a/proto/opencdc/v1/opencdc.proto +++ b/proto/opencdc/v1/opencdc.proto @@ -13,7 +13,8 @@ option (metadata_created_at) = "opencdc.createdAt"; option (metadata_read_at) = "opencdc.readAt"; option (metadata_version) = "opencdc.version"; option (metadata_collection) = "opencdc.collection"; -option (metadata_schema_id) = "opencdc.schema.id"; +option (metadata_schema_name) = "opencdc.schema.name"; +option (metadata_schema_version) = "opencdc.schema.version"; option (metadata_schema_type) = "opencdc.schema.type"; // We are (ab)using custom file options to define constants. @@ -41,12 +42,15 @@ extend google.protobuf.FileOptions { // Metadata field "opencdc.collection" can contain the name of the collection // where the record originated from and/or where it should be stored. string metadata_collection = 10003; - // Metadata field "opencdc.schema.id" contains the ID of the schema of + // Metadata field "opencdc.schema.name" contains the name of the schema of // the record's .Payload.After field. - string metadata_schema_id = 10004; + string metadata_schema_name = 10004; + // Metadata field "opencdc.schema.version" contains the version of the schema of + // the record's .Payload.After field. + string metadata_schema_version = 10005; // Metadata field "opencdc.schema.type" contains the type of the schema of // the record's .Payload.After field (currently, the only supported value is "avro") - string metadata_schema_type = 10005; + string metadata_schema_type = 10006; } // Operation defines what triggered the creation of a record. From 061edf8e3a04e5befbc52d1671b09d55294a5255 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Thu, 13 Jun 2024 11:50:04 +0200 Subject: [PATCH 02/21] attach schema to record --- opencdc/errors.go | 4 +++- opencdc/metadata.go | 19 +++++++++++++++---- opencdc/record.go | 7 +++++++ schema/instance.go | 4 +++- schema/type_string.go | 24 ++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 schema/type_string.go diff --git a/opencdc/errors.go b/opencdc/errors.go index 4b47984..2defe4a 100644 --- a/opencdc/errors.go +++ b/opencdc/errors.go @@ -25,7 +25,9 @@ var ( // ErrUnknownOperation is returned when trying to parse an Operation string // and encountering an unknown operation. ErrUnknownOperation = errors.New("unknown operation") - + // ErrUnknownSchemaType is returned when trying to parse an Operation string + // and encountering an unknown operation. + ErrUnknownSchemaType = errors.New("unknown schema type") // ErrInvalidProtoDataType is returned when trying to convert a proto data // type to raw or structured data. ErrInvalidProtoDataType = errors.New("invalid proto data type") diff --git a/opencdc/metadata.go b/opencdc/metadata.go index 398d23b..0e17ddc 100644 --- a/opencdc/metadata.go +++ b/opencdc/metadata.go @@ -16,6 +16,7 @@ package opencdc import ( "fmt" + "github.com/conduitio/conduit-commons/schema" "strconv" "time" ) @@ -278,13 +279,23 @@ func (m Metadata) SetSchemaVersion(version int) { // GetSchemaType returns the value for key MetadataSchemaType. // If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetSchemaType() (string, error) { - return m.getValue(MetadataSchemaType) +func (m Metadata) GetSchemaType() (schema.Type, error) { + typeString, err := m.getValue(MetadataSchemaType) + if err != nil { + return 0, err + } + + switch typeString { + case schema.TypeAvro.String(): + return schema.TypeAvro, nil + default: + return 0, fmt.Errorf("%q: %w", typeString, ErrUnknownSchemaType) + } } // SetSchemaType sets the metadata value for key MetadataSchemaType. -func (m Metadata) SetSchemaType(typeStr string) { - m[MetadataSchemaType] = typeStr +func (m Metadata) SetSchemaType(t schema.Type) { + m[MetadataSchemaType] = t.String() } // getValue returns the value for a specific key. If the value does not exist or diff --git a/opencdc/record.go b/opencdc/record.go index cc7b8a0..3785a98 100644 --- a/opencdc/record.go +++ b/opencdc/record.go @@ -17,6 +17,7 @@ package opencdc import ( "bytes" "fmt" + "github.com/conduitio/conduit-commons/schema" "github.com/goccy/go-json" ) @@ -147,3 +148,9 @@ func (r Record) Clone() Record { } return clone } + +func (r Record) AttachSchema(s schema.Instance) { + r.Metadata.SetSchemaType(schema.TypeAvro) + r.Metadata.SetSchemaName(s.Name) + r.Metadata.SetSchemaVersion(s.Version) +} diff --git a/schema/instance.go b/schema/instance.go index c35b33f..f766039 100644 --- a/schema/instance.go +++ b/schema/instance.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:generate stringer -type=Type -linecomment + package schema type Type int32 @@ -23,7 +25,7 @@ const ( type Instance struct { ID string Name string - Version int32 + Version int Type Type Bytes []byte } diff --git a/schema/type_string.go b/schema/type_string.go new file mode 100644 index 0000000..6429cb5 --- /dev/null +++ b/schema/type_string.go @@ -0,0 +1,24 @@ +// Code generated by "stringer -type=Type -linecomment"; DO NOT EDIT. + +package schema + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[TypeAvro-1] +} + +const _Type_name = "TypeAvro" + +var _Type_index = [...]uint8{0, 8} + +func (i Type) String() string { + i -= 1 + if i < 0 || i >= Type(len(_Type_index)-1) { + return "Type(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _Type_name[_Type_index[i]:_Type_index[i+1]] +} From 1322ada9f180b491fe5e92546a652e2d2a57636f Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Thu, 20 Jun 2024 12:52:08 +0200 Subject: [PATCH 03/21] move function --- opencdc/record.go | 8 -------- schema/record.go | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 schema/record.go diff --git a/opencdc/record.go b/opencdc/record.go index 3785a98..0bf7cf3 100644 --- a/opencdc/record.go +++ b/opencdc/record.go @@ -17,8 +17,6 @@ package opencdc import ( "bytes" "fmt" - "github.com/conduitio/conduit-commons/schema" - "github.com/goccy/go-json" ) @@ -148,9 +146,3 @@ func (r Record) Clone() Record { } return clone } - -func (r Record) AttachSchema(s schema.Instance) { - r.Metadata.SetSchemaType(schema.TypeAvro) - r.Metadata.SetSchemaName(s.Name) - r.Metadata.SetSchemaVersion(s.Version) -} diff --git a/schema/record.go b/schema/record.go new file mode 100644 index 0000000..9198017 --- /dev/null +++ b/schema/record.go @@ -0,0 +1,23 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import "github.com/conduitio/conduit-commons/opencdc" + +func AttachToRecord(r opencdc.Record, s Instance) { + r.Metadata.SetSchemaType(TypeAvro) + r.Metadata.SetSchemaName(s.Name) + r.Metadata.SetSchemaVersion(s.Version) +} From a603d71499e096301ad70c8977da89fa6236dd65 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Thu, 20 Jun 2024 14:06:16 +0200 Subject: [PATCH 04/21] move function --- opencdc/metadata.go | 19 ++++--------------- opencdc/record.go | 1 + schema/record.go | 2 +- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/opencdc/metadata.go b/opencdc/metadata.go index 0e17ddc..062562a 100644 --- a/opencdc/metadata.go +++ b/opencdc/metadata.go @@ -16,7 +16,6 @@ package opencdc import ( "fmt" - "github.com/conduitio/conduit-commons/schema" "strconv" "time" ) @@ -279,23 +278,13 @@ func (m Metadata) SetSchemaVersion(version int) { // GetSchemaType returns the value for key MetadataSchemaType. // If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetSchemaType() (schema.Type, error) { - typeString, err := m.getValue(MetadataSchemaType) - if err != nil { - return 0, err - } - - switch typeString { - case schema.TypeAvro.String(): - return schema.TypeAvro, nil - default: - return 0, fmt.Errorf("%q: %w", typeString, ErrUnknownSchemaType) - } +func (m Metadata) GetSchemaType() (string, error) { + return m.getValue(MetadataSchemaType) } // SetSchemaType sets the metadata value for key MetadataSchemaType. -func (m Metadata) SetSchemaType(t schema.Type) { - m[MetadataSchemaType] = t.String() +func (m Metadata) SetSchemaType(t string) { + m[MetadataSchemaType] = t } // getValue returns the value for a specific key. If the value does not exist or diff --git a/opencdc/record.go b/opencdc/record.go index 0bf7cf3..cc7b8a0 100644 --- a/opencdc/record.go +++ b/opencdc/record.go @@ -17,6 +17,7 @@ package opencdc import ( "bytes" "fmt" + "github.com/goccy/go-json" ) diff --git a/schema/record.go b/schema/record.go index 9198017..a025b0e 100644 --- a/schema/record.go +++ b/schema/record.go @@ -17,7 +17,7 @@ package schema import "github.com/conduitio/conduit-commons/opencdc" func AttachToRecord(r opencdc.Record, s Instance) { - r.Metadata.SetSchemaType(TypeAvro) + r.Metadata.SetSchemaType(TypeAvro.String()) r.Metadata.SetSchemaName(s.Name) r.Metadata.SetSchemaVersion(s.Version) } From d77306383c6f6e6612e1b034c99d0cd69357f0c6 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Thu, 20 Jun 2024 14:29:39 +0200 Subject: [PATCH 05/21] key and payload schema --- opencdc/metadata.go | 108 +++++++++++++----- opencdc/metadata_test.go | 19 ++-- proto/opencdc/v1/opencdc.pb.go | 194 ++++++++++++++++++++++----------- proto/opencdc/v1/opencdc.proto | 36 ++++-- schema/record.go | 12 +- 5 files changed, 255 insertions(+), 114 deletions(-) diff --git a/opencdc/metadata.go b/opencdc/metadata.go index 062562a..fcceda9 100644 --- a/opencdc/metadata.go +++ b/opencdc/metadata.go @@ -43,15 +43,26 @@ const ( // MetadataCollection is a Record.Metadata key for the name of the collection // where the record originated from and/or where it should be stored. MetadataCollection = "opencdc.collection" - // MetadataSchemaName is a Record.Metadata key for the name of the schema of - // the record's .Payload.After field. - MetadataSchemaName = "opencdc.schema.name" - // MetadataSchemaVersion is a Record.Metadata key for the version of the schema of - // the record's .Payload.After field. - MetadataSchemaVersion = "opencdc.schema.version" - // MetadataSchemaType is a Record.Metadata key for the type of the schema of - // the record's .Payload.After field. - MetadataSchemaType = "opencdc.schema.type" + + // MetadataKeySchemaName is a Record.Metadata key for the name of the schema of + // the record's .Key field. + MetadataKeySchemaName = "opencdc.key.schema.name" + // MetadataKeySchemaVersion is a Record.Metadata key for the version of the schema of + // the record's .Key field. + MetadataKeySchemaVersion = "opencdc.key.schema.version" + // MetadataKeySchemaType is a Record.Metadata key for the type of the schema of + // the record's .Key field. + MetadataKeySchemaType = "opencdc.key.schema.type" + + // MetadataPayloadSchemaName is a Record.Metadata key for the name of the schema of + // the record's .Payload field. + MetadataPayloadSchemaName = "opencdc.payload.schema.name" + // MetadataPayloadSchemaVersion is a Record.Metadata key for the version of the schema of + // the record's .Payload field. + MetadataPayloadSchemaVersion = "opencdc.payload.schema.version" + // MetadataPayloadSchemaType is a Record.Metadata key for the type of the schema of + // the record's .Payload field. + MetadataPayloadSchemaType = "opencdc.payload.schema.type" // MetadataConduitSourcePluginName is a Record.Metadata key for the name of // the source plugin that created this record. @@ -244,21 +255,64 @@ func (m Metadata) SetConduitDLQNackNodeID(id string) { m[MetadataConduitDLQNackNodeID] = id } -// GetSchemaName returns the value for key MetadataSchemaName. +// GetKeySchemaName returns the value for key MetadataKeySchemaName. +// If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. +func (m Metadata) GetKeySchemaName() (string, error) { + return m.getValue(MetadataKeySchemaName) +} + +// SetKeySchemaName sets the metadata value for key MetadataKeySchemaName. +func (m Metadata) SetKeySchemaName(name string) { + m[MetadataKeySchemaName] = name +} + +// GetKeySchemaVersion returns the value for key MetadataKeySchemaVersion. +// If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. +func (m Metadata) GetKeySchemaVersion() (int, error) { + vs, err := m.getValue(MetadataKeySchemaVersion) + if err != nil { + return 0, err + } + + v, err := strconv.Atoi(vs) + if err != nil { + return 0, fmt.Errorf("invalid version %q: %w", vs, err) + } + + return v, nil +} + +// SetKeySchemaVersion sets the metadata value for key MetadataKeySchemaVersion. +func (m Metadata) SetKeySchemaVersion(version int) { + m[MetadataKeySchemaVersion] = strconv.Itoa(version) +} + +// GetKeySchemaType returns the value for key MetadataKeySchemaType. +// If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. +func (m Metadata) GetKeySchemaType() (string, error) { + return m.getValue(MetadataKeySchemaType) +} + +// SetKeySchemaType sets the metadata value for key MetadataKeySchemaType. +func (m Metadata) SetKeySchemaType(t string) { + m[MetadataKeySchemaType] = t +} + +// GetPayloadSchemaName returns the value for key MetadataPayloadSchemaName. // If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetSchemaName() (string, error) { - return m.getValue(MetadataSchemaName) +func (m Metadata) GetPayloadSchemaName() (string, error) { + return m.getValue(MetadataPayloadSchemaName) } -// SetSchemaName sets the metadata value for key MetadataSchemaName. -func (m Metadata) SetSchemaName(name string) { - m[MetadataSchemaName] = name +// SetPayloadSchemaName sets the metadata value for key MetadataPayloadSchemaName. +func (m Metadata) SetPayloadSchemaName(name string) { + m[MetadataPayloadSchemaName] = name } -// GetSchemaVersion returns the value for key MetadataSchemaVersion. +// GetPayloadSchemaVersion returns the value for key MetadataPayloadSchemaVersion. // If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetSchemaVersion() (int, error) { - vs, err := m.getValue(MetadataSchemaVersion) +func (m Metadata) GetPayloadSchemaVersion() (int, error) { + vs, err := m.getValue(MetadataPayloadSchemaVersion) if err != nil { return 0, err } @@ -271,20 +325,20 @@ func (m Metadata) GetSchemaVersion() (int, error) { return v, nil } -// SetSchemaVersion sets the metadata value for key MetadataSchemaVersion. -func (m Metadata) SetSchemaVersion(version int) { - m[MetadataSchemaVersion] = strconv.Itoa(version) +// SetPayloadSchemaVersion sets the metadata value for key MetadataPayloadSchemaVersion. +func (m Metadata) SetPayloadSchemaVersion(version int) { + m[MetadataPayloadSchemaVersion] = strconv.Itoa(version) } -// GetSchemaType returns the value for key MetadataSchemaType. +// GetPayloadSchemaType returns the value for key MetadataPayloadSchemaType. // If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetSchemaType() (string, error) { - return m.getValue(MetadataSchemaType) +func (m Metadata) GetPayloadSchemaType() (string, error) { + return m.getValue(MetadataPayloadSchemaType) } -// SetSchemaType sets the metadata value for key MetadataSchemaType. -func (m Metadata) SetSchemaType(t string) { - m[MetadataSchemaType] = t +// SetPayloadSchemaType sets the metadata value for key MetadataPayloadSchemaType. +func (m Metadata) SetPayloadSchemaType(t string) { + m[MetadataPayloadSchemaType] = t } // getValue returns the value for a specific key. If the value does not exist or diff --git a/opencdc/metadata_test.go b/opencdc/metadata_test.go index ceef5e9..b2084c8 100644 --- a/opencdc/metadata_test.go +++ b/opencdc/metadata_test.go @@ -25,14 +25,17 @@ import ( func TestMetadataConstants(t *testing.T) { wantMapping := map[string]*protoimpl.ExtensionInfo{ - OpenCDCVersion: opencdcv1.E_OpencdcVersion, - MetadataOpenCDCVersion: opencdcv1.E_MetadataVersion, - MetadataCreatedAt: opencdcv1.E_MetadataCreatedAt, - MetadataReadAt: opencdcv1.E_MetadataReadAt, - MetadataCollection: opencdcv1.E_MetadataCollection, - MetadataSchemaName: opencdcv1.E_MetadataSchemaName, - MetadataSchemaVersion: opencdcv1.E_MetadataSchemaVersion, - MetadataSchemaType: opencdcv1.E_MetadataSchemaType, + OpenCDCVersion: opencdcv1.E_OpencdcVersion, + MetadataOpenCDCVersion: opencdcv1.E_MetadataVersion, + MetadataCreatedAt: opencdcv1.E_MetadataCreatedAt, + MetadataReadAt: opencdcv1.E_MetadataReadAt, + MetadataCollection: opencdcv1.E_MetadataCollection, + MetadataKeySchemaName: opencdcv1.E_MetadataKeySchemaName, + MetadataKeySchemaVersion: opencdcv1.E_MetadataKeySchemaVersion, + MetadataKeySchemaType: opencdcv1.E_MetadataKeySchemaType, + MetadataPayloadSchemaName: opencdcv1.E_MetadataPayloadSchemaName, + MetadataPayloadSchemaVersion: opencdcv1.E_MetadataPayloadSchemaVersion, + MetadataPayloadSchemaType: opencdcv1.E_MetadataPayloadSchemaType, MetadataConduitSourcePluginName: metadatav1.E_MetadataConduitSourcePluginName, MetadataConduitSourcePluginVersion: metadatav1.E_MetadataConduitSourcePluginVersion, diff --git a/proto/opencdc/v1/opencdc.pb.go b/proto/opencdc/v1/opencdc.pb.go index dd104f2..06914f1 100644 --- a/proto/opencdc/v1/opencdc.pb.go +++ b/proto/opencdc/v1/opencdc.pb.go @@ -367,24 +367,48 @@ var file_opencdc_v1_opencdc_proto_extTypes = []protoimpl.ExtensionInfo{ ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), Field: 10004, - Name: "opencdc.v1.metadata_schema_name", - Tag: "bytes,10004,opt,name=metadata_schema_name", + Name: "opencdc.v1.metadata_key_schema_name", + Tag: "bytes,10004,opt,name=metadata_key_schema_name", Filename: "opencdc/v1/opencdc.proto", }, { ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), Field: 10005, - Name: "opencdc.v1.metadata_schema_version", - Tag: "bytes,10005,opt,name=metadata_schema_version", + Name: "opencdc.v1.metadata_key_schema_version", + Tag: "bytes,10005,opt,name=metadata_key_schema_version", Filename: "opencdc/v1/opencdc.proto", }, { ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), Field: 10006, - Name: "opencdc.v1.metadata_schema_type", - Tag: "bytes,10006,opt,name=metadata_schema_type", + Name: "opencdc.v1.metadata_key_schema_type", + Tag: "bytes,10006,opt,name=metadata_key_schema_type", + Filename: "opencdc/v1/opencdc.proto", + }, + { + ExtendedType: (*descriptorpb.FileOptions)(nil), + ExtensionType: (*string)(nil), + Field: 10007, + Name: "opencdc.v1.metadata_payload_schema_name", + Tag: "bytes,10007,opt,name=metadata_payload_schema_name", + Filename: "opencdc/v1/opencdc.proto", + }, + { + ExtendedType: (*descriptorpb.FileOptions)(nil), + ExtensionType: (*string)(nil), + Field: 10008, + Name: "opencdc.v1.metadata_payload_schema_version", + Tag: "bytes,10008,opt,name=metadata_payload_schema_version", + Filename: "opencdc/v1/opencdc.proto", + }, + { + ExtendedType: (*descriptorpb.FileOptions)(nil), + ExtensionType: (*string)(nil), + Field: 10009, + Name: "opencdc.v1.metadata_payload_schema_type", + Tag: "bytes,10009,opt,name=metadata_payload_schema_type", Filename: "opencdc/v1/opencdc.proto", }, } @@ -422,21 +446,36 @@ var ( // // optional string metadata_collection = 10003; E_MetadataCollection = &file_opencdc_v1_opencdc_proto_extTypes[4] - // Metadata field "opencdc.schema.name" contains the name of the schema of - // the record's .Payload.After field. + // Metadata field "opencdc.key.schema.name" contains the name of the schema of + // the record's .Key field. // - // optional string metadata_schema_name = 10004; - E_MetadataSchemaName = &file_opencdc_v1_opencdc_proto_extTypes[5] - // Metadata field "opencdc.schema.version" contains the version of the schema of - // the record's .Payload.After field. + // optional string metadata_key_schema_name = 10004; + E_MetadataKeySchemaName = &file_opencdc_v1_opencdc_proto_extTypes[5] + // Metadata field "opencdc.key.schema.version" contains the version of the schema of + // the record's .Key field. // - // optional string metadata_schema_version = 10005; - E_MetadataSchemaVersion = &file_opencdc_v1_opencdc_proto_extTypes[6] - // Metadata field "opencdc.schema.type" contains the type of the schema of - // the record's .Payload.After field (currently, the only supported value is "avro") + // optional string metadata_key_schema_version = 10005; + E_MetadataKeySchemaVersion = &file_opencdc_v1_opencdc_proto_extTypes[6] + // Metadata field "opencdc.key.schema.type" contains the type of the schema of + // the record's .Key field (currently, the only supported value is "avro") // - // optional string metadata_schema_type = 10006; - E_MetadataSchemaType = &file_opencdc_v1_opencdc_proto_extTypes[7] + // optional string metadata_key_schema_type = 10006; + E_MetadataKeySchemaType = &file_opencdc_v1_opencdc_proto_extTypes[7] + // Metadata field "opencdc.payload.schema.name" contains the name of the schema of + // the record's .Payload fields. + // + // optional string metadata_payload_schema_name = 10007; + E_MetadataPayloadSchemaName = &file_opencdc_v1_opencdc_proto_extTypes[8] + // Metadata field "opencdc.payload.schema.version" contains the version of the schema of + // the record's .Payload fields. + // + // optional string metadata_payload_schema_version = 10008; + E_MetadataPayloadSchemaVersion = &file_opencdc_v1_opencdc_proto_extTypes[9] + // Metadata field "opencdc.payload.schema.type" contains the type of the schema of + // the record's .Payload fields (currently, the only supported value is "avro") + // + // optional string metadata_payload_schema_type = 10009; + E_MetadataPayloadSchemaType = &file_opencdc_v1_opencdc_proto_extTypes[10] ) var File_opencdc_v1_opencdc_proto protoreflect.FileDescriptor @@ -511,44 +550,70 @@ var file_opencdc_v1_opencdc_proto_rawDesc = []byte{ 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x93, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x43, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x4f, 0x0a, 0x14, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x94, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x4e, 0x61, 0x6d, 0x65, 0x3a, 0x55, 0x0a, 0x17, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x56, 0x0a, 0x18, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x95, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x3a, 0x4f, 0x0a, 0x14, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x96, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, - 0x70, 0x65, 0x42, 0xc6, 0x02, 0xfa, 0xf0, 0x04, 0x02, 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, - 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, - 0x04, 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x64, 0x41, 0x74, 0x92, 0xf1, 0x04, 0x0e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x72, - 0x65, 0x61, 0x64, 0x41, 0x74, 0x9a, 0xf1, 0x04, 0x12, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, - 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x13, 0x6f, - 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, - 0x6d, 0x65, 0xaa, 0xf1, 0x04, 0x16, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xb2, 0xf1, 0x04, 0x13, - 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, - 0x79, 0x70, 0x65, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, - 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, - 0x74, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, - 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, - 0x64, 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, - 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, - 0x63, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, - 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, - 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x6f, 0x6e, 0x73, 0x18, 0x94, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x4e, 0x61, 0x6d, + 0x65, 0x3a, 0x5c, 0x0a, 0x1b, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6b, 0x65, + 0x79, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x95, + 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4b, + 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, + 0x56, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x5f, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, + 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x96, 0x4e, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x15, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x5e, 0x0a, 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x97, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x4e, 0x61, 0x6d, 0x65, 0x3a, 0x64, 0x0a, 0x1f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, + 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x98, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x5e, 0x0a, + 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x99, 0x4e, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x19, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0xb2, 0x03, + 0xfa, 0xf0, 0x04, 0x02, 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, + 0x63, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, 0x04, 0x11, 0x6f, 0x70, 0x65, + 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x92, 0xf1, + 0x04, 0x0e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x72, 0x65, 0x61, 0x64, 0x41, 0x74, + 0x9a, 0xf1, 0x04, 0x12, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x17, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, + 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, + 0x65, 0xaa, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, + 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xb2, + 0xf1, 0x04, 0x17, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0xba, 0xf1, 0x04, 0x1b, 0x6f, 0x70, + 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0xc2, 0xf1, 0x04, 0x1e, 0x6f, 0x70, 0x65, + 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xca, 0xf1, 0x04, 0x1b, 0x6f, + 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, + 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, + 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, + 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, + 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, + 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, + 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, + 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, + 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -587,13 +652,16 @@ var file_opencdc_v1_opencdc_proto_depIdxs = []int32{ 6, // 9: opencdc.v1.metadata_created_at:extendee -> google.protobuf.FileOptions 6, // 10: opencdc.v1.metadata_read_at:extendee -> google.protobuf.FileOptions 6, // 11: opencdc.v1.metadata_collection:extendee -> google.protobuf.FileOptions - 6, // 12: opencdc.v1.metadata_schema_name:extendee -> google.protobuf.FileOptions - 6, // 13: opencdc.v1.metadata_schema_version:extendee -> google.protobuf.FileOptions - 6, // 14: opencdc.v1.metadata_schema_type:extendee -> google.protobuf.FileOptions - 15, // [15:15] is the sub-list for method output_type - 15, // [15:15] is the sub-list for method input_type - 15, // [15:15] is the sub-list for extension type_name - 7, // [7:15] is the sub-list for extension extendee + 6, // 12: opencdc.v1.metadata_key_schema_name:extendee -> google.protobuf.FileOptions + 6, // 13: opencdc.v1.metadata_key_schema_version:extendee -> google.protobuf.FileOptions + 6, // 14: opencdc.v1.metadata_key_schema_type:extendee -> google.protobuf.FileOptions + 6, // 15: opencdc.v1.metadata_payload_schema_name:extendee -> google.protobuf.FileOptions + 6, // 16: opencdc.v1.metadata_payload_schema_version:extendee -> google.protobuf.FileOptions + 6, // 17: opencdc.v1.metadata_payload_schema_type:extendee -> google.protobuf.FileOptions + 18, // [18:18] is the sub-list for method output_type + 18, // [18:18] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 7, // [7:18] is the sub-list for extension extendee 0, // [0:7] is the sub-list for field type_name } @@ -651,7 +719,7 @@ func file_opencdc_v1_opencdc_proto_init() { RawDescriptor: file_opencdc_v1_opencdc_proto_rawDesc, NumEnums: 1, NumMessages: 4, - NumExtensions: 8, + NumExtensions: 11, NumServices: 0, }, GoTypes: file_opencdc_v1_opencdc_proto_goTypes, diff --git a/proto/opencdc/v1/opencdc.proto b/proto/opencdc/v1/opencdc.proto index 2b19cc1..74f2d78 100644 --- a/proto/opencdc/v1/opencdc.proto +++ b/proto/opencdc/v1/opencdc.proto @@ -13,9 +13,12 @@ option (metadata_created_at) = "opencdc.createdAt"; option (metadata_read_at) = "opencdc.readAt"; option (metadata_version) = "opencdc.version"; option (metadata_collection) = "opencdc.collection"; -option (metadata_schema_name) = "opencdc.schema.name"; -option (metadata_schema_version) = "opencdc.schema.version"; -option (metadata_schema_type) = "opencdc.schema.type"; +option (metadata_key_schema_name) = "opencdc.key.schema.name"; +option (metadata_key_schema_version) = "opencdc.key.schema.version"; +option (metadata_key_schema_type) = "opencdc.key.schema.type"; +option (metadata_payload_schema_name) = "opencdc.payload.schema.name"; +option (metadata_payload_schema_version) = "opencdc.payload.schema.version"; +option (metadata_payload_schema_type) = "opencdc.payload.schema.type"; // We are (ab)using custom file options to define constants. // See https://github.com/protocolbuffers/protobuf/issues/3520#issuecomment-323613839 @@ -42,15 +45,24 @@ extend google.protobuf.FileOptions { // Metadata field "opencdc.collection" can contain the name of the collection // where the record originated from and/or where it should be stored. string metadata_collection = 10003; - // Metadata field "opencdc.schema.name" contains the name of the schema of - // the record's .Payload.After field. - string metadata_schema_name = 10004; - // Metadata field "opencdc.schema.version" contains the version of the schema of - // the record's .Payload.After field. - string metadata_schema_version = 10005; - // Metadata field "opencdc.schema.type" contains the type of the schema of - // the record's .Payload.After field (currently, the only supported value is "avro") - string metadata_schema_type = 10006; + // Metadata field "opencdc.key.schema.name" contains the name of the schema of + // the record's .Key field. + string metadata_key_schema_name = 10004; + // Metadata field "opencdc.key.schema.version" contains the version of the schema of + // the record's .Key field. + string metadata_key_schema_version = 10005; + // Metadata field "opencdc.key.schema.type" contains the type of the schema of + // the record's .Key field (currently, the only supported value is "avro") + string metadata_key_schema_type = 10006; + // Metadata field "opencdc.payload.schema.name" contains the name of the schema of + // the record's .Payload fields. + string metadata_payload_schema_name = 10007; + // Metadata field "opencdc.payload.schema.version" contains the version of the schema of + // the record's .Payload fields. + string metadata_payload_schema_version = 10008; + // Metadata field "opencdc.payload.schema.type" contains the type of the schema of + // the record's .Payload fields (currently, the only supported value is "avro") + string metadata_payload_schema_type = 10009; } // Operation defines what triggered the creation of a record. diff --git a/schema/record.go b/schema/record.go index a025b0e..a23f69e 100644 --- a/schema/record.go +++ b/schema/record.go @@ -16,8 +16,12 @@ package schema import "github.com/conduitio/conduit-commons/opencdc" -func AttachToRecord(r opencdc.Record, s Instance) { - r.Metadata.SetSchemaType(TypeAvro.String()) - r.Metadata.SetSchemaName(s.Name) - r.Metadata.SetSchemaVersion(s.Version) +func AttachToRecord(r opencdc.Record, keySchema, payloadSchema Instance) { + r.Metadata.SetKeySchemaType(keySchema.Type.String()) + r.Metadata.SetKeySchemaName(keySchema.Name) + r.Metadata.SetKeySchemaVersion(keySchema.Version) + + r.Metadata.SetPayloadSchemaType(payloadSchema.Type.String()) + r.Metadata.SetPayloadSchemaName(payloadSchema.Name) + r.Metadata.SetPayloadSchemaVersion(payloadSchema.Version) } From a0d1e2e06e88b6ba0f3cd22f52a5706c4d71dd76 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Fri, 21 Jun 2024 12:17:42 +0200 Subject: [PATCH 06/21] rename --- schema/instance.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/instance.go b/schema/instance.go index f766039..73fc95c 100644 --- a/schema/instance.go +++ b/schema/instance.go @@ -24,8 +24,8 @@ const ( type Instance struct { ID string - Name string - Version int + Subject string + Version int32 Type Type Bytes []byte } From bcad84fa5ac99ee3a77f9caa68dedcf3264283e2 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Fri, 21 Jun 2024 20:15:08 +0200 Subject: [PATCH 07/21] fix --- schema/instance.go | 2 +- schema/record.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/instance.go b/schema/instance.go index 73fc95c..65832bf 100644 --- a/schema/instance.go +++ b/schema/instance.go @@ -25,7 +25,7 @@ const ( type Instance struct { ID string Subject string - Version int32 + Version int Type Type Bytes []byte } diff --git a/schema/record.go b/schema/record.go index a23f69e..f1e40e0 100644 --- a/schema/record.go +++ b/schema/record.go @@ -18,10 +18,10 @@ import "github.com/conduitio/conduit-commons/opencdc" func AttachToRecord(r opencdc.Record, keySchema, payloadSchema Instance) { r.Metadata.SetKeySchemaType(keySchema.Type.String()) - r.Metadata.SetKeySchemaName(keySchema.Name) + r.Metadata.SetKeySchemaName(keySchema.Subject) r.Metadata.SetKeySchemaVersion(keySchema.Version) r.Metadata.SetPayloadSchemaType(payloadSchema.Type.String()) - r.Metadata.SetPayloadSchemaName(payloadSchema.Name) + r.Metadata.SetPayloadSchemaName(payloadSchema.Subject) r.Metadata.SetPayloadSchemaVersion(payloadSchema.Version) } From baa462282a64c578f02fb14234be359987127965 Mon Sep 17 00:00:00 2001 From: Haris Osmanagic Date: Mon, 24 Jun 2024 19:34:53 +0200 Subject: [PATCH 08/21] remove ID field --- schema/instance.go | 1 - 1 file changed, 1 deletion(-) diff --git a/schema/instance.go b/schema/instance.go index 65832bf..d7d3cee 100644 --- a/schema/instance.go +++ b/schema/instance.go @@ -23,7 +23,6 @@ const ( ) type Instance struct { - ID string Subject string Version int Type Type From 612a82822712fcbf714c242ed507c0a2df51bc71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 13:43:57 +0200 Subject: [PATCH 09/21] split methods --- schema/record.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/schema/record.go b/schema/record.go index f1e40e0..84c5d6b 100644 --- a/schema/record.go +++ b/schema/record.go @@ -16,11 +16,13 @@ package schema import "github.com/conduitio/conduit-commons/opencdc" -func AttachToRecord(r opencdc.Record, keySchema, payloadSchema Instance) { +func AttachKeySchemaToRecord(r opencdc.Record, keySchema Instance) { r.Metadata.SetKeySchemaType(keySchema.Type.String()) r.Metadata.SetKeySchemaName(keySchema.Subject) r.Metadata.SetKeySchemaVersion(keySchema.Version) +} +func AttachPayloadSchemaToRecord(r opencdc.Record, payloadSchema Instance) { r.Metadata.SetPayloadSchemaType(payloadSchema.Type.String()) r.Metadata.SetPayloadSchemaName(payloadSchema.Subject) r.Metadata.SetPayloadSchemaVersion(payloadSchema.Version) From d2ae397ac8a324701dd5f092731f56782e45880f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 13:50:03 +0200 Subject: [PATCH 10/21] update type --- schema/instance.go | 2 +- schema/type_string.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/instance.go b/schema/instance.go index d7d3cee..91a2691 100644 --- a/schema/instance.go +++ b/schema/instance.go @@ -19,7 +19,7 @@ package schema type Type int32 const ( - TypeAvro Type = iota + 1 + TypeAvro Type = iota + 1 // avro ) type Instance struct { diff --git a/schema/type_string.go b/schema/type_string.go index 6429cb5..8add5fa 100644 --- a/schema/type_string.go +++ b/schema/type_string.go @@ -11,9 +11,9 @@ func _() { _ = x[TypeAvro-1] } -const _Type_name = "TypeAvro" +const _Type_name = "avro" -var _Type_index = [...]uint8{0, 8} +var _Type_index = [...]uint8{0, 4} func (i Type) String() string { i -= 1 From 162339aa92cfbba4c62c09c1b1de61ccbf0614c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 13:51:56 +0200 Subject: [PATCH 11/21] remove unused error --- opencdc/errors.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/opencdc/errors.go b/opencdc/errors.go index 2defe4a..bb352a3 100644 --- a/opencdc/errors.go +++ b/opencdc/errors.go @@ -25,9 +25,6 @@ var ( // ErrUnknownOperation is returned when trying to parse an Operation string // and encountering an unknown operation. ErrUnknownOperation = errors.New("unknown operation") - // ErrUnknownSchemaType is returned when trying to parse an Operation string - // and encountering an unknown operation. - ErrUnknownSchemaType = errors.New("unknown schema type") // ErrInvalidProtoDataType is returned when trying to convert a proto data // type to raw or structured data. ErrInvalidProtoDataType = errors.New("invalid proto data type") From 20efa2b33ee80c327818950bafae9f6915db298b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 15:05:16 +0200 Subject: [PATCH 12/21] add schema proto --- proto/schema/v1/schema.pb.go | 235 +++++++++++++++++++++++++++++++++++ proto/schema/v1/schema.proto | 16 +++ 2 files changed, 251 insertions(+) create mode 100644 proto/schema/v1/schema.pb.go create mode 100644 proto/schema/v1/schema.proto diff --git a/proto/schema/v1/schema.pb.go b/proto/schema/v1/schema.pb.go new file mode 100644 index 0000000..1f21c50 --- /dev/null +++ b/proto/schema/v1/schema.pb.go @@ -0,0 +1,235 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc (unknown) +// source: schema/v1/schema.proto + +package schemav1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Schema_Type int32 + +const ( + Schema_TYPE_UNSPECIFIED Schema_Type = 0 + Schema_TYPE_AVRO Schema_Type = 1 +) + +// Enum value maps for Schema_Type. +var ( + Schema_Type_name = map[int32]string{ + 0: "TYPE_UNSPECIFIED", + 1: "TYPE_AVRO", + } + Schema_Type_value = map[string]int32{ + "TYPE_UNSPECIFIED": 0, + "TYPE_AVRO": 1, + } +) + +func (x Schema_Type) Enum() *Schema_Type { + p := new(Schema_Type) + *p = x + return p +} + +func (x Schema_Type) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Schema_Type) Descriptor() protoreflect.EnumDescriptor { + return file_schema_v1_schema_proto_enumTypes[0].Descriptor() +} + +func (Schema_Type) Type() protoreflect.EnumType { + return &file_schema_v1_schema_proto_enumTypes[0] +} + +func (x Schema_Type) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Schema_Type.Descriptor instead. +func (Schema_Type) EnumDescriptor() ([]byte, []int) { + return file_schema_v1_schema_proto_rawDescGZIP(), []int{0, 0} +} + +type Schema struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Subject string `protobuf:"bytes,2,opt,name=subject,proto3" json:"subject,omitempty"` + Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + Type Schema_Type `protobuf:"varint,4,opt,name=type,proto3,enum=schema.v1.Schema_Type" json:"type,omitempty"` + // The schema contents + Bytes []byte `protobuf:"bytes,5,opt,name=bytes,proto3" json:"bytes,omitempty"` +} + +func (x *Schema) Reset() { + *x = Schema{} + if protoimpl.UnsafeEnabled { + mi := &file_schema_v1_schema_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Schema) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Schema) ProtoMessage() {} + +func (x *Schema) ProtoReflect() protoreflect.Message { + mi := &file_schema_v1_schema_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Schema.ProtoReflect.Descriptor instead. +func (*Schema) Descriptor() ([]byte, []int) { + return file_schema_v1_schema_proto_rawDescGZIP(), []int{0} +} + +func (x *Schema) GetSubject() string { + if x != nil { + return x.Subject + } + return "" +} + +func (x *Schema) GetVersion() int32 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *Schema) GetType() Schema_Type { + if x != nil { + return x.Type + } + return Schema_TYPE_UNSPECIFIED +} + +func (x *Schema) GetBytes() []byte { + if x != nil { + return x.Bytes + } + return nil +} + +var File_schema_v1_schema_proto protoreflect.FileDescriptor + +var file_schema_v1_schema_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2f, 0x76, 0x31, 0x2f, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x76, 0x31, 0x22, 0xab, 0x01, 0x0a, 0x06, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x18, + 0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x16, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x22, 0x2b, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, + 0x01, 0x42, 0xa0, 0x01, 0x0a, 0x0d, 0x63, 0x6f, 0x6d, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x50, 0x01, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, + 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, + 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2f, 0x76, 0x31, 0x3b, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x76, + 0x31, 0xa2, 0x02, 0x03, 0x53, 0x58, 0x58, 0xaa, 0x02, 0x09, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x56, 0x31, 0xca, 0x02, 0x09, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5c, 0x56, 0x31, 0xe2, + 0x02, 0x15, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0a, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_schema_v1_schema_proto_rawDescOnce sync.Once + file_schema_v1_schema_proto_rawDescData = file_schema_v1_schema_proto_rawDesc +) + +func file_schema_v1_schema_proto_rawDescGZIP() []byte { + file_schema_v1_schema_proto_rawDescOnce.Do(func() { + file_schema_v1_schema_proto_rawDescData = protoimpl.X.CompressGZIP(file_schema_v1_schema_proto_rawDescData) + }) + return file_schema_v1_schema_proto_rawDescData +} + +var file_schema_v1_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_schema_v1_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_schema_v1_schema_proto_goTypes = []interface{}{ + (Schema_Type)(0), // 0: schema.v1.Schema.Type + (*Schema)(nil), // 1: schema.v1.Schema +} +var file_schema_v1_schema_proto_depIdxs = []int32{ + 0, // 0: schema.v1.Schema.type:type_name -> schema.v1.Schema.Type + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_schema_v1_schema_proto_init() } +func file_schema_v1_schema_proto_init() { + if File_schema_v1_schema_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_schema_v1_schema_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Schema); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_schema_v1_schema_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_schema_v1_schema_proto_goTypes, + DependencyIndexes: file_schema_v1_schema_proto_depIdxs, + EnumInfos: file_schema_v1_schema_proto_enumTypes, + MessageInfos: file_schema_v1_schema_proto_msgTypes, + }.Build() + File_schema_v1_schema_proto = out.File + file_schema_v1_schema_proto_rawDesc = nil + file_schema_v1_schema_proto_goTypes = nil + file_schema_v1_schema_proto_depIdxs = nil +} diff --git a/proto/schema/v1/schema.proto b/proto/schema/v1/schema.proto new file mode 100644 index 0000000..d3adbdc --- /dev/null +++ b/proto/schema/v1/schema.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package schema.v1; + +message Schema { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_AVRO = 1; + } + + string subject = 2; + int32 version = 3; + Type type = 4; + // The schema contents + bytes bytes = 5; +} From bc90bb2ad91bd431373c0eb5a263bb304373dc4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 15:09:06 +0200 Subject: [PATCH 13/21] update proto --- proto/opencdc/v1/opencdc.pb.go | 40 ++++++++++++++++------------------ proto/opencdc/v1/opencdc.proto | 1 - proto/schema/v1/schema.pb.go | 1 + proto/schema/v1/schema.proto | 1 + 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/proto/opencdc/v1/opencdc.pb.go b/proto/opencdc/v1/opencdc.pb.go index 06914f1..430be1e 100644 --- a/proto/opencdc/v1/opencdc.pb.go +++ b/proto/opencdc/v1/opencdc.pb.go @@ -585,7 +585,7 @@ var file_opencdc_v1_opencdc_proto_rawDesc = []byte{ 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x99, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0xb2, 0x03, + 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0x97, 0x03, 0xfa, 0xf0, 0x04, 0x02, 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, 0x04, 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x92, 0xf1, @@ -594,26 +594,24 @@ var file_opencdc_v1_opencdc_proto_rawDesc = []byte{ 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x17, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0xaa, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xb2, - 0xf1, 0x04, 0x17, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0xba, 0xf1, 0x04, 0x1b, 0x6f, 0x70, - 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0xc2, 0xf1, 0x04, 0x1e, 0x6f, 0x70, 0x65, - 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xca, 0xf1, 0x04, 0x1b, 0x6f, - 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, - 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, - 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, - 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, - 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, - 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, - 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, - 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, - 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xba, + 0xf1, 0x04, 0x1b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0xc2, 0xf1, + 0x04, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0xca, 0xf1, 0x04, 0x1b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x0a, + 0x0e, 0x63, 0x6f, 0x6d, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x31, 0x42, + 0x0c, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, + 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x64, + 0x75, 0x69, 0x74, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x2d, 0x63, 0x6f, + 0x6d, 0x6d, 0x6f, 0x6e, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x70, 0x65, 0x6e, + 0x63, 0x64, 0x63, 0x2f, 0x76, 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x76, 0x31, + 0xa2, 0x02, 0x03, 0x4f, 0x58, 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, + 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, + 0xe2, 0x02, 0x16, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, + 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, 0x70, 0x65, 0x6e, + 0x63, 0x64, 0x63, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/opencdc/v1/opencdc.proto b/proto/opencdc/v1/opencdc.proto index 74f2d78..85c9cb1 100644 --- a/proto/opencdc/v1/opencdc.proto +++ b/proto/opencdc/v1/opencdc.proto @@ -15,7 +15,6 @@ option (metadata_version) = "opencdc.version"; option (metadata_collection) = "opencdc.collection"; option (metadata_key_schema_name) = "opencdc.key.schema.name"; option (metadata_key_schema_version) = "opencdc.key.schema.version"; -option (metadata_key_schema_type) = "opencdc.key.schema.type"; option (metadata_payload_schema_name) = "opencdc.payload.schema.name"; option (metadata_payload_schema_version) = "opencdc.payload.schema.version"; option (metadata_payload_schema_type) = "opencdc.payload.schema.type"; diff --git a/proto/schema/v1/schema.pb.go b/proto/schema/v1/schema.pb.go index 1f21c50..1a9bca1 100644 --- a/proto/schema/v1/schema.pb.go +++ b/proto/schema/v1/schema.pb.go @@ -66,6 +66,7 @@ func (Schema_Type) EnumDescriptor() ([]byte, []int) { return file_schema_v1_schema_proto_rawDescGZIP(), []int{0, 0} } +// Schema is a representation of a schema in the schema registry. type Schema struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/proto/schema/v1/schema.proto b/proto/schema/v1/schema.proto index d3adbdc..47e079d 100644 --- a/proto/schema/v1/schema.proto +++ b/proto/schema/v1/schema.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package schema.v1; +// Schema is a representation of a schema in the schema registry. message Schema { enum Type { TYPE_UNSPECIFIED = 0; From 80847d887d518509500277756b0ccea222bdf77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 15:31:10 +0200 Subject: [PATCH 14/21] bring key schema type back --- proto/opencdc/v1/opencdc.pb.go | 40 ++++++++++++++++++---------------- proto/opencdc/v1/opencdc.proto | 1 + 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/proto/opencdc/v1/opencdc.pb.go b/proto/opencdc/v1/opencdc.pb.go index 430be1e..06914f1 100644 --- a/proto/opencdc/v1/opencdc.pb.go +++ b/proto/opencdc/v1/opencdc.pb.go @@ -585,7 +585,7 @@ var file_opencdc_v1_opencdc_proto_rawDesc = []byte{ 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x99, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0x97, 0x03, + 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0xb2, 0x03, 0xfa, 0xf0, 0x04, 0x02, 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, 0x04, 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x92, 0xf1, @@ -594,24 +594,26 @@ var file_opencdc_v1_opencdc_proto_rawDesc = []byte{ 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x17, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0xaa, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xba, - 0xf1, 0x04, 0x1b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0xc2, 0xf1, - 0x04, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0xca, 0xf1, 0x04, 0x1b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x0a, - 0x0e, 0x63, 0x6f, 0x6d, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x31, 0x42, - 0x0c, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, - 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x64, - 0x75, 0x69, 0x74, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x2d, 0x63, 0x6f, - 0x6d, 0x6d, 0x6f, 0x6e, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x70, 0x65, 0x6e, - 0x63, 0x64, 0x63, 0x2f, 0x76, 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x76, 0x31, - 0xa2, 0x02, 0x03, 0x4f, 0x58, 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, - 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, - 0xe2, 0x02, 0x16, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, - 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, 0x70, 0x65, 0x6e, - 0x63, 0x64, 0x63, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xb2, + 0xf1, 0x04, 0x17, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0xba, 0xf1, 0x04, 0x1b, 0x6f, 0x70, + 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0xc2, 0xf1, 0x04, 0x1e, 0x6f, 0x70, 0x65, + 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xca, 0xf1, 0x04, 0x1b, 0x6f, + 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, + 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, + 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, + 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, + 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, + 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, + 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, + 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, + 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/opencdc/v1/opencdc.proto b/proto/opencdc/v1/opencdc.proto index 85c9cb1..74f2d78 100644 --- a/proto/opencdc/v1/opencdc.proto +++ b/proto/opencdc/v1/opencdc.proto @@ -15,6 +15,7 @@ option (metadata_version) = "opencdc.version"; option (metadata_collection) = "opencdc.collection"; option (metadata_key_schema_name) = "opencdc.key.schema.name"; option (metadata_key_schema_version) = "opencdc.key.schema.version"; +option (metadata_key_schema_type) = "opencdc.key.schema.type"; option (metadata_payload_schema_name) = "opencdc.payload.schema.name"; option (metadata_payload_schema_version) = "opencdc.payload.schema.version"; option (metadata_payload_schema_type) = "opencdc.payload.schema.type"; From af0f2b06a1eb7d81c1dbd93d703bd9b504618192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 16:52:23 +0200 Subject: [PATCH 15/21] refactor instance in favor of schema type --- schema/record.go | 16 ++++++++-------- schema/{instance.go => schema.go} | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) rename schema/{instance.go => schema.go} (97%) diff --git a/schema/record.go b/schema/record.go index 84c5d6b..1d8d388 100644 --- a/schema/record.go +++ b/schema/record.go @@ -16,14 +16,14 @@ package schema import "github.com/conduitio/conduit-commons/opencdc" -func AttachKeySchemaToRecord(r opencdc.Record, keySchema Instance) { - r.Metadata.SetKeySchemaType(keySchema.Type.String()) - r.Metadata.SetKeySchemaName(keySchema.Subject) - r.Metadata.SetKeySchemaVersion(keySchema.Version) +func AttachKeySchemaToRecord(r opencdc.Record, schema Schema) { + r.Metadata.SetKeySchemaType(schema.Type.String()) + r.Metadata.SetKeySchemaName(schema.Subject) + r.Metadata.SetKeySchemaVersion(schema.Version) } -func AttachPayloadSchemaToRecord(r opencdc.Record, payloadSchema Instance) { - r.Metadata.SetPayloadSchemaType(payloadSchema.Type.String()) - r.Metadata.SetPayloadSchemaName(payloadSchema.Subject) - r.Metadata.SetPayloadSchemaVersion(payloadSchema.Version) +func AttachPayloadSchemaToRecord(r opencdc.Record, schema Schema) { + r.Metadata.SetPayloadSchemaType(schema.Type.String()) + r.Metadata.SetPayloadSchemaName(schema.Subject) + r.Metadata.SetPayloadSchemaVersion(schema.Version) } diff --git a/schema/instance.go b/schema/schema.go similarity index 97% rename from schema/instance.go rename to schema/schema.go index 91a2691..456db57 100644 --- a/schema/instance.go +++ b/schema/schema.go @@ -22,7 +22,7 @@ const ( TypeAvro Type = iota + 1 // avro ) -type Instance struct { +type Schema struct { Subject string Version int Type Type From a49d8fb7bd71114ae2b9e3d031ca6264437e1a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 16:55:12 +0200 Subject: [PATCH 16/21] better naming --- schema/record.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/schema/record.go b/schema/record.go index 1d8d388..e1d6c30 100644 --- a/schema/record.go +++ b/schema/record.go @@ -16,14 +16,14 @@ package schema import "github.com/conduitio/conduit-commons/opencdc" -func AttachKeySchemaToRecord(r opencdc.Record, schema Schema) { - r.Metadata.SetKeySchemaType(schema.Type.String()) - r.Metadata.SetKeySchemaName(schema.Subject) - r.Metadata.SetKeySchemaVersion(schema.Version) +func AttachKeySchemaToRecord(r opencdc.Record, s Schema) { + r.Metadata.SetKeySchemaType(s.Type.String()) + r.Metadata.SetKeySchemaName(s.Subject) + r.Metadata.SetKeySchemaVersion(s.Version) } -func AttachPayloadSchemaToRecord(r opencdc.Record, schema Schema) { - r.Metadata.SetPayloadSchemaType(schema.Type.String()) - r.Metadata.SetPayloadSchemaName(schema.Subject) - r.Metadata.SetPayloadSchemaVersion(schema.Version) +func AttachPayloadSchemaToRecord(r opencdc.Record, s Schema) { + r.Metadata.SetPayloadSchemaType(s.Type.String()) + r.Metadata.SetPayloadSchemaName(s.Subject) + r.Metadata.SetPayloadSchemaVersion(s.Version) } From d5a441ebd80aed9e78322c727121a4fe2edfda89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 17:36:08 +0200 Subject: [PATCH 17/21] use subject and version on metadata --- opencdc/metadata.go | 60 ++++---------- opencdc/metadata_test.go | 6 +- proto/opencdc/v1/opencdc.pb.go | 146 ++++++++++++++++----------------- proto/opencdc/v1/opencdc.proto | 14 ++-- 4 files changed, 96 insertions(+), 130 deletions(-) diff --git a/opencdc/metadata.go b/opencdc/metadata.go index fcceda9..23eb42c 100644 --- a/opencdc/metadata.go +++ b/opencdc/metadata.go @@ -44,25 +44,19 @@ const ( // where the record originated from and/or where it should be stored. MetadataCollection = "opencdc.collection" - // MetadataKeySchemaName is a Record.Metadata key for the name of the schema of + // MetadataKeySchemaSubject is a Record.Metadata key for the name of the schema of // the record's .Key field. - MetadataKeySchemaName = "opencdc.key.schema.name" + MetadataKeySchemaSubject = "opencdc.key.schema.subject" // MetadataKeySchemaVersion is a Record.Metadata key for the version of the schema of // the record's .Key field. MetadataKeySchemaVersion = "opencdc.key.schema.version" - // MetadataKeySchemaType is a Record.Metadata key for the type of the schema of - // the record's .Key field. - MetadataKeySchemaType = "opencdc.key.schema.type" - // MetadataPayloadSchemaName is a Record.Metadata key for the name of the schema of + // MetadataPayloadSchemaSubject is a Record.Metadata key for the name of the schema of // the record's .Payload field. - MetadataPayloadSchemaName = "opencdc.payload.schema.name" + MetadataPayloadSchemaSubject = "opencdc.payload.schema.subject" // MetadataPayloadSchemaVersion is a Record.Metadata key for the version of the schema of // the record's .Payload field. MetadataPayloadSchemaVersion = "opencdc.payload.schema.version" - // MetadataPayloadSchemaType is a Record.Metadata key for the type of the schema of - // the record's .Payload field. - MetadataPayloadSchemaType = "opencdc.payload.schema.type" // MetadataConduitSourcePluginName is a Record.Metadata key for the name of // the source plugin that created this record. @@ -255,15 +249,15 @@ func (m Metadata) SetConduitDLQNackNodeID(id string) { m[MetadataConduitDLQNackNodeID] = id } -// GetKeySchemaName returns the value for key MetadataKeySchemaName. +// GetKeySchemaSubject returns the value for key MetadataKeySchemaSubject. // If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetKeySchemaName() (string, error) { - return m.getValue(MetadataKeySchemaName) +func (m Metadata) GetKeySchemaSubject() (string, error) { + return m.getValue(MetadataKeySchemaSubject) } -// SetKeySchemaName sets the metadata value for key MetadataKeySchemaName. -func (m Metadata) SetKeySchemaName(name string) { - m[MetadataKeySchemaName] = name +// SetKeySchemaSubject sets the metadata value for key MetadataKeySchemaSubject. +func (m Metadata) SetKeySchemaSubject(subject string) { + m[MetadataKeySchemaSubject] = subject } // GetKeySchemaVersion returns the value for key MetadataKeySchemaVersion. @@ -287,26 +281,15 @@ func (m Metadata) SetKeySchemaVersion(version int) { m[MetadataKeySchemaVersion] = strconv.Itoa(version) } -// GetKeySchemaType returns the value for key MetadataKeySchemaType. -// If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetKeySchemaType() (string, error) { - return m.getValue(MetadataKeySchemaType) -} - -// SetKeySchemaType sets the metadata value for key MetadataKeySchemaType. -func (m Metadata) SetKeySchemaType(t string) { - m[MetadataKeySchemaType] = t -} - -// GetPayloadSchemaName returns the value for key MetadataPayloadSchemaName. +// GetPayloadSchemaSubject returns the value for key MetadataPayloadSchemaSubject. // If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetPayloadSchemaName() (string, error) { - return m.getValue(MetadataPayloadSchemaName) +func (m Metadata) GetPayloadSchemaSubject() (string, error) { + return m.getValue(MetadataPayloadSchemaSubject) } -// SetPayloadSchemaName sets the metadata value for key MetadataPayloadSchemaName. -func (m Metadata) SetPayloadSchemaName(name string) { - m[MetadataPayloadSchemaName] = name +// SetPayloadSchemaSubject sets the metadata value for key MetadataPayloadSchemaSubject. +func (m Metadata) SetPayloadSchemaSubject(subject string) { + m[MetadataPayloadSchemaSubject] = subject } // GetPayloadSchemaVersion returns the value for key MetadataPayloadSchemaVersion. @@ -330,17 +313,6 @@ func (m Metadata) SetPayloadSchemaVersion(version int) { m[MetadataPayloadSchemaVersion] = strconv.Itoa(version) } -// GetPayloadSchemaType returns the value for key MetadataPayloadSchemaType. -// If the value does not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetPayloadSchemaType() (string, error) { - return m.getValue(MetadataPayloadSchemaType) -} - -// SetPayloadSchemaType sets the metadata value for key MetadataPayloadSchemaType. -func (m Metadata) SetPayloadSchemaType(t string) { - m[MetadataPayloadSchemaType] = t -} - // getValue returns the value for a specific key. If the value does not exist or // is empty the function returns ErrMetadataFieldNotFound. func (m Metadata) getValue(key string) (string, error) { diff --git a/opencdc/metadata_test.go b/opencdc/metadata_test.go index b2084c8..4d9ba90 100644 --- a/opencdc/metadata_test.go +++ b/opencdc/metadata_test.go @@ -30,12 +30,10 @@ func TestMetadataConstants(t *testing.T) { MetadataCreatedAt: opencdcv1.E_MetadataCreatedAt, MetadataReadAt: opencdcv1.E_MetadataReadAt, MetadataCollection: opencdcv1.E_MetadataCollection, - MetadataKeySchemaName: opencdcv1.E_MetadataKeySchemaName, + MetadataKeySchemaSubject: opencdcv1.E_MetadataKeySchemaSubject, MetadataKeySchemaVersion: opencdcv1.E_MetadataKeySchemaVersion, - MetadataKeySchemaType: opencdcv1.E_MetadataKeySchemaType, - MetadataPayloadSchemaName: opencdcv1.E_MetadataPayloadSchemaName, + MetadataPayloadSchemaSubject: opencdcv1.E_MetadataPayloadSchemaSubject, MetadataPayloadSchemaVersion: opencdcv1.E_MetadataPayloadSchemaVersion, - MetadataPayloadSchemaType: opencdcv1.E_MetadataPayloadSchemaType, MetadataConduitSourcePluginName: metadatav1.E_MetadataConduitSourcePluginName, MetadataConduitSourcePluginVersion: metadatav1.E_MetadataConduitSourcePluginVersion, diff --git a/proto/opencdc/v1/opencdc.pb.go b/proto/opencdc/v1/opencdc.pb.go index 06914f1..c287db6 100644 --- a/proto/opencdc/v1/opencdc.pb.go +++ b/proto/opencdc/v1/opencdc.pb.go @@ -367,8 +367,8 @@ var file_opencdc_v1_opencdc_proto_extTypes = []protoimpl.ExtensionInfo{ ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), Field: 10004, - Name: "opencdc.v1.metadata_key_schema_name", - Tag: "bytes,10004,opt,name=metadata_key_schema_name", + Name: "opencdc.v1.metadata_key_schema_subject", + Tag: "bytes,10004,opt,name=metadata_key_schema_subject", Filename: "opencdc/v1/opencdc.proto", }, { @@ -391,8 +391,8 @@ var file_opencdc_v1_opencdc_proto_extTypes = []protoimpl.ExtensionInfo{ ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), Field: 10007, - Name: "opencdc.v1.metadata_payload_schema_name", - Tag: "bytes,10007,opt,name=metadata_payload_schema_name", + Name: "opencdc.v1.metadata_payload_schema_subject", + Tag: "bytes,10007,opt,name=metadata_payload_schema_subject", Filename: "opencdc/v1/opencdc.proto", }, { @@ -446,11 +446,11 @@ var ( // // optional string metadata_collection = 10003; E_MetadataCollection = &file_opencdc_v1_opencdc_proto_extTypes[4] - // Metadata field "opencdc.key.schema.name" contains the name of the schema of + // Metadata field "opencdc.key.schema.subject" contains the name of the schema of // the record's .Key field. // - // optional string metadata_key_schema_name = 10004; - E_MetadataKeySchemaName = &file_opencdc_v1_opencdc_proto_extTypes[5] + // optional string metadata_key_schema_subject = 10004; + E_MetadataKeySchemaSubject = &file_opencdc_v1_opencdc_proto_extTypes[5] // Metadata field "opencdc.key.schema.version" contains the version of the schema of // the record's .Key field. // @@ -461,11 +461,11 @@ var ( // // optional string metadata_key_schema_type = 10006; E_MetadataKeySchemaType = &file_opencdc_v1_opencdc_proto_extTypes[7] - // Metadata field "opencdc.payload.schema.name" contains the name of the schema of + // Metadata field "opencdc.payload.schema.subject" contains the name of the schema of // the record's .Payload fields. // - // optional string metadata_payload_schema_name = 10007; - E_MetadataPayloadSchemaName = &file_opencdc_v1_opencdc_proto_extTypes[8] + // optional string metadata_payload_schema_subject = 10007; + E_MetadataPayloadSchemaSubject = &file_opencdc_v1_opencdc_proto_extTypes[8] // Metadata field "opencdc.payload.schema.version" contains the version of the schema of // the record's .Payload fields. // @@ -550,70 +550,68 @@ var file_opencdc_v1_opencdc_proto_rawDesc = []byte{ 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x93, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x43, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x56, 0x0a, 0x18, 0x6d, 0x65, 0x74, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x5c, 0x0a, 0x1b, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x94, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x4e, 0x61, 0x6d, - 0x65, 0x3a, 0x5c, 0x0a, 0x1b, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6b, 0x65, - 0x79, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x5f, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x94, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x3a, 0x5c, 0x0a, 0x1b, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x95, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x56, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x96, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x64, 0x0a, + 0x1f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x95, - 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4b, - 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, - 0x56, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x5f, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, - 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x96, 0x4e, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x15, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x5e, 0x0a, 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x97, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x4e, 0x61, 0x6d, 0x65, 0x3a, 0x64, 0x0a, 0x1f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, - 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x98, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x5e, 0x0a, - 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x99, 0x4e, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x19, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0xb2, 0x03, - 0xfa, 0xf0, 0x04, 0x02, 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, - 0x63, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, 0x04, 0x11, 0x6f, 0x70, 0x65, - 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x92, 0xf1, - 0x04, 0x0e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x72, 0x65, 0x61, 0x64, 0x41, 0x74, - 0x9a, 0xf1, 0x04, 0x12, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x17, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, - 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, - 0x65, 0xaa, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xb2, - 0xf1, 0x04, 0x17, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0xba, 0xf1, 0x04, 0x1b, 0x6f, 0x70, - 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0xc2, 0xf1, 0x04, 0x1e, 0x6f, 0x70, 0x65, - 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xca, 0xf1, 0x04, 0x1b, 0x6f, - 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, - 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, - 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, - 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, - 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, - 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, - 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, - 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, - 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x97, + 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x53, 0x75, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x3a, 0x64, 0x0a, 0x1f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, + 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x98, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1c, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x5e, 0x0a, 0x1c, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x99, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0xfe, 0x02, 0xfa, 0xf0, 0x04, 0x02, + 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, 0x04, 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, + 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x92, 0xf1, 0x04, 0x0e, 0x6f, 0x70, + 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x72, 0x65, 0x61, 0x64, 0x41, 0x74, 0x9a, 0xf1, 0x04, 0x12, + 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, + 0x79, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0xaa, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xba, 0xf1, + 0x04, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0xc2, 0xf1, 0x04, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, + 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, + 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, + 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, + 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, + 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, + 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, + 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, + 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, + 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -652,10 +650,10 @@ var file_opencdc_v1_opencdc_proto_depIdxs = []int32{ 6, // 9: opencdc.v1.metadata_created_at:extendee -> google.protobuf.FileOptions 6, // 10: opencdc.v1.metadata_read_at:extendee -> google.protobuf.FileOptions 6, // 11: opencdc.v1.metadata_collection:extendee -> google.protobuf.FileOptions - 6, // 12: opencdc.v1.metadata_key_schema_name:extendee -> google.protobuf.FileOptions + 6, // 12: opencdc.v1.metadata_key_schema_subject:extendee -> google.protobuf.FileOptions 6, // 13: opencdc.v1.metadata_key_schema_version:extendee -> google.protobuf.FileOptions 6, // 14: opencdc.v1.metadata_key_schema_type:extendee -> google.protobuf.FileOptions - 6, // 15: opencdc.v1.metadata_payload_schema_name:extendee -> google.protobuf.FileOptions + 6, // 15: opencdc.v1.metadata_payload_schema_subject:extendee -> google.protobuf.FileOptions 6, // 16: opencdc.v1.metadata_payload_schema_version:extendee -> google.protobuf.FileOptions 6, // 17: opencdc.v1.metadata_payload_schema_type:extendee -> google.protobuf.FileOptions 18, // [18:18] is the sub-list for method output_type diff --git a/proto/opencdc/v1/opencdc.proto b/proto/opencdc/v1/opencdc.proto index 74f2d78..b5ef909 100644 --- a/proto/opencdc/v1/opencdc.proto +++ b/proto/opencdc/v1/opencdc.proto @@ -13,12 +13,10 @@ option (metadata_created_at) = "opencdc.createdAt"; option (metadata_read_at) = "opencdc.readAt"; option (metadata_version) = "opencdc.version"; option (metadata_collection) = "opencdc.collection"; -option (metadata_key_schema_name) = "opencdc.key.schema.name"; +option (metadata_key_schema_subject) = "opencdc.key.schema.subject"; option (metadata_key_schema_version) = "opencdc.key.schema.version"; -option (metadata_key_schema_type) = "opencdc.key.schema.type"; -option (metadata_payload_schema_name) = "opencdc.payload.schema.name"; +option (metadata_payload_schema_subject) = "opencdc.payload.schema.subject"; option (metadata_payload_schema_version) = "opencdc.payload.schema.version"; -option (metadata_payload_schema_type) = "opencdc.payload.schema.type"; // We are (ab)using custom file options to define constants. // See https://github.com/protocolbuffers/protobuf/issues/3520#issuecomment-323613839 @@ -45,18 +43,18 @@ extend google.protobuf.FileOptions { // Metadata field "opencdc.collection" can contain the name of the collection // where the record originated from and/or where it should be stored. string metadata_collection = 10003; - // Metadata field "opencdc.key.schema.name" contains the name of the schema of + // Metadata field "opencdc.key.schema.subject" contains the name of the schema of // the record's .Key field. - string metadata_key_schema_name = 10004; + string metadata_key_schema_subject = 10004; // Metadata field "opencdc.key.schema.version" contains the version of the schema of // the record's .Key field. string metadata_key_schema_version = 10005; // Metadata field "opencdc.key.schema.type" contains the type of the schema of // the record's .Key field (currently, the only supported value is "avro") string metadata_key_schema_type = 10006; - // Metadata field "opencdc.payload.schema.name" contains the name of the schema of + // Metadata field "opencdc.payload.schema.subject" contains the name of the schema of // the record's .Payload fields. - string metadata_payload_schema_name = 10007; + string metadata_payload_schema_subject = 10007; // Metadata field "opencdc.payload.schema.version" contains the version of the schema of // the record's .Payload fields. string metadata_payload_schema_version = 10008; From 333baee1b34a269688b033e3bb49fc1eaff9abe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 17:38:25 +0200 Subject: [PATCH 18/21] update comments --- opencdc/metadata.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencdc/metadata.go b/opencdc/metadata.go index 23eb42c..f2686c6 100644 --- a/opencdc/metadata.go +++ b/opencdc/metadata.go @@ -44,14 +44,14 @@ const ( // where the record originated from and/or where it should be stored. MetadataCollection = "opencdc.collection" - // MetadataKeySchemaSubject is a Record.Metadata key for the name of the schema of + // MetadataKeySchemaSubject is a Record.Metadata key for the subject of the schema of // the record's .Key field. MetadataKeySchemaSubject = "opencdc.key.schema.subject" // MetadataKeySchemaVersion is a Record.Metadata key for the version of the schema of // the record's .Key field. MetadataKeySchemaVersion = "opencdc.key.schema.version" - // MetadataPayloadSchemaSubject is a Record.Metadata key for the name of the schema of + // MetadataPayloadSchemaSubject is a Record.Metadata key for the subject of the schema of // the record's .Payload field. MetadataPayloadSchemaSubject = "opencdc.payload.schema.subject" // MetadataPayloadSchemaVersion is a Record.Metadata key for the version of the schema of From b8367b52bd0383be06059ca0ee32a7e6f7f91d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 17:40:20 +0200 Subject: [PATCH 19/21] fix --- schema/record.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/schema/record.go b/schema/record.go index e1d6c30..df181c6 100644 --- a/schema/record.go +++ b/schema/record.go @@ -17,13 +17,11 @@ package schema import "github.com/conduitio/conduit-commons/opencdc" func AttachKeySchemaToRecord(r opencdc.Record, s Schema) { - r.Metadata.SetKeySchemaType(s.Type.String()) - r.Metadata.SetKeySchemaName(s.Subject) + r.Metadata.SetKeySchemaSubject(s.Subject) r.Metadata.SetKeySchemaVersion(s.Version) } func AttachPayloadSchemaToRecord(r opencdc.Record, s Schema) { - r.Metadata.SetPayloadSchemaType(s.Type.String()) - r.Metadata.SetPayloadSchemaName(s.Subject) + r.Metadata.SetPayloadSchemaSubject(s.Subject) r.Metadata.SetPayloadSchemaVersion(s.Version) } From baa257030fbb04a2ba63e19928525bcb41a0d34f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 17:44:09 +0200 Subject: [PATCH 20/21] update proto --- proto/opencdc/v1/opencdc.pb.go | 150 ++++++++++++--------------------- proto/opencdc/v1/opencdc.proto | 16 ++-- 2 files changed, 60 insertions(+), 106 deletions(-) diff --git a/proto/opencdc/v1/opencdc.pb.go b/proto/opencdc/v1/opencdc.pb.go index c287db6..29290bf 100644 --- a/proto/opencdc/v1/opencdc.pb.go +++ b/proto/opencdc/v1/opencdc.pb.go @@ -91,7 +91,7 @@ type Record struct { // Position uniquely identifies the record. Position []byte `protobuf:"bytes,1,opt,name=position,proto3" json:"position,omitempty"` - // Operation defines what triggered the creation of a record. There are four + // Operation defines what triggered thmetadata_key_schema_typee creation of a record. There are four // possibilities: create, update, delete or snapshot. The first three // operations are encountered during normal CDC operation, while "snapshot" is // meant to represent records during an initial load. Depending on the @@ -383,32 +383,16 @@ var file_opencdc_v1_opencdc_proto_extTypes = []protoimpl.ExtensionInfo{ ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), Field: 10006, - Name: "opencdc.v1.metadata_key_schema_type", - Tag: "bytes,10006,opt,name=metadata_key_schema_type", - Filename: "opencdc/v1/opencdc.proto", - }, - { - ExtendedType: (*descriptorpb.FileOptions)(nil), - ExtensionType: (*string)(nil), - Field: 10007, Name: "opencdc.v1.metadata_payload_schema_subject", - Tag: "bytes,10007,opt,name=metadata_payload_schema_subject", + Tag: "bytes,10006,opt,name=metadata_payload_schema_subject", Filename: "opencdc/v1/opencdc.proto", }, { ExtendedType: (*descriptorpb.FileOptions)(nil), ExtensionType: (*string)(nil), - Field: 10008, + Field: 10007, Name: "opencdc.v1.metadata_payload_schema_version", - Tag: "bytes,10008,opt,name=metadata_payload_schema_version", - Filename: "opencdc/v1/opencdc.proto", - }, - { - ExtendedType: (*descriptorpb.FileOptions)(nil), - ExtensionType: (*string)(nil), - Field: 10009, - Name: "opencdc.v1.metadata_payload_schema_type", - Tag: "bytes,10009,opt,name=metadata_payload_schema_type", + Tag: "bytes,10007,opt,name=metadata_payload_schema_version", Filename: "opencdc/v1/opencdc.proto", }, } @@ -446,7 +430,7 @@ var ( // // optional string metadata_collection = 10003; E_MetadataCollection = &file_opencdc_v1_opencdc_proto_extTypes[4] - // Metadata field "opencdc.key.schema.subject" contains the name of the schema of + // Metadata field "opencdc.key.schema.subject" contains the subject of the schema of // the record's .Key field. // // optional string metadata_key_schema_subject = 10004; @@ -456,26 +440,16 @@ var ( // // optional string metadata_key_schema_version = 10005; E_MetadataKeySchemaVersion = &file_opencdc_v1_opencdc_proto_extTypes[6] - // Metadata field "opencdc.key.schema.type" contains the type of the schema of - // the record's .Key field (currently, the only supported value is "avro") - // - // optional string metadata_key_schema_type = 10006; - E_MetadataKeySchemaType = &file_opencdc_v1_opencdc_proto_extTypes[7] - // Metadata field "opencdc.payload.schema.subject" contains the name of the schema of + // Metadata field "opencdc.payload.schema.subject" contains the subject of the schema of // the record's .Payload fields. // - // optional string metadata_payload_schema_subject = 10007; - E_MetadataPayloadSchemaSubject = &file_opencdc_v1_opencdc_proto_extTypes[8] + // optional string metadata_payload_schema_subject = 10006; + E_MetadataPayloadSchemaSubject = &file_opencdc_v1_opencdc_proto_extTypes[7] // Metadata field "opencdc.payload.schema.version" contains the version of the schema of // the record's .Payload fields. // - // optional string metadata_payload_schema_version = 10008; - E_MetadataPayloadSchemaVersion = &file_opencdc_v1_opencdc_proto_extTypes[9] - // Metadata field "opencdc.payload.schema.type" contains the type of the schema of - // the record's .Payload fields (currently, the only supported value is "avro") - // - // optional string metadata_payload_schema_type = 10009; - E_MetadataPayloadSchemaType = &file_opencdc_v1_opencdc_proto_extTypes[10] + // optional string metadata_payload_schema_version = 10007; + E_MetadataPayloadSchemaVersion = &file_opencdc_v1_opencdc_proto_extTypes[8] ) var File_opencdc_v1_opencdc_proto protoreflect.FileDescriptor @@ -562,56 +536,44 @@ var file_opencdc_v1_opencdc_proto_rawDesc = []byte{ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x95, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x56, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x96, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x4b, 0x65, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x64, 0x0a, - 0x1f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x97, - 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x53, 0x75, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x3a, 0x64, 0x0a, 0x1f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, - 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x98, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1c, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x5e, 0x0a, 0x1c, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x99, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, 0xfe, 0x02, 0xfa, 0xf0, 0x04, 0x02, - 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, 0x04, 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, - 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x92, 0xf1, 0x04, 0x0e, 0x6f, 0x70, - 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x72, 0x65, 0x61, 0x64, 0x41, 0x74, 0x9a, 0xf1, 0x04, 0x12, - 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, - 0x79, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0xaa, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xba, 0xf1, - 0x04, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0xc2, 0xf1, 0x04, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, - 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, - 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, - 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, - 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, - 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, - 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, - 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, - 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, - 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, - 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x64, 0x0a, 0x1f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x5f, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x96, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1c, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x3a, 0x64, 0x0a, 0x1f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x97, 0x4e, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x1c, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x42, 0xfe, 0x02, 0xfa, 0xf0, 0x04, 0x02, 0x76, 0x31, 0x82, 0xf1, 0x04, 0x0f, 0x6f, 0x70, + 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x8a, 0xf1, 0x04, + 0x11, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x41, 0x74, 0x92, 0xf1, 0x04, 0x0e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x72, 0x65, + 0x61, 0x64, 0x41, 0x74, 0x9a, 0xf1, 0x04, 0x12, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, + 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0xa2, 0xf1, 0x04, 0x1a, 0x6f, 0x70, + 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xaa, 0xf1, 0x04, 0x1a, 0x6f, 0x70, 0x65, 0x6e, + 0x63, 0x64, 0x63, 0x2e, 0x6b, 0x65, 0x79, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0xb2, 0xf1, 0x04, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, + 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xba, 0xf1, 0x04, 0x1e, 0x6f, 0x70, 0x65, 0x6e, + 0x63, 0x64, 0x63, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x2e, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, + 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, + 0x63, 0x64, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x69, 0x6f, + 0x2f, 0x63, 0x6f, 0x6e, 0x64, 0x75, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x73, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2f, 0x76, + 0x31, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4f, 0x58, + 0x58, 0xaa, 0x02, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x2e, 0x56, 0x31, 0xca, 0x02, + 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x16, 0x4f, 0x70, + 0x65, 0x6e, 0x63, 0x64, 0x63, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x4f, 0x70, 0x65, 0x6e, 0x63, 0x64, 0x63, 0x3a, 0x3a, + 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -652,14 +614,12 @@ var file_opencdc_v1_opencdc_proto_depIdxs = []int32{ 6, // 11: opencdc.v1.metadata_collection:extendee -> google.protobuf.FileOptions 6, // 12: opencdc.v1.metadata_key_schema_subject:extendee -> google.protobuf.FileOptions 6, // 13: opencdc.v1.metadata_key_schema_version:extendee -> google.protobuf.FileOptions - 6, // 14: opencdc.v1.metadata_key_schema_type:extendee -> google.protobuf.FileOptions - 6, // 15: opencdc.v1.metadata_payload_schema_subject:extendee -> google.protobuf.FileOptions - 6, // 16: opencdc.v1.metadata_payload_schema_version:extendee -> google.protobuf.FileOptions - 6, // 17: opencdc.v1.metadata_payload_schema_type:extendee -> google.protobuf.FileOptions - 18, // [18:18] is the sub-list for method output_type - 18, // [18:18] is the sub-list for method input_type - 18, // [18:18] is the sub-list for extension type_name - 7, // [7:18] is the sub-list for extension extendee + 6, // 14: opencdc.v1.metadata_payload_schema_subject:extendee -> google.protobuf.FileOptions + 6, // 15: opencdc.v1.metadata_payload_schema_version:extendee -> google.protobuf.FileOptions + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 7, // [7:16] is the sub-list for extension extendee 0, // [0:7] is the sub-list for field type_name } @@ -717,7 +677,7 @@ func file_opencdc_v1_opencdc_proto_init() { RawDescriptor: file_opencdc_v1_opencdc_proto_rawDesc, NumEnums: 1, NumMessages: 4, - NumExtensions: 11, + NumExtensions: 9, NumServices: 0, }, GoTypes: file_opencdc_v1_opencdc_proto_goTypes, diff --git a/proto/opencdc/v1/opencdc.proto b/proto/opencdc/v1/opencdc.proto index b5ef909..bdf38a7 100644 --- a/proto/opencdc/v1/opencdc.proto +++ b/proto/opencdc/v1/opencdc.proto @@ -43,24 +43,18 @@ extend google.protobuf.FileOptions { // Metadata field "opencdc.collection" can contain the name of the collection // where the record originated from and/or where it should be stored. string metadata_collection = 10003; - // Metadata field "opencdc.key.schema.subject" contains the name of the schema of + // Metadata field "opencdc.key.schema.subject" contains the subject of the schema of // the record's .Key field. string metadata_key_schema_subject = 10004; // Metadata field "opencdc.key.schema.version" contains the version of the schema of // the record's .Key field. string metadata_key_schema_version = 10005; - // Metadata field "opencdc.key.schema.type" contains the type of the schema of - // the record's .Key field (currently, the only supported value is "avro") - string metadata_key_schema_type = 10006; - // Metadata field "opencdc.payload.schema.subject" contains the name of the schema of + // Metadata field "opencdc.payload.schema.subject" contains the subject of the schema of // the record's .Payload fields. - string metadata_payload_schema_subject = 10007; + string metadata_payload_schema_subject = 10006; // Metadata field "opencdc.payload.schema.version" contains the version of the schema of // the record's .Payload fields. - string metadata_payload_schema_version = 10008; - // Metadata field "opencdc.payload.schema.type" contains the type of the schema of - // the record's .Payload fields (currently, the only supported value is "avro") - string metadata_payload_schema_type = 10009; + string metadata_payload_schema_version = 10007; } // Operation defines what triggered the creation of a record. @@ -82,7 +76,7 @@ message Record { // Position uniquely identifies the record. bytes position = 1; - // Operation defines what triggered the creation of a record. There are four + // Operation defines what triggered thmetadata_key_schema_typee creation of a record. There are four // possibilities: create, update, delete or snapshot. The first three // operations are encountered during normal CDC operation, while "snapshot" is // meant to represent records during an initial load. Depending on the From 5961cf87605f6143216b50b7f266ea9feb9ed916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 28 Jun 2024 17:45:33 +0200 Subject: [PATCH 21/21] fix comment --- proto/opencdc/v1/opencdc.pb.go | 2 +- proto/opencdc/v1/opencdc.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/proto/opencdc/v1/opencdc.pb.go b/proto/opencdc/v1/opencdc.pb.go index 29290bf..3e1cd70 100644 --- a/proto/opencdc/v1/opencdc.pb.go +++ b/proto/opencdc/v1/opencdc.pb.go @@ -91,7 +91,7 @@ type Record struct { // Position uniquely identifies the record. Position []byte `protobuf:"bytes,1,opt,name=position,proto3" json:"position,omitempty"` - // Operation defines what triggered thmetadata_key_schema_typee creation of a record. There are four + // Operation defines what triggered the creation of a record. There are four // possibilities: create, update, delete or snapshot. The first three // operations are encountered during normal CDC operation, while "snapshot" is // meant to represent records during an initial load. Depending on the diff --git a/proto/opencdc/v1/opencdc.proto b/proto/opencdc/v1/opencdc.proto index bdf38a7..1862505 100644 --- a/proto/opencdc/v1/opencdc.proto +++ b/proto/opencdc/v1/opencdc.proto @@ -76,7 +76,7 @@ message Record { // Position uniquely identifies the record. bytes position = 1; - // Operation defines what triggered thmetadata_key_schema_typee creation of a record. There are four + // Operation defines what triggered the creation of a record. There are four // possibilities: create, update, delete or snapshot. The first three // operations are encountered during normal CDC operation, while "snapshot" is // meant to represent records during an initial load. Depending on the