diff --git a/karapace/config.py b/karapace/config.py index c87275a8f..eaece2d9b 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -169,6 +169,7 @@ class NameStrategy(Enum): topic_name = "topic_name" record_name = "record_name" topic_record_name = "topic_record_name" + no_validation = "no_validation_strategy" def parse_env_value(value: str) -> str | int | bool: diff --git a/karapace/in_memory_database.py b/karapace/in_memory_database.py index 222e38046..07eae56b9 100644 --- a/karapace/in_memory_database.py +++ b/karapace/in_memory_database.py @@ -7,9 +7,10 @@ from __future__ import annotations from dataclasses import dataclass, field +from karapace.config import NameStrategy from karapace.schema_models import SchemaVersion, TypedSchema from karapace.schema_references import Reference, Referents -from karapace.typing import ResolvedVersion, SchemaId, Subject +from karapace.typing import ResolvedVersion, SchemaId, Subject, TopicName from threading import Lock, RLock from typing import Iterable, Sequence @@ -32,6 +33,7 @@ def __init__(self) -> None: self.schemas: dict[SchemaId, TypedSchema] = {} self.schema_lock_thread = RLock() self.referenced_by: dict[tuple[Subject, ResolvedVersion], Referents] = {} + self.topic_validation_strategies: dict[TopicName, NameStrategy] = {} # Content based deduplication of schemas. This is used to reduce memory # usage when the same schema is produce multiple times to the same or @@ -229,6 +231,15 @@ def find_subject_schemas(self, *, subject: Subject, include_deleted: bool) -> di if schema_version.deleted is False } + def get_topic_strategy(self, *, topic_name: TopicName) -> NameStrategy: + if topic_name not in self.topic_validation_strategies: + return NameStrategy.topic_name + + return self.topic_validation_strategies[topic_name] + + def override_topic_strategy(self, *, topic_name: TopicName, name_strategy: NameStrategy) -> None: + self.topic_validation_strategies[topic_name] = name_strategy + def delete_subject(self, *, subject: Subject, version: ResolvedVersion) -> None: with self.schema_lock_thread: for schema_version in self.subjects[subject].schemas.values(): diff --git a/karapace/kafka_rest_apis/__init__.py b/karapace/kafka_rest_apis/__init__.py index c63194e52..8d9102b5a 100644 --- a/karapace/kafka_rest_apis/__init__.py +++ b/karapace/kafka_rest_apis/__init__.py @@ -13,7 +13,7 @@ TopicAuthorizationFailedError, UnknownTopicOrPartitionError, ) -from karapace.config import Config, create_client_ssl_context +from karapace.config import Config, create_client_ssl_context, NameStrategy from karapace.errors import InvalidSchema from karapace.kafka_rest_apis.admin import KafkaRestAdminClient from karapace.kafka_rest_apis.authentication import ( @@ -29,7 +29,7 @@ from karapace.schema_models import TypedSchema, ValidatedTypedSchema from karapace.schema_type import SchemaType from karapace.serialization import InvalidMessageSchema, InvalidPayload, SchemaRegistrySerializer, SchemaRetrievalError -from karapace.typing import SchemaId, Subject +from karapace.typing import SchemaId, Subject, TopicName from karapace.utils import convert_to_int, json_encode, KarapaceKafkaClient from typing import Callable, Dict, List, Optional, Tuple, Union @@ -773,18 +773,20 @@ async def get_schema_id( SchemaId(int(data[f"{prefix}_schema_id"])) if f"{prefix}_schema_id" in data else None ) schema_str = data.get(f"{prefix}_schema") + naming_strategy = await self.serializer.get_topic_strategy_name(topic_name=TopicName(topic)) if schema_id is None and schema_str is None: raise InvalidSchema() if schema_id is None: parsed_schema = ValidatedTypedSchema.parse(schema_type, schema_str) - subject_name = self.serializer.get_subject_name(topic, parsed_schema, prefix, schema_type) + + subject_name = self.serializer.get_subject_name(topic, parsed_schema, prefix, schema_type, naming_strategy) schema_id = await self._query_schema_id_from_cache_or_registry(parsed_schema, schema_str, subject_name) else: def subject_not_included(schema: TypedSchema, subjects: List[Subject]) -> bool: - subject = self.serializer.get_subject_name(topic, schema, prefix, schema_type) + subject = self.serializer.get_subject_name(topic, schema, prefix, schema_type, naming_strategy) return subject not in subjects parsed_schema, valid_subjects = await self._query_schema_and_subjects( @@ -792,7 +794,11 @@ def subject_not_included(schema: TypedSchema, subjects: List[Subject]) -> bool: need_new_call=subject_not_included, ) - if self.config["name_strategy_validation"] and subject_not_included(parsed_schema, valid_subjects): + if ( + self.config["name_strategy_validation"] + and naming_strategy != NameStrategy.no_validation + and subject_not_included(parsed_schema, valid_subjects) + ): raise InvalidSchema() return schema_id diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 3dec4a887..f4c78330f 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -8,6 +8,7 @@ from avro.schema import Schema as AvroSchema from contextlib import closing, ExitStack +from enum import Enum from jsonschema.validators import Draft7Validator from kafka import KafkaConsumer, TopicPartition from kafka.admin import KafkaAdminClient, NewTopic @@ -20,7 +21,7 @@ TopicAlreadyExistsError, ) from karapace import constants -from karapace.config import Config +from karapace.config import Config, NameStrategy from karapace.dependency import Dependency from karapace.errors import InvalidReferences, InvalidSchema from karapace.in_memory_database import InMemoryDatabase @@ -31,7 +32,7 @@ from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_references import LatestVersionReference, Reference, reference_from_mapping, Referents from karapace.statsd import StatsClient -from karapace.typing import JsonObject, ResolvedVersion, SchemaId, Subject +from karapace.typing import JsonObject, ResolvedVersion, SchemaId, Subject, TopicName from karapace.utils import json_decode, JSONDecodeError, KarapaceKafkaClient from threading import Event, Thread from typing import Final, Mapping, Sequence @@ -58,6 +59,14 @@ METRIC_SUBJECT_DATA_SCHEMA_VERSIONS_GAUGE: Final = "karapace_schema_reader_subject_data_schema_versions" +class MessageType(Enum): + config = "CONFIG" + schema = "SCHEMA" + delete_subject = "DELETE_SUBJECT" + schema_strategy = "SCHEMA_STRATEGY" + no_operation = "NOOP" + + def _create_consumer_from_config(config: Config) -> KafkaConsumer: # Group not set on purpose, all consumers read the same data session_timeout_ms = config["session_timeout_ms"] @@ -429,6 +438,11 @@ def _handle_msg_delete_subject(self, key: dict, value: dict | None) -> None: # LOG.info("Deleting subject: %r, value: %r", subject, value) self.database.delete_subject(subject=subject, version=version) + def _handle_msg_schema_strategy(self, key: dict, value: dict | None) -> None: # pylint: disable=unused-argument + assert isinstance(value, dict) + topic, strategy = value["topic"], value["strategy"] + self.database.override_topic_strategy(topic_name=TopicName(topic), name_strategy=NameStrategy(strategy)) + def _handle_msg_schema_hard_delete(self, key: dict) -> None: subject, version = key["subject"], key["version"] @@ -522,14 +536,27 @@ def _handle_msg_schema(self, key: dict, value: dict | None) -> None: self.database.insert_referenced_by(subject=ref.subject, version=ref.version, schema_id=schema_id) def handle_msg(self, key: dict, value: dict | None) -> None: - if key["keytype"] == "CONFIG": - self._handle_msg_config(key, value) - elif key["keytype"] == "SCHEMA": - self._handle_msg_schema(key, value) - elif key["keytype"] == "DELETE_SUBJECT": - self._handle_msg_delete_subject(key, value) - elif key["keytype"] == "NOOP": # for spec completeness - pass + if "keytype" in key: + try: + message_type = MessageType(key["keytype"]) + + if message_type == MessageType.config: + self._handle_msg_config(key, value) + elif message_type == MessageType.schema: + self._handle_msg_schema(key, value) + elif message_type == MessageType.delete_subject: + self._handle_msg_delete_subject(key, value) + elif message_type == MessageType.schema_strategy: + self._handle_msg_schema_strategy(key, value) + elif message_type == MessageType.no_operation: + pass + except ValueError: + LOG.error("The message %s-%s has been discarded because the %s is not managed", key, value, key["keytype"]) + + else: + LOG.error( + "The message %s-%s has been discarded because doesn't contain the `keytype` key in the key", key, value + ) def remove_referenced_by( self, diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 867eeb633..6e54b0517 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -7,7 +7,7 @@ from contextlib import AsyncExitStack, closing from karapace.compatibility import check_compatibility, CompatibilityModes from karapace.compatibility.jsonschema.checks import is_incompatible -from karapace.config import Config +from karapace.config import Config, NameStrategy from karapace.dependency import Dependency from karapace.errors import ( IncompatibleSchema, @@ -27,9 +27,9 @@ from karapace.messaging import KarapaceProducer from karapace.offset_watcher import OffsetWatcher from karapace.schema_models import ParsedTypedSchema, SchemaType, SchemaVersion, TypedSchema, ValidatedTypedSchema -from karapace.schema_reader import KafkaSchemaReader +from karapace.schema_reader import KafkaSchemaReader, MessageType from karapace.schema_references import LatestVersionReference, Reference -from karapace.typing import JsonObject, ResolvedVersion, SchemaId, Subject, Version +from karapace.typing import JsonObject, ResolvedVersion, SchemaId, Subject, TopicName, Version from typing import Mapping, Sequence import asyncio @@ -466,6 +466,19 @@ def send_schema_message( value = None self.producer.send_message(key=key, value=value) + def get_validation_strategy_for_topic(self, *, topic_name: TopicName) -> NameStrategy: + return self.database.get_topic_strategy(topic_name=topic_name) + + def send_validation_strategy_for_topic( + self, + *, + topic_name: TopicName, + validation_strategy: NameStrategy, + ) -> None: + key = {"topic": topic_name, "keytype": MessageType.schema_strategy.value, "magic": 0} + value = {"strategy": validation_strategy.value, "topic": topic_name} + self.producer.send_message(key=key, value=value) + def send_config_message(self, compatibility_level: CompatibilityModes, subject: Subject | None = None) -> None: key = {"subject": subject, "magic": 0, "keytype": "CONFIG"} value = {"compatibilityLevel": compatibility_level.value} diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index f4d22cd78..d9a43aa09 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -11,7 +11,7 @@ from karapace.auth import HTTPAuthorizer, Operation, User from karapace.compatibility import check_compatibility, CompatibilityModes from karapace.compatibility.jsonschema.checks import is_incompatible -from karapace.config import Config +from karapace.config import Config, NameStrategy from karapace.errors import ( IncompatibleSchema, InvalidReferences, @@ -28,13 +28,13 @@ SubjectSoftDeletedException, VersionNotFoundException, ) -from karapace.karapace import KarapaceBase +from karapace.karapace import empty_response, KarapaceBase from karapace.protobuf.exception import ProtobufUnresolvedDependencyException from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE, SERVER_NAME from karapace.schema_models import ParsedTypedSchema, SchemaType, SchemaVersion, TypedSchema, ValidatedTypedSchema from karapace.schema_references import LatestVersionReference, Reference, reference_from_mapping from karapace.schema_registry import KarapaceSchemaRegistry, validate_version -from karapace.typing import JsonData, JsonObject, ResolvedVersion, SchemaId +from karapace.typing import JsonData, JsonObject, ResolvedVersion, SchemaId, TopicName from karapace.utils import JSONDecodeError from typing import Any @@ -301,6 +301,23 @@ def _add_schema_registry_routes(self) -> None: json_body=False, auth=self._auth, ) + self.route( + "/topic//name_strategy", + callback=self.subject_validation_strategy_get, + method="GET", + schema_request=True, + json_body=False, + auth=None, + ) + self.route( + "/topic//name_strategy/", + callback=self.subject_validation_strategy_set, + method="POST", + schema_request=True, + with_request=True, + json_body=False, + auth=None, + ) async def close(self) -> None: async with AsyncExitStack() as stack: @@ -985,6 +1002,38 @@ def _validate_schema_type(self, content_type: str, data: JsonData) -> SchemaType ) return schema_type + def _validate_topic_name(self, topic: str) -> TopicName: + valid_topic_names = self.schema_registry.schema_reader.admin_client.list_topics() + + if topic in valid_topic_names: + return TopicName(topic) + + self.r( + body={ + "error_code": SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value, + "message": f"The topic {topic} isn't existing, proceed with creating it first", + }, + content_type=JSON_CONTENT_TYPE, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + + def _validate_name_strategy(self, name_strategy: str) -> NameStrategy: + try: + strategy = NameStrategy(name_strategy) + return strategy + except ValueError: + valid_strategies = [strategy.value for strategy in NameStrategy] + error_message = f"Invalid name strategy: {name_strategy}, valid values are {valid_strategies}" + + self.r( + body={ + "error_code": SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value, + "message": error_message, + }, + content_type=JSON_CONTENT_TYPE, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + def _validate_schema_key(self, content_type: str, body: dict) -> None: if "schema" not in body: self.r( @@ -1238,6 +1287,44 @@ async def subject_post( url = f"{master_url}/subjects/{subject}/versions" await self._forward_request_remote(request=request, body=body, url=url, content_type=content_type, method="POST") + async def subject_validation_strategy_get(self, content_type: str, *, topic: str) -> None: + strategy_name = self.schema_registry.get_validation_strategy_for_topic(topic_name=TopicName(topic)).value + reply = {"strategy": strategy_name} + self.r(reply, content_type) + + async def subject_validation_strategy_set( + self, + content_type: str, + request: HTTPRequest, + *, + topic: str, + strategy: str, + ) -> None: + # proceeding with the strategy first since it's cheaper + strategy_name = self._validate_name_strategy(strategy) + # real validation of the topic name commented, do we need to do that? does it make sense? + topic_name = TopicName(topic) # self._validate_topic_name(topic) + + are_we_master, master_url = await self.schema_registry.get_master() + if are_we_master: + self.schema_registry.send_validation_strategy_for_topic( + topic_name=topic_name, + validation_strategy=strategy_name, + ) + empty_response() + else: + # I don't really like it, in theory we should parse the URL and change only the host portion while + # keeping the rest the same + url = f"{master_url}/topic/{topic}/name_strategy" + + await self._forward_request_remote( + request=request, + body=None, + url=url, + content_type=content_type, + method="POST", + ) + def get_schema_id_if_exists(self, *, subject: str, schema: TypedSchema, include_deleted: bool) -> SchemaId | None: schema_id = self.schema_registry.database.get_schema_id_if_exists( subject=subject, schema=schema, include_deleted=include_deleted diff --git a/karapace/serialization.py b/karapace/serialization.py index 29dc51a6c..25b073b73 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -9,13 +9,14 @@ from google.protobuf.message import DecodeError from jsonschema import ValidationError from karapace.client import Client +from karapace.config import NameStrategy from karapace.dependency import Dependency from karapace.errors import InvalidReferences from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter from karapace.schema_models import InvalidSchema, ParsedTypedSchema, SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_references import LatestVersionReference, Reference, reference_from_mapping -from karapace.typing import ResolvedVersion, SchemaId, Subject +from karapace.typing import ResolvedVersion, SchemaId, Subject, TopicName from karapace.utils import json_decode, json_encode from typing import Any, Callable, Dict, List, MutableMapping, Optional, Set, Tuple from urllib.parse import quote @@ -71,10 +72,15 @@ def topic_record_name_strategy(topic_name: str, record_name: str) -> str: return topic_name + "-" + record_name +def no_validation_strategy(topic_name: str, record_name: str) -> str: + return f"__auto_registration_anonymous_{topic_record_name_strategy(topic_name, record_name)}" + + NAME_STRATEGIES = { - "topic_name": topic_name_strategy, - "record_name": record_name_strategy, - "topic_record_name": topic_record_name_strategy, + NameStrategy.topic_name: topic_name_strategy, + NameStrategy.record_name: record_name_strategy, + NameStrategy.topic_record_name: topic_record_name_strategy, + NameStrategy.no_validation: no_validation_strategy, } @@ -103,7 +109,7 @@ async def post_new_schema( raise SchemaRetrievalError(result.json()) return SchemaId(result.json()["id"]) - async def _get_schema_r( + async def _get_schema_recursive( self, subject: Subject, explored_schemas: Set[Tuple[Subject, Optional[ResolvedVersion]]], @@ -131,7 +137,7 @@ async def _get_schema_r( references = [Reference.from_dict(data) for data in json_result["references"]] dependencies = {} for reference in references: - _, schema, version = await self._get_schema_r(reference.subject, explored_schemas, reference.version) + _, schema, version = await self._get_schema_recursive(reference.subject, explored_schemas, reference.version) dependencies[reference.name] = Dependency( name=reference.name, subject=reference.subject, version=version, target_schema=schema ) @@ -174,7 +180,7 @@ async def get_schema( - ValidatedTypedSchema: The retrieved schema, validated and typed. - ResolvedVersion: The version of the schema that was retrieved. """ - return await self._get_schema_r(subject, set(), version) + return await self._get_schema_recursive(subject, set(), version) async def get_schema_for_id(self, schema_id: SchemaId) -> Tuple[TypedSchema, List[Subject]]: result = await self.client.get(f"schemas/ids/{schema_id}", params={"includeSubjects": "True"}) @@ -243,12 +249,11 @@ def __init__( else: registry_url = f"http://{self.config['registry_host']}:{self.config['registry_port']}" registry_client = SchemaRegistryClient(registry_url, session_auth=session_auth) - name_strategy = config.get("name_strategy", "topic_name") - self.subject_name_strategy = NAME_STRATEGIES.get(name_strategy, topic_name_strategy) self.registry_client: Optional[SchemaRegistryClient] = registry_client self.ids_to_schemas: Dict[int, TypedSchema] = {} self.ids_to_subjects: MutableMapping[int, List[Subject]] = TTLCache(maxsize=10000, ttl=600) self.schemas_to_ids: Dict[str, SchemaId] = {} + self._topic_strategy_cache: MutableMapping[TopicName, NameStrategy] = TTLCache(maxsize=10000, ttl=600) async def close(self) -> None: if self.registry_client: @@ -261,6 +266,7 @@ def get_subject_name( schema: TypedSchema, subject_type: str, schema_type: SchemaType, + naming_strategy: NameStrategy, ) -> Subject: namespace = "dummy" if schema_type is SchemaType.AVRO: @@ -272,7 +278,19 @@ def get_subject_name( if schema_type is SchemaType.PROTOBUF: namespace = "" - return Subject(f"{self.subject_name_strategy(topic_name, namespace)}-{subject_type}") + naming_strategy = NAME_STRATEGIES[naming_strategy] + + return Subject(f"{naming_strategy(topic_name, namespace)}-{subject_type}") + + async def get_topic_strategy_name(self, topic_name: TopicName) -> NameStrategy: + assert self.registry_client, "must not call this method after the object is closed." + if topic_name in self._topic_strategy_cache: + return self._topic_strategy_cache[topic_name] + result = await self.registry_client.client.get(f"topic/{topic_name}/name_strategy") + + strategy = NameStrategy(result.json()["strategy"]) + self._topic_strategy_cache[topic_name] = strategy + return strategy async def get_schema_for_subject(self, subject: Subject) -> TypedSchema: assert self.registry_client, "must not call this method after the object is closed." diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 4e325a2a0..0ceb53e3c 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -7,6 +7,7 @@ from http import HTTPStatus from kafka import KafkaProducer from karapace.client import Client +from karapace.config import NameStrategy from karapace.rapu import is_success from karapace.schema_registry_apis import SchemaErrorMessages from karapace.utils import json_encode @@ -1079,6 +1080,25 @@ async def assert_schema_versions_failed(client: Client, trail: str, schema_id: i assert res.status_code == response_code +async def test_default_name_strategy(registry_async_client: Client) -> None: + res = await registry_async_client.get("/topic/foo/name_strategy") + assert res.ok + assert res.json() == {"strategy": NameStrategy.topic_name.value} + + +async def test_set_name_strategy(registry_async_client: Client) -> None: + res = await registry_async_client.get("/topic/foo/name_strategy") + assert res.ok + assert res.json() == {"strategy": NameStrategy.topic_name.value} + + res = await registry_async_client.post(f"/topic/foo/name_strategy/{NameStrategy.record_name.value}", json={}) + assert res.ok + + res = await registry_async_client.get("/topic/foo/name_strategy") + assert res.ok + assert res.json() == {"strategy": NameStrategy.record_name.value} + + async def register_schema(registry_async_client: Client, trail, subject: str, schema_str: str) -> Tuple[int, int]: # Register to get the id res = await registry_async_client.post(