diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 5fd6c856d..d0fe34428 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -16,8 +16,8 @@ from kpops.component_handlers.schema_handler.schema_handler import SchemaHandler from kpops.component_handlers.topic.handler import TopicHandler from kpops.component_handlers.topic.proxy_wrapper import ProxyWrapper -from kpops.components.base_components.models.resource import Resource from kpops.config import KpopsConfig +from kpops.manifests.kubernetes import KubernetesManifest from kpops.pipeline import ( Pipeline, PipelineGenerator, @@ -79,7 +79,7 @@ def manifest_deploy( environment: str | None = None, verbose: bool = True, operation_mode: OperationMode = OperationMode.MANIFEST, -) -> Iterator[Resource]: +) -> Iterator[tuple[KubernetesManifest, ...]]: pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 0d6af160e..27afb1d94 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -227,7 +227,7 @@ def deploy( ) for resource in resources: for rendered_manifest in resource: - print_yaml(rendered_manifest) + print_yaml(rendered_manifest.model_dump()) @app.command(help="Destroy pipeline steps") diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index 3fd38df20..d4b519c27 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -20,13 +20,11 @@ RepoAuthFlags, Version, ) -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.manifests.kubernetes import KubernetesManifest if TYPE_CHECKING: from collections.abc import Iterable, Iterator - from kpops.components.base_components.models.resource import Resource - log = logging.getLogger("Helm") @@ -162,7 +160,7 @@ def template( namespace: str, values: dict[str, Any], flags: HelmTemplateFlags | None = None, - ) -> Resource: + ) -> tuple[KubernetesManifest, ...]: """From Helm: Render chart templates locally and display the output. Any values that would normally be looked up or retrieved in-cluster will @@ -193,7 +191,7 @@ def template( command.extend(flags.to_command()) output = self.__execute(command) manifests = KubernetesManifest.from_yaml(output) - return list(manifests) + return tuple(manifests) def get_manifest(self, release_name: str, namespace: str) -> Iterable[HelmTemplate]: command = [ diff --git a/kpops/component_handlers/helm_wrapper/helm_diff.py b/kpops/component_handlers/helm_wrapper/helm_diff.py index e90edc433..5004ee52c 100644 --- a/kpops/component_handlers/helm_wrapper/helm_diff.py +++ b/kpops/component_handlers/helm_wrapper/helm_diff.py @@ -2,7 +2,7 @@ from collections.abc import Iterable, Iterator from kpops.component_handlers.helm_wrapper.model import HelmDiffConfig, HelmTemplate -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.dict_differ import Change, render_diff log = logging.getLogger("HelmDiff") @@ -16,7 +16,7 @@ def __init__(self, config: HelmDiffConfig) -> None: def calculate_changes( current_release: Iterable[HelmTemplate], new_release: Iterable[HelmTemplate], - ) -> Iterator[Change[KubernetesManifest, KubernetesManifest]]: + ) -> Iterator[Change[KubernetesManifest | None, KubernetesManifest | None]]: """Compare 2 releases and generate a Change object for each difference. :param current_release: Iterable containing HelmTemplate objects for the current release @@ -33,12 +33,15 @@ def calculate_changes( new_resource = new_release_index.pop(current_resource.filepath, None) yield Change( current_resource.manifest, - new_resource.manifest if new_resource else KubernetesManifest(), + new_resource.manifest if new_resource else None, ) # collect added files for new_resource in new_release_index.values(): - yield Change(KubernetesManifest(), new_resource.manifest) + yield Change( + None, + new_resource.manifest, + ) def log_helm_diff( self, @@ -48,8 +51,8 @@ def log_helm_diff( ) -> None: for change in self.calculate_changes(current_release, new_release): if diff := render_diff( - change.old_value.data, - change.new_value.data, + change.old_value.model_dump() if change.old_value else {}, + change.new_value.model_dump() if change.new_value else {}, ignore=self.config.ignore, ): logger.info("\n" + diff) diff --git a/kpops/component_handlers/helm_wrapper/model.py b/kpops/component_handlers/helm_wrapper/model.py index b81328e46..d23f2c299 100644 --- a/kpops/component_handlers/helm_wrapper/model.py +++ b/kpops/component_handlers/helm_wrapper/model.py @@ -6,7 +6,7 @@ from typing_extensions import override from kpops.component_handlers.helm_wrapper.exception import ParseError -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import DescConfigModel diff --git a/kpops/component_handlers/helm_wrapper/utils.py b/kpops/component_handlers/helm_wrapper/utils.py index aa618f6a1..add4b8bcc 100644 --- a/kpops/component_handlers/helm_wrapper/utils.py +++ b/kpops/component_handlers/helm_wrapper/utils.py @@ -1,5 +1,5 @@ -from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN from kpops.component_handlers.kubernetes.utils import trim +from kpops.manifests.kubernetes import K8S_LABEL_MAX_LEN RELEASE_NAME_MAX_LEN = 53 diff --git a/kpops/component_handlers/kubernetes/model.py b/kpops/component_handlers/kubernetes/model.py deleted file mode 100644 index 5970de1bf..000000000 --- a/kpops/component_handlers/kubernetes/model.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -import json -from collections import UserDict -from collections.abc import Iterator - -import yaml - -from kpops.utils.types import JsonType - -K8S_LABEL_MAX_LEN = 63 - - -class KubernetesManifest(UserDict[str, JsonType]): - """Representation of a Kubernetes API object as YAML/JSON mapping.""" - - @classmethod - def from_yaml( - cls, /, content: str - ) -> Iterator[KubernetesManifest]: # TODO: typing.Self for Python 3.11+ - manifests: Iterator[dict[str, JsonType]] = yaml.load_all(content, yaml.Loader) - for manifest in manifests: - yield cls(manifest) - - @classmethod - def from_json( - cls, /, content: str - ) -> KubernetesManifest: # TODO: typing.Self for Python 3.11+ - manifest: dict[str, JsonType] = json.loads(content) - return cls(manifest) diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index a2658b0e3..38d768359 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -22,14 +22,13 @@ create_helm_name_override, create_helm_release_name, ) -from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN from kpops.components.base_components.kubernetes_app import ( KubernetesApp, KubernetesAppValues, ) -from kpops.components.base_components.models.resource import Resource from kpops.config import get_config from kpops.manifests.argo import ArgoSyncWave, enrich_annotations +from kpops.manifests.kubernetes import K8S_LABEL_MAX_LEN, KubernetesManifest from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import exclude_by_name @@ -144,7 +143,7 @@ def template_flags(self) -> HelmTemplateFlags: ) @override - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: sync_wave = ArgoSyncWave(sync_wave=1) diff --git a/kpops/components/base_components/models/resource.py b/kpops/components/base_components/models/resource.py index e6867081d..e69de29bb 100644 --- a/kpops/components/base_components/models/resource.py +++ b/kpops/components/base_components/models/resource.py @@ -1,5 +0,0 @@ -from collections.abc import Mapping -from typing import Any, TypeAlias - -# representation of final resource for component, e.g. a list of Kubernetes manifests -Resource: TypeAlias = list[Mapping[str, Any]] diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 28445fe17..b37a1da83 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -13,7 +13,6 @@ FromTopic, InputTopicTypes, ) -from kpops.components.base_components.models.resource import Resource from kpops.components.base_components.models.to_section import ( ToSection, ) @@ -22,6 +21,7 @@ OutputTopicTypes, TopicConfig, ) +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr @@ -229,21 +229,21 @@ def inflate(self) -> list[PipelineComponent]: """ return [self] - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: """Render Kubernetes manifests for deploy.""" - return [] + return () - def manifest_destroy(self) -> Resource: + def manifest_destroy(self) -> tuple[KubernetesManifest, ...]: """Render Kubernetes manifests resources for destroy.""" - return [] + return () - def manifest_reset(self) -> Resource: + def manifest_reset(self) -> tuple[KubernetesManifest, ...]: """Render Kubernetes manifests resources for reset.""" - return [] + return () - def manifest_clean(self) -> Resource: + def manifest_clean(self) -> tuple[KubernetesManifest, ...]: """Render Kubernetes manifests resources for clean.""" - return [] + return () async def deploy(self, dry_run: bool) -> None: """Deploy component, e.g. to Kubernetes cluster. diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index c31393a19..47d78d75b 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -6,7 +6,6 @@ from kpops.api import OperationMode from kpops.components.base_components.kafka_app import KafkaAppCleaner -from kpops.components.base_components.models.resource import Resource from kpops.components.common.app_type import AppType from kpops.components.common.topic import ( KafkaTopic, @@ -20,6 +19,7 @@ from kpops.config import get_config from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.manifests.argo import ArgoHook, enrich_annotations +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr log = logging.getLogger("ProducerApp") @@ -36,11 +36,12 @@ def helm_chart(self) -> str: ) @override - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: post_delete = ArgoHook.POST_DELETE values = enrich_annotations(values, post_delete.key, post_delete.value) + return self.helm.template( self.helm_release_name, self.helm_chart, @@ -144,11 +145,12 @@ async def clean(self, dry_run: bool) -> None: await super().clean(dry_run) await self._cleaner.clean(dry_run) - def manifest_deploy(self) -> Resource: + @override + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: manifests = super().manifest_deploy() operation_mode = get_config().operation_mode if operation_mode is OperationMode.ARGO: - manifests.extend(self._cleaner.manifest_deploy()) + manifests = manifests + self._cleaner.manifest_deploy() return manifests diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 39677bfe2..cdffbcd35 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -8,7 +8,6 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp from kpops.components.base_components.kafka_app import KafkaAppCleaner -from kpops.components.base_components.models.resource import Resource from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.base import ( @@ -20,6 +19,7 @@ from kpops.config import get_config from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.manifests.argo import ArgoHook, enrich_annotations +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr log = logging.getLogger("StreamsApp") @@ -49,7 +49,7 @@ async def clean(self, dry_run: bool) -> None: await self.clean_pvcs(dry_run) @override - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: post_delete = ArgoHook.POST_DELETE @@ -174,13 +174,13 @@ async def clean(self, dry_run: bool) -> None: await self._cleaner.clean(dry_run) @override - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: manifests = super().manifest_deploy() if get_config().operation_mode is OperationMode.ARGO: - manifests.extend(self._cleaner.manifest_deploy()) + manifests = manifests + self._cleaner.manifest_deploy() return manifests @override - def manifest_clean(self) -> Resource: - return [] + def manifest_clean(self) -> tuple[KubernetesManifest, ...]: + return () diff --git a/kpops/manifests/kubernetes.py b/kpops/manifests/kubernetes.py new file mode 100644 index 000000000..342bcf585 --- /dev/null +++ b/kpops/manifests/kubernetes.py @@ -0,0 +1,80 @@ +from collections.abc import Iterator +from typing import Any + +import pydantic +import yaml +from pydantic import ConfigDict, Field +from typing_extensions import override + +from kpops.utils.pydantic import CamelCaseConfigModel, by_alias + +K8S_LABEL_MAX_LEN = 63 + + +class ObjectMeta(CamelCaseConfigModel): + """Metadata for all Kubernetes objects. + + https://gtsystem.github.io/lightkube-models/1.19/models/meta_v1/#objectmeta + + """ + + annotations: dict[str, str] | None = None + creation_timestamp: str | None = Field( + default=None, description="Timestamp in RFC3339 format" + ) + finalizers: list[str] | None = None + labels: dict[str, str] | None = None + name: str | None = None + namespace: str | None = None + resource_version: str | None = None + uid: str | None = None + + model_config = ConfigDict(extra="allow") + + @pydantic.model_serializer(mode="wrap", when_used="always") + def serialize_model( + self, + default_serialize_handler: pydantic.SerializerFunctionWrapHandler, + info: pydantic.SerializationInfo, + ) -> dict[str, Any]: + result = default_serialize_handler(self) + return { + by_alias(self, name): value + for name, value in result.items() + if name in self.model_fields_set + } + + +class KubernetesManifest(CamelCaseConfigModel): + api_version: str + kind: str + metadata: ObjectMeta + _required: set[str] = pydantic.PrivateAttr({"api_version", "kind"}) + + model_config = ConfigDict(extra="allow") + + @classmethod + def from_yaml( + cls, /, content: str + ) -> Iterator["KubernetesManifest"]: # TODO: typing.Self for Python 3.11+ + manifests: Iterator[dict[str, Any]] = yaml.load_all(content, yaml.Loader) + for manifest in manifests: + yield cls(**manifest) + + @pydantic.model_serializer(mode="wrap", when_used="always") + def serialize_model( + self, + default_serialize_handler: pydantic.SerializerFunctionWrapHandler, + info: pydantic.SerializationInfo, + ) -> dict[str, Any]: + include = self._required | self.model_fields_set + result = default_serialize_handler(self) + return { + by_alias(self, name): value + for name, value in result.items() + if name in include + } + + @override + def model_dump(self, **_: Any) -> dict[str, Any]: + return super().model_dump(mode="json") diff --git a/tests/component_handlers/helm_wrapper/test_dry_run_handler.py b/tests/component_handlers/helm_wrapper/test_dry_run_handler.py index 0e05ad09f..b33411001 100644 --- a/tests/component_handlers/helm_wrapper/test_dry_run_handler.py +++ b/tests/component_handlers/helm_wrapper/test_dry_run_handler.py @@ -8,7 +8,7 @@ from kpops.component_handlers.helm_wrapper.dry_run_handler import DryRunHandler from kpops.component_handlers.helm_wrapper.model import HelmTemplate -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.manifests.kubernetes import KubernetesManifest log = Logger("TestLogger") @@ -35,7 +35,14 @@ def test_should_print_helm_diff_when_release_is_new( ): helm_mock.get_manifest.return_value = iter(()) new_release = iter( - [HelmTemplate(Path("path.yaml"), KubernetesManifest({"a": 1}))] + [ + HelmTemplate( + Path("path.yaml"), + KubernetesManifest.model_validate( + {"apiVersion": "v1", "kind": "Deployment", "metadata": {}} + ), + ) + ] ) mock_load_manifest = mocker.patch( "kpops.component_handlers.helm_wrapper.dry_run_handler.Helm.load_manifest", @@ -61,12 +68,24 @@ def test_should_print_helm_diff_when_release_exists( caplog: LogCaptureFixture, ): current_release = [ - HelmTemplate(Path("path.yaml"), KubernetesManifest({"a": 1})) + HelmTemplate( + Path("path.yaml"), + KubernetesManifest.model_validate( + {"apiVersion": "v1", "kind": "Deployment", "metadata": {}} + ), + ) ] helm_mock.get_manifest.return_value = iter(current_release) new_release = iter( - [HelmTemplate(Path("path.yaml"), KubernetesManifest({"a": 1}))] + [ + HelmTemplate( + Path("path.yaml"), + KubernetesManifest.model_validate( + {"apiVersion": "v1", "kind": "Deployment", "metadata": {}} + ), + ) + ] ) mock_load_manifest = mocker.patch( "kpops.component_handlers.helm_wrapper.dry_run_handler.Helm.load_manifest", diff --git a/tests/component_handlers/helm_wrapper/test_helm_diff.py b/tests/component_handlers/helm_wrapper/test_helm_diff.py index ce64ec4ae..8373b5f0c 100644 --- a/tests/component_handlers/helm_wrapper/test_helm_diff.py +++ b/tests/component_handlers/helm_wrapper/test_helm_diff.py @@ -6,7 +6,7 @@ from kpops.component_handlers.helm_wrapper.helm_diff import HelmDiff from kpops.component_handlers.helm_wrapper.model import HelmDiffConfig, HelmTemplate -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.manifests.kubernetes import KubernetesManifest, ObjectMeta from kpops.utils.dict_differ import Change logger = logging.getLogger("TestHelmDiff") @@ -18,11 +18,34 @@ def helm_diff(self) -> HelmDiff: return HelmDiff(HelmDiffConfig()) def test_calculate_changes_unchanged(self, helm_diff: HelmDiff): - templates = [HelmTemplate(Path("a.yaml"), KubernetesManifest())] + templates = [ + HelmTemplate( + Path("a.yaml"), + KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({}), + } + ), + ) + ] assert list(helm_diff.calculate_changes(templates, templates)) == [ Change( - old_value={}, - new_value={}, + old_value=KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({}), + } + ), + new_value=KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({}), + } + ), ), ] @@ -31,26 +54,86 @@ def test_calculate_changes_matching(self, helm_diff: HelmDiff): assert list( helm_diff.calculate_changes( [ - HelmTemplate(Path("a.yaml"), KubernetesManifest({"a": 1})), - HelmTemplate(Path("b.yaml"), KubernetesManifest({"b": 1})), + HelmTemplate( + Path("a.yaml"), + KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"a": "1"}), + } + ), + ), + HelmTemplate( + Path("b.yaml"), + KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"b": "1"}), + } + ), + ), ], [ - HelmTemplate(Path("a.yaml"), KubernetesManifest({"a": 2})), - HelmTemplate(Path("c.yaml"), KubernetesManifest({"c": 1})), + HelmTemplate( + Path("a.yaml"), + KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"a": "2"}), + } + ), + ), + HelmTemplate( + Path("c.yaml"), + KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"c": "1"}), + } + ), + ), ], ) ) == [ Change( - old_value={"a": 1}, - new_value={"a": 2}, + old_value=KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"a": "1"}), + } + ), + new_value=KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"a": "2"}), + } + ), ), Change( - old_value={"b": 1}, - new_value={}, + old_value=KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"b": "1"}), + } + ), + new_value=None, ), Change( - old_value={}, - new_value={"c": 1}, + old_value=None, + new_value=KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"c": "1"}), + } + ), ), ] @@ -58,12 +141,30 @@ def test_calculate_changes_new_release(self, helm_diff: HelmDiff): # test no current release assert list( helm_diff.calculate_changes( - (), [HelmTemplate(Path("a.yaml"), KubernetesManifest({"a": 1}))] + (), + [ + HelmTemplate( + Path("a.yaml"), + KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"a": "1"}), + } + ), + ) + ], ) ) == [ Change( - old_value={}, - new_value={"a": 1}, + old_value=None, + new_value=KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"a": "1"}), + } + ), ), ] @@ -71,6 +172,24 @@ def test_log_helm_diff(self, helm_diff: HelmDiff, caplog: LogCaptureFixture): helm_diff.log_helm_diff( logger, (), - [HelmTemplate(Path("a.yaml"), KubernetesManifest({"a": 1}))], + [ + HelmTemplate( + Path("a.yaml"), + KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": ObjectMeta.model_validate({"a": "1"}), + } + ), + ) + ], ) - assert caplog.messages == ["\n\x1b[32m+ a: 1\n\x1b[0m"] + assert caplog.messages == [ + "\n" + "\x1b[32m+ apiVersion: v1\n" + "\x1b[0m\x1b[32m+ kind: Deployment\n" + "\x1b[0m\x1b[32m+ metadata:\n" + "\x1b[0m\x1b[32m+ a: '1'\n" + "\x1b[0m" + ] diff --git a/tests/component_handlers/helm_wrapper/test_helm_wrapper.py b/tests/component_handlers/helm_wrapper/test_helm_wrapper.py index 23c855251..3d80e59b2 100644 --- a/tests/component_handlers/helm_wrapper/test_helm_wrapper.py +++ b/tests/component_handlers/helm_wrapper/test_helm_wrapper.py @@ -13,12 +13,12 @@ HelmConfig, HelmTemplateFlags, HelmUpgradeInstallFlags, - KubernetesManifest, ParseError, RepoAuthFlags, Version, ) from kpops.components.common.app_type import AppType +from kpops.manifests.kubernetes import KubernetesManifest class TestHelmWrapper: @@ -292,7 +292,7 @@ def test_validate_console_output(self): def test_helm_template(self): path = Path("test2.yaml") - manifest = KubernetesManifest( + manifest = KubernetesManifest.model_validate( { "apiVersion": "v1", "kind": "ServiceAccount", @@ -309,12 +309,16 @@ def test_load_manifest_with_no_notes(self): MANIFEST: --- # Source: chart/templates/test3a.yaml - data: - - a: 1 - - b: 2 + apiVersion: v1 + kind: Pod + metadata: + name: test-3a --- # Source: chart/templates/test3b.yaml - foo: bar + apiVersion: v1 + kind: Pod + metadata: + name: test-3b """ ) helm_templates = list(Helm.load_manifest(stdout)) @@ -323,11 +327,21 @@ def test_load_manifest_with_no_notes(self): isinstance(helm_template, HelmTemplate) for helm_template in helm_templates ) assert helm_templates[0].filepath == Path("chart/templates/test3a.yaml") - assert helm_templates[0].manifest == KubernetesManifest( - {"data": [{"a": 1}, {"b": 2}]} + assert helm_templates[0].manifest == KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "test-3a"}, + } ) assert helm_templates[1].filepath == Path("chart/templates/test3b.yaml") - assert helm_templates[1].manifest == KubernetesManifest({"foo": "bar"}) + assert helm_templates[1].manifest == KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "test-3b"}, + } + ) def test_raise_parse_error_when_helm_content_is_invalid(self): stdout = dedent( @@ -372,12 +386,16 @@ def test_load_manifest(self): MANIFEST: --- # Source: chart/templates/test3a.yaml - data: - - a: 1 - - b: 2 + apiVersion: v1 + kind: Pod + metadata: + name: test-3a --- # Source: chart/templates/test3b.yaml - foo: bar + apiVersion: v1 + kind: Pod + metadata: + name: test-3b NOTES: 1. Get the application URL by running these commands: @@ -393,20 +411,31 @@ def test_load_manifest(self): isinstance(helm_template, HelmTemplate) for helm_template in helm_templates ) assert helm_templates[0].filepath == Path("chart/templates/test3a.yaml") - assert helm_templates[0].manifest == KubernetesManifest( - {"data": [{"a": 1}, {"b": 2}]} + assert helm_templates[0].manifest == KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "test-3a"}, + } ) assert helm_templates[1].filepath == Path("chart/templates/test3b.yaml") - assert helm_templates[1].manifest == KubernetesManifest({"foo": "bar"}) + assert helm_templates[1].manifest == KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "test-3b"}, + } + ) def test_helm_get_manifest(self, helm: Helm, mock_execute: MagicMock): mock_execute.return_value = dedent( """ --- # Source: chart/templates/test.yaml - data: - - a: 1 - - b: 2 + apiVersion: v1 + kind: Pod + metadata: + name: my-pod """ ) helm_templates = list(helm.get_manifest("test-release", "test-namespace")) @@ -422,8 +451,12 @@ def test_helm_get_manifest(self, helm: Helm, mock_execute: MagicMock): ) assert len(helm_templates) == 1 assert helm_templates[0].filepath == Path("chart/templates/test.yaml") - assert helm_templates[0].manifest == KubernetesManifest( - {"data": [{"a": 1}, {"b": 2}]} + assert helm_templates[0].manifest == KubernetesManifest.model_validate( + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "my-pod"}, + } ) mock_execute.side_effect = ReleaseNotFoundException() diff --git a/tests/component_handlers/kubernetes/model.py b/tests/component_handlers/kubernetes/model.py index 334c1f937..06c492def 100644 --- a/tests/component_handlers/kubernetes/model.py +++ b/tests/component_handlers/kubernetes/model.py @@ -2,7 +2,7 @@ import pytest -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.manifests.kubernetes import KubernetesManifest class TestKubernetesManifest: @@ -23,7 +23,7 @@ class TestKubernetesManifest: ), [ KubernetesManifest( - { + **{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"labels": {"foo": "bar"}}, diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py index 0d0649747..f72514299 100644 --- a/tests/components/test_helm_app.py +++ b/tests/components/test_helm_app.py @@ -9,8 +9,8 @@ HelmUpgradeInstallFlags, RepoAuthFlags, ) -from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN from kpops.components.base_components.helm_app import HelmApp, HelmAppValues +from kpops.manifests.kubernetes import K8S_LABEL_MAX_LEN from kpops.utils.colorify import magentaify diff --git a/tests/manifest/__init__.py b/tests/manifests/__init__.py similarity index 100% rename from tests/manifest/__init__.py rename to tests/manifests/__init__.py diff --git a/tests/manifest/test_argo_enricher.py b/tests/manifests/test_argo_enricher.py similarity index 100% rename from tests/manifest/test_argo_enricher.py rename to tests/manifests/test_argo_enricher.py diff --git a/tests/manifests/test_kubernetes_model.py b/tests/manifests/test_kubernetes_model.py new file mode 100644 index 000000000..d1fbcd623 --- /dev/null +++ b/tests/manifests/test_kubernetes_model.py @@ -0,0 +1,125 @@ +from textwrap import dedent + +import pytest + +from kpops.manifests.kubernetes import KubernetesManifest, ObjectMeta + + +class TestCRD(KubernetesManifest): + api_version: str = "v1" + kind: str = "TestCRD" + + +@pytest.fixture +def crd_manifest() -> TestCRD: + return TestCRD(metadata=ObjectMeta.model_validate({"foo": "bar"})) + + +@pytest.fixture +def example_manifest() -> KubernetesManifest: + """Fixture providing an example KubernetesManifest instance.""" + metadata = ObjectMeta( + name="example", + namespace="default", + labels={"app": "test"}, + ) + return KubernetesManifest( + api_version="v1", + kind="Deployment", + metadata=metadata, + ) + + +def test_serialize_model_include_required_fields(crd_manifest: TestCRD): + """Test that the serialize_model method excludes unset fields.""" + serialized = crd_manifest.model_dump() + expected_serialized = { + "apiVersion": "v1", + "kind": "TestCRD", + "metadata": {"foo": "bar"}, + } + assert serialized == expected_serialized + + +def test_serialize_model_excludes_none(example_manifest: KubernetesManifest): + """Test that the serialize_model method excludes unset fields.""" + serialized = example_manifest.model_dump() + expected_serialized = { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": { + "name": "example", + "namespace": "default", + "labels": {"app": "test"}, + }, + } + assert serialized == expected_serialized + + +def test_serialize_model_includes_required_fields(): + """Test that required fields are always included in serialization.""" + metadata = ObjectMeta(name="example", namespace="default") + manifest = KubernetesManifest(api_version="v1", kind="Pod", metadata=metadata) + serialized = manifest.model_dump() + assert "apiVersion" in serialized + assert "kind" in serialized + assert "metadata" in serialized + + +def test_from_yaml_parsing(): + """Test the from_yaml method parses YAML into KubernetesManifest objects.""" + yaml_content = dedent(""" + --- + apiVersion: v1 + kind: Service + metadata: + name: test-service + namespace: test-namespace + + --- + apiVersion: v1 + kind: Pod + metadata: + name: test-pod + namespace: test-namespace + """) + manifests = list(KubernetesManifest.from_yaml(yaml_content)) + assert len(manifests) == 2 + assert manifests[0].api_version == "v1" + assert manifests[0].kind == "Service" + assert manifests[0].metadata.name == "test-service" + assert manifests[1].kind == "Pod" + assert manifests[1].metadata.name == "test-pod" + + +def test_model_dump_json_output(example_manifest: KubernetesManifest): + """Test the model_dump method for JSON output.""" + dumped = example_manifest.model_dump() + expected_dumped = { + "apiVersion": "v1", + "kind": "Deployment", + "metadata": { + "name": "example", + "namespace": "default", + "labels": {"app": "test"}, + }, + } + assert dumped == expected_dumped + + +def test_objectmeta_serialization(): + """Test ObjectMeta serialization with optional fields.""" + metadata = ObjectMeta( + name="example", + namespace="default", + labels={"app": "test"}, + annotations=None, # This field should be included + ) + serialized = metadata.model_dump() + expected_serialized = { + "annotations": None, + "name": "example", + "namespace": "default", + "labels": {"app": "test"}, + } + assert serialized == expected_serialized diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml new file mode 100644 index 000000000..f7086850d --- /dev/null +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml @@ -0,0 +1,107 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: resources-manifest-pipeline-my-producer-app + chart: producer-app-3.0.3 + release: resources-manifest-pipeline-my-producer-app + name: resources-manifest-pipeline-my-producer-app +spec: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-manifest-pipeline-my-producer-app + release: resources-manifest-pipeline-my-producer-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-manifest-pipeline-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + consumerGroup: my-streams-app-id + labels: + app: resources-manifest-pipeline-my-streams-app + chart: streams-app-3.0.3 + release: resources-manifest-pipeline-my-streams-app + name: resources-manifest-pipeline-my-streams-app +spec: + replicas: 1 + selector: + matchLabels: + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app + template: + metadata: + labels: + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_VOLATILE_GROUP_INSTANCE_ID + value: 'true' + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_INPUT_TOPICS + value: my-input-topic + - name: APP_INPUT_PATTERN + value: my-input-pattern + - name: APP_OUTPUT_TOPIC + value: my-output-topic + - name: APP_ERROR_TOPIC + value: resources-manifest-pipeline-my-streams-app-error + - name: APP_LABELED_OUTPUT_TOPICS + value: my-output-topic-label=my-labeled-topic-output, + - name: APP_LABELED_INPUT_TOPICS + value: my-input-topic-label=my-labeled-input-topic, + - name: APP_LABELED_INPUT_PATTERNS + value: my-input-topic-labeled-pattern=my-labeled-input-pattern, + - name: APP_APPLICATION_ID + value: my-streams-app-id + - name: JAVA_TOOL_OPTIONS + value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-streams-app-image:1.0.0 + imagePullPolicy: Always + name: resources-manifest-pipeline-my-streams-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + terminationGracePeriodSeconds: 300 + diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources deleted file mode 100644 index 723353c93..000000000 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ /dev/null @@ -1,108 +0,0 @@ -- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest - data: - apiVersion: batch/v1 - kind: Job - metadata: - labels: - app: resources-manifest-pipeline-my-producer-app - chart: producer-app-3.0.3 - release: resources-manifest-pipeline-my-producer-app - name: resources-manifest-pipeline-my-producer-app - spec: - backoffLimit: 6 - template: - metadata: - labels: - app: resources-manifest-pipeline-my-producer-app - release: resources-manifest-pipeline-my-producer-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_OUTPUT_TOPIC - value: my-producer-app-output-topic - - name: APP_LABELED_OUTPUT_TOPICS - value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, - - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-producer-image:1.0.0 - imagePullPolicy: Always - name: resources-manifest-pipeline-my-producer-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - restartPolicy: OnFailure ---- -- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest - data: - apiVersion: apps/v1 - kind: Deployment - metadata: - annotations: - consumerGroup: my-streams-app-id - labels: - app: resources-manifest-pipeline-my-streams-app - chart: streams-app-3.0.3 - release: resources-manifest-pipeline-my-streams-app - name: resources-manifest-pipeline-my-streams-app - spec: - replicas: 1 - selector: - matchLabels: - app: resources-manifest-pipeline-my-streams-app - release: resources-manifest-pipeline-my-streams-app - template: - metadata: - labels: - app: resources-manifest-pipeline-my-streams-app - release: resources-manifest-pipeline-my-streams-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_VOLATILE_GROUP_INSTANCE_ID - value: 'true' - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_INPUT_TOPICS - value: my-input-topic - - name: APP_INPUT_PATTERN - value: my-input-pattern - - name: APP_OUTPUT_TOPIC - value: my-output-topic - - name: APP_ERROR_TOPIC - value: resources-manifest-pipeline-my-streams-app-error - - name: APP_LABELED_OUTPUT_TOPICS - value: my-output-topic-label=my-labeled-topic-output, - - name: APP_LABELED_INPUT_TOPICS - value: my-input-topic-label=my-labeled-input-topic, - - name: APP_LABELED_INPUT_PATTERNS - value: my-input-topic-labeled-pattern=my-labeled-input-pattern, - - name: APP_APPLICATION_ID - value: my-streams-app-id - - name: JAVA_TOOL_OPTIONS - value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false - -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-streams-app-image:1.0.0 - imagePullPolicy: Always - name: resources-manifest-pipeline-my-streams-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - terminationGracePeriodSeconds: 300 diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 983859a44..65c132cf4 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -3,7 +3,7 @@ from unittest.mock import ANY, MagicMock import pytest -import yaml +from _pytest.capture import CaptureFixture from pytest_mock import MockerFixture from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner @@ -13,6 +13,8 @@ from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.model import HelmConfig, Version from kpops.const.file_type import PIPELINE_YAML +from kpops.manifests.kubernetes import KubernetesManifest +from kpops.utils.yaml import print_yaml MANIFEST_YAML = "manifest.yaml" @@ -123,7 +125,7 @@ def test_manifest_command(self, snapshot: Snapshot): assert result.exit_code == 0, result.stdout snapshot.assert_match(result.stdout, MANIFEST_YAML) - def test_python_api(self, snapshot: Snapshot): + def test_python_api(self, capsys: CaptureFixture, snapshot: Snapshot): generator = kpops.manifest_deploy( RESOURCE_PATH / "manifest-pipeline" / PIPELINE_YAML, environment="development", @@ -131,7 +133,13 @@ def test_python_api(self, snapshot: Snapshot): assert isinstance(generator, Iterator) resources = list(generator) assert len(resources) == 2 - snapshot.assert_match(yaml.dump_all(resources), "resources") + for resource in resources: + for manifest in resource: + assert isinstance(manifest, KubernetesManifest) + print_yaml(manifest.model_dump()) + + captured = capsys.readouterr() + snapshot.assert_match(captured.out, MANIFEST_YAML) def test_streams_bootstrap(self, snapshot: Snapshot): result = runner.invoke(