From 813854a2c4a5f0a8e1d0370699a7203a5f10998c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 12 Dec 2024 18:32:56 +0100 Subject: [PATCH 01/28] Create annotated list type that serializes to `None` if empty --- docs/docs/schema/defaults.json | 19 ++++-------- docs/docs/schema/pipeline.json | 19 ++++-------- kpops/components/common/kubernetes_model.py | 29 +++++++++++++++---- .../test_streams_bootstrap.py | 10 ++++++- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index aef4b99f2..ceaad6bf1 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -960,20 +960,13 @@ "description": "Node affinity is a group of node affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/PreferredSchedulingTerm" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, + "default": [], "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding *weight* to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", - "title": "Preferredduringschedulingignoredduringexecution" + "items": { + "$ref": "#/$defs/PreferredSchedulingTerm" + }, + "title": "Preferredduringschedulingignoredduringexecution", + "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "anyOf": [ diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 29afa02e6..0b05d6c24 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -667,20 +667,13 @@ "description": "Node affinity is a group of node affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/PreferredSchedulingTerm" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, + "default": [], "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding *weight* to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", - "title": "Preferredduringschedulingignoredduringexecution" + "items": { + "$ref": "#/$defs/PreferredSchedulingTerm" + }, + "title": "Preferredduringschedulingignoredduringexecution", + "type": "array" }, "requiredDuringSchedulingIgnoredDuringExecution": { "anyOf": [ diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index e2fd0b7a6..9f231f1e7 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -1,7 +1,7 @@ from __future__ import annotations import enum -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING, Annotated, TypeVar import pydantic from pydantic import Field, model_validator @@ -130,6 +130,25 @@ class PreferredSchedulingTerm(DescConfigModel, CamelCaseConfigModel): weight: Weight = Field(description=describe_attr("weight", __doc__)) +_T = TypeVar("_T") + + +def serialize_list_to_optional( + value: list[_T], + default_serialize_handler: pydantic.SerializerFunctionWrapHandler, + info: pydantic.SerializationInfo, +) -> list[_T] | None: + result = default_serialize_handler(value) + return result or None + + +OptionalList = Annotated[ + list[_T], + pydantic.WrapSerializer(serialize_list_to_optional), + "list that is serialized to None if empty", +] + + class NodeAffinity(DescConfigModel, CamelCaseConfigModel): """Node affinity is a group of node affinity scheduling rules. @@ -143,10 +162,10 @@ class NodeAffinity(DescConfigModel, CamelCaseConfigModel): "required_during_scheduling_ignored_during_execution", __doc__ ), ) - preferred_during_scheduling_ignored_during_execution: ( - list[PreferredSchedulingTerm] | None - ) = Field( - default=None, + preferred_during_scheduling_ignored_during_execution: OptionalList[ + PreferredSchedulingTerm + ] = Field( + default=[], description=describe_attr( "preferred_during_scheduling_ignored_during_execution", __doc__ ), diff --git a/tests/components/streams_bootstrap/test_streams_bootstrap.py b/tests/components/streams_bootstrap/test_streams_bootstrap.py index d7deebae4..5c1efc9bf 100644 --- a/tests/components/streams_bootstrap/test_streams_bootstrap.py +++ b/tests/components/streams_bootstrap/test_streams_bootstrap.py @@ -12,7 +12,7 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.common.kubernetes_model import ResourceDefinition +from kpops.components.common.kubernetes_model import NodeAffinity, ResourceDefinition from kpops.components.streams_bootstrap.base import StreamsBootstrap from kpops.components.streams_bootstrap.model import StreamsBootstrapValues @@ -179,3 +179,11 @@ def test_resource_definition( ): with expectation: assert ResourceDefinition.model_validate(input) + + def test_affinity(self): + node_affinity = NodeAffinity() + assert node_affinity.preferred_during_scheduling_ignored_during_execution == [] + assert node_affinity.model_dump(by_alias=True) == { + "requiredDuringSchedulingIgnoredDuringExecution": None, + "preferredDuringSchedulingIgnoredDuringExecution": None, + } From 6e7448f764e86bc91ffbd1548c8783d3b16f1ed1 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 12 Dec 2024 18:34:56 +0100 Subject: [PATCH 02/28] Expand test --- .../test_streams_bootstrap.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/components/streams_bootstrap/test_streams_bootstrap.py b/tests/components/streams_bootstrap/test_streams_bootstrap.py index 5c1efc9bf..3141143a3 100644 --- a/tests/components/streams_bootstrap/test_streams_bootstrap.py +++ b/tests/components/streams_bootstrap/test_streams_bootstrap.py @@ -12,7 +12,12 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.common.kubernetes_model import NodeAffinity, ResourceDefinition +from kpops.components.common.kubernetes_model import ( + NodeAffinity, + NodeSelectorTerm, + PreferredSchedulingTerm, + ResourceDefinition, +) from kpops.components.streams_bootstrap.base import StreamsBootstrap from kpops.components.streams_bootstrap.model import StreamsBootstrapValues @@ -180,10 +185,26 @@ def test_resource_definition( with expectation: assert ResourceDefinition.model_validate(input) - def test_affinity(self): + def test_node_affinity(self): node_affinity = NodeAffinity() assert node_affinity.preferred_during_scheduling_ignored_during_execution == [] assert node_affinity.model_dump(by_alias=True) == { "requiredDuringSchedulingIgnoredDuringExecution": None, "preferredDuringSchedulingIgnoredDuringExecution": None, } + + node_affinity.preferred_during_scheduling_ignored_during_execution.append( + PreferredSchedulingTerm(preference=NodeSelectorTerm(), weight=1) + ) + assert node_affinity.model_dump(by_alias=True) == { + "requiredDuringSchedulingIgnoredDuringExecution": None, + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "preference": { + "matchExpressions": None, + "matchFields": None, + }, + "weight": 1, + } + ], + } From 903434e21cf77b37162aafbcbd633cf7c1a29004 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 12 Dec 2024 18:41:53 +0100 Subject: [PATCH 03/28] Make it more generic --- kpops/components/common/kubernetes_model.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index 9f231f1e7..5b60b789a 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -133,19 +133,19 @@ class PreferredSchedulingTerm(DescConfigModel, CamelCaseConfigModel): _T = TypeVar("_T") -def serialize_list_to_optional( - value: list[_T], +def serialize_to_optional( + value: _T, default_serialize_handler: pydantic.SerializerFunctionWrapHandler, info: pydantic.SerializationInfo, -) -> list[_T] | None: +) -> _T | None: result = default_serialize_handler(value) return result or None -OptionalList = Annotated[ - list[_T], - pydantic.WrapSerializer(serialize_list_to_optional), - "list that is serialized to None if empty", +SerializeAsOptional = Annotated[ + _T, + pydantic.WrapSerializer(serialize_to_optional), + "Optional that is serialized to None if falsy", ] @@ -162,8 +162,8 @@ class NodeAffinity(DescConfigModel, CamelCaseConfigModel): "required_during_scheduling_ignored_during_execution", __doc__ ), ) - preferred_during_scheduling_ignored_during_execution: OptionalList[ - PreferredSchedulingTerm + preferred_during_scheduling_ignored_during_execution: SerializeAsOptional[ + list[PreferredSchedulingTerm] ] = Field( default=[], description=describe_attr( From 111948e5593ef1155b8143058e6716509c9d39ae Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 12 Dec 2024 19:05:19 +0100 Subject: [PATCH 04/28] Improve schema --- docs/docs/schema/defaults.json | 17 ++++++++++++----- docs/docs/schema/pipeline.json | 17 ++++++++++++----- kpops/components/common/kubernetes_model.py | 17 +++++++++++++++-- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index ceaad6bf1..368e0329e 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -960,13 +960,20 @@ "description": "Node affinity is a group of node affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/PreferredSchedulingTerm" + }, + "type": "array" + }, + { + "type": "null" + } + ], "default": [], "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding *weight* to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", - "items": { - "$ref": "#/$defs/PreferredSchedulingTerm" - }, - "title": "Preferredduringschedulingignoredduringexecution", - "type": "array" + "title": "Preferredduringschedulingignoredduringexecution" }, "requiredDuringSchedulingIgnoredDuringExecution": { "anyOf": [ diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 0b05d6c24..b70a96f73 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -667,13 +667,20 @@ "description": "Node affinity is a group of node affinity scheduling rules.", "properties": { "preferredDuringSchedulingIgnoredDuringExecution": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/PreferredSchedulingTerm" + }, + "type": "array" + }, + { + "type": "null" + } + ], "default": [], "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding *weight* to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", - "items": { - "$ref": "#/$defs/PreferredSchedulingTerm" - }, - "title": "Preferredduringschedulingignoredduringexecution", - "type": "array" + "title": "Preferredduringschedulingignoredduringexecution" }, "requiredDuringSchedulingIgnoredDuringExecution": { "anyOf": [ diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index 5b60b789a..8a817c4a7 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -1,10 +1,11 @@ from __future__ import annotations import enum -from typing import TYPE_CHECKING, Annotated, TypeVar +from typing import TYPE_CHECKING, Annotated, Any, TypeVar import pydantic -from pydantic import Field, model_validator +from pydantic import Field, GetCoreSchemaHandler, model_validator +from pydantic_core import core_schema from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import CamelCaseConfigModel, DescConfigModel @@ -142,9 +143,21 @@ def serialize_to_optional( return result or None +class OptionalSchema: + def __get_pydantic_core_schema__( + self, + source: type[Any], + handler: GetCoreSchemaHandler, + ) -> core_schema.CoreSchema: + schema = handler(source) + # wrap generated schema in nullable + return core_schema.NullableSchema(type="nullable", schema=schema) + + SerializeAsOptional = Annotated[ _T, pydantic.WrapSerializer(serialize_to_optional), + OptionalSchema(), "Optional that is serialized to None if falsy", ] From 372c6be267c1f2f6b71d423e3a15b66d6089a378 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 12 Dec 2024 20:59:34 +0100 Subject: [PATCH 05/28] Move to Pydantic utils --- kpops/components/common/kubernetes_model.py | 42 ++++---------------- kpops/utils/pydantic.py | 44 ++++++++++++++++++++- 2 files changed, 49 insertions(+), 37 deletions(-) diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index 8a817c4a7..0091f2de4 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -1,14 +1,17 @@ from __future__ import annotations import enum -from typing import TYPE_CHECKING, Annotated, Any, TypeVar +from typing import TYPE_CHECKING, Annotated import pydantic -from pydantic import Field, GetCoreSchemaHandler, model_validator -from pydantic_core import core_schema +from pydantic import Field, model_validator from kpops.utils.docstring import describe_attr -from kpops.utils.pydantic import CamelCaseConfigModel, DescConfigModel +from kpops.utils.pydantic import ( + CamelCaseConfigModel, + DescConfigModel, + SerializeAsOptional, +) if TYPE_CHECKING: try: @@ -131,37 +134,6 @@ class PreferredSchedulingTerm(DescConfigModel, CamelCaseConfigModel): weight: Weight = Field(description=describe_attr("weight", __doc__)) -_T = TypeVar("_T") - - -def serialize_to_optional( - value: _T, - default_serialize_handler: pydantic.SerializerFunctionWrapHandler, - info: pydantic.SerializationInfo, -) -> _T | None: - result = default_serialize_handler(value) - return result or None - - -class OptionalSchema: - def __get_pydantic_core_schema__( - self, - source: type[Any], - handler: GetCoreSchemaHandler, - ) -> core_schema.CoreSchema: - schema = handler(source) - # wrap generated schema in nullable - return core_schema.NullableSchema(type="nullable", schema=schema) - - -SerializeAsOptional = Annotated[ - _T, - pydantic.WrapSerializer(serialize_to_optional), - OptionalSchema(), - "Optional that is serialized to None if falsy", -] - - class NodeAffinity(DescConfigModel, CamelCaseConfigModel): """Node affinity is a group of node affinity scheduling rules. diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index 3b3fb28ae..09f028dca 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -1,11 +1,20 @@ import json import logging from pathlib import Path -from typing import Any +from typing import Annotated, Any import humps -from pydantic import BaseModel, ConfigDict, Field +from pydantic import ( + BaseModel, + ConfigDict, + Field, + GetCoreSchemaHandler, + SerializationInfo, + SerializerFunctionWrapHandler, + WrapSerializer, +) from pydantic.fields import FieldInfo +from pydantic_core import core_schema from pydantic_settings import PydanticBaseSettingsSource from typing_extensions import TypeVar, override @@ -224,3 +233,34 @@ def __call__(self) -> dict[str, Any]: if field_value is not None: d[field_key] = field_value return d + + +_T = TypeVar("_T") + + +def serialize_to_optional( + value: _T, + default_serialize_handler: SerializerFunctionWrapHandler, + info: SerializationInfo, +) -> _T | None: + result = default_serialize_handler(value) + return result or None + + +class OptionalSchema: + def __get_pydantic_core_schema__( + self, + source: type[Any], + handler: GetCoreSchemaHandler, + ) -> core_schema.CoreSchema: + schema = handler(source) + # wrap generated schema in nullable + return core_schema.NullableSchema(type="nullable", schema=schema) + + +SerializeAsOptional = Annotated[ + _T, + WrapSerializer(serialize_to_optional), + OptionalSchema(), + "Optional that is serialized to None if falsy", +] From c5c2876099840175ec6476c383bdafcfd71a35a8 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 12 Dec 2024 21:04:45 +0100 Subject: [PATCH 06/28] Refactor other optional collection types --- docs/docs/schema/defaults.json | 18 ++++----- docs/docs/schema/pipeline.json | 18 ++++----- kpops/components/common/kubernetes_model.py | 44 ++++++++++----------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 368e0329e..da763feb5 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -893,7 +893,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "title": "Matchexpressions" }, @@ -909,7 +909,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is *key*, the operator is *In*, and the values array contains only *value*. The requirements are ANDed.", "title": "Matchlabels" } @@ -1065,7 +1065,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "A list of node selector requirements by node's labels.", "title": "Matchexpressions" }, @@ -1081,7 +1081,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "A list of node selector requirements by node's fields.", "title": "Matchfields" } @@ -1168,7 +1168,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding weight to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "title": "Preferredduringschedulingignoredduringexecution" }, @@ -1184,7 +1184,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "title": "Requiredduringschedulingignoredduringexecution" } @@ -1219,7 +1219,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).", "title": "Matchlabelkeys" }, @@ -1235,7 +1235,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).", "title": "Mismatchlabelkeys" }, @@ -1263,7 +1263,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Namespaces" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index b70a96f73..00df083fd 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -600,7 +600,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", "title": "Matchexpressions" }, @@ -616,7 +616,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is *key*, the operator is *In*, and the values array contains only *value*. The requirements are ANDed.", "title": "Matchlabels" } @@ -772,7 +772,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "A list of node selector requirements by node's labels.", "title": "Matchexpressions" }, @@ -788,7 +788,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "A list of node selector requirements by node's fields.", "title": "Matchfields" } @@ -828,7 +828,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding weight to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", "title": "Preferredduringschedulingignoredduringexecution" }, @@ -844,7 +844,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", "title": "Requiredduringschedulingignoredduringexecution" } @@ -879,7 +879,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).", "title": "Matchlabelkeys" }, @@ -895,7 +895,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).", "title": "Mismatchlabelkeys" }, @@ -923,7 +923,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Namespaces" }, diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index 0091f2de4..05d4006c9 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -102,11 +102,11 @@ class NodeSelectorTerm(DescConfigModel, CamelCaseConfigModel): :param match_fields: A list of node selector requirements by node's fields. """ - match_expressions: list[NodeSelectorRequirement] | None = Field( - default=None, description=describe_attr("match_expressions", __doc__) + match_expressions: SerializeAsOptional[list[NodeSelectorRequirement]] = Field( + default=[], description=describe_attr("match_expressions", __doc__) ) - match_fields: list[NodeSelectorRequirement] | None = Field( - default=None, description=describe_attr("match_fields", __doc__) + match_fields: SerializeAsOptional[list[NodeSelectorRequirement]] = Field( + default=[], description=describe_attr("match_fields", __doc__) ) @@ -201,12 +201,12 @@ class LabelSelector(DescConfigModel, CamelCaseConfigModel): :param match_expressions: matchExpressions is a list of label selector requirements. The requirements are ANDed. """ - match_labels: dict[str, str] | None = Field( - default=None, + match_labels: SerializeAsOptional[dict[str, str]] = Field( + default={}, description=describe_attr("match_labels", __doc__), ) - match_expressions: list[LabelSelectorRequirement] | None = Field( - default=None, + match_expressions: SerializeAsOptional[list[LabelSelectorRequirement]] = Field( + default=[], description=describe_attr("match_expressions", __doc__), ) @@ -226,19 +226,19 @@ class PodAffinityTerm(DescConfigModel, CamelCaseConfigModel): default=None, description=describe_attr("label_selector", __doc__), ) - match_label_keys: list[str] | None = Field( - default=None, + match_label_keys: SerializeAsOptional[list[str]] = Field( + default=[], description=describe_attr("match_label_keys", __doc__), ) - mismatch_label_keys: list[str] | None = Field( - default=None, + mismatch_label_keys: SerializeAsOptional[list[str]] = Field( + default=[], description=describe_attr("mismatch_label_keys", __doc__), ) topology_key: str = Field( description=describe_attr("topology_key", __doc__), ) - namespaces: list[str] | None = Field( - default=None, + namespaces: SerializeAsOptional[list[str]] = Field( + default=[], description=describe_attr("namespaces", __doc__), ) namespace_selector: LabelSelector | None = Field( @@ -269,18 +269,18 @@ class PodAffinity(DescConfigModel, CamelCaseConfigModel): :param preferred_during_scheduling_ignored_during_execution: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding weight to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. """ - required_during_scheduling_ignored_during_execution: ( - list[PodAffinityTerm] | None - ) = Field( - default=None, + required_during_scheduling_ignored_during_execution: SerializeAsOptional[ + list[PodAffinityTerm] + ] = Field( + default=[], description=describe_attr( "required_during_scheduling_ignored_during_execution", __doc__ ), ) - preferred_during_scheduling_ignored_during_execution: ( - list[WeightedPodAffinityTerm] | None - ) = Field( - default=None, + preferred_during_scheduling_ignored_during_execution: SerializeAsOptional[ + list[WeightedPodAffinityTerm] + ] = Field( + default=[], description=describe_attr( "preferred_during_scheduling_ignored_during_execution", __doc__ ), From f716bcb5f4130fab75574cb23d217c62983acae4 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 12 Dec 2024 21:05:58 +0100 Subject: [PATCH 07/28] Rename schema class --- kpops/utils/pydantic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index 09f028dca..7c49a8349 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -247,7 +247,7 @@ def serialize_to_optional( return result or None -class OptionalSchema: +class WrapNullableSchema: def __get_pydantic_core_schema__( self, source: type[Any], @@ -261,6 +261,6 @@ def __get_pydantic_core_schema__( SerializeAsOptional = Annotated[ _T, WrapSerializer(serialize_to_optional), - OptionalSchema(), + WrapNullableSchema(), "Optional that is serialized to None if falsy", ] From 306019c7fd573553801622efb99b1b2862b4425e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 12:17:50 +0100 Subject: [PATCH 08/28] Apply to StreamsBootstrapValues --- docs/docs/schema/defaults.json | 129 ++++++++------------ docs/docs/schema/pipeline.json | 86 ++++++------- kpops/components/streams_bootstrap/model.py | 53 ++++---- 3 files changed, 117 insertions(+), 151 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index da763feb5..6e30bde5f 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1624,7 +1624,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of command line arguments passed to the streams app.", "title": "Commandline" }, @@ -1666,7 +1666,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom environment variables.", "title": "Env" }, @@ -1692,7 +1692,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of files to mount for the app. File will be mounted as $value.mountPath/$key. $value.content denotes file content (recommended to be used with --set-file).", "title": "Files" }, @@ -1728,7 +1728,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Imagepullsecrets" }, @@ -1775,7 +1775,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Livenessprobe" }, @@ -1805,7 +1805,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom annotations to attach to the pod spec.", "title": "Podannotations" }, @@ -1821,7 +1821,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom labels to attach to the pod spec.", "title": "Podlabels" }, @@ -1837,7 +1837,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Ports" }, @@ -1850,7 +1850,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Readinessprobe" }, @@ -1903,7 +1903,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Mount existing secrets as volumes", "title": "Secretfilesrefs" }, @@ -1916,7 +1916,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Inject existing secrets as environment variables. Map key is used as environment variable name. Value consists of secret name and key.", "title": "Secretrefs" }, @@ -1932,7 +1932,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom secret environment variables. Prefix with configurationEnvPrefix in order to pass secrets to command line or prefix with KAFKA_ to pass secrets to Kafka Streams configuration.", "title": "Secrets" }, @@ -1975,20 +1975,13 @@ "title": "Suspend" }, "tolerations": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/Toleration" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "title": "Tolerations" + "items": { + "$ref": "#/$defs/Toleration" + }, + "title": "Tolerations", + "type": "array" }, "ttlSecondsAfterFinished": { "anyOf": [ @@ -2726,7 +2719,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of command line arguments passed to the streams app.", "title": "Commandline" }, @@ -2755,7 +2748,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom environment variables.", "title": "Env" }, @@ -2768,7 +2761,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of files to mount for the app. File will be mounted as $value.mountPath/$key. $value.content denotes file content (recommended to be used with --set-file).", "title": "Files" }, @@ -2804,7 +2797,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Imagepullsecrets" }, @@ -2863,7 +2856,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Livenessprobe" }, @@ -2905,7 +2898,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom annotations to attach to the pod spec.", "title": "Podannotations" }, @@ -2921,7 +2914,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom labels to attach to the pod spec.", "title": "Podlabels" }, @@ -2937,7 +2930,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Ports" }, @@ -2962,7 +2955,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Readinessprobe" }, @@ -2990,7 +2983,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Mount existing secrets as volumes", "title": "Secretfilesrefs" }, @@ -3003,7 +2996,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Inject existing secrets as environment variables. Map key is used as environment variable name. Value consists of secret name and key.", "title": "Secretrefs" }, @@ -3019,7 +3012,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom secret environment variables. Prefix with configurationEnvPrefix in order to pass secrets to command line or prefix with KAFKA_ to pass secrets to Kafka Streams configuration.", "title": "Secrets" }, @@ -3055,20 +3048,13 @@ "title": "Terminationgraceperiodseconds" }, "tolerations": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/Toleration" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "title": "Tolerations" + "items": { + "$ref": "#/$defs/Toleration" + }, + "title": "Tolerations", + "type": "array" } }, "required": [ @@ -3338,7 +3324,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of command line arguments passed to the streams app.", "title": "Commandline" }, @@ -3367,7 +3353,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom environment variables.", "title": "Env" }, @@ -3380,7 +3366,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of files to mount for the app. File will be mounted as $value.mountPath/$key. $value.content denotes file content (recommended to be used with --set-file).", "title": "Files" }, @@ -3416,7 +3402,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Imagepullsecrets" }, @@ -3463,7 +3449,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Livenessprobe" }, @@ -3493,7 +3479,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom annotations to attach to the pod spec.", "title": "Podannotations" }, @@ -3509,7 +3495,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom labels to attach to the pod spec.", "title": "Podlabels" }, @@ -3525,7 +3511,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Ports" }, @@ -3538,7 +3524,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Readinessprobe" }, @@ -3566,7 +3552,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Mount existing secrets as volumes", "title": "Secretfilesrefs" }, @@ -3579,7 +3565,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Inject existing secrets as environment variables. Map key is used as environment variable name. Value consists of secret name and key.", "title": "Secretrefs" }, @@ -3595,7 +3581,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom secret environment variables. Prefix with configurationEnvPrefix in order to pass secrets to command line or prefix with KAFKA_ to pass secrets to Kafka Streams configuration.", "title": "Secrets" }, @@ -3612,20 +3598,13 @@ "description": "" }, "tolerations": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/Toleration" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "title": "Tolerations" + "items": { + "$ref": "#/$defs/Toleration" + }, + "title": "Tolerations", + "type": "array" } }, "required": [ diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 00df083fd..9a94b1567 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1284,7 +1284,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of command line arguments passed to the streams app.", "title": "Commandline" }, @@ -1326,7 +1326,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom environment variables.", "title": "Env" }, @@ -1352,7 +1352,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of files to mount for the app. File will be mounted as $value.mountPath/$key. $value.content denotes file content (recommended to be used with --set-file).", "title": "Files" }, @@ -1388,7 +1388,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Imagepullsecrets" }, @@ -1435,7 +1435,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Livenessprobe" }, @@ -1465,7 +1465,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom annotations to attach to the pod spec.", "title": "Podannotations" }, @@ -1481,7 +1481,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom labels to attach to the pod spec.", "title": "Podlabels" }, @@ -1497,7 +1497,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Ports" }, @@ -1510,7 +1510,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Readinessprobe" }, @@ -1563,7 +1563,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Mount existing secrets as volumes", "title": "Secretfilesrefs" }, @@ -1576,7 +1576,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Inject existing secrets as environment variables. Map key is used as environment variable name. Value consists of secret name and key.", "title": "Secretrefs" }, @@ -1592,7 +1592,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom secret environment variables. Prefix with configurationEnvPrefix in order to pass secrets to command line or prefix with KAFKA_ to pass secrets to Kafka Streams configuration.", "title": "Secrets" }, @@ -1635,20 +1635,13 @@ "title": "Suspend" }, "tolerations": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/Toleration" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "title": "Tolerations" + "items": { + "$ref": "#/$defs/Toleration" + }, + "title": "Tolerations", + "type": "array" }, "ttlSecondsAfterFinished": { "anyOf": [ @@ -2386,7 +2379,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of command line arguments passed to the streams app.", "title": "Commandline" }, @@ -2415,7 +2408,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom environment variables.", "title": "Env" }, @@ -2428,7 +2421,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of files to mount for the app. File will be mounted as $value.mountPath/$key. $value.content denotes file content (recommended to be used with --set-file).", "title": "Files" }, @@ -2464,7 +2457,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Imagepullsecrets" }, @@ -2523,7 +2516,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Livenessprobe" }, @@ -2565,7 +2558,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom annotations to attach to the pod spec.", "title": "Podannotations" }, @@ -2581,7 +2574,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Map of custom labels to attach to the pod spec.", "title": "Podlabels" }, @@ -2597,7 +2590,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "", "title": "Ports" }, @@ -2622,7 +2615,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core", "title": "Readinessprobe" }, @@ -2650,7 +2643,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Mount existing secrets as volumes", "title": "Secretfilesrefs" }, @@ -2663,7 +2656,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Inject existing secrets as environment variables. Map key is used as environment variable name. Value consists of secret name and key.", "title": "Secretrefs" }, @@ -2679,7 +2672,7 @@ "type": "null" } ], - "default": null, + "default": {}, "description": "Custom secret environment variables. Prefix with configurationEnvPrefix in order to pass secrets to command line or prefix with KAFKA_ to pass secrets to Kafka Streams configuration.", "title": "Secrets" }, @@ -2715,20 +2708,13 @@ "title": "Terminationgraceperiodseconds" }, "tolerations": { - "anyOf": [ - { - "items": { - "$ref": "#/$defs/Toleration" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "title": "Tolerations" + "items": { + "$ref": "#/$defs/Toleration" + }, + "title": "Tolerations", + "type": "array" } }, "required": [ diff --git a/kpops/components/streams_bootstrap/model.py b/kpops/components/streams_bootstrap/model.py index 2bff9b6c2..282d48c68 100644 --- a/kpops/components/streams_bootstrap/model.py +++ b/kpops/components/streams_bootstrap/model.py @@ -11,6 +11,7 @@ ImagePullPolicy, ProtocolSchema, Resources, + SerializeAsOptional, ServiceType, Toleration, ) @@ -132,8 +133,8 @@ class StreamsBootstrapValues(HelmAppValues): description=describe_attr("image_pull_policy", __doc__), ) - image_pull_secrets: list[dict[str, str]] | None = Field( - default=None, + image_pull_secrets: SerializeAsOptional[list[dict[str, str]]] = Field( + default=[], description=describe_attr("image_pull_secret", __doc__), ) @@ -146,8 +147,8 @@ class StreamsBootstrapValues(HelmAppValues): description=describe_attr("resources", __doc__), ) - ports: list[PortConfig] | None = Field( - default=None, + ports: SerializeAsOptional[list[PortConfig]] = Field( + default=[], description=describe_attr("ports", __doc__), ) @@ -161,33 +162,33 @@ class StreamsBootstrapValues(HelmAppValues): description=describe_attr("configuration_env_prefix", __doc__), ) - command_line: dict[str, str | bool | int] | None = Field( - default=None, + command_line: SerializeAsOptional[dict[str, str | bool | int]] = Field( + default={}, description=describe_attr("command_line", __doc__), ) - env: dict[str, str] | None = Field( - default=None, + env: SerializeAsOptional[dict[str, str]] = Field( + default={}, description=describe_attr("env", __doc__), ) - secrets: dict[str, str] | None = Field( - default=None, + secrets: SerializeAsOptional[dict[str, str]] = Field( + default={}, description=describe_attr("secrets", __doc__), ) - secret_refs: dict[str, Any] | None = Field( - default=None, + secret_refs: SerializeAsOptional[dict[str, Any]] = Field( + default={}, description=describe_attr("secret_refs", __doc__), ) - secret_files_refs: list[str] | None = Field( - default=None, + secret_files_refs: SerializeAsOptional[list[str]] = Field( + default=[], description=describe_attr("secret_files_refs", __doc__), ) - files: dict[str, Any] | None = Field( - default=None, + files: SerializeAsOptional[dict[str, Any]] = Field( + default={}, description=describe_attr("files", __doc__), ) @@ -196,23 +197,23 @@ class StreamsBootstrapValues(HelmAppValues): description=describe_attr("java_options", __doc__), ) - pod_annotations: dict[str, str] | None = Field( - default=None, + pod_annotations: SerializeAsOptional[dict[str, str]] = Field( + default={}, description=describe_attr("pod_annotations", __doc__), ) - pod_labels: dict[str, str] | None = Field( - default=None, + pod_labels: SerializeAsOptional[dict[str, str]] = Field( + default={}, description=describe_attr("pod_labels", __doc__), ) - liveness_probe: dict[str, Any] | None = Field( - default=None, + liveness_probe: SerializeAsOptional[dict[str, Any]] = Field( + default={}, description=describe_attr("liveness_probe", __doc__), ) - readiness_probe: dict[str, Any] | None = Field( - default=None, + readiness_probe: SerializeAsOptional[dict[str, Any]] = Field( + default={}, description=describe_attr("readiness_probe", __doc__), ) @@ -221,8 +222,8 @@ class StreamsBootstrapValues(HelmAppValues): description=describe_attr("affinity", __doc__), ) - tolerations: list[Toleration] | None = Field( - default=None, + tolerations: list[Toleration] = Field( + default=[], description=describe_attr("tolerations", __doc__), ) From e74bcf4152eadf6d3070455c47ccbe59cdeec895 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 12:20:46 +0100 Subject: [PATCH 09/28] Apply to StreamsBootstrapValues --- docs/docs/schema/defaults.json | 8 ++++---- docs/docs/schema/pipeline.json | 8 ++++---- .../streams_bootstrap/streams/model.py | 17 +++++++++-------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 6e30bde5f..8682166f0 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -295,7 +295,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "List of JMX metric rules.", "title": "Metricrules" }, @@ -3882,7 +3882,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "List of additional KEDA triggers, see https://keda.sh/docs/latest/scalers/", "title": "Additionaltriggers" }, @@ -3930,7 +3930,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "List of auto-generated Kafka Streams topics used by the streams app", "title": "Internaltopics" }, @@ -4011,7 +4011,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "List of topics used by the streams app", "title": "Topics" } diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 9a94b1567..f2a4d1d56 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -295,7 +295,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "List of JMX metric rules.", "title": "Metricrules" }, @@ -2992,7 +2992,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "List of additional KEDA triggers, see https://keda.sh/docs/latest/scalers/", "title": "Additionaltriggers" }, @@ -3040,7 +3040,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "List of auto-generated Kafka Streams topics used by the streams app", "title": "Internaltopics" }, @@ -3121,7 +3121,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "List of topics used by the streams app", "title": "Topics" } diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py index 63fe1f10e..10f72abc8 100644 --- a/kpops/components/streams_bootstrap/streams/model.py +++ b/kpops/components/streams_bootstrap/streams/model.py @@ -8,6 +8,7 @@ from kpops.components.common.kubernetes_model import ( ImagePullPolicy, Resources, + SerializeAsOptional, ) from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.components.streams_bootstrap.model import ( @@ -191,16 +192,16 @@ class StreamsAppAutoScaling(CamelCaseConfigModel, DescConfigModel): title="Idle replica count", description=describe_attr("idle_replicas", __doc__), ) - internal_topics: list[str] | None = Field( - default=None, + internal_topics: SerializeAsOptional[list[str]] = Field( + default=[], description=describe_attr("internal_topics", __doc__), ) - topics: list[str] | None = Field( - default=None, + topics: SerializeAsOptional[list[str]] = Field( + default=[], description=describe_attr("topics", __doc__), ) - additional_triggers: list[str] | None = Field( - default=None, + additional_triggers: SerializeAsOptional[list[str]] = Field( + default=[], description=describe_attr("additional_triggers", __doc__), ) model_config = ConfigDict(extra="allow") @@ -289,8 +290,8 @@ class JMXConfig(CamelCaseConfigModel, DescConfigModel): description=describe_attr("port", __doc__), ) - metric_rules: list[str] | None = Field( - default=None, + metric_rules: SerializeAsOptional[list[str]] = Field( + default=[], description=describe_attr("metric_rules", __doc__), ) From 7503e4b564d30b38ae6b7c05b98fd18125b007f2 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 12:29:42 +0100 Subject: [PATCH 10/28] Fix --- docs/docs/schema/defaults.json | 51 +++++++++++++++------ docs/docs/schema/pipeline.json | 34 ++++++++++---- kpops/components/streams_bootstrap/model.py | 2 +- 3 files changed, 61 insertions(+), 26 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 8682166f0..734e2ba2a 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1975,13 +1975,20 @@ "title": "Suspend" }, "tolerations": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Toleration" + }, + "type": "array" + }, + { + "type": "null" + } + ], "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "items": { - "$ref": "#/$defs/Toleration" - }, - "title": "Tolerations", - "type": "array" + "title": "Tolerations" }, "ttlSecondsAfterFinished": { "anyOf": [ @@ -3048,13 +3055,20 @@ "title": "Terminationgraceperiodseconds" }, "tolerations": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Toleration" + }, + "type": "array" + }, + { + "type": "null" + } + ], "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "items": { - "$ref": "#/$defs/Toleration" - }, - "title": "Tolerations", - "type": "array" + "title": "Tolerations" } }, "required": [ @@ -3598,13 +3612,20 @@ "description": "" }, "tolerations": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Toleration" + }, + "type": "array" + }, + { + "type": "null" + } + ], "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "items": { - "$ref": "#/$defs/Toleration" - }, - "title": "Tolerations", - "type": "array" + "title": "Tolerations" } }, "required": [ diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index f2a4d1d56..1075dda76 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1635,13 +1635,20 @@ "title": "Suspend" }, "tolerations": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Toleration" + }, + "type": "array" + }, + { + "type": "null" + } + ], "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "items": { - "$ref": "#/$defs/Toleration" - }, - "title": "Tolerations", - "type": "array" + "title": "Tolerations" }, "ttlSecondsAfterFinished": { "anyOf": [ @@ -2708,13 +2715,20 @@ "title": "Terminationgraceperiodseconds" }, "tolerations": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Toleration" + }, + "type": "array" + }, + { + "type": "null" + } + ], "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", - "items": { - "$ref": "#/$defs/Toleration" - }, - "title": "Tolerations", - "type": "array" + "title": "Tolerations" } }, "required": [ diff --git a/kpops/components/streams_bootstrap/model.py b/kpops/components/streams_bootstrap/model.py index 282d48c68..5d758718e 100644 --- a/kpops/components/streams_bootstrap/model.py +++ b/kpops/components/streams_bootstrap/model.py @@ -222,7 +222,7 @@ class StreamsBootstrapValues(HelmAppValues): description=describe_attr("affinity", __doc__), ) - tolerations: list[Toleration] = Field( + tolerations: SerializeAsOptional[list[Toleration]] = Field( default=[], description=describe_attr("tolerations", __doc__), ) From ebf19a288736eb0483084171e0b66dab34e3f005 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 12:43:27 +0100 Subject: [PATCH 11/28] Skip serialization step for cleaner instantiation --- .../streams_bootstrap/producer/producer_app.py | 9 ++++++--- .../components/streams_bootstrap/streams/streams_app.py | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 44fe2e592..be445079d 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -63,9 +63,12 @@ def is_cron_job(self) -> bool: @computed_field @cached_property def _cleaner(self) -> ProducerAppCleaner: - return ProducerAppCleaner( - **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}) - ) + kwargs = { + name: getattr(self, name) + for name in self.model_fields_set + if name not in {"_cleaner", "from_", "to"} + } + return ProducerAppCleaner.model_validate(kwargs) @override def apply_to_outputs(self, name: str, topic: TopicConfig) -> None: diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index c0a21e7b6..6e5d8c037 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -67,9 +67,12 @@ class StreamsApp(StreamsBootstrap): @computed_field @cached_property def _cleaner(self) -> StreamsAppCleaner: - return StreamsAppCleaner( - **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}) - ) + kwargs = { + name: getattr(self, name) + for name in self.model_fields_set + if name not in {"_cleaner", "from_", "to"} + } + return StreamsAppCleaner.model_validate(kwargs) @property @override From 209137592cb22335ca041ae608fe136d4d5b7319 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 13:13:32 +0100 Subject: [PATCH 12/28] Add failing test --- tests/utils/test_pydantic.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 525db63b0..82bdfa8f4 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -1,7 +1,9 @@ from typing import Any import pytest +from pydantic import BaseModel +from kpops.components.common.kubernetes_model import SerializeAsOptional from kpops.utils.pydantic import to_dash, to_dot, to_snake, to_str @@ -64,3 +66,13 @@ def test_to_dot(input: str, expected: str): ) def test_to_str(input: Any, expected: str): assert to_str(input) == expected + + +def test_serialize_as_optional(): + class Model(BaseModel): + foo: SerializeAsOptional[list[str]] = [] + + assert Model().model_dump() == {"foo": None} + assert Model().model_dump(exclude_defaults=True) == {} + assert Model().model_dump(exclude_unset=True) == {} + assert Model().model_dump(exclude_none=True) == {} From 7a1f4b04e672747689c9ecc035c0d343f35f61b2 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 13:56:23 +0100 Subject: [PATCH 13/28] Try refactor to include serializer function --- kpops/utils/pydantic.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index 7c49a8349..b9362f309 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -9,9 +9,7 @@ ConfigDict, Field, GetCoreSchemaHandler, - SerializationInfo, SerializerFunctionWrapHandler, - WrapSerializer, ) from pydantic.fields import FieldInfo from pydantic_core import core_schema @@ -241,7 +239,6 @@ def __call__(self) -> dict[str, Any]: def serialize_to_optional( value: _T, default_serialize_handler: SerializerFunctionWrapHandler, - info: SerializationInfo, ) -> _T | None: result = default_serialize_handler(value) return result or None @@ -255,12 +252,19 @@ def __get_pydantic_core_schema__( ) -> core_schema.CoreSchema: schema = handler(source) # wrap generated schema in nullable - return core_schema.NullableSchema(type="nullable", schema=schema) + return core_schema.NullableSchema( + type="nullable", + schema=schema, + serialization=core_schema.wrap_serializer_function_ser_schema( + serialize_to_optional, + schema=core_schema.nullable_schema(schema), + ), + ) + return schema SerializeAsOptional = Annotated[ _T, - WrapSerializer(serialize_to_optional), WrapNullableSchema(), "Optional that is serialized to None if falsy", ] From 5917f8c2df60dd4950b502b6f3a648ce06754af3 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 13:59:13 +0100 Subject: [PATCH 14/28] Cosmetic --- tests/utils/test_pydantic.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 82bdfa8f4..c764a5d50 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -72,7 +72,8 @@ def test_serialize_as_optional(): class Model(BaseModel): foo: SerializeAsOptional[list[str]] = [] - assert Model().model_dump() == {"foo": None} - assert Model().model_dump(exclude_defaults=True) == {} - assert Model().model_dump(exclude_unset=True) == {} - assert Model().model_dump(exclude_none=True) == {} + model = Model() + assert model.model_dump() == {"foo": None} + assert model.model_dump(exclude_defaults=True) == {} + assert model.model_dump(exclude_unset=True) == {} + assert model.model_dump(exclude_none=True) == {} From 32de61b688fac29fabec2c86ed67138e7da5929c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 14:06:42 +0100 Subject: [PATCH 15/28] Implement and explain workaround in test --- tests/utils/test_pydantic.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index c764a5d50..0c00bdf8a 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -1,10 +1,11 @@ from typing import Any +import pydantic import pytest -from pydantic import BaseModel +from pydantic import BaseModel, model_serializer from kpops.components.common.kubernetes_model import SerializeAsOptional -from kpops.utils.pydantic import to_dash, to_dot, to_snake, to_str +from kpops.utils.pydantic import exclude_by_value, to_dash, to_dot, to_snake, to_str @pytest.mark.parametrize( @@ -72,8 +73,21 @@ def test_serialize_as_optional(): class Model(BaseModel): foo: SerializeAsOptional[list[str]] = [] + # HACK: workaround for exclude_none, which is otherwise evaluated too early + @model_serializer(mode="wrap", when_used="always") + def serialize_model( + self, + default_serialize_handler: pydantic.SerializerFunctionWrapHandler, + info: pydantic.SerializationInfo, + ) -> dict[str, Any]: + result = default_serialize_handler(self) + if info.exclude_none: + return exclude_by_value(result, None) + return result + model = Model() assert model.model_dump() == {"foo": None} assert model.model_dump(exclude_defaults=True) == {} assert model.model_dump(exclude_unset=True) == {} + # this would fail without custom model_serializer assert model.model_dump(exclude_none=True) == {} From bf2f6b661f1100b0ceb34b6aa78872df1fff1a30 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 14:18:10 +0100 Subject: [PATCH 16/28] Apply workaround --- kpops/components/common/kubernetes_model.py | 5 ++-- kpops/utils/pydantic.py | 17 ++++++++++++ .../test_streams_bootstrap.py | 9 +++---- tests/utils/test_pydantic.py | 26 +++++++------------ 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index 05d4006c9..051f928b9 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -11,6 +11,7 @@ CamelCaseConfigModel, DescConfigModel, SerializeAsOptional, + SerializeAsOptionalModel, ) if TYPE_CHECKING: @@ -95,7 +96,7 @@ def validate_values(self) -> Self: return self -class NodeSelectorTerm(DescConfigModel, CamelCaseConfigModel): +class NodeSelectorTerm(SerializeAsOptionalModel, DescConfigModel, CamelCaseConfigModel): """A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. :param match_expressions: A list of node selector requirements by node's labels. @@ -134,7 +135,7 @@ class PreferredSchedulingTerm(DescConfigModel, CamelCaseConfigModel): weight: Weight = Field(description=describe_attr("weight", __doc__)) -class NodeAffinity(DescConfigModel, CamelCaseConfigModel): +class NodeAffinity(SerializeAsOptionalModel, DescConfigModel, CamelCaseConfigModel): """Node affinity is a group of node affinity scheduling rules. :param required_during_scheduling_ignored_during_execution: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index b9362f309..20bb64108 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -9,7 +9,9 @@ ConfigDict, Field, GetCoreSchemaHandler, + SerializationInfo, SerializerFunctionWrapHandler, + model_serializer, ) from pydantic.fields import FieldInfo from pydantic_core import core_schema @@ -267,4 +269,19 @@ def __get_pydantic_core_schema__( _T, WrapNullableSchema(), "Optional that is serialized to None if falsy", + "requires inheriting from SerializeAsOptionalModel for `model_dump(exclude_none=True)` to work", ] + + +class SerializeAsOptionalModel(BaseModel): + # HACK: workaround for exclude_none, which is otherwise evaluated too early + @model_serializer(mode="wrap", when_used="always") + def serialize_model( + self, + default_serialize_handler: SerializerFunctionWrapHandler, + info: SerializationInfo, + ) -> dict[str, Any]: + result = default_serialize_handler(self) + if info.exclude_none: + return exclude_by_value(result, None) + return result diff --git a/tests/components/streams_bootstrap/test_streams_bootstrap.py b/tests/components/streams_bootstrap/test_streams_bootstrap.py index 3141143a3..b2d9b6f77 100644 --- a/tests/components/streams_bootstrap/test_streams_bootstrap.py +++ b/tests/components/streams_bootstrap/test_streams_bootstrap.py @@ -192,18 +192,15 @@ def test_node_affinity(self): "requiredDuringSchedulingIgnoredDuringExecution": None, "preferredDuringSchedulingIgnoredDuringExecution": None, } + assert node_affinity.model_dump(by_alias=True, exclude_none=True) == {} node_affinity.preferred_during_scheduling_ignored_during_execution.append( PreferredSchedulingTerm(preference=NodeSelectorTerm(), weight=1) ) - assert node_affinity.model_dump(by_alias=True) == { - "requiredDuringSchedulingIgnoredDuringExecution": None, + assert node_affinity.model_dump(by_alias=True, exclude_none=True) == { "preferredDuringSchedulingIgnoredDuringExecution": [ { - "preference": { - "matchExpressions": None, - "matchFields": None, - }, + "preference": {}, "weight": 1, } ], diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 0c00bdf8a..ce91203ae 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -1,11 +1,15 @@ from typing import Any -import pydantic import pytest -from pydantic import BaseModel, model_serializer from kpops.components.common.kubernetes_model import SerializeAsOptional -from kpops.utils.pydantic import exclude_by_value, to_dash, to_dot, to_snake, to_str +from kpops.utils.pydantic import ( + SerializeAsOptionalModel, + to_dash, + to_dot, + to_snake, + to_str, +) @pytest.mark.parametrize( @@ -70,24 +74,12 @@ def test_to_str(input: Any, expected: str): def test_serialize_as_optional(): - class Model(BaseModel): + class Model(SerializeAsOptionalModel): foo: SerializeAsOptional[list[str]] = [] - # HACK: workaround for exclude_none, which is otherwise evaluated too early - @model_serializer(mode="wrap", when_used="always") - def serialize_model( - self, - default_serialize_handler: pydantic.SerializerFunctionWrapHandler, - info: pydantic.SerializationInfo, - ) -> dict[str, Any]: - result = default_serialize_handler(self) - if info.exclude_none: - return exclude_by_value(result, None) - return result - model = Model() assert model.model_dump() == {"foo": None} assert model.model_dump(exclude_defaults=True) == {} assert model.model_dump(exclude_unset=True) == {} - # this would fail without custom model_serializer + # this would fail without inheriting from SerializeAsOptionalModel assert model.model_dump(exclude_none=True) == {} From 3ad6c9cfefbc51f25cda53670a7610e5e680684a Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 14:31:00 +0100 Subject: [PATCH 17/28] Link to upstream issue with potential solution for `exclude_none` --- kpops/utils/pydantic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index 20bb64108..1fff4e0e0 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -241,9 +241,12 @@ def __call__(self) -> dict[str, Any]: def serialize_to_optional( value: _T, default_serialize_handler: SerializerFunctionWrapHandler, + # info: SerializationInfo, ) -> _T | None: - result = default_serialize_handler(value) - return result or None + return default_serialize_handler(value) or None + # TODO: another potential solution, depends on https://github.com/pydantic/pydantic/issues/6969 + # if not result and info.exclude_none: + # raise PydanticOmit class WrapNullableSchema: @@ -262,7 +265,6 @@ def __get_pydantic_core_schema__( schema=core_schema.nullable_schema(schema), ), ) - return schema SerializeAsOptional = Annotated[ From 0c3ccb8bf10b8e300bcd604f932e239d687842f3 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 14:55:33 +0100 Subject: [PATCH 18/28] Validate `None` to default --- kpops/utils/pydantic.py | 13 +++++++++++-- tests/utils/test_pydantic.py | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index 1fff4e0e0..47293bdad 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -6,6 +6,7 @@ import humps from pydantic import ( BaseModel, + BeforeValidator, ConfigDict, Field, GetCoreSchemaHandler, @@ -14,7 +15,7 @@ model_serializer, ) from pydantic.fields import FieldInfo -from pydantic_core import core_schema +from pydantic_core import PydanticUseDefault, core_schema from pydantic_settings import PydanticBaseSettingsSource from typing_extensions import TypeVar, override @@ -238,6 +239,12 @@ def __call__(self) -> dict[str, Any]: _T = TypeVar("_T") +def validate_optional_to_default(value: Any | None) -> Any: + if value is None: + raise PydanticUseDefault + return value + + def serialize_to_optional( value: _T, default_serialize_handler: SerializerFunctionWrapHandler, @@ -270,7 +277,9 @@ def __get_pydantic_core_schema__( SerializeAsOptional = Annotated[ _T, WrapNullableSchema(), - "Optional that is serialized to None if falsy", + BeforeValidator(validate_optional_to_default), + "Optional that is serialized to `None` if falsy", + "similarly an input of `None` is translated to its default during validation", "requires inheriting from SerializeAsOptionalModel for `model_dump(exclude_none=True)` to work", ] diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index ce91203ae..ca6b79a11 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -78,8 +78,12 @@ class Model(SerializeAsOptionalModel): foo: SerializeAsOptional[list[str]] = [] model = Model() + assert model.foo == [] assert model.model_dump() == {"foo": None} assert model.model_dump(exclude_defaults=True) == {} assert model.model_dump(exclude_unset=True) == {} # this would fail without inheriting from SerializeAsOptionalModel assert model.model_dump(exclude_none=True) == {} + + model = Model.model_validate({"foo": None}) + assert model.foo == [] From e271eb202a368e800b38bc8915c3173e3b3ba418 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 15:38:27 +0100 Subject: [PATCH 19/28] Expand test --- tests/utils/test_pydantic.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index ca6b79a11..6bd9f0759 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -75,15 +75,38 @@ def test_to_str(input: Any, expected: str): def test_serialize_as_optional(): class Model(SerializeAsOptionalModel): - foo: SerializeAsOptional[list[str]] = [] + optional_list: SerializeAsOptional[list[str]] = [] + optional_dict: SerializeAsOptional[dict[str, str]] = {} model = Model() - assert model.foo == [] - assert model.model_dump() == {"foo": None} + assert model.optional_list == [] + assert model.optional_dict == {} + assert model.model_dump() == {"optional_list": None, "optional_dict": None} assert model.model_dump(exclude_defaults=True) == {} assert model.model_dump(exclude_unset=True) == {} # this would fail without inheriting from SerializeAsOptionalModel assert model.model_dump(exclude_none=True) == {} - model = Model.model_validate({"foo": None}) - assert model.foo == [] + model = Model.model_validate({"optional_list": None, "optional_dict": None}) + assert model.optional_list == [] + assert model.optional_dict == {} + + model = Model(optional_list=["el"], optional_dict={"foo": "bar"}) + assert model.optional_list == ["el"] + assert model.optional_dict == {"foo": "bar"} + assert model.model_dump() == { + "optional_list": ["el"], + "optional_dict": {"foo": "bar"}, + } + assert model.model_dump(exclude_defaults=True) == { + "optional_list": ["el"], + "optional_dict": {"foo": "bar"}, + } + assert model.model_dump(exclude_unset=True) == { + "optional_list": ["el"], + "optional_dict": {"foo": "bar"}, + } + assert model.model_dump(exclude_none=True) == { + "optional_list": ["el"], + "optional_dict": {"foo": "bar"}, + } From afd6ac2330b03a1647dc257e86e60edf5cd5843a Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 16:10:09 +0100 Subject: [PATCH 20/28] Serialize StreamsBootstrapValues correctly --- kpops/components/streams_bootstrap/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kpops/components/streams_bootstrap/model.py b/kpops/components/streams_bootstrap/model.py index 5d758718e..bc70e37e5 100644 --- a/kpops/components/streams_bootstrap/model.py +++ b/kpops/components/streams_bootstrap/model.py @@ -20,6 +20,7 @@ from kpops.utils.pydantic import ( CamelCaseConfigModel, DescConfigModel, + SerializeAsOptionalModel, exclude_by_value, exclude_defaults, ) @@ -94,7 +95,7 @@ class JavaOptions(CamelCaseConfigModel, DescConfigModel): ) -class StreamsBootstrapValues(HelmAppValues): +class StreamsBootstrapValues(SerializeAsOptionalModel, HelmAppValues): """Base value class for all streams bootstrap related components. :param image: Docker image of the Kafka producer app. From 549697c199638d79931423de3bbfd1767d21d9ae Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 16:16:16 +0100 Subject: [PATCH 21/28] Add test for exclude_by_value --- tests/utils/test_pydantic.py | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/utils/test_pydantic.py b/tests/utils/test_pydantic.py index 6bd9f0759..5d6be4a66 100644 --- a/tests/utils/test_pydantic.py +++ b/tests/utils/test_pydantic.py @@ -5,6 +5,7 @@ from kpops.components.common.kubernetes_model import SerializeAsOptional from kpops.utils.pydantic import ( SerializeAsOptionalModel, + exclude_by_value, to_dash, to_dot, to_snake, @@ -73,6 +74,64 @@ def test_to_str(input: Any, expected: str): assert to_str(input) == expected +@pytest.mark.parametrize( + ("dumped_model", "excluded_values", "expected"), + [ + pytest.param( + {}, + (), + {}, + ), + pytest.param( + {}, + (None,), + {}, + ), + pytest.param( + {"foo": 0, "bar": 1}, + (), + {"foo": 0, "bar": 1}, + ), + pytest.param( + {"foo": 0, "bar": 1}, + (None,), + {"foo": 0, "bar": 1}, + ), + pytest.param( + {"foo": 0, "bar": 1}, + (0,), + {"bar": 1}, + ), + pytest.param( + {"foo": 0, "bar": 1}, + (1,), + {"foo": 0}, + ), + pytest.param( + {"foo": None}, + (None,), + {}, + ), + pytest.param( + {"foo": None, "bar": 0}, + (None,), + {"bar": 0}, + ), + pytest.param( + {"foo": None, "bar": 0}, + (None, 0), + {}, + ), + ], +) +def test_exclude_by_value( + dumped_model: dict[str, Any], + excluded_values: tuple[Any, ...], + expected: dict[str, Any], +): + assert exclude_by_value(dumped_model, *excluded_values) == expected + + def test_serialize_as_optional(): class Model(SerializeAsOptionalModel): optional_list: SerializeAsOptional[list[str]] = [] From e55dfc938c4aa17d3899410d21a449136a42605b Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 17:04:23 +0100 Subject: [PATCH 22/28] Inherit from SerializeAsOptionalModel --- kpops/components/common/kubernetes_model.py | 6 +++--- kpops/components/streams_bootstrap/streams/model.py | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index 051f928b9..4353b6848 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -195,7 +195,7 @@ def validate_values(self) -> Self: return self -class LabelSelector(DescConfigModel, CamelCaseConfigModel): +class LabelSelector(SerializeAsOptionalModel, DescConfigModel, CamelCaseConfigModel): """A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. :param match_labels: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is *key*, the operator is *In*, and the values array contains only *value*. The requirements are ANDed. @@ -212,7 +212,7 @@ class LabelSelector(DescConfigModel, CamelCaseConfigModel): ) -class PodAffinityTerm(DescConfigModel, CamelCaseConfigModel): +class PodAffinityTerm(SerializeAsOptionalModel, DescConfigModel, CamelCaseConfigModel): """Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running. :param label_selector: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. @@ -263,7 +263,7 @@ class WeightedPodAffinityTerm(DescConfigModel, CamelCaseConfigModel): ) -class PodAffinity(DescConfigModel, CamelCaseConfigModel): +class PodAffinity(SerializeAsOptionalModel, DescConfigModel, CamelCaseConfigModel): """Pod affinity is a group of inter pod affinity scheduling rules. :param required_during_scheduling_ignored_during_execution: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py index 10f72abc8..c9626a22f 100644 --- a/kpops/components/streams_bootstrap/streams/model.py +++ b/kpops/components/streams_bootstrap/streams/model.py @@ -19,6 +19,7 @@ from kpops.utils.pydantic import ( CamelCaseConfigModel, DescConfigModel, + SerializeAsOptionalModel, ) @@ -118,7 +119,9 @@ def add_labeled_input_topics(self, label: str, topics: list[KafkaTopic]) -> None ) -class StreamsAppAutoScaling(CamelCaseConfigModel, DescConfigModel): +class StreamsAppAutoScaling( + SerializeAsOptionalModel, CamelCaseConfigModel, DescConfigModel +): """Kubernetes Event-driven Autoscaling config. :param enabled: Whether to enable auto-scaling using KEDA., defaults to False @@ -278,7 +281,7 @@ class PrometheusJMXExporterConfig(CamelCaseConfigModel, DescConfigModel): ) -class JMXConfig(CamelCaseConfigModel, DescConfigModel): +class JMXConfig(SerializeAsOptionalModel, CamelCaseConfigModel, DescConfigModel): """JMX configuration options. :param port: The jmx port which JMX style metrics are exposed. From 0ba91ca6f46a76c6c52a8042cfd2cce2bf590cc5 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 18:34:29 +0100 Subject: [PATCH 23/28] Extend snapshot test with affinity --- .../resources/streams-bootstrap/pipeline.yaml | 13 ++++++++ .../test_streams_bootstrap/pipeline.yaml | 30 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/tests/pipeline/resources/streams-bootstrap/pipeline.yaml b/tests/pipeline/resources/streams-bootstrap/pipeline.yaml index ad6308a4a..b2c9fae90 100644 --- a/tests/pipeline/resources/streams-bootstrap/pipeline.yaml +++ b/tests/pipeline/resources/streams-bootstrap/pipeline.yaml @@ -20,6 +20,19 @@ imageTag: "1.0.0" kafka: applicationId: "my-streams-app-id" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: foo + operator: Exists + weight: 2 + - preference: + matchExpressions: + - key: bar + operator: DoesNotExist + weight: 1 commandLine: CONVERT_XML: true javaOptions: diff --git a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml index d72426d8d..4741ad72f 100644 --- a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml @@ -85,6 +85,21 @@ suffix: -clean type: streams-app-cleaner values: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: foo + operator: Exists + values: [] + weight: 2 + - preference: + matchExpressions: + - key: bar + operator: DoesNotExist + values: [] + weight: 1 commandLine: CONVERT_XML: true files: @@ -162,6 +177,21 @@ value_schema: com.bakdata.kafka.DeadLetter type: my-streams-app values: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: foo + operator: Exists + values: [] + weight: 2 + - preference: + matchExpressions: + - key: bar + operator: DoesNotExist + values: [] + weight: 1 commandLine: CONVERT_XML: true files: From ec8841feea7063ff88846c95d4fce2e73b45270c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 16 Dec 2024 18:41:53 +0100 Subject: [PATCH 24/28] Update snapshot --- .../test_streams_bootstrap/manifest.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml index 075777bf9..84354ab8e 100644 --- a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml @@ -124,6 +124,19 @@ spec: app: resources-streams-bootstrap-my-streams-app release: resources-streams-bootstrap-my-streams-app spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: foo + operator: Exists + weight: 2 + - preference: + matchExpressions: + - key: bar + operator: DoesNotExist + weight: 1 containers: - env: - name: ENV_PREFIX From 92c9698bbcec5b36befc698fdeaec31d41ae932f Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Tue, 17 Dec 2024 16:29:26 +0100 Subject: [PATCH 25/28] Apply SerializeAsOptional to streams-bootstrap v2 --- docs/docs/schema/defaults.json | 6 +++--- docs/docs/schema/pipeline.json | 4 ++-- kpops/components/streams_bootstrap_v2/base.py | 8 +++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 5f6917aa7..2132826af 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1590,7 +1590,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", "title": "Tolerations" } @@ -2739,7 +2739,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", "title": "Tolerations" } @@ -3391,7 +3391,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", "title": "Tolerations" } diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 69aa92aae..af9302c82 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1250,7 +1250,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", "title": "Tolerations" } @@ -2399,7 +2399,7 @@ "type": "null" } ], - "default": null, + "default": [], "description": "Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling.", "title": "Tolerations" } diff --git a/kpops/components/streams_bootstrap_v2/base.py b/kpops/components/streams_bootstrap_v2/base.py index 27fc1e9b6..0cfe9b0d0 100644 --- a/kpops/components/streams_bootstrap_v2/base.py +++ b/kpops/components/streams_bootstrap_v2/base.py @@ -17,9 +17,11 @@ from kpops.utils.pydantic import ( CamelCaseConfigModel, DescConfigModel, + SerializeAsOptionalModel, exclude_by_value, exclude_defaults, ) +from tests.utils.test_pydantic import SerializeAsOptional if TYPE_CHECKING: try: @@ -97,7 +99,7 @@ def serialize_model( ) -class StreamsBootstrapV2Values(HelmAppValues): +class StreamsBootstrapV2Values(SerializeAsOptionalModel, HelmAppValues): """Base value class for all streams bootstrap v2 related components. :param image_tag: Docker image tag of the streams-bootstrap-v2 app. @@ -120,8 +122,8 @@ class StreamsBootstrapV2Values(HelmAppValues): description=describe_attr("affinity", __doc__), ) - tolerations: list[Toleration] | None = Field( - default=None, + tolerations: SerializeAsOptional[list[Toleration]] = Field( + default=[], description=describe_attr("tolerations", __doc__), ) From 302016897b1340aaa1ae3747cc72de746ddba38e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Tue, 17 Dec 2024 15:39:23 +0100 Subject: [PATCH 26/28] Fix allow optional resources requests and limits (#570) --- docs/docs/schema/defaults.json | 16 ++++++++++------ docs/docs/schema/pipeline.json | 16 ++++++++++------ kpops/components/common/kubernetes_model.py | 10 ++++++++-- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 2132826af..fb978b0d9 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -2383,26 +2383,30 @@ "description": "Model representing the resource specifications for a Kubernetes container.", "properties": { "limits": { - "allOf": [ + "anyOf": [ { "$ref": "#/$defs/ResourceDefinition" + }, + { + "type": "null" } ], + "default": null, "description": "The maximum resource limits for the container." }, "requests": { - "allOf": [ + "anyOf": [ { "$ref": "#/$defs/ResourceDefinition" + }, + { + "type": "null" } ], + "default": null, "description": "The minimum resource requirements for the container." } }, - "required": [ - "requests", - "limits" - ], "title": "Resources", "type": "object" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index af9302c82..4afa89596 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -2043,26 +2043,30 @@ "description": "Model representing the resource specifications for a Kubernetes container.", "properties": { "limits": { - "allOf": [ + "anyOf": [ { "$ref": "#/$defs/ResourceDefinition" + }, + { + "type": "null" } ], + "default": null, "description": "The maximum resource limits for the container." }, "requests": { - "allOf": [ + "anyOf": [ { "$ref": "#/$defs/ResourceDefinition" + }, + { + "type": "null" } ], + "default": null, "description": "The minimum resource requirements for the container." } }, - "required": [ - "requests", - "limits" - ], "title": "Resources", "type": "object" }, diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index 6c5f63a0f..967182218 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -389,5 +389,11 @@ class Resources(DescConfigModel): :param limits: The maximum resource limits for the container. """ - requests: ResourceDefinition = Field(description=describe_attr("requests", __doc__)) - limits: ResourceDefinition = Field(description=describe_attr("limits", __doc__)) + requests: ResourceDefinition | None = Field( + default=None, + description=describe_attr("requests", __doc__), + ) + limits: ResourceDefinition | None = Field( + default=None, + description=describe_attr("limits", __doc__), + ) From b14738f25b820d03dd10874eef1f9798837122fd Mon Sep 17 00:00:00 2001 From: bakdata-bots Date: Tue, 17 Dec 2024 14:50:23 +0000 Subject: [PATCH 27/28] =?UTF-8?q?Bump=20version=208.3.1=20=E2=86=92=208.3.?= =?UTF-8?q?2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/docs/user/changelog.md | 11 +++++++++++ kpops/const/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/docs/user/changelog.md b/docs/docs/user/changelog.md index 60732b3a9..f8f29b546 100644 --- a/docs/docs/user/changelog.md +++ b/docs/docs/user/changelog.md @@ -1,4 +1,15 @@ # Changelog +## [8.3.2](https://github.com/bakdata/kpops/releases/tag/8.3.2) - Release Date: [2024-12-17] + +### 🐛 Fixes + +- Fix allow optional resources requests and limits - [#570](https://github.com/bakdata/kpops/pull/570) + + + + + + ## [8.3.1](https://github.com/bakdata/kpops/releases/tag/8.3.1) - Release Date: [2024-12-17] ### 🐛 Fixes diff --git a/kpops/const/__init__.py b/kpops/const/__init__.py index a7e85fa09..2ca3e6814 100644 --- a/kpops/const/__init__.py +++ b/kpops/const/__init__.py @@ -1,3 +1,3 @@ -__version__ = "8.3.1" +__version__ = "8.3.2" KPOPS = "KPOps" KPOPS_MODULE = "kpops." diff --git a/pyproject.toml b/pyproject.toml index 1ad7eee87..cd31e4f71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "kpops" -version = "8.3.1" +version = "8.3.2" description = "KPOps is a tool to deploy Kafka pipelines to Kubernetes" authors = ["bakdata "] license = "MIT" From 286be69de998130dc86505bb186433973d1c14df Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Tue, 17 Dec 2024 16:45:38 +0100 Subject: [PATCH 28/28] Fix import --- kpops/components/streams_bootstrap_v2/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpops/components/streams_bootstrap_v2/base.py b/kpops/components/streams_bootstrap_v2/base.py index 0cfe9b0d0..2fa44354b 100644 --- a/kpops/components/streams_bootstrap_v2/base.py +++ b/kpops/components/streams_bootstrap_v2/base.py @@ -17,11 +17,11 @@ from kpops.utils.pydantic import ( CamelCaseConfigModel, DescConfigModel, + SerializeAsOptional, SerializeAsOptionalModel, exclude_by_value, exclude_defaults, ) -from tests.utils.test_pydantic import SerializeAsOptional if TYPE_CHECKING: try: