From 0b12c0bfa49dbee23a37b968f1e4d5967bce6391 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 1 Aug 2024 09:18:50 +0200 Subject: [PATCH 01/23] Add support for streams-bootstrap-v3 --- .../dependencies/kpops_structure.yaml | 40 + docs/docs/schema/defaults.json | 1278 ++++++++++++----- docs/docs/schema/pipeline.json | 1056 ++++++++++---- examples | 2 +- kpops/components/common/topic.py | 3 + .../streams_bootstrap_v3/__init__.py | 6 + .../streams_bootstrap_v3/app_type.py | 8 + .../streams_bootstrap_v3/kafka_app.py | 150 ++ .../streams_bootstrap_v3/producer/__init__.py | 0 .../streams_bootstrap_v3/producer/model.py | 25 + .../producer/producer_app.py | 121 ++ .../streams_bootstrap_v3/streams/__init__.py | 0 .../streams_bootstrap_v3/streams/model.py | 266 ++++ .../streams/streams_app.py | 153 ++ tests/api/test_registry.py | 5 + 15 files changed, 2519 insertions(+), 594 deletions(-) create mode 100644 kpops/components/streams_bootstrap_v3/__init__.py create mode 100644 kpops/components/streams_bootstrap_v3/app_type.py create mode 100644 kpops/components/streams_bootstrap_v3/kafka_app.py create mode 100644 kpops/components/streams_bootstrap_v3/producer/__init__.py create mode 100644 kpops/components/streams_bootstrap_v3/producer/model.py create mode 100644 kpops/components/streams_bootstrap_v3/producer/producer_app.py create mode 100644 kpops/components/streams_bootstrap_v3/streams/__init__.py create mode 100644 kpops/components/streams_bootstrap_v3/streams/model.py create mode 100644 kpops/components/streams_bootstrap_v3/streams/streams_app.py diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 8705b752d..ac4e98715 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -60,6 +60,15 @@ kpops_components_fields: - values - repo_config - version + producer-app-v3: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version streams-app: - name - prefix @@ -69,6 +78,15 @@ kpops_components_fields: - values - repo_config - version + streams-app-v3: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version streams-bootstrap: - name - prefix @@ -133,6 +151,17 @@ kpops_components_inheritance_ref: - kubernetes-app - pipeline-component - base-defaults-component + producer-app-v3: + bases: + - kafka-app + - streams-bootstrap + parents: + - kafka-app + - streams-bootstrap + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component streams-app: bases: - kafka-app @@ -144,6 +173,17 @@ kpops_components_inheritance_ref: - kubernetes-app - pipeline-component - base-defaults-component + streams-app-v3: + bases: + - kafka-app-v3 + - streams-bootstrap + parents: + - kafka-app-v3 + - streams-bootstrap + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component streams-bootstrap: bases: - helm-app diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index e728d5fba..580d36362 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -888,7 +888,7 @@ "values": { "allOf": [ { - "$ref": "#/$defs/ProducerAppValues" + "$ref": "#/$defs/kpops__components__streams_bootstrap__producer__model__ProducerAppValues" } ], "description": "streams-bootstrap Helm values" @@ -916,77 +916,76 @@ "title": "ProducerApp", "type": "object" }, - "ProducerAppValues": { + "ProducerAppV3": { "additionalProperties": true, - "description": "Settings specific to producers.", + "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" + }, + "name": { + "description": "Component name", + "title": "Name", "type": "string" }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" }, - "streams": { + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { "allOf": [ { - "$ref": "#/$defs/ProducerStreamsConfig" + "$ref": "#/$defs/HelmRepoConfig" } ], - "description": "Kafka Streams settings" - } - }, - "required": [ - "streams" - ], - "title": "ProducerAppValues", - "type": "object" - }, - "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" + "description": "Configuration of the Helm chart repo to be used for deploying the component" }, - "outputTopic": { + "to": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "Output topic" + "description": "Topic(s) into which the component will write output" }, - "schema_registry_url": { + "type": { + "const": "producer-app-v3", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__producer__model__ProducerAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { "anyOf": [ { "type": "string" @@ -995,15 +994,17 @@ "type": "null" } ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" + "default": "3.0.0", + "title": "Version" } }, "required": [ - "brokers" + "name", + "namespace", + "values", + "type" ], - "title": "ProducerStreamsConfig", + "title": "ProducerAppV3", "type": "object" }, "RepoAuthFlags": { @@ -1144,7 +1145,7 @@ "values": { "allOf": [ { - "$ref": "#/$defs/StreamsAppValues" + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppValues" } ], "description": "streams-bootstrap Helm values" @@ -1172,204 +1173,139 @@ "title": "StreamsApp", "type": "object" }, - "StreamsAppAutoScaling": { + "StreamsAppV3": { "additionalProperties": true, - "description": "Kubernetes Event-driven Autoscaling config.", + "description": "StreamsApp component that configures a streams-bootstrap app.", "properties": { - "consumerGroup": { + "from": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/FromSection" }, { "type": "null" } ], "default": null, - "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag. Mandatory to set when auto-scaling is enabled.", - "title": "Consumer group" + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "cooldownPeriod": { - "default": 300, - "description": "The period to wait after the last trigger reported active before scaling the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod", - "title": "Cooldown period", - "type": "integer" + "name": { + "description": "Component name", + "title": "Name", + "type": "string" }, - "enabled": { - "default": false, - "description": "", - "title": "Enabled", - "type": "boolean" + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" }, - "idleReplicas": { + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { "anyOf": [ { - "type": "integer" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", - "title": "Idle replica count" + "description": "Topic(s) into which the component will write output" }, - "lagThreshold": { + "type": { + "const": "streams-app-v3", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { "anyOf": [ { - "type": "integer" + "type": "string" }, { "type": "null" } ], - "default": null, - "description": "Average target value to trigger scaling actions. Mandatory to set when auto-scaling is enabled.", - "title": "Lag threshold" - }, - "maxReplicas": { - "default": 1, - "description": "This setting is passed to the HPA definition that KEDA will create for a given resource and holds the maximum number of replicas of the target resouce. https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount", - "title": "Max replica count", - "type": "integer" - }, - "minReplicas": { - "default": 0, - "description": "Minimum number of replicas KEDA will scale the resource down to. \"https://keda.sh/docs/2.9/concepts/scaling-deployments/#minreplicacount\"", - "title": "Min replica count", - "type": "integer" - }, - "offsetResetPolicy": { - "default": "earliest", - "description": "The offset reset policy for the consumer if the consumer group is not yet subscribed to a partition.", - "title": "Offset reset policy", - "type": "string" - }, - "pollingInterval": { - "default": 30, - "description": "This is the interval to check each trigger on. https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval", - "title": "Polling interval", - "type": "integer" - }, - "topics": { - "default": [], - "description": "List of auto-generated Kafka Streams topics used by the streams app.", - "items": { - "type": "string" - }, - "title": "Topics", - "type": "array" + "default": "3.0.0", + "title": "Version" } }, - "title": "StreamsAppAutoScaling", + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "StreamsAppV3", "type": "object" }, - "StreamsAppValues": { + "StreamsBootstrap": { "additionalProperties": true, - "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "description": "Base for components with a streams-bootstrap Helm chart.", "properties": { - "autoscaling": { + "from": { "anyOf": [ { - "$ref": "#/$defs/StreamsAppAutoScaling" + "$ref": "#/$defs/FromSection" }, { "type": "null" } ], "default": null, - "description": "Kubernetes event-driven autoscaling config" + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", + "name": { + "description": "Component name", + "title": "Name", "type": "string" }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - }, - "persistence": { - "allOf": [ - { - "$ref": "#/$defs/PersistenceConfig" - } - ], - "default": { - "enabled": false, - "size": null, - "storage_class": null - }, - "description": "" - }, - "statefulSet": { - "default": false, - "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", - "title": "Statefulset", - "type": "boolean" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/StreamsConfig" - } - ], - "description": "streams-bootstrap streams section" - } - }, - "required": [ - "streams" - ], - "title": "StreamsAppValues", - "type": "object" - }, - "StreamsBootstrap": { - "additionalProperties": true, - "description": "Base for components with a streams-bootstrap Helm chart.", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ { "$ref": "#/$defs/HelmRepoConfig" } @@ -1457,130 +1393,6 @@ "title": "StreamsBootstrapValues", "type": "object" }, - "StreamsConfig": { - "additionalProperties": true, - "description": "Streams Bootstrap streams section.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "config": { - "default": {}, - "description": "Configuration", - "title": "Config", - "type": "object" - }, - "deleteOutput": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Whether the output topics with their associated schemas and the consumer group should be deleted during the cleanup", - "title": "Deleteoutput" - }, - "errorTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Error topic" - }, - "extraInputPatterns": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra input patterns", - "title": "Extrainputpatterns", - "type": "object" - }, - "extraInputTopics": { - "additionalProperties": { - "items": { - "type": "string" - }, - "type": "array" - }, - "default": {}, - "description": "Extra input topics", - "title": "Extrainputtopics", - "type": "object" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, - "inputPattern": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Input pattern", - "title": "Inputpattern" - }, - "inputTopics": { - "default": [], - "description": "Input topics", - "items": { - "type": "string" - }, - "title": "Inputtopics", - "type": "array" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "StreamsConfig", - "type": "object" - }, "ToSection": { "additionalProperties": false, "description": "Holds multiple output topics.", @@ -1708,25 +1520,779 @@ }, "title": "TopicConfig", "type": "object" - } - }, - "properties": { - "helm-app": { - "$ref": "#/$defs/HelmApp" - }, - "kafka-app": { - "$ref": "#/$defs/KafkaApp" - }, - "kafka-connector": { - "$ref": "#/$defs/KafkaConnector" - }, - "kafka-sink-connector": { - "$ref": "#/$defs/KafkaSinkConnector" }, - "kafka-source-connector": { - "$ref": "#/$defs/KafkaSourceConnector" + "kpops__components__streams_bootstrap__producer__model__ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap__producer__model__ProducerStreamsConfig" + } + ], + "description": "Kafka Streams settings" + } + }, + "required": [ + "streams" + ], + "title": "ProducerAppValues", + "type": "object" }, - "kubernetes-app": { + "kpops__components__streams_bootstrap__producer__model__ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" + }, + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "brokers" + ], + "title": "ProducerStreamsConfig", + "type": "object" + }, + "kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling": { + "additionalProperties": true, + "description": "Kubernetes Event-driven Autoscaling config.", + "properties": { + "consumerGroup": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag. Mandatory to set when auto-scaling is enabled.", + "title": "Consumer group" + }, + "cooldownPeriod": { + "default": 300, + "description": "The period to wait after the last trigger reported active before scaling the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod", + "title": "Cooldown period", + "type": "integer" + }, + "enabled": { + "default": false, + "description": "", + "title": "Enabled", + "type": "boolean" + }, + "idleReplicas": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", + "title": "Idle replica count" + }, + "lagThreshold": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Average target value to trigger scaling actions. Mandatory to set when auto-scaling is enabled.", + "title": "Lag threshold" + }, + "maxReplicas": { + "default": 1, + "description": "This setting is passed to the HPA definition that KEDA will create for a given resource and holds the maximum number of replicas of the target resouce. https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount", + "title": "Max replica count", + "type": "integer" + }, + "minReplicas": { + "default": 0, + "description": "Minimum number of replicas KEDA will scale the resource down to. \"https://keda.sh/docs/2.9/concepts/scaling-deployments/#minreplicacount\"", + "title": "Min replica count", + "type": "integer" + }, + "offsetResetPolicy": { + "default": "earliest", + "description": "The offset reset policy for the consumer if the consumer group is not yet subscribed to a partition.", + "title": "Offset reset policy", + "type": "string" + }, + "pollingInterval": { + "default": 30, + "description": "This is the interval to check each trigger on. https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval", + "title": "Polling interval", + "type": "integer" + }, + "topics": { + "default": [], + "description": "List of auto-generated Kafka Streams topics used by the streams app.", + "items": { + "type": "string" + }, + "title": "Topics", + "type": "array" + } + }, + "title": "StreamsAppAutoScaling", + "type": "object" + }, + "kpops__components__streams_bootstrap__streams__model__StreamsAppValues": { + "additionalProperties": true, + "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "properties": { + "autoscaling": { + "anyOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kubernetes event-driven autoscaling config" + }, + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "persistence": { + "allOf": [ + { + "$ref": "#/$defs/PersistenceConfig" + } + ], + "default": { + "enabled": false, + "size": null, + "storage_class": null + }, + "description": "" + }, + "statefulSet": { + "default": false, + "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", + "title": "Statefulset", + "type": "boolean" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsConfig" + } + ], + "description": "streams-bootstrap streams section" + } + }, + "required": [ + "streams" + ], + "title": "StreamsAppValues", + "type": "object" + }, + "kpops__components__streams_bootstrap__streams__model__StreamsConfig": { + "additionalProperties": true, + "description": "Streams Bootstrap streams section.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" + }, + "config": { + "default": {}, + "description": "Configuration", + "title": "Config", + "type": "object" + }, + "deleteOutput": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether the output topics with their associated schemas and the consumer group should be deleted during the cleanup", + "title": "Deleteoutput" + }, + "errorTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Error topic" + }, + "extraInputPatterns": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra input patterns", + "title": "Extrainputpatterns", + "type": "object" + }, + "extraInputTopics": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "default": {}, + "description": "Extra input topics", + "title": "Extrainputtopics", + "type": "object" + }, + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" + }, + "inputPattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Input pattern", + "title": "Inputpattern" + }, + "inputTopics": { + "default": [], + "description": "Input topics", + "items": { + "type": "string" + }, + "title": "Inputtopics", + "type": "array" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "brokers" + ], + "title": "StreamsConfig", + "type": "object" + }, + "kpops__components__streams_bootstrap_v3__producer__model__ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__producer__model__ProducerStreamsConfig" + } + ], + "description": "" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + } + }, + "required": [ + "kafka" + ], + "title": "ProducerAppValues", + "type": "object" + }, + "kpops__components__streams_bootstrap_v3__producer__model__ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerStreamsConfig", + "type": "object" + }, + "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling": { + "additionalProperties": true, + "description": "Kubernetes Event-driven Autoscaling config.", + "properties": { + "cooldownPeriod": { + "default": 300, + "description": "The period to wait after the last trigger reported active before scaling the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod", + "title": "Cooldown period", + "type": "integer" + }, + "enabled": { + "default": false, + "description": "", + "title": "Enabled", + "type": "boolean" + }, + "idleReplicas": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", + "title": "Idle replica count" + }, + "lagThreshold": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Average target value to trigger scaling actions. Mandatory to set when auto-scaling is enabled.", + "title": "Lag threshold" + }, + "maxReplicas": { + "default": 1, + "description": "This setting is passed to the HPA definition that KEDA will create for a given resource and holds the maximum number of replicas of the target resouce. https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount", + "title": "Max replica count", + "type": "integer" + }, + "minReplicas": { + "default": 0, + "description": "Minimum number of replicas KEDA will scale the resource down to. \"https://keda.sh/docs/2.9/concepts/scaling-deployments/#minreplicacount\"", + "title": "Min replica count", + "type": "integer" + }, + "offsetResetPolicy": { + "default": "earliest", + "description": "The offset reset policy for the consumer if the consumer group is not yet subscribed to a partition.", + "title": "Offset reset policy", + "type": "string" + }, + "pollingInterval": { + "default": 30, + "description": "This is the interval to check each trigger on. https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval", + "title": "Polling interval", + "type": "integer" + }, + "topics": { + "default": [], + "description": "List of auto-generated Kafka Streams topics used by the streams app.", + "items": { + "type": "string" + }, + "title": "Topics", + "type": "array" + } + }, + "title": "StreamsAppAutoScaling", + "type": "object" + }, + "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppValues": { + "additionalProperties": true, + "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "properties": { + "autoscaling": { + "anyOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kubernetes event-driven autoscaling config" + }, + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" + } + ], + "description": "" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "persistence": { + "allOf": [ + { + "$ref": "#/$defs/PersistenceConfig" + } + ], + "default": { + "enabled": false, + "size": null, + "storage_class": null + }, + "description": "" + }, + "statefulSet": { + "default": false, + "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", + "title": "Statefulset", + "type": "boolean" + } + }, + "required": [ + "kafka" + ], + "title": "StreamsAppValues", + "type": "object" + }, + "kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig": { + "additionalProperties": true, + "description": "Streams Bootstrap kafka section.", + "properties": { + "applicationId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique application ID for Kafka Streams. Required for auto-scaling", + "title": "Unique application ID" + }, + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "config": { + "default": {}, + "description": "Configuration", + "title": "Config", + "type": "object" + }, + "deleteOutput": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether the output topics with their associated schemas and the consumer group should be deleted during the cleanup", + "title": "Deleteoutput" + }, + "errorTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Error topic" + }, + "inputPattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Input pattern", + "title": "Inputpattern" + }, + "inputTopics": { + "default": [], + "description": "Input topics", + "items": { + "type": "string" + }, + "title": "Inputtopics", + "type": "array" + }, + "labeledInputPatterns": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "", + "title": "Labeledinputpatterns", + "type": "object" + }, + "labeledInputTopics": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "default": {}, + "description": "", + "title": "Labeledinputtopics", + "type": "object" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "StreamsConfig", + "type": "object" + } + }, + "properties": { + "helm-app": { + "$ref": "#/$defs/HelmApp" + }, + "kafka-app": { + "$ref": "#/$defs/KafkaApp" + }, + "kafka-connector": { + "$ref": "#/$defs/KafkaConnector" + }, + "kafka-sink-connector": { + "$ref": "#/$defs/KafkaSinkConnector" + }, + "kafka-source-connector": { + "$ref": "#/$defs/KafkaSourceConnector" + }, + "kubernetes-app": { "$ref": "#/$defs/KubernetesApp" }, "pipeline-component": { @@ -1735,9 +2301,15 @@ "producer-app": { "$ref": "#/$defs/ProducerApp" }, + "producer-app-v3": { + "$ref": "#/$defs/ProducerAppV3" + }, "streams-app": { "$ref": "#/$defs/StreamsApp" }, + "streams-app-v3": { + "$ref": "#/$defs/StreamsAppV3" + }, "streams-bootstrap": { "$ref": "#/$defs/StreamsBootstrap" } @@ -1752,7 +2324,9 @@ "pipeline-component", "producer-app", "streams-app", - "streams-bootstrap" + "streams-bootstrap", + "producer-app-v3", + "streams-app-v3" ], "title": "DefaultsSchema", "type": "object" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index e2acabfdd..1667ed41f 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -556,7 +556,7 @@ "values": { "allOf": [ { - "$ref": "#/$defs/ProducerAppValues" + "$ref": "#/$defs/kpops__components__streams_bootstrap__producer__model__ProducerAppValues" } ], "description": "streams-bootstrap Helm values" @@ -584,77 +584,76 @@ "title": "ProducerApp", "type": "object" }, - "ProducerAppValues": { + "ProducerAppV3": { "additionalProperties": true, - "description": "Settings specific to producers.", + "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" + }, + "name": { + "description": "Component name", + "title": "Name", "type": "string" }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" }, - "streams": { + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { "allOf": [ { - "$ref": "#/$defs/ProducerStreamsConfig" + "$ref": "#/$defs/HelmRepoConfig" } ], - "description": "Kafka Streams settings" - } - }, - "required": [ - "streams" - ], - "title": "ProducerAppValues", - "type": "object" - }, - "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" + "description": "Configuration of the Helm chart repo to be used for deploying the component" }, - "outputTopic": { + "to": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "Output topic" + "description": "Topic(s) into which the component will write output" }, - "schema_registry_url": { + "type": { + "const": "producer-app-v3", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__producer__model__ProducerAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { "anyOf": [ { "type": "string" @@ -663,15 +662,17 @@ "type": "null" } ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" + "default": "3.0.0", + "title": "Version" } }, "required": [ - "brokers" + "name", + "namespace", + "values", + "type" ], - "title": "ProducerStreamsConfig", + "title": "ProducerAppV3", "type": "object" }, "RepoAuthFlags": { @@ -812,39 +813,696 @@ "values": { "allOf": [ { - "$ref": "#/$defs/StreamsAppValues" + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "StreamsApp", + "type": "object" + }, + "StreamsAppV3": { + "additionalProperties": true, + "description": "StreamsApp component that configures a streams-bootstrap app.", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "streams-app-v3", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "3.0.0", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "StreamsAppV3", + "type": "object" + }, + "ToSection": { + "additionalProperties": false, + "description": "Holds multiple output topics.", + "properties": { + "models": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Data models", + "title": "Models", + "type": "object" + }, + "topics": { + "additionalProperties": { + "$ref": "#/$defs/TopicConfig" + }, + "default": {}, + "description": "Output topics", + "title": "Topics", + "type": "object" + } + }, + "title": "ToSection", + "type": "object" + }, + "TopicConfig": { + "additionalProperties": false, + "description": "Configure an output topic.", + "properties": { + "configs": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "default": {}, + "description": "Topic configs", + "title": "Configs", + "type": "object" + }, + "key_schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Key schema class name", + "title": "Key schema" + }, + "partitions_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number of partitions into which the topic is divided", + "title": "Partitions count" + }, + "replication_factor": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Replication factor of the topic", + "title": "Replication factor" + }, + "role": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Custom identifier belonging to one or multiple topics, provide only if `type` is `extra`", + "title": "Role" + }, + "type": { + "anyOf": [ + { + "$ref": "#/$defs/OutputTopicTypes" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic type", + "title": "Topic type" + }, + "value_schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Value schema class name", + "title": "Value schema" + } + }, + "title": "TopicConfig", + "type": "object" + }, + "kpops__components__streams_bootstrap__producer__model__ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap__producer__model__ProducerStreamsConfig" + } + ], + "description": "Kafka Streams settings" + } + }, + "required": [ + "streams" + ], + "title": "ProducerAppValues", + "type": "object" + }, + "kpops__components__streams_bootstrap__producer__model__ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" + }, + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "brokers" + ], + "title": "ProducerStreamsConfig", + "type": "object" + }, + "kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling": { + "additionalProperties": true, + "description": "Kubernetes Event-driven Autoscaling config.", + "properties": { + "consumerGroup": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag. Mandatory to set when auto-scaling is enabled.", + "title": "Consumer group" + }, + "cooldownPeriod": { + "default": 300, + "description": "The period to wait after the last trigger reported active before scaling the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod", + "title": "Cooldown period", + "type": "integer" + }, + "enabled": { + "default": false, + "description": "", + "title": "Enabled", + "type": "boolean" + }, + "idleReplicas": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", + "title": "Idle replica count" + }, + "lagThreshold": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Average target value to trigger scaling actions. Mandatory to set when auto-scaling is enabled.", + "title": "Lag threshold" + }, + "maxReplicas": { + "default": 1, + "description": "This setting is passed to the HPA definition that KEDA will create for a given resource and holds the maximum number of replicas of the target resouce. https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount", + "title": "Max replica count", + "type": "integer" + }, + "minReplicas": { + "default": 0, + "description": "Minimum number of replicas KEDA will scale the resource down to. \"https://keda.sh/docs/2.9/concepts/scaling-deployments/#minreplicacount\"", + "title": "Min replica count", + "type": "integer" + }, + "offsetResetPolicy": { + "default": "earliest", + "description": "The offset reset policy for the consumer if the consumer group is not yet subscribed to a partition.", + "title": "Offset reset policy", + "type": "string" + }, + "pollingInterval": { + "default": 30, + "description": "This is the interval to check each trigger on. https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval", + "title": "Polling interval", + "type": "integer" + }, + "topics": { + "default": [], + "description": "List of auto-generated Kafka Streams topics used by the streams app.", + "items": { + "type": "string" + }, + "title": "Topics", + "type": "array" + } + }, + "title": "StreamsAppAutoScaling", + "type": "object" + }, + "kpops__components__streams_bootstrap__streams__model__StreamsAppValues": { + "additionalProperties": true, + "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "properties": { + "autoscaling": { + "anyOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kubernetes event-driven autoscaling config" + }, + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "persistence": { + "allOf": [ + { + "$ref": "#/$defs/PersistenceConfig" + } + ], + "default": { + "enabled": false, + "size": null, + "storage_class": null + }, + "description": "" + }, + "statefulSet": { + "default": false, + "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", + "title": "Statefulset", + "type": "boolean" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsConfig" + } + ], + "description": "streams-bootstrap streams section" + } + }, + "required": [ + "streams" + ], + "title": "StreamsAppValues", + "type": "object" + }, + "kpops__components__streams_bootstrap__streams__model__StreamsConfig": { + "additionalProperties": true, + "description": "Streams Bootstrap streams section.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" + }, + "config": { + "default": {}, + "description": "Configuration", + "title": "Config", + "type": "object" + }, + "deleteOutput": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether the output topics with their associated schemas and the consumer group should be deleted during the cleanup", + "title": "Deleteoutput" + }, + "errorTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Error topic" + }, + "extraInputPatterns": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra input patterns", + "title": "Extrainputpatterns", + "type": "object" + }, + "extraInputTopics": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "default": {}, + "description": "Extra input topics", + "title": "Extrainputtopics", + "type": "object" + }, + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" + }, + "inputPattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Input pattern", + "title": "Inputpattern" + }, + "inputTopics": { + "default": [], + "description": "Input topics", + "items": { + "type": "string" + }, + "title": "Inputtopics", + "type": "array" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "brokers" + ], + "title": "StreamsConfig", + "type": "object" + }, + "kpops__components__streams_bootstrap_v3__producer__model__ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__producer__model__ProducerStreamsConfig" } ], - "description": "streams-bootstrap Helm values" + "description": "" }, - "version": { + "nameOverride": { "anyOf": [ { + "maxLength": 63, "type": "string" }, { "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" } }, "required": [ - "name", - "namespace", - "values", - "type" + "kafka" ], - "title": "StreamsApp", + "title": "ProducerAppValues", "type": "object" }, - "StreamsAppAutoScaling": { + "kpops__components__streams_bootstrap_v3__producer__model__ProducerStreamsConfig": { "additionalProperties": true, - "description": "Kubernetes Event-driven Autoscaling config.", + "description": "Kafka Streams settings specific to Producer.", "properties": { - "consumerGroup": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { "anyOf": [ { "type": "string" @@ -854,9 +1512,32 @@ } ], "default": null, - "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag. Mandatory to set when auto-scaling is enabled.", - "title": "Consumer group" + "description": "Output topic" }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerStreamsConfig", + "type": "object" + }, + "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling": { + "additionalProperties": true, + "description": "Kubernetes Event-driven Autoscaling config.", + "properties": { "cooldownPeriod": { "default": 300, "description": "The period to wait after the last trigger reported active before scaling the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod", @@ -932,14 +1613,14 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "StreamsAppValues": { + "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppValues": { "additionalProperties": true, "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", "properties": { "autoscaling": { "anyOf": [ { - "$ref": "#/$defs/StreamsAppAutoScaling" + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling" }, { "type": "null" @@ -955,6 +1636,14 @@ "title": "Imagetag", "type": "string" }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" + } + ], + "description": "" + }, "nameOverride": { "anyOf": [ { @@ -987,29 +1676,34 @@ "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", "title": "Statefulset", "type": "boolean" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/StreamsConfig" - } - ], - "description": "streams-bootstrap streams section" } }, "required": [ - "streams" + "kafka" ], "title": "StreamsAppValues", "type": "object" }, - "StreamsConfig": { + "kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig": { "additionalProperties": true, - "description": "Streams Bootstrap streams section.", + "description": "Streams Bootstrap kafka section.", "properties": { - "brokers": { + "applicationId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique application ID for Kafka Streams. Required for auto-scaling", + "title": "Unique application ID" + }, + "bootstrapServers": { "description": "Brokers", - "title": "Brokers", + "title": "Bootstrapservers", "type": "string" }, "config": { @@ -1043,36 +1737,6 @@ "default": null, "description": "Error topic" }, - "extraInputPatterns": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra input patterns", - "title": "Extrainputpatterns", - "type": "object" - }, - "extraInputTopics": { - "additionalProperties": { - "items": { - "type": "string" - }, - "type": "array" - }, - "default": {}, - "description": "Extra input topics", - "title": "Extrainputtopics", - "type": "object" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, "inputPattern": { "anyOf": [ { @@ -1095,124 +1759,37 @@ "title": "Inputtopics", "type": "array" }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "StreamsConfig", - "type": "object" - }, - "ToSection": { - "additionalProperties": false, - "description": "Holds multiple output topics.", - "properties": { - "models": { + "labeledInputPatterns": { "additionalProperties": { "type": "string" }, "default": {}, - "description": "Data models", - "title": "Models", + "description": "", + "title": "Labeledinputpatterns", "type": "object" }, - "topics": { + "labeledInputTopics": { "additionalProperties": { - "$ref": "#/$defs/TopicConfig" + "items": { + "type": "string" + }, + "type": "array" }, "default": {}, - "description": "Output topics", - "title": "Topics", + "description": "", + "title": "Labeledinputtopics", "type": "object" - } - }, - "title": "ToSection", - "type": "object" - }, - "TopicConfig": { - "additionalProperties": false, - "description": "Configure an output topic.", - "properties": { - "configs": { + }, + "labeledOutputTopics": { "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] + "type": "string" }, "default": {}, - "description": "Topic configs", - "title": "Configs", + "description": "Extra output topics", + "title": "Labeledoutputtopics", "type": "object" }, - "key_schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Key schema class name", - "title": "Key schema" - }, - "partitions_count": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Number of partitions into which the topic is divided", - "title": "Partitions count" - }, - "replication_factor": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Replication factor of the topic", - "title": "Replication factor" - }, - "role": { + "outputTopic": { "anyOf": [ { "type": "string" @@ -1222,23 +1799,9 @@ } ], "default": null, - "description": "Custom identifier belonging to one or multiple topics, provide only if `type` is `extra`", - "title": "Role" - }, - "type": { - "anyOf": [ - { - "$ref": "#/$defs/OutputTopicTypes" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic type", - "title": "Topic type" + "description": "Output topic" }, - "value_schema": { + "schema_registry_url": { "anyOf": [ { "type": "string" @@ -1248,11 +1811,14 @@ } ], "default": null, - "description": "Value schema class name", - "title": "Value schema" + "description": "URL of the schema registry", + "title": "Schema Registry Url" } }, - "title": "TopicConfig", + "required": [ + "bootstrapServers" + ], + "title": "StreamsConfig", "type": "object" } }, @@ -1263,7 +1829,9 @@ "kafka-sink-connector": "#/$defs/KafkaSinkConnector", "kafka-source-connector": "#/$defs/KafkaSourceConnector", "producer-app": "#/$defs/ProducerApp", - "streams-app": "#/$defs/StreamsApp" + "producer-app-v3": "#/$defs/ProducerAppV3", + "streams-app": "#/$defs/StreamsApp", + "streams-app-v3": "#/$defs/StreamsAppV3" }, "propertyName": "type" }, @@ -1282,6 +1850,12 @@ }, { "$ref": "#/$defs/StreamsApp" + }, + { + "$ref": "#/$defs/ProducerAppV3" + }, + { + "$ref": "#/$defs/StreamsAppV3" } ] }, diff --git a/examples b/examples index 2e7b20516..f7613426d 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 2e7b20516fce3658b64789e48fd18c61198456a3 +Subproject commit f7613426dffe5a1d6332c9e3cc7f0bfb23396e68 diff --git a/kpops/components/common/topic.py b/kpops/components/common/topic.py index 3f4431b23..2084aa503 100644 --- a/kpops/components/common/topic.py +++ b/kpops/components/common/topic.py @@ -60,7 +60,10 @@ class TopicConfig(DescConfigModel): configs: dict[str, str | int] = Field( default={}, description=describe_attr("configs", __doc__) ) + # TODO: We can rename this but this would be a breaking change role: str | None = Field(default=None, description=describe_attr("role", __doc__)) + # TODO: Alternatively, we can define label and use both. Double checks everywhere. + # label: str | None = Field(default=None, description=describe_attr("label", __doc__)) model_config = ConfigDict( extra="forbid", diff --git a/kpops/components/streams_bootstrap_v3/__init__.py b/kpops/components/streams_bootstrap_v3/__init__.py new file mode 100644 index 000000000..adb6a8c92 --- /dev/null +++ b/kpops/components/streams_bootstrap_v3/__init__.py @@ -0,0 +1,6 @@ +from kpops.components.common.streams_bootstrap import StreamsBootstrap + +from .producer.producer_app import ProducerAppV3 +from .streams.streams_app import StreamsAppV3 + +__all__ = ("StreamsBootstrap", "StreamsAppV3", "ProducerAppV3") diff --git a/kpops/components/streams_bootstrap_v3/app_type.py b/kpops/components/streams_bootstrap_v3/app_type.py new file mode 100644 index 000000000..982ad07fa --- /dev/null +++ b/kpops/components/streams_bootstrap_v3/app_type.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class AppType(Enum): + STREAMS_APP = "streams-app" + PRODUCER_APP = "producer-app" + CLEANUP_STREAMS_APP = "streams-app-cleanup-job" + CLEANUP_PRODUCER_APP = "producer-app-cleanup-job" diff --git a/kpops/components/streams_bootstrap_v3/kafka_app.py b/kpops/components/streams_bootstrap_v3/kafka_app.py new file mode 100644 index 000000000..1cf518098 --- /dev/null +++ b/kpops/components/streams_bootstrap_v3/kafka_app.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import logging +from abc import ABC +from typing import Any + +import pydantic +from pydantic import AliasChoices, ConfigDict, Field +from typing_extensions import override + +from kpops.component_handlers import get_handlers +from kpops.components.base_components.cleaner import Cleaner +from kpops.components.base_components.helm_app import HelmAppValues +from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.common.topic import KafkaTopic, KafkaTopicStr +from kpops.config import get_config +from kpops.utils.docstring import describe_attr +from kpops.utils.pydantic import ( + CamelCaseConfigModel, + DescConfigModel, + exclude_by_value, + exclude_defaults, +) + +log = logging.getLogger("KafkaApp") + + +class KafkaStreamsConfig(CamelCaseConfigModel, DescConfigModel): + """Kafka Streams config. + + :param bootstrap_servers: Brokers + :param schema_registry_url: URL of the schema registry, defaults to None + :param labeled_output_topics: Extra output topics + :param output_topic: Output topic, defaults to None + """ + + bootstrap_servers: str = Field( + default=..., description=describe_attr("bootstrap_servers", __doc__) + ) + schema_registry_url: str | None = Field( + default=None, + validation_alias=AliasChoices( + "schema_registry_url", "schemaRegistryUrl" + ), # TODO: same for other camelcase fields, avoids duplicates during enrichment + description=describe_attr("schema_registry_url", __doc__), + ) + labeled_output_topics: dict[str, KafkaTopicStr] = Field( + default={}, description=describe_attr("labeled_output_topics", __doc__) + ) + output_topic: KafkaTopicStr | None = Field( + default=None, + description=describe_attr("output_topic", __doc__), + json_schema_extra={}, + ) + + model_config = ConfigDict(extra="allow") + + @pydantic.field_validator("labeled_output_topics", mode="before") + @classmethod + def deserialize_extra_output_topics( + cls, extra_output_topics: dict[str, str] | Any + ) -> dict[str, KafkaTopic] | Any: + if isinstance(extra_output_topics, dict): + return { + role: KafkaTopic(name=topic_name) + for role, topic_name in extra_output_topics.items() + } + return extra_output_topics + + @pydantic.field_serializer("labeled_output_topics") + def serialize_extra_output_topics( + self, extra_topics: dict[str, KafkaTopic] + ) -> dict[str, str]: + return {role: topic.name for role, topic in extra_topics.items()} + + # TODO(Ivan Yordanov): Currently hacky and potentially unsafe. Find cleaner solution + @pydantic.model_serializer(mode="wrap", when_used="always") + def serialize_model( + self, + default_serialize_handler: pydantic.SerializerFunctionWrapHandler, + info: pydantic.SerializationInfo, + ) -> dict[str, Any]: + return exclude_defaults( + self, exclude_by_value(default_serialize_handler(self), None) + ) + + +class KafkaAppValues(HelmAppValues): + """Settings specific to Kafka Apps. + + :param kafka: Kafka streams config + """ + + kafka: KafkaStreamsConfig = Field( + default=..., description=describe_attr("streams", __doc__) + ) + + +class KafkaAppCleaner(Cleaner, StreamsBootstrap, ABC): + """Helm app for resetting and cleaning a streams-bootstrap app.""" + + from_: None = None + to: None = None + + @property + @override + def helm_chart(self) -> str: + raise NotImplementedError + + @override + async def clean(self, dry_run: bool) -> None: + """Clean an app using a cleanup job. + + :param dry_run: Dry run command + """ + log.info(f"Uninstall old cleanup job for {self.helm_release_name}") + await self.destroy(dry_run) + + log.info(f"Init cleanup job for {self.helm_release_name}") + await self.deploy(dry_run) + + if not get_config().retain_clean_jobs: + log.info(f"Uninstall cleanup job for {self.helm_release_name}") + await self.destroy(dry_run) + + +class KafkaAppV3(PipelineComponent, ABC): + """Base component for Kafka-based components. + + Producer or streaming apps should inherit from this class. + + :param values: Application-specific settings + """ + + values: KafkaAppValues = Field( + default=..., + description=describe_attr("values", __doc__), + ) + + @override + async def deploy(self, dry_run: bool) -> None: + if self.to: + for topic in self.to.kafka_topics: + await get_handlers().topic_handler.create_topic(topic, dry_run=dry_run) + + if schema_handler := get_handlers().schema_handler: + await schema_handler.submit_schemas(to_section=self.to, dry_run=dry_run) + + await super().deploy(dry_run) diff --git a/kpops/components/streams_bootstrap_v3/producer/__init__.py b/kpops/components/streams_bootstrap_v3/producer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kpops/components/streams_bootstrap_v3/producer/model.py b/kpops/components/streams_bootstrap_v3/producer/model.py new file mode 100644 index 000000000..b117e0c2c --- /dev/null +++ b/kpops/components/streams_bootstrap_v3/producer/model.py @@ -0,0 +1,25 @@ +from pydantic import ConfigDict, Field + +from kpops.components.common.streams_bootstrap import StreamsBootstrapValues +from kpops.components.streams_bootstrap_v3.kafka_app import ( + KafkaAppValues, + KafkaStreamsConfig, +) +from kpops.utils.docstring import describe_attr + + +class ProducerStreamsConfig(KafkaStreamsConfig): + """Kafka Streams settings specific to Producer.""" + + +class ProducerAppValues(StreamsBootstrapValues, KafkaAppValues): + """Settings specific to producers. + + :param kafka: Kafka Streams settings + """ + + kafka: ProducerStreamsConfig = Field( + default=..., description=describe_attr("streams", __doc__) + ) + + model_config = ConfigDict(extra="allow") diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap_v3/producer/producer_app.py new file mode 100644 index 000000000..e7416b8c2 --- /dev/null +++ b/kpops/components/streams_bootstrap_v3/producer/producer_app.py @@ -0,0 +1,121 @@ +import logging +from functools import cached_property + +from pydantic import Field, computed_field +from typing_extensions import override + +from kpops.components.base_components.kafka_app import ( + KafkaApp, + KafkaAppCleaner, +) +from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.common.topic import ( + KafkaTopic, + OutputTopicTypes, + TopicConfig, +) +from kpops.components.streams_bootstrap_v3.app_type import AppType +from kpops.components.streams_bootstrap_v3.producer.model import ProducerAppValues +from kpops.utils.docstring import describe_attr + +log = logging.getLogger("ProducerApp") +STREAMS_BOOTSTRAP_V3 = "3.0.0" + + +class ProducerAppCleaner(KafkaAppCleaner): + values: ProducerAppValues + + @property + @override + def helm_chart(self) -> str: + return ( + f"{self.repo_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" + ) + + +class ProducerAppV3(KafkaApp, StreamsBootstrap): + """Producer component. + + This producer holds configuration to use as values for the streams-bootstrap + producer Helm chart. + + Note that the producer does not support error topics. + + :param values: streams-bootstrap Helm values + :param from_: Producer doesn't support FromSection, defaults to None + """ + + values: ProducerAppValues = Field( + default=..., + description=describe_attr("values", __doc__), + ) + from_: None = Field( + default=None, + alias="from", + title="From", + description=describe_attr("from_", __doc__), + ) + + version: str | None = STREAMS_BOOTSTRAP_V3 + + @computed_field + @cached_property + def _cleaner(self) -> ProducerAppCleaner: + return ProducerAppCleaner( + **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}) + ) + + @override + def apply_to_outputs(self, name: str, topic: TopicConfig) -> None: + match topic.type: + case OutputTopicTypes.ERROR: + msg = "Producer apps do not support error topics" + raise ValueError(msg) + case _: + super().apply_to_outputs(name, topic) + + @property + @override + def output_topic(self) -> KafkaTopic | None: + return self.values.kafka.output_topic + + @property + @override + def extra_output_topics(self) -> dict[str, KafkaTopic]: + return self.values.kafka.labeled_output_topics + + @override + def set_output_topic(self, topic: KafkaTopic) -> None: + self.values.kafka.output_topic = topic + + @override + def add_extra_output_topic(self, topic: KafkaTopic, label: str) -> None: + self.values.kafka.labeled_output_topics[label] = topic + + @property + @override + def helm_chart(self) -> str: + return f"{self.repo_config.repository_name}/{AppType.PRODUCER_APP.value}" + + async def reset(self, dry_run: bool) -> None: + """Reset not necessary, since producer app has no consumer group offsets.""" + await super().reset(dry_run) + + @override + async def destroy(self, dry_run: bool) -> None: + cluster_values = await self.helm.get_values( + self.namespace, self.helm_release_name + ) + if cluster_values: + log.debug("Fetched Helm chart values from cluster") + name_override = self._cleaner.helm_name_override + self._cleaner.values = self.values.model_validate(cluster_values) + self._cleaner.values.name_override = name_override + + await super().destroy(dry_run) + + @override + async def clean(self, dry_run: bool) -> None: + """Destroy and clean.""" + await super().clean(dry_run) + await self._cleaner.clean(dry_run) diff --git a/kpops/components/streams_bootstrap_v3/streams/__init__.py b/kpops/components/streams_bootstrap_v3/streams/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py new file mode 100644 index 000000000..21be904a2 --- /dev/null +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +from typing import Any + +import pydantic +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from kpops.api.exception import ValidationError +from kpops.components.common.streams_bootstrap import StreamsBootstrapValues +from kpops.components.common.topic import KafkaTopic, KafkaTopicStr +from kpops.components.streams_bootstrap_v3.kafka_app import ( + KafkaAppValues, + KafkaStreamsConfig, +) +from kpops.utils.docstring import describe_attr +from kpops.utils.pydantic import ( + CamelCaseConfigModel, + DescConfigModel, +) + + +class StreamsConfig(KafkaStreamsConfig): + """Streams Bootstrap kafka section. + + :param application_id: Unique application ID for Kafka Streams. Required for auto-scaling + :param input_topics: Input topics, defaults to [] + :param input_pattern: Input pattern, defaults to None + :param labeled_input_topics: Extra input topics, defaults to {} + :param labeled_input_patterns: Extra input patterns, defaults to {} + :param error_topic: Error topic, defaults to None + :param config: Configuration, defaults to {} + :param delete_output: Whether the output topics with their associated schemas and the consumer group should be deleted during the cleanup, defaults to None + """ + + application_id: str | None = Field( + default=None, + title="Unique application ID", + description=describe_attr("application_id", __doc__), + ) + input_topics: list[KafkaTopicStr] = Field( + default=[], description=describe_attr("input_topics", __doc__) + ) + input_pattern: str | None = Field( + default=None, description=describe_attr("input_pattern", __doc__) + ) + labeled_input_topics: dict[str, list[KafkaTopicStr]] = Field( + default={}, description=describe_attr("extra_input_topics", __doc__) + ) + labeled_input_patterns: dict[str, str] = Field( + default={}, description=describe_attr("extra_input_patterns", __doc__) + ) + error_topic: KafkaTopicStr | None = Field( + default=None, description=describe_attr("error_topic", __doc__) + ) + config: dict[str, Any] = Field( + default={}, description=describe_attr("config", __doc__) + ) + delete_output: bool | None = Field( + default=None, description=describe_attr("delete_output", __doc__) + ) + + @pydantic.field_validator("input_topics", mode="before") + @classmethod + def deserialize_input_topics( + cls, input_topics: list[str] | Any + ) -> list[KafkaTopic] | Any: + if isinstance(input_topics, list): + return [KafkaTopic(name=topic_name) for topic_name in input_topics] + return input_topics + + @pydantic.field_validator("labeled_input_topics", mode="before") + @classmethod + def deserialize_labeled_input_topics( + cls, extra_input_topics: dict[str, str] | Any + ) -> dict[str, list[KafkaTopic]] | Any: + if isinstance(extra_input_topics, dict): + return { + role: [KafkaTopic(name=topic_name) for topic_name in topics] + for role, topics in extra_input_topics.items() + } + return extra_input_topics + + @pydantic.field_serializer("input_topics") + def serialize_topics(self, topics: list[KafkaTopic]) -> list[str]: + return [topic.name for topic in topics] + + @pydantic.field_serializer("labeled_input_patterns") + def serialize_labeled_input_topics( + self, extra_topics: dict[str, list[KafkaTopic]] + ) -> dict[str, list[str]]: + return { + role: self.serialize_topics(topics) for role, topics in extra_topics.items() + } + + def add_input_topics(self, topics: list[KafkaTopic]) -> None: + """Add given topics to the list of input topics. + + Ensures no duplicate topics in the list. + + :param topics: Input topics + """ + self.input_topics = KafkaTopic.deduplicate(self.input_topics + topics) + + def add_labeled_input_topics(self, label: str, topics: list[KafkaTopic]) -> None: + """Add given labeled topics that share a label to the list of extra input topics. + + Ensures no duplicate topics in the list. + + :param topics: Extra input topics + :param label: Topic role + """ + self.labeled_input_topics[label] = KafkaTopic.deduplicate( + self.labeled_input_topics.get(label, []) + topics + ) + + +class StreamsAppAutoScaling(CamelCaseConfigModel, DescConfigModel): + """Kubernetes Event-driven Autoscaling config. + + :param enabled: Whether to enable auto-scaling using KEDA., defaults to False + :param lag_threshold: Average target value to trigger scaling actions. + Mandatory to set when auto-scaling is enabled. + :param polling_interval: This is the interval to check each trigger on. + https://keda.sh/docs/2.9/concepts/scaling-deployments/#pollinginterval, + defaults to 30 + :param cooldown_period: The period to wait after the last trigger reported + active before scaling the resource back to 0. + https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod, + defaults to 300 + :param offset_reset_policy: The offset reset policy for the consumer if the + consumer group is not yet subscribed to a partition., + defaults to "earliest" + :param min_replicas: Minimum number of replicas KEDA will scale the resource down to. + "https://keda.sh/docs/2.9/concepts/scaling-deployments/#minreplicacount", + defaults to 0 + :param max_replicas: This setting is passed to the HPA definition that KEDA + will create for a given resource and holds the maximum number of replicas + of the target resouce. + https://keda.sh/docs/2.9/concepts/scaling-deployments/#maxreplicacount, + defaults to 1 + :param idle_replicas: If this property is set, KEDA will scale the resource + down to this number of replicas. + https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount, + defaults to None + :param topics: List of auto-generated Kafka Streams topics used by the streams app., + defaults to [] + """ + + enabled: bool = Field( + default=False, + description=describe_attr("streams", __doc__), + ) + lag_threshold: int | None = Field( + default=None, + title="Lag threshold", + description=describe_attr("lag_threshold", __doc__), + ) + polling_interval: int = Field( + default=30, + title="Polling interval", + description=describe_attr("polling_interval", __doc__), + ) + cooldown_period: int = Field( + default=300, + title="Cooldown period", + description=describe_attr("cooldown_period", __doc__), + ) + offset_reset_policy: str = Field( + default="earliest", + title="Offset reset policy", + description=describe_attr("offset_reset_policy", __doc__), + ) + min_replicas: int = Field( + default=0, + title="Min replica count", + description=describe_attr("min_replicas", __doc__), + ) + max_replicas: int = Field( + default=1, + title="Max replica count", + description=describe_attr("max_replicas", __doc__), + ) + idle_replicas: int | None = Field( + default=None, + title="Idle replica count", + description=describe_attr("idle_replicas", __doc__), + ) + topics: list[str] = Field( + default=[], + description=describe_attr("topics", __doc__), + ) + model_config = ConfigDict(extra="allow") + + @model_validator(mode="after") + def validate_mandatory_fields_are_set( + self: StreamsAppAutoScaling, + ) -> StreamsAppAutoScaling: # TODO: typing.Self for Python 3.11+ + if self.enabled and self.lag_threshold is None: + msg = ( + "If app.autoscaling.enabled is set to true, " + "the fields app.autoscaling.consumer_group and app.autoscaling.lag_threshold should be set." + ) + raise ValidationError(msg) + return self + + +class PersistenceConfig(BaseModel): + """streams-bootstrap persistence configurations. + + :param enabled: Whether to use a persistent volume to store the state of the streams app. + :param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet. + :param storage_class: Storage class to use for the persistent volume. + """ + + enabled: bool = Field( + default=False, + description="Whether to use a persistent volume to store the state of the streams app. ", + ) + size: str | None = Field( + default=None, + description="The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + ) + storage_class: str | None = Field( + default=None, + description="Storage class to use for the persistent volume.", + ) + + @model_validator(mode="after") + def validate_mandatory_fields_are_set( + self: PersistenceConfig, + ) -> PersistenceConfig: # TODO: typing.Self for Python 3.11+ + if self.enabled and self.size is None: + msg = ( + "If app.persistence.enabled is set to true, " + "the field app.persistence.size needs to be set." + ) + raise ValidationError(msg) + return self + + +class StreamsAppValues(StreamsBootstrapValues, KafkaAppValues): + """streams-bootstrap app configurations. + + The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart. + + :param kafka: streams-bootstrap kafka section + :param autoscaling: Kubernetes event-driven autoscaling config, defaults to None + """ + + kafka: StreamsConfig = Field( + default=..., + description=describe_attr("streams", __doc__), + ) + autoscaling: StreamsAppAutoScaling | None = Field( + default=None, + description=describe_attr("autoscaling", __doc__), + ) + stateful_set: bool = Field( + default=False, + description="Whether to use a Statefulset instead of a Deployment to deploy the streams app.", + ) + persistence: PersistenceConfig = Field( + default=PersistenceConfig(), + description=describe_attr("persistence", __doc__), + ) + model_config = ConfigDict(extra="allow") diff --git a/kpops/components/streams_bootstrap_v3/streams/streams_app.py b/kpops/components/streams_bootstrap_v3/streams/streams_app.py new file mode 100644 index 000000000..6b334fd44 --- /dev/null +++ b/kpops/components/streams_bootstrap_v3/streams/streams_app.py @@ -0,0 +1,153 @@ +import logging +from functools import cached_property + +from pydantic import Field, computed_field +from typing_extensions import override + +from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler +from kpops.components.base_components.helm_app import HelmApp +from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.common.topic import KafkaTopic +from kpops.components.streams_bootstrap_v3.app_type import AppType +from kpops.components.streams_bootstrap_v3.kafka_app import KafkaAppCleaner, KafkaAppV3 +from kpops.components.streams_bootstrap_v3.streams.model import ( + StreamsAppValues, +) +from kpops.utils.docstring import describe_attr + +log = logging.getLogger("StreamsApp") + +STREAMS_BOOTSTRAP_V3 = "3.0.0" + + +class StreamsAppCleaner(KafkaAppCleaner): + from_: None = None + to: None = None + values: StreamsAppValues + + @property + @override + def helm_chart(self) -> str: + return f"{self.repo_config.repository_name}/{AppType.CLEANUP_STREAMS_APP.value}" + + @override + async def reset(self, dry_run: bool) -> None: + self.values.kafka.delete_output = False + await super().clean(dry_run) + + @override + async def clean(self, dry_run: bool) -> None: + self.values.kafka.delete_output = True + await super().clean(dry_run) + + if self.values.stateful_set and self.values.persistence.enabled: + await self.clean_pvcs(dry_run) + + async def clean_pvcs(self, dry_run: bool) -> None: + app_full_name = super(HelmApp, self).full_name + pvc_handler = await PVCHandler.create(app_full_name, self.namespace) + if dry_run: + pvc_names = await pvc_handler.list_pvcs() + log.info(f"Deleting the PVCs {pvc_names} for StatefulSet '{app_full_name}'") + else: + log.info(f"Deleting the PVCs for StatefulSet '{app_full_name}'") + await pvc_handler.delete_pvcs() + + +class StreamsAppV3(KafkaAppV3, StreamsBootstrap): + """StreamsApp component that configures a streams-bootstrap app. + + :param values: streams-bootstrap Helm values + """ + + values: StreamsAppValues = Field( + default=..., + description=describe_attr("values", __doc__), + ) + + version: str | None = STREAMS_BOOTSTRAP_V3 + + @computed_field + @cached_property + def _cleaner(self) -> StreamsAppCleaner: + return StreamsAppCleaner( + **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}) + ) + + @property + @override + def input_topics(self) -> list[KafkaTopic]: + return self.values.kafka.input_topics + + @property + @override + def extra_input_topics(self) -> dict[str, list[KafkaTopic]]: + return self.values.kafka.labeled_input_topics + + @property + @override + def output_topic(self) -> KafkaTopic | None: + return self.values.kafka.output_topic + + @property + @override + def extra_output_topics(self) -> dict[str, KafkaTopic]: + return self.values.kafka.labeled_output_topics + + @override + def add_input_topics(self, topics: list[KafkaTopic]) -> None: + self.values.kafka.add_input_topics(topics) + + @override + def add_extra_input_topics(self, label: str, topics: list[KafkaTopic]) -> None: + self.values.kafka.add_labeled_input_topics(label, topics) + + @override + def set_input_pattern(self, name: str) -> None: + self.values.kafka.input_pattern = name + + @override + def add_extra_input_pattern(self, label: str, topic: str) -> None: + self.values.kafka.labeled_input_patterns[label] = topic + + @override + def set_output_topic(self, topic: KafkaTopic) -> None: + self.values.kafka.output_topic = topic + + @override + def set_error_topic(self, topic: KafkaTopic) -> None: + self.values.kafka.error_topic = topic + + @override + def add_extra_output_topic(self, topic: KafkaTopic, label: str) -> None: + self.values.kafka.labeled_output_topics[label] = topic + + @property + @override + def helm_chart(self) -> str: + return f"{self.repo_config.repository_name}/{AppType.STREAMS_APP.value}" + + @override + async def destroy(self, dry_run: bool) -> None: + cluster_values = await self.helm.get_values( + self.namespace, self.helm_release_name + ) + if cluster_values: + log.debug("Fetched Helm chart values from cluster") + name_override = self._cleaner.helm_name_override + self._cleaner.values = self.values.model_validate(cluster_values) + self._cleaner.values.name_override = name_override + + await super().destroy(dry_run) + + @override + async def reset(self, dry_run: bool) -> None: + """Destroy and reset.""" + await super().reset(dry_run) + await self._cleaner.reset(dry_run) + + @override + async def clean(self, dry_run: bool) -> None: + """Destroy and clean.""" + await super().clean(dry_run) + await self._cleaner.clean(dry_run) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index ed496e0c8..609b41702 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -21,6 +21,7 @@ from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp +from kpops.components.streams_bootstrap_v3 import ProducerAppV3, StreamsAppV3 from tests.cli.resources.custom_module import CustomSchemaProvider @@ -50,6 +51,7 @@ def test_iter_namespace(): "kpops.components.base_components", "kpops.components.common", "kpops.components.streams_bootstrap", + "kpops.components.streams_bootstrap_v3", "kpops.components.test_components", ] @@ -61,6 +63,7 @@ def test_iter_component_modules(): "kpops.components.base_components", "kpops.components.common", "kpops.components.streams_bootstrap", + "kpops.components.streams_bootstrap_v3", "kpops.components.test_components", ] @@ -99,7 +102,9 @@ def test_registry(): "kubernetes-app": KubernetesApp, "pipeline-component": PipelineComponent, "producer-app": ProducerApp, + "producer-app-v3": ProducerAppV3, "streams-app": StreamsApp, + "streams-app-v3": StreamsAppV3, "streams-bootstrap": StreamsBootstrap, } for _type, _class in registry._classes.items(): From 6a3f10e9ac83c5c3860eb5840ef1048ebad155a1 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 1 Aug 2024 09:37:11 +0200 Subject: [PATCH 02/23] fix submodules --- examples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples b/examples index f7613426d..2e7b20516 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f7613426dffe5a1d6332c9e3cc7f0bfb23396e68 +Subproject commit 2e7b20516fce3658b64789e48fd18c61198456a3 From a43152ec897ddea28776eba73ffa8d44e24d772f Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 1 Aug 2024 12:48:43 +0200 Subject: [PATCH 03/23] add tests --- .../dependencies/kpops_structure.yaml | 4 +- .../streams_bootstrap_v3/kafka_app.py | 14 +- .../producer/producer_app.py | 4 +- .../streams_bootstrap_v3/streams/model.py | 9 +- .../streams-bootstrap-v3/defaults.yaml | 41 +++++ .../streams-bootstrap-v3/pipeline.yaml | 52 ++++++ .../pipeline.yaml | 165 ++++++++++++++++++ .../test_streams_bootstrap_v3/pipeline.yaml | 163 +++++++++++++++++ .../test_streams_bootstrap_v3/manifest.yaml | 165 ++++++++++++++++++ tests/pipeline/test_components/__init__.py | 2 + tests/pipeline/test_components/components.py | 7 + tests/pipeline/test_generate.py | 28 +++ tests/pipeline/test_manifest.py | 18 +- 13 files changed, 656 insertions(+), 16 deletions(-) create mode 100644 tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml create mode 100644 tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml create mode 100644 tests/pipeline/snapshots/test_generate/test_manifest_streams_bootstrap_v3/pipeline.yaml create mode 100644 tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml create mode 100644 tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index ac4e98715..97876ad87 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -153,10 +153,10 @@ kpops_components_inheritance_ref: - base-defaults-component producer-app-v3: bases: - - kafka-app + - kafka-app-v3 - streams-bootstrap parents: - - kafka-app + - kafka-app-v3 - streams-bootstrap - helm-app - kubernetes-app diff --git a/kpops/components/streams_bootstrap_v3/kafka_app.py b/kpops/components/streams_bootstrap_v3/kafka_app.py index 1cf518098..998851367 100644 --- a/kpops/components/streams_bootstrap_v3/kafka_app.py +++ b/kpops/components/streams_bootstrap_v3/kafka_app.py @@ -59,20 +59,20 @@ class KafkaStreamsConfig(CamelCaseConfigModel, DescConfigModel): @pydantic.field_validator("labeled_output_topics", mode="before") @classmethod def deserialize_extra_output_topics( - cls, extra_output_topics: dict[str, str] | Any + cls, labeled_output_topics: dict[str, str] | Any ) -> dict[str, KafkaTopic] | Any: - if isinstance(extra_output_topics, dict): + if isinstance(labeled_output_topics, dict): return { - role: KafkaTopic(name=topic_name) - for role, topic_name in extra_output_topics.items() + label: KafkaTopic(name=topic_name) + for label, topic_name in labeled_output_topics.items() } - return extra_output_topics + return labeled_output_topics @pydantic.field_serializer("labeled_output_topics") def serialize_extra_output_topics( - self, extra_topics: dict[str, KafkaTopic] + self, labeled_output_topics: dict[str, KafkaTopic] ) -> dict[str, str]: - return {role: topic.name for role, topic in extra_topics.items()} + return {label: topic.name for label, topic in labeled_output_topics.items()} # TODO(Ivan Yordanov): Currently hacky and potentially unsafe. Find cleaner solution @pydantic.model_serializer(mode="wrap", when_used="always") diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap_v3/producer/producer_app.py index e7416b8c2..7536e5bc6 100644 --- a/kpops/components/streams_bootstrap_v3/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v3/producer/producer_app.py @@ -5,7 +5,6 @@ from typing_extensions import override from kpops.components.base_components.kafka_app import ( - KafkaApp, KafkaAppCleaner, ) from kpops.components.common.streams_bootstrap import StreamsBootstrap @@ -15,6 +14,7 @@ TopicConfig, ) from kpops.components.streams_bootstrap_v3.app_type import AppType +from kpops.components.streams_bootstrap_v3.kafka_app import KafkaAppV3 from kpops.components.streams_bootstrap_v3.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr @@ -33,7 +33,7 @@ def helm_chart(self) -> str: ) -class ProducerAppV3(KafkaApp, StreamsBootstrap): +class ProducerAppV3(KafkaAppV3, StreamsBootstrap): """Producer component. This producer holds configuration to use as values for the streams-bootstrap diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index 21be904a2..bfff1f988 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -75,8 +75,8 @@ def deserialize_labeled_input_topics( ) -> dict[str, list[KafkaTopic]] | Any: if isinstance(extra_input_topics, dict): return { - role: [KafkaTopic(name=topic_name) for topic_name in topics] - for role, topics in extra_input_topics.items() + label: [KafkaTopic(name=topic_name) for topic_name in topics] + for label, topics in extra_input_topics.items() } return extra_input_topics @@ -89,7 +89,8 @@ def serialize_labeled_input_topics( self, extra_topics: dict[str, list[KafkaTopic]] ) -> dict[str, list[str]]: return { - role: self.serialize_topics(topics) for role, topics in extra_topics.items() + label: self.serialize_topics(topics) + for label, topics in extra_topics.items() } def add_input_topics(self, topics: list[KafkaTopic]) -> None: @@ -107,7 +108,7 @@ def add_labeled_input_topics(self, label: str, topics: list[KafkaTopic]) -> None Ensures no duplicate topics in the list. :param topics: Extra input topics - :param label: Topic role + :param label: Topic label """ self.labeled_input_topics[label] = KafkaTopic.deduplicate( self.labeled_input_topics.get(label, []) + topics diff --git a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml new file mode 100644 index 000000000..02b82f405 --- /dev/null +++ b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml @@ -0,0 +1,41 @@ +pipeline-component: + name: ${component.type} + +kubernetes-app: + namespace: example-namespace + +kafka-app-v3: + values: + kafka: + bootstrapServers: ${config.kafka_brokers} + schemaRegistryUrl: ${config.schema_registry.url} + version: "3.0.0-SNAPSHOT" + +producer-app-v3: {} # inherits from kafka-app + +streams-app-v3: # inherits from kafka-app + values: + kafka: + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + to: + topics: + ${error_topic_name}: + type: error + value_schema: com.bakdata.kafka.DeadLetter + partitions_count: 1 + configs: + cleanup.policy: compact,delete + +kafka-sink-connector: + config: + batch.size: "2000" + behavior.on.malformed.documents: "warn" + behavior.on.null.values: "delete" + connection.compression: "true" + connector.class: "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector" + key.ignore: "false" + linger.ms: "5000" + max.buffered.records: "20000" + read.timeout.ms: "120000" + tasks.max: "1" diff --git a/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml b/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml new file mode 100644 index 000000000..8f35ff5a9 --- /dev/null +++ b/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml @@ -0,0 +1,52 @@ +- type: my-producer-app + values: + image: "my-registry/my-producer-image" + imageTag: "1.0.0" + commandLine: + FAKE_ARG: "fake-arg-value" + schedule: "30 3/8 * * *" + + to: + topics: + my-producer-app-output-topic: + type: output + my-labeled-producer-app-topic-output: + role: my-producer-app-output-topic-label + + +- type: my-streams-app + values: + image: "my-registry/my-streams-app-image" + imageTag: "1.0.0" + kafka: + applicationId: "my-streams-app-id" + commandLine: + CONVERT_XML: true + resources: + limits: + memory: 2G + requests: + memory: 2G + from: + topics: + my-input-topic: + type: input + my-labeled-input-topic: + role: my-input-topic-label + my-input-pattern: + type: pattern + # TODO: not working, for some reason the extra pattern is not serialized as Topic object but as str + # pydantic_core._pydantic_core.PydanticSerializationError: + # Error calling function `serialize_labeled_input_topics`: AttributeError: 'str' object has no attribute 'name' +# my-labeled-input-pattern: +# type: pattern +# role: my-input-topic-labeled-pattern + + to: + topics: + my-output-topic: + type: output + my-error-topic: + type: error + my-labeled-topic-output: + role: my-output-topic-label diff --git a/tests/pipeline/snapshots/test_generate/test_manifest_streams_bootstrap_v3/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_manifest_streams_bootstrap_v3/pipeline.yaml new file mode 100644 index 000000000..81e609c5c --- /dev/null +++ b/tests/pipeline/snapshots/test_generate/test_manifest_streams_bootstrap_v3/pipeline.yaml @@ -0,0 +1,165 @@ +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + labels: + app: resources-streams-bootstrap-v3-my-producer-app + chart: producer-app-3.0.0-SNAPSHOT + release: resources-streams-bootstrap-v3-my-producer-app + name: resources-streams-bootstrap-v3-my-producer-app +spec: + concurrencyPolicy: Replace + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-streams-bootstrap-v3-my-producer-app + release: resources-streams-bootstrap-v3-my-producer-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: APP_FAKE_ARG + value: fake-arg-value + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-v3-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure + schedule: 30 3/8 * * * + successfulJobsHistoryLimit: 1 + suspend: false + +--- +apiVersion: v1 +data: + jmx-kafka-streams-app-prometheus.yml: "jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi\n\ + lowercaseOutputName: true\nlowercaseOutputLabelNames: true\nssl: false\nrules:\n\ + \ - pattern: \".*\"\n" +kind: ConfigMap +metadata: + labels: + app: resources-streams-bootstrap-v3-my-streams-app + chart: streams-app-3.0.0-SNAPSHOT + heritage: Helm + release: resources-streams-bootstrap-v3-my-streams-app + name: resources-streams-bootstrap-v3-my-streams-app-jmx-configmap + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + consumerGroup: my-streams-app-id + labels: + app: resources-streams-bootstrap-v3-my-streams-app + chart: streams-app-3.0.0-SNAPSHOT + release: resources-streams-bootstrap-v3-my-streams-app + name: resources-streams-bootstrap-v3-my-streams-app +spec: + replicas: 1 + selector: + matchLabels: + app: resources-streams-bootstrap-v3-my-streams-app + release: resources-streams-bootstrap-v3-my-streams-app + template: + metadata: + annotations: + prometheus.io/port: '5556' + prometheus.io/scrape: 'true' + labels: + app: resources-streams-bootstrap-v3-my-streams-app + release: resources-streams-bootstrap-v3-my-streams-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR + value: com.bakdata.kafka.MurmurHashIdGenerator + - name: KAFKA_JMX_PORT + value: '5555' + - name: APP_VOLATILE_GROUP_INSTANCE_ID + value: 'true' + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_INPUT_TOPICS + value: my-input-topic + - name: APP_INPUT_PATTERN + value: my-input-pattern + - name: APP_OUTPUT_TOPIC + value: my-output-topic + - name: APP_ERROR_TOPIC + value: resources-streams-bootstrap-v3-my-streams-app-error + - name: APP_LABELED_OUTPUT_TOPICS + value: my-output-topic-label=my-labeled-topic-output, + - name: APP_LABELED_INPUT_TOPICS + value: my-input-topic-label=my-labeled-input-topic, + - name: APP_APPLICATION_ID + value: my-streams-app-id + - name: APP_CONVERT_XML + value: 'true' + - name: JAVA_TOOL_OPTIONS + value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-streams-app-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-v3-my-streams-app + ports: + - containerPort: 5555 + name: jmx + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 2G + - command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - '5556' + - /etc/jmx-streams-app/jmx-kafka-streams-app-prometheus.yml + image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 + name: prometheus-jmx-exporter + ports: + - containerPort: 5556 + resources: + limits: + cpu: 300m + memory: 2G + requests: + cpu: 100m + memory: 500Mi + volumeMounts: + - mountPath: /etc/jmx-streams-app + name: jmx-config + terminationGracePeriodSeconds: 300 + volumes: + - configMap: + name: resources-streams-bootstrap-v3-my-streams-app-jmx-configmap + name: jmx-config + diff --git a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml new file mode 100644 index 000000000..fccf0dc84 --- /dev/null +++ b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml @@ -0,0 +1,163 @@ +- _cleaner: + name: my-producer-app + namespace: example-namespace + prefix: resources-streams-bootstrap-v3- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: + commandLine: + FAKE_ARG: fake-arg-value + image: my-registry/my-producer-image + imageTag: 1.0.0 + kafka: + bootstrapServers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + labeledOutputTopics: + my-producer-app-output-topic-label: my-labeled-producer-app-topic-output + outputTopic: my-producer-app-output-topic + schemaRegistryUrl: http://localhost:8081/ + schedule: 30 3/8 * * * + version: 3.0.0-SNAPSHOT + name: my-producer-app + namespace: example-namespace + prefix: resources-streams-bootstrap-v3- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + my-labeled-producer-app-topic-output: + configs: {} + role: my-producer-app-output-topic-label + my-producer-app-output-topic: + configs: {} + type: output + type: my-producer-app + values: + commandLine: + FAKE_ARG: fake-arg-value + image: my-registry/my-producer-image + imageTag: 1.0.0 + kafka: + bootstrapServers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + labeledOutputTopics: + my-producer-app-output-topic-label: my-labeled-producer-app-topic-output + outputTopic: my-producer-app-output-topic + schemaRegistryUrl: http://localhost:8081/ + schedule: 30 3/8 * * * + version: 3.0.0-SNAPSHOT +- _cleaner: + name: my-streams-app + namespace: example-namespace + prefix: resources-streams-bootstrap-v3- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: + commandLine: + CONVERT_XML: true + image: my-registry/my-streams-app-image + imageTag: 1.0.0 + kafka: + applicationId: my-streams-app-id + bootstrapServers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-streams-bootstrap-v3-my-streams-app-error + inputPattern: my-input-pattern + inputTopics: + - my-input-topic + labeledInputTopics: + my-input-topic-label: + - my-labeled-input-topic + labeledOutputTopics: + my-output-topic-label: my-labeled-topic-output + outputTopic: my-output-topic + schemaRegistryUrl: http://localhost:8081/ + persistence: + enabled: false + resources: + limits: + memory: 2G + requests: + memory: 2G + statefulSet: false + version: 3.0.0-SNAPSHOT + from: + components: {} + topics: + my-input-pattern: + type: pattern + my-input-topic: + type: input + my-labeled-input-topic: + role: my-input-topic-label + name: my-streams-app + namespace: example-namespace + prefix: resources-streams-bootstrap-v3- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + my-error-topic: + configs: {} + type: error + my-labeled-topic-output: + configs: {} + role: my-output-topic-label + my-output-topic: + configs: {} + type: output + resources-streams-bootstrap-v3-my-streams-app-error: + configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: my-streams-app + values: + commandLine: + CONVERT_XML: true + image: my-registry/my-streams-app-image + imageTag: 1.0.0 + kafka: + applicationId: my-streams-app-id + bootstrapServers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-streams-bootstrap-v3-my-streams-app-error + inputPattern: my-input-pattern + inputTopics: + - my-input-topic + labeledInputTopics: + my-input-topic-label: + - my-labeled-input-topic + labeledOutputTopics: + my-output-topic-label: my-labeled-topic-output + outputTopic: my-output-topic + schemaRegistryUrl: http://localhost:8081/ + persistence: + enabled: false + resources: + limits: + memory: 2G + requests: + memory: 2G + statefulSet: false + version: 3.0.0-SNAPSHOT + diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml new file mode 100644 index 000000000..81e609c5c --- /dev/null +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml @@ -0,0 +1,165 @@ +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + labels: + app: resources-streams-bootstrap-v3-my-producer-app + chart: producer-app-3.0.0-SNAPSHOT + release: resources-streams-bootstrap-v3-my-producer-app + name: resources-streams-bootstrap-v3-my-producer-app +spec: + concurrencyPolicy: Replace + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-streams-bootstrap-v3-my-producer-app + release: resources-streams-bootstrap-v3-my-producer-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: APP_FAKE_ARG + value: fake-arg-value + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-v3-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure + schedule: 30 3/8 * * * + successfulJobsHistoryLimit: 1 + suspend: false + +--- +apiVersion: v1 +data: + jmx-kafka-streams-app-prometheus.yml: "jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi\n\ + lowercaseOutputName: true\nlowercaseOutputLabelNames: true\nssl: false\nrules:\n\ + \ - pattern: \".*\"\n" +kind: ConfigMap +metadata: + labels: + app: resources-streams-bootstrap-v3-my-streams-app + chart: streams-app-3.0.0-SNAPSHOT + heritage: Helm + release: resources-streams-bootstrap-v3-my-streams-app + name: resources-streams-bootstrap-v3-my-streams-app-jmx-configmap + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + consumerGroup: my-streams-app-id + labels: + app: resources-streams-bootstrap-v3-my-streams-app + chart: streams-app-3.0.0-SNAPSHOT + release: resources-streams-bootstrap-v3-my-streams-app + name: resources-streams-bootstrap-v3-my-streams-app +spec: + replicas: 1 + selector: + matchLabels: + app: resources-streams-bootstrap-v3-my-streams-app + release: resources-streams-bootstrap-v3-my-streams-app + template: + metadata: + annotations: + prometheus.io/port: '5556' + prometheus.io/scrape: 'true' + labels: + app: resources-streams-bootstrap-v3-my-streams-app + release: resources-streams-bootstrap-v3-my-streams-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR + value: com.bakdata.kafka.MurmurHashIdGenerator + - name: KAFKA_JMX_PORT + value: '5555' + - name: APP_VOLATILE_GROUP_INSTANCE_ID + value: 'true' + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_INPUT_TOPICS + value: my-input-topic + - name: APP_INPUT_PATTERN + value: my-input-pattern + - name: APP_OUTPUT_TOPIC + value: my-output-topic + - name: APP_ERROR_TOPIC + value: resources-streams-bootstrap-v3-my-streams-app-error + - name: APP_LABELED_OUTPUT_TOPICS + value: my-output-topic-label=my-labeled-topic-output, + - name: APP_LABELED_INPUT_TOPICS + value: my-input-topic-label=my-labeled-input-topic, + - name: APP_APPLICATION_ID + value: my-streams-app-id + - name: APP_CONVERT_XML + value: 'true' + - name: JAVA_TOOL_OPTIONS + value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-streams-app-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-v3-my-streams-app + ports: + - containerPort: 5555 + name: jmx + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 2G + - command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - '5556' + - /etc/jmx-streams-app/jmx-kafka-streams-app-prometheus.yml + image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 + name: prometheus-jmx-exporter + ports: + - containerPort: 5556 + resources: + limits: + cpu: 300m + memory: 2G + requests: + cpu: 100m + memory: 500Mi + volumeMounts: + - mountPath: /etc/jmx-streams-app + name: jmx-config + terminationGracePeriodSeconds: 300 + volumes: + - configMap: + name: resources-streams-bootstrap-v3-my-streams-app-jmx-configmap + name: jmx-config + diff --git a/tests/pipeline/test_components/__init__.py b/tests/pipeline/test_components/__init__.py index cbf849307..2dcf7e825 100644 --- a/tests/pipeline/test_components/__init__.py +++ b/tests/pipeline/test_components/__init__.py @@ -1,6 +1,8 @@ from tests.pipeline.test_components.components import ( Converter, Filter, + MyProducerApp, + MyStreamsApp, ScheduledProducer, ShouldInflate, SimpleInflateConnectors, diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 8557c6b2e..fd3cc08ee 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -14,6 +14,13 @@ from kpops.components.common.topic import OutputTopicTypes, TopicConfig from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp +from kpops.components.streams_bootstrap_v3 import ProducerAppV3, StreamsAppV3 + + +class MyProducerApp(ProducerAppV3): ... + + +class MyStreamsApp(StreamsAppV3): ... class ScheduledProducer(ProducerApp): ... diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index c8aac3f0d..1a5845502 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -867,3 +867,31 @@ def test_substitution_in_resetter(self): enriched_pipeline[0]["_resetter"]["values"]["imageTag"] == "override-default-image-tag" ) + + def test_streams_bootstrap_v3(self, snapshot: Snapshot): + result = runner.invoke( + app, + [ + "generate", + str(RESOURCE_PATH / "streams-bootstrap-v3" / PIPELINE_YAML), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.stdout + + snapshot.assert_match(result.stdout, PIPELINE_YAML) + + def test_manifest_streams_bootstrap_v3(self, snapshot: Snapshot): + result = runner.invoke( + app, + [ + "manifest", + str(RESOURCE_PATH / "streams-bootstrap-v3" / PIPELINE_YAML), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.stdout + + snapshot.assert_match(result.stdout, PIPELINE_YAML) diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 45f3e3a94..b58cb6a67 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -11,12 +11,16 @@ from kpops.cli.main import app from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.model import HelmConfig, Version +from kpops.const.file_type import PIPELINE_YAML + +MANIFEST_YAML = "manifest.yaml" runner = CliRunner() RESOURCE_PATH = Path(__file__).parent / "resources" +@pytest.mark.usefixtures("mock_env", "load_yaml_file_clear_cache", "custom_components") class TestManifest: @pytest.fixture() def mock_execute(self, mocker: MockerFixture) -> MagicMock: @@ -110,7 +114,7 @@ def test_manifest_command(self, snapshot: Snapshot): catch_exceptions=False, ) assert result.exit_code == 0, result.stdout - snapshot.assert_match(result.stdout, "manifest.yaml") + snapshot.assert_match(result.stdout, MANIFEST_YAML) def test_python_api(self, snapshot: Snapshot): resources = kpops.manifest( @@ -120,3 +124,15 @@ def test_python_api(self, snapshot: Snapshot): assert isinstance(resources, list) assert len(resources) == 2 snapshot.assert_match(yaml.dump_all(resources), "resources") + + def test_streams_bootstrap_v3(self, snapshot: Snapshot): + result = runner.invoke( + app, + [ + "manifest", + str(RESOURCE_PATH / "streams-bootstrap-v3" / PIPELINE_YAML), + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.stdout + snapshot.assert_match(result.stdout, MANIFEST_YAML) From f3dbaeac83590f97d2c610b47ba65b547c0160cc Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 1 Aug 2024 12:51:54 +0200 Subject: [PATCH 04/23] Update files --- .../pipeline.yaml | 165 ------------------ tests/pipeline/test_generate.py | 14 -- 2 files changed, 179 deletions(-) delete mode 100644 tests/pipeline/snapshots/test_generate/test_manifest_streams_bootstrap_v3/pipeline.yaml diff --git a/tests/pipeline/snapshots/test_generate/test_manifest_streams_bootstrap_v3/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_manifest_streams_bootstrap_v3/pipeline.yaml deleted file mode 100644 index 81e609c5c..000000000 --- a/tests/pipeline/snapshots/test_generate/test_manifest_streams_bootstrap_v3/pipeline.yaml +++ /dev/null @@ -1,165 +0,0 @@ ---- -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - labels: - app: resources-streams-bootstrap-v3-my-producer-app - chart: producer-app-3.0.0-SNAPSHOT - release: resources-streams-bootstrap-v3-my-producer-app - name: resources-streams-bootstrap-v3-my-producer-app -spec: - concurrencyPolicy: Replace - failedJobsHistoryLimit: 1 - jobTemplate: - spec: - backoffLimit: 6 - template: - metadata: - labels: - app: resources-streams-bootstrap-v3-my-producer-app - release: resources-streams-bootstrap-v3-my-producer-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_OUTPUT_TOPIC - value: my-producer-app-output-topic - - name: APP_LABELED_OUTPUT_TOPICS - value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, - - name: APP_FAKE_ARG - value: fake-arg-value - - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-producer-image:1.0.0 - imagePullPolicy: Always - name: resources-streams-bootstrap-v3-my-producer-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - restartPolicy: OnFailure - schedule: 30 3/8 * * * - successfulJobsHistoryLimit: 1 - suspend: false - ---- -apiVersion: v1 -data: - jmx-kafka-streams-app-prometheus.yml: "jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi\n\ - lowercaseOutputName: true\nlowercaseOutputLabelNames: true\nssl: false\nrules:\n\ - \ - pattern: \".*\"\n" -kind: ConfigMap -metadata: - labels: - app: resources-streams-bootstrap-v3-my-streams-app - chart: streams-app-3.0.0-SNAPSHOT - heritage: Helm - release: resources-streams-bootstrap-v3-my-streams-app - name: resources-streams-bootstrap-v3-my-streams-app-jmx-configmap - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - consumerGroup: my-streams-app-id - labels: - app: resources-streams-bootstrap-v3-my-streams-app - chart: streams-app-3.0.0-SNAPSHOT - release: resources-streams-bootstrap-v3-my-streams-app - name: resources-streams-bootstrap-v3-my-streams-app -spec: - replicas: 1 - selector: - matchLabels: - app: resources-streams-bootstrap-v3-my-streams-app - release: resources-streams-bootstrap-v3-my-streams-app - template: - metadata: - annotations: - prometheus.io/port: '5556' - prometheus.io/scrape: 'true' - labels: - app: resources-streams-bootstrap-v3-my-streams-app - release: resources-streams-bootstrap-v3-my-streams-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR - value: com.bakdata.kafka.MurmurHashIdGenerator - - name: KAFKA_JMX_PORT - value: '5555' - - name: APP_VOLATILE_GROUP_INSTANCE_ID - value: 'true' - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_INPUT_TOPICS - value: my-input-topic - - name: APP_INPUT_PATTERN - value: my-input-pattern - - name: APP_OUTPUT_TOPIC - value: my-output-topic - - name: APP_ERROR_TOPIC - value: resources-streams-bootstrap-v3-my-streams-app-error - - name: APP_LABELED_OUTPUT_TOPICS - value: my-output-topic-label=my-labeled-topic-output, - - name: APP_LABELED_INPUT_TOPICS - value: my-input-topic-label=my-labeled-input-topic, - - name: APP_APPLICATION_ID - value: my-streams-app-id - - name: APP_CONVERT_XML - value: 'true' - - name: JAVA_TOOL_OPTIONS - value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false - -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-streams-app-image:1.0.0 - imagePullPolicy: Always - name: resources-streams-bootstrap-v3-my-streams-app - ports: - - containerPort: 5555 - name: jmx - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 2G - - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - '5556' - - /etc/jmx-streams-app/jmx-kafka-streams-app-prometheus.yml - image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 - name: prometheus-jmx-exporter - ports: - - containerPort: 5556 - resources: - limits: - cpu: 300m - memory: 2G - requests: - cpu: 100m - memory: 500Mi - volumeMounts: - - mountPath: /etc/jmx-streams-app - name: jmx-config - terminationGracePeriodSeconds: 300 - volumes: - - configMap: - name: resources-streams-bootstrap-v3-my-streams-app-jmx-configmap - name: jmx-config - diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 1a5845502..184534ad3 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -881,17 +881,3 @@ def test_streams_bootstrap_v3(self, snapshot: Snapshot): assert result.exit_code == 0, result.stdout snapshot.assert_match(result.stdout, PIPELINE_YAML) - - def test_manifest_streams_bootstrap_v3(self, snapshot: Snapshot): - result = runner.invoke( - app, - [ - "manifest", - str(RESOURCE_PATH / "streams-bootstrap-v3" / PIPELINE_YAML), - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0, result.stdout - - snapshot.assert_match(result.stdout, PIPELINE_YAML) From dc55119d6f60943c9518d44e1aef9f5b90b60f84 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 1 Aug 2024 12:55:27 +0200 Subject: [PATCH 05/23] Update files --- kpops/components/streams_bootstrap_v3/kafka_app.py | 4 ++-- .../components/streams_bootstrap_v3/streams/model.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/kpops/components/streams_bootstrap_v3/kafka_app.py b/kpops/components/streams_bootstrap_v3/kafka_app.py index 998851367..00de5fc3b 100644 --- a/kpops/components/streams_bootstrap_v3/kafka_app.py +++ b/kpops/components/streams_bootstrap_v3/kafka_app.py @@ -58,7 +58,7 @@ class KafkaStreamsConfig(CamelCaseConfigModel, DescConfigModel): @pydantic.field_validator("labeled_output_topics", mode="before") @classmethod - def deserialize_extra_output_topics( + def deserialize_labeled_output_topics( cls, labeled_output_topics: dict[str, str] | Any ) -> dict[str, KafkaTopic] | Any: if isinstance(labeled_output_topics, dict): @@ -69,7 +69,7 @@ def deserialize_extra_output_topics( return labeled_output_topics @pydantic.field_serializer("labeled_output_topics") - def serialize_extra_output_topics( + def serialize_labeled_output_topics( self, labeled_output_topics: dict[str, KafkaTopic] ) -> dict[str, str]: return {label: topic.name for label, topic in labeled_output_topics.items()} diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index bfff1f988..5df5c41cf 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -71,14 +71,14 @@ def deserialize_input_topics( @pydantic.field_validator("labeled_input_topics", mode="before") @classmethod def deserialize_labeled_input_topics( - cls, extra_input_topics: dict[str, str] | Any + cls, labeled_input_topics: dict[str, str] | Any ) -> dict[str, list[KafkaTopic]] | Any: - if isinstance(extra_input_topics, dict): + if isinstance(labeled_input_topics, dict): return { label: [KafkaTopic(name=topic_name) for topic_name in topics] - for label, topics in extra_input_topics.items() + for label, topics in labeled_input_topics.items() } - return extra_input_topics + return labeled_input_topics @pydantic.field_serializer("input_topics") def serialize_topics(self, topics: list[KafkaTopic]) -> list[str]: @@ -86,11 +86,11 @@ def serialize_topics(self, topics: list[KafkaTopic]) -> list[str]: @pydantic.field_serializer("labeled_input_patterns") def serialize_labeled_input_topics( - self, extra_topics: dict[str, list[KafkaTopic]] + self, labeled_input_topics: dict[str, list[KafkaTopic]] ) -> dict[str, list[str]]: return { label: self.serialize_topics(topics) - for label, topics in extra_topics.items() + for label, topics in labeled_input_topics.items() } def add_input_topics(self, topics: list[KafkaTopic]) -> None: From 10e91e83f1465e4ee764f7905208bb9d78289c81 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Fri, 2 Aug 2024 09:46:42 +0200 Subject: [PATCH 06/23] add v3 base class and remove kafka app --- .../dependencies/kpops_structure.yaml | 12 +- docs/docs/schema/defaults.json | 208 +++++++++--------- docs/docs/schema/pipeline.json | 208 +++++++++--------- .../{kafka_app.py => base.py} | 115 ++++++---- .../streams_bootstrap_v3/producer/model.py | 13 +- .../producer/producer_app.py | 7 +- .../streams_bootstrap_v3/streams/model.py | 23 +- .../streams/streams_app.py | 10 +- .../streams-bootstrap-v3/defaults.yaml | 2 +- .../streams-bootstrap-v3/pipeline.yaml | 9 +- .../test_streams_bootstrap_v3/pipeline.yaml | 7 + .../test_streams_bootstrap_v3/manifest.yaml | 2 + 12 files changed, 323 insertions(+), 293 deletions(-) rename kpops/components/streams_bootstrap_v3/{kafka_app.py => base.py} (59%) diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 97876ad87..597babcf8 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -153,11 +153,9 @@ kpops_components_inheritance_ref: - base-defaults-component producer-app-v3: bases: - - kafka-app-v3 - - streams-bootstrap + - streams-bootstrap-v3 parents: - - kafka-app-v3 - - streams-bootstrap + - streams-bootstrap-v3 - helm-app - kubernetes-app - pipeline-component @@ -175,11 +173,9 @@ kpops_components_inheritance_ref: - base-defaults-component streams-app-v3: bases: - - kafka-app-v3 - - streams-bootstrap + - streams-bootstrap-v3 parents: - - kafka-app-v3 - - streams-bootstrap + - streams-bootstrap-v3 - helm-app - kubernetes-app - pipeline-component diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 580d36362..0976f2945 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1007,6 +1007,106 @@ "title": "ProducerAppV3", "type": "object" }, + "ProducerConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerConfig", + "type": "object" + }, + "ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" + }, + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "brokers" + ], + "title": "ProducerStreamsConfig", + "type": "object" + }, "RepoAuthFlags": { "description": "Authorisation-related flags for `helm repo`.", "properties": { @@ -1549,7 +1649,7 @@ "streams": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap__producer__model__ProducerStreamsConfig" + "$ref": "#/$defs/ProducerStreamsConfig" } ], "description": "Kafka Streams settings" @@ -1561,56 +1661,6 @@ "title": "ProducerAppValues", "type": "object" }, - "kpops__components__streams_bootstrap__producer__model__ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "ProducerStreamsConfig", - "type": "object" - }, "kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", @@ -1912,7 +1962,7 @@ "kafka": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__producer__model__ProducerStreamsConfig" + "$ref": "#/$defs/ProducerConfig" } ], "description": "" @@ -1938,56 +1988,6 @@ "title": "ProducerAppValues", "type": "object" }, - "kpops__components__streams_bootstrap_v3__producer__model__ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "bootstrapServers": { - "description": "Brokers", - "title": "Bootstrapservers", - "type": "string" - }, - "labeledOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Labeledoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "bootstrapServers" - ], - "title": "ProducerStreamsConfig", - "type": "object" - }, "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", @@ -2218,7 +2218,7 @@ "type": "string" }, "default": {}, - "description": "", + "description": "Extra input patterns", "title": "Labeledinputpatterns", "type": "object" }, @@ -2230,7 +2230,7 @@ "type": "array" }, "default": {}, - "description": "", + "description": "Extra input topics", "title": "Labeledinputtopics", "type": "object" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 1667ed41f..5a0b255d3 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -675,6 +675,106 @@ "title": "ProducerAppV3", "type": "object" }, + "ProducerConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerConfig", + "type": "object" + }, + "ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" + }, + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "brokers" + ], + "title": "ProducerStreamsConfig", + "type": "object" + }, "RepoAuthFlags": { "description": "Authorisation-related flags for `helm repo`.", "properties": { @@ -1095,7 +1195,7 @@ "streams": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap__producer__model__ProducerStreamsConfig" + "$ref": "#/$defs/ProducerStreamsConfig" } ], "description": "Kafka Streams settings" @@ -1107,56 +1207,6 @@ "title": "ProducerAppValues", "type": "object" }, - "kpops__components__streams_bootstrap__producer__model__ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "ProducerStreamsConfig", - "type": "object" - }, "kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", @@ -1458,7 +1508,7 @@ "kafka": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__producer__model__ProducerStreamsConfig" + "$ref": "#/$defs/ProducerConfig" } ], "description": "" @@ -1484,56 +1534,6 @@ "title": "ProducerAppValues", "type": "object" }, - "kpops__components__streams_bootstrap_v3__producer__model__ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "bootstrapServers": { - "description": "Brokers", - "title": "Bootstrapservers", - "type": "string" - }, - "labeledOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Labeledoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "bootstrapServers" - ], - "title": "ProducerStreamsConfig", - "type": "object" - }, "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", @@ -1764,7 +1764,7 @@ "type": "string" }, "default": {}, - "description": "", + "description": "Extra input patterns", "title": "Labeledinputpatterns", "type": "object" }, @@ -1776,7 +1776,7 @@ "type": "array" }, "default": {}, - "description": "", + "description": "Extra input topics", "title": "Labeledinputtopics", "type": "object" }, diff --git a/kpops/components/streams_bootstrap_v3/kafka_app.py b/kpops/components/streams_bootstrap_v3/base.py similarity index 59% rename from kpops/components/streams_bootstrap_v3/kafka_app.py rename to kpops/components/streams_bootstrap_v3/base.py index 00de5fc3b..a6be0f08a 100644 --- a/kpops/components/streams_bootstrap_v3/kafka_app.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -2,17 +2,15 @@ import logging from abc import ABC -from typing import Any +from typing import TYPE_CHECKING, Any import pydantic from pydantic import AliasChoices, ConfigDict, Field from typing_extensions import override -from kpops.component_handlers import get_handlers +from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.components.base_components.cleaner import Cleaner -from kpops.components.base_components.helm_app import HelmAppValues -from kpops.components.base_components.pipeline_component import PipelineComponent -from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.config import get_config from kpops.utils.docstring import describe_attr @@ -23,10 +21,75 @@ exclude_defaults, ) -log = logging.getLogger("KafkaApp") +if TYPE_CHECKING: + try: + from typing import Self # pyright: ignore[reportAttributeAccessIssue] + except ImportError: + from typing_extensions import Self -class KafkaStreamsConfig(CamelCaseConfigModel, DescConfigModel): +STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( + repository_name="bakdata-streams-bootstrap", + url="https://bakdata.github.io/streams-bootstrap/", +) +STREAMS_BOOTSTRAP_VERSION = "3.0.0" + +# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names +IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" + +log = logging.getLogger("StreamsBootstrapV3") + + +class StreamsBootstrapV3Values(HelmAppValues): + """Base value class for all streams bootstrap related components. + + :param image_tag: Docker image tag of the streams-bootstrap app. + """ + + image_tag: str = Field( + default="latest", + pattern=IMAGE_TAG_PATTERN, + description=describe_attr("image_tag", __doc__), + ) + + kafka: KafkaConfig = Field(default=..., description=describe_attr("kafka", __doc__)) + + +class StreamsBootstrapV3(HelmApp, ABC): + """Base for components with a streams-bootstrap Helm chart. + + :param values: streams-bootstrap Helm values + :param repo_config: Configuration of the Helm chart repo to be used for + deploying the component, defaults to streams-bootstrap Helm repo + :param version: Helm chart version, defaults to "3.0.0" + """ + + values: StreamsBootstrapV3Values = Field( + default_factory=StreamsBootstrapV3Values, + description=describe_attr("values", __doc__), + ) + + repo_config: HelmRepoConfig = Field( + default=STREAMS_BOOTSTRAP_HELM_REPO, + description=describe_attr("repo_config", __doc__), + ) + + # TODO: validate that version is higher than 3.x.x + version: str | None = Field( + default=STREAMS_BOOTSTRAP_VERSION, + description=describe_attr("version", __doc__), + ) + + @pydantic.model_validator(mode="after") + def warning_for_latest_image_tag(self) -> Self: + if self.validate_ and self.values.image_tag == "latest": + log.warning( + f"The image tag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable image tag." + ) + return self + + +class KafkaConfig(CamelCaseConfigModel, DescConfigModel): """Kafka Streams config. :param bootstrap_servers: Brokers @@ -86,18 +149,7 @@ def serialize_model( ) -class KafkaAppValues(HelmAppValues): - """Settings specific to Kafka Apps. - - :param kafka: Kafka streams config - """ - - kafka: KafkaStreamsConfig = Field( - default=..., description=describe_attr("streams", __doc__) - ) - - -class KafkaAppCleaner(Cleaner, StreamsBootstrap, ABC): +class StreamsBootstrapV3Cleaner(Cleaner, StreamsBootstrapV3, ABC): """Helm app for resetting and cleaning a streams-bootstrap app.""" from_: None = None @@ -123,28 +175,3 @@ async def clean(self, dry_run: bool) -> None: if not get_config().retain_clean_jobs: log.info(f"Uninstall cleanup job for {self.helm_release_name}") await self.destroy(dry_run) - - -class KafkaAppV3(PipelineComponent, ABC): - """Base component for Kafka-based components. - - Producer or streaming apps should inherit from this class. - - :param values: Application-specific settings - """ - - values: KafkaAppValues = Field( - default=..., - description=describe_attr("values", __doc__), - ) - - @override - async def deploy(self, dry_run: bool) -> None: - if self.to: - for topic in self.to.kafka_topics: - await get_handlers().topic_handler.create_topic(topic, dry_run=dry_run) - - if schema_handler := get_handlers().schema_handler: - await schema_handler.submit_schemas(to_section=self.to, dry_run=dry_run) - - await super().deploy(dry_run) diff --git a/kpops/components/streams_bootstrap_v3/producer/model.py b/kpops/components/streams_bootstrap_v3/producer/model.py index b117e0c2c..220a39982 100644 --- a/kpops/components/streams_bootstrap_v3/producer/model.py +++ b/kpops/components/streams_bootstrap_v3/producer/model.py @@ -1,24 +1,23 @@ from pydantic import ConfigDict, Field -from kpops.components.common.streams_bootstrap import StreamsBootstrapValues -from kpops.components.streams_bootstrap_v3.kafka_app import ( - KafkaAppValues, - KafkaStreamsConfig, +from kpops.components.streams_bootstrap_v3.base import ( + KafkaConfig, + StreamsBootstrapV3Values, ) from kpops.utils.docstring import describe_attr -class ProducerStreamsConfig(KafkaStreamsConfig): +class ProducerConfig(KafkaConfig): """Kafka Streams settings specific to Producer.""" -class ProducerAppValues(StreamsBootstrapValues, KafkaAppValues): +class ProducerAppValues(StreamsBootstrapV3Values): """Settings specific to producers. :param kafka: Kafka Streams settings """ - kafka: ProducerStreamsConfig = Field( + kafka: ProducerConfig = Field( default=..., description=describe_attr("streams", __doc__) ) diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap_v3/producer/producer_app.py index 7536e5bc6..41e932997 100644 --- a/kpops/components/streams_bootstrap_v3/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v3/producer/producer_app.py @@ -7,14 +7,15 @@ from kpops.components.base_components.kafka_app import ( KafkaAppCleaner, ) -from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, ) from kpops.components.streams_bootstrap_v3.app_type import AppType -from kpops.components.streams_bootstrap_v3.kafka_app import KafkaAppV3 +from kpops.components.streams_bootstrap_v3.base import ( + StreamsBootstrapV3, +) from kpops.components.streams_bootstrap_v3.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr @@ -33,7 +34,7 @@ def helm_chart(self) -> str: ) -class ProducerAppV3(KafkaAppV3, StreamsBootstrap): +class ProducerAppV3(StreamsBootstrapV3): """Producer component. This producer holds configuration to use as values for the streams-bootstrap diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index 5df5c41cf..9a5883592 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -6,11 +6,10 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from kpops.api.exception import ValidationError -from kpops.components.common.streams_bootstrap import StreamsBootstrapValues from kpops.components.common.topic import KafkaTopic, KafkaTopicStr -from kpops.components.streams_bootstrap_v3.kafka_app import ( - KafkaAppValues, - KafkaStreamsConfig, +from kpops.components.streams_bootstrap_v3.base import ( + KafkaConfig, + StreamsBootstrapV3Values, ) from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( @@ -19,7 +18,7 @@ ) -class StreamsConfig(KafkaStreamsConfig): +class StreamsConfig(KafkaConfig): """Streams Bootstrap kafka section. :param application_id: Unique application ID for Kafka Streams. Required for auto-scaling @@ -44,10 +43,10 @@ class StreamsConfig(KafkaStreamsConfig): default=None, description=describe_attr("input_pattern", __doc__) ) labeled_input_topics: dict[str, list[KafkaTopicStr]] = Field( - default={}, description=describe_attr("extra_input_topics", __doc__) + default={}, description=describe_attr("labeled_input_topics", __doc__) ) labeled_input_patterns: dict[str, str] = Field( - default={}, description=describe_attr("extra_input_patterns", __doc__) + default={}, description=describe_attr("labeled_input_patterns", __doc__) ) error_topic: KafkaTopicStr | None = Field( default=None, description=describe_attr("error_topic", __doc__) @@ -71,7 +70,7 @@ def deserialize_input_topics( @pydantic.field_validator("labeled_input_topics", mode="before") @classmethod def deserialize_labeled_input_topics( - cls, labeled_input_topics: dict[str, str] | Any + cls, labeled_input_topics: Any ) -> dict[str, list[KafkaTopic]] | Any: if isinstance(labeled_input_topics, dict): return { @@ -81,10 +80,10 @@ def deserialize_labeled_input_topics( return labeled_input_topics @pydantic.field_serializer("input_topics") - def serialize_topics(self, topics: list[KafkaTopic]) -> list[str]: - return [topic.name for topic in topics] + def serialize_topics(self, input_topics: list[KafkaTopic]) -> list[str]: + return [topic.name for topic in input_topics] - @pydantic.field_serializer("labeled_input_patterns") + @pydantic.field_serializer("labeled_input_topics") def serialize_labeled_input_topics( self, labeled_input_topics: dict[str, list[KafkaTopic]] ) -> dict[str, list[str]]: @@ -239,7 +238,7 @@ def validate_mandatory_fields_are_set( return self -class StreamsAppValues(StreamsBootstrapValues, KafkaAppValues): +class StreamsAppValues(StreamsBootstrapV3Values): """streams-bootstrap app configurations. The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart. diff --git a/kpops/components/streams_bootstrap_v3/streams/streams_app.py b/kpops/components/streams_bootstrap_v3/streams/streams_app.py index 6b334fd44..325d3c726 100644 --- a/kpops/components/streams_bootstrap_v3/streams/streams_app.py +++ b/kpops/components/streams_bootstrap_v3/streams/streams_app.py @@ -6,10 +6,12 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp -from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap_v3.app_type import AppType -from kpops.components.streams_bootstrap_v3.kafka_app import KafkaAppCleaner, KafkaAppV3 +from kpops.components.streams_bootstrap_v3.base import ( + StreamsBootstrapV3, + StreamsBootstrapV3Cleaner, +) from kpops.components.streams_bootstrap_v3.streams.model import ( StreamsAppValues, ) @@ -20,7 +22,7 @@ STREAMS_BOOTSTRAP_V3 = "3.0.0" -class StreamsAppCleaner(KafkaAppCleaner): +class StreamsAppCleaner(StreamsBootstrapV3Cleaner): from_: None = None to: None = None values: StreamsAppValues @@ -54,7 +56,7 @@ async def clean_pvcs(self, dry_run: bool) -> None: await pvc_handler.delete_pvcs() -class StreamsAppV3(KafkaAppV3, StreamsBootstrap): +class StreamsAppV3(StreamsBootstrapV3): """StreamsApp component that configures a streams-bootstrap app. :param values: streams-bootstrap Helm values diff --git a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml index 02b82f405..b88979e76 100644 --- a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml +++ b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml @@ -4,7 +4,7 @@ pipeline-component: kubernetes-app: namespace: example-namespace -kafka-app-v3: +streams-bootstrap-v3: values: kafka: bootstrapServers: ${config.kafka_brokers} diff --git a/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml b/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml index 8f35ff5a9..ae03216b3 100644 --- a/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml +++ b/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml @@ -35,12 +35,9 @@ role: my-input-topic-label my-input-pattern: type: pattern - # TODO: not working, for some reason the extra pattern is not serialized as Topic object but as str - # pydantic_core._pydantic_core.PydanticSerializationError: - # Error calling function `serialize_labeled_input_topics`: AttributeError: 'str' object has no attribute 'name' -# my-labeled-input-pattern: -# type: pattern -# role: my-input-topic-labeled-pattern + my-labeled-input-pattern: + type: pattern + role: my-input-topic-labeled-pattern to: topics: diff --git a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml index fccf0dc84..503bacae3 100644 --- a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml @@ -78,6 +78,8 @@ inputPattern: my-input-pattern inputTopics: - my-input-topic + labeledInputPatterns: + my-input-topic-labeled-pattern: my-labeled-input-pattern labeledInputTopics: my-input-topic-label: - my-labeled-input-topic @@ -101,6 +103,9 @@ type: pattern my-input-topic: type: input + my-labeled-input-pattern: + role: my-input-topic-labeled-pattern + type: pattern my-labeled-input-topic: role: my-input-topic-label name: my-streams-app @@ -144,6 +149,8 @@ inputPattern: my-input-pattern inputTopics: - my-input-topic + labeledInputPatterns: + my-input-topic-labeled-pattern: my-labeled-input-pattern labeledInputTopics: my-input-topic-label: - my-labeled-input-topic diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml index 81e609c5c..19eb16b15 100644 --- a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml @@ -113,6 +113,8 @@ spec: value: my-output-topic-label=my-labeled-topic-output, - name: APP_LABELED_INPUT_TOPICS value: my-input-topic-label=my-labeled-input-topic, + - name: APP_LABELED_INPUT_PATTERNS + value: my-input-topic-labeled-pattern=my-labeled-input-pattern, - name: APP_APPLICATION_ID value: my-streams-app-id - name: APP_CONVERT_XML From 63c6f9c34a5278b56e31cd629287dcb0fd7ff643 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Fri, 2 Aug 2024 09:57:19 +0200 Subject: [PATCH 07/23] Update files --- kpops/components/streams_bootstrap_v3/base.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index a6be0f08a..5e56c9402 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -8,6 +8,7 @@ from pydantic import AliasChoices, ConfigDict, Field from typing_extensions import override +from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.components.base_components.cleaner import Cleaner from kpops.components.base_components.helm_app import HelmApp, HelmAppValues @@ -80,6 +81,17 @@ class StreamsBootstrapV3(HelmApp, ABC): description=describe_attr("version", __doc__), ) + @override + async def deploy(self, dry_run: bool) -> None: + if self.to: + for topic in self.to.kafka_topics: + await get_handlers().topic_handler.create_topic(topic, dry_run=dry_run) + + if schema_handler := get_handlers().schema_handler: + await schema_handler.submit_schemas(to_section=self.to, dry_run=dry_run) + + await super().deploy(dry_run) + @pydantic.model_validator(mode="after") def warning_for_latest_image_tag(self) -> Self: if self.validate_ and self.values.image_tag == "latest": From 757701f76807b7ad4ba41283fcb3b79c042625c8 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Fri, 2 Aug 2024 10:51:04 +0200 Subject: [PATCH 08/23] remove values from kafka app --- ...aults_pipeline_component_dependencies.yaml | 1 - .../dependencies/kpops_structure.yaml | 3 +- .../pipeline_component_dependencies.yaml | 1 - .../pipeline-components/kafka-app.yaml | 6 -- .../pipeline-components/pipeline.yaml | 6 -- .../pipeline-defaults/defaults-kafka-app.yaml | 6 -- .../resources/pipeline-defaults/defaults.yaml | 6 -- docs/docs/schema/defaults.json | 98 +------------------ docs/docs/schema/pipeline.json | 4 +- kpops/components/base_components/kafka_app.py | 21 +--- .../streams_bootstrap/producer/model.py | 3 +- .../producer/producer_app.py | 2 +- .../streams_bootstrap/streams/model.py | 3 +- .../streams_bootstrap/streams/streams_app.py | 2 +- kpops/components/streams_bootstrap_v3/base.py | 48 +-------- .../streams_bootstrap_v3/producer/model.py | 2 +- .../producer/producer_app.py | 6 +- .../streams_bootstrap_v3/streams/model.py | 2 +- .../streams/streams_app.py | 4 +- tests/pipeline/resources/defaults.yaml | 2 +- .../streams-bootstrap-v3/defaults.yaml | 23 +---- tests/pipeline/test_generate.py | 1 + 22 files changed, 24 insertions(+), 226 deletions(-) diff --git a/docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml b/docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml index 20d30f041..e31f0946f 100644 --- a/docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml @@ -5,7 +5,6 @@ kafka-app.yaml: - prefix.yaml - from_.yaml - to.yaml -- values-kafka-app.yaml kafka-connector.yaml: - prefix.yaml - from_.yaml diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 597babcf8..830445552 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -13,7 +13,6 @@ kpops_components_fields: - prefix - from_ - to - - values kafka-connector: - name - prefix @@ -156,6 +155,7 @@ kpops_components_inheritance_ref: - streams-bootstrap-v3 parents: - streams-bootstrap-v3 + - kafka-app - helm-app - kubernetes-app - pipeline-component @@ -176,6 +176,7 @@ kpops_components_inheritance_ref: - streams-bootstrap-v3 parents: - streams-bootstrap-v3 + - kafka-app - helm-app - kubernetes-app - pipeline-component diff --git a/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml b/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml index 0ff8788ad..4e5c7b800 100644 --- a/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml @@ -10,7 +10,6 @@ kafka-app.yaml: - prefix.yaml - from_.yaml - to.yaml -- values-kafka-app.yaml kafka-connector.yaml: - prefix.yaml - from_.yaml diff --git a/docs/docs/resources/pipeline-components/kafka-app.yaml b/docs/docs/resources/pipeline-components/kafka-app.yaml index 806662ef3..bbc042086 100644 --- a/docs/docs/resources/pipeline-components/kafka-app.yaml +++ b/docs/docs/resources/pipeline-components/kafka-app.yaml @@ -44,9 +44,3 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - values: # required - streams: # required - brokers: ${config.kafka_brokers} # required - schemaRegistryUrl: ${config.schema_registry.url} - nameOverride: override-with-this-name # kafka-app-specific - imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app diff --git a/docs/docs/resources/pipeline-components/pipeline.yaml b/docs/docs/resources/pipeline-components/pipeline.yaml index 835c989f1..4c8379c0f 100644 --- a/docs/docs/resources/pipeline-components/pipeline.yaml +++ b/docs/docs/resources/pipeline-components/pipeline.yaml @@ -105,12 +105,6 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - values: # required - streams: # required - brokers: ${config.kafka_brokers} # required - schemaRegistryUrl: ${config.schema_registry.url} - nameOverride: override-with-this-name # kafka-app-specific - imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app # Kafka sink connector - type: kafka-sink-connector name: kafka-sink-connector # required diff --git a/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml index 2e4c1dc82..cbd45977b 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml @@ -45,9 +45,3 @@ kafka-app: cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - values: # required - streams: # required - brokers: ${config.kafka_brokers} # required - schemaRegistryUrl: ${config.schema_registry.url} - nameOverride: override-with-this-name # kafka-app-specific - imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app diff --git a/docs/docs/resources/pipeline-defaults/defaults.yaml b/docs/docs/resources/pipeline-defaults/defaults.yaml index 2eead5c37..820f06307 100644 --- a/docs/docs/resources/pipeline-defaults/defaults.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults.yaml @@ -64,12 +64,6 @@ kafka-app: cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - values: # required - streams: # required - brokers: ${config.kafka_brokers} # required - schemaRegistryUrl: ${config.schema_registry.url} - nameOverride: override-with-this-name # kafka-app-specific - imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app # Kafka connector # # Parent of: KafkaSinkConnector, KafkaSourceConnector diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 0976f2945..cbfabb521 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -257,56 +257,14 @@ ], "default": null, "description": "Topic(s) into which the component will write output" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/KafkaAppValues" - } - ], - "description": "Application-specific settings" } }, "required": [ - "name", - "values" + "name" ], "title": "KafkaApp", "type": "object" }, - "KafkaAppValues": { - "additionalProperties": true, - "description": "Settings specific to Kafka Apps.", - "properties": { - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/KafkaStreamsConfig" - } - ], - "description": "Kafka streams config" - } - }, - "required": [ - "streams" - ], - "title": "KafkaAppValues", - "type": "object" - }, "KafkaConnector": { "additionalProperties": true, "description": "Base class for all Kafka connectors.\nShould only be used to set defaults", @@ -610,56 +568,6 @@ "title": "KafkaSourceConnector", "type": "object" }, - "KafkaStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams config.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "KafkaStreamsConfig", - "type": "object" - }, "KubernetesApp": { "additionalProperties": true, "description": "Base class for all Kubernetes apps.\nAll built-in components are Kubernetes apps, except for the Kafka connectors.", @@ -1965,7 +1873,7 @@ "$ref": "#/$defs/ProducerConfig" } ], - "description": "" + "description": "Kafka Streams settings" }, "nameOverride": { "anyOf": [ @@ -2096,7 +2004,7 @@ "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" } ], - "description": "" + "description": "streams-bootstrap kafka section" }, "nameOverride": { "anyOf": [ diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 5a0b255d3..25b89d6c6 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1511,7 +1511,7 @@ "$ref": "#/$defs/ProducerConfig" } ], - "description": "" + "description": "Kafka Streams settings" }, "nameOverride": { "anyOf": [ @@ -1642,7 +1642,7 @@ "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" } ], - "description": "" + "description": "streams-bootstrap kafka section" }, "nameOverride": { "anyOf": [ diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 6ab89ed6e..97ab46843 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -10,9 +10,7 @@ from kpops.component_handlers import get_handlers from kpops.components.base_components.cleaner import Cleaner -from kpops.components.base_components.helm_app import HelmAppValues from kpops.components.base_components.pipeline_component import PipelineComponent -from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.config import get_config from kpops.utils.docstring import describe_attr @@ -84,18 +82,7 @@ def serialize_model( ) -class KafkaAppValues(HelmAppValues): - """Settings specific to Kafka Apps. - - :param streams: Kafka streams config - """ - - streams: KafkaStreamsConfig = Field( - default=..., description=describe_attr("streams", __doc__) - ) - - -class KafkaAppCleaner(Cleaner, StreamsBootstrap, ABC): +class KafkaAppCleaner(Cleaner, ABC): """Helm app for resetting and cleaning a streams-bootstrap app.""" from_: None = None @@ -128,14 +115,8 @@ class KafkaApp(PipelineComponent, ABC): Producer or streaming apps should inherit from this class. - :param values: Application-specific settings """ - values: KafkaAppValues = Field( - default=..., - description=describe_attr("values", __doc__), - ) - @override async def deploy(self, dry_run: bool) -> None: if self.to: diff --git a/kpops/components/streams_bootstrap/producer/model.py b/kpops/components/streams_bootstrap/producer/model.py index 1cbdf495c..e8079ec53 100644 --- a/kpops/components/streams_bootstrap/producer/model.py +++ b/kpops/components/streams_bootstrap/producer/model.py @@ -1,7 +1,6 @@ from pydantic import ConfigDict, Field from kpops.components.base_components.kafka_app import ( - KafkaAppValues, KafkaStreamsConfig, ) from kpops.components.common.streams_bootstrap import StreamsBootstrapValues @@ -12,7 +11,7 @@ class ProducerStreamsConfig(KafkaStreamsConfig): """Kafka Streams settings specific to Producer.""" -class ProducerAppValues(StreamsBootstrapValues, KafkaAppValues): +class ProducerAppValues(StreamsBootstrapValues): """Settings specific to producers. :param streams: Kafka Streams settings diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 564487616..b2373b7d8 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -21,7 +21,7 @@ log = logging.getLogger("ProducerApp") -class ProducerAppCleaner(KafkaAppCleaner): +class ProducerAppCleaner(KafkaAppCleaner, StreamsBootstrap): values: ProducerAppValues @property diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py index 299d17759..8a1ea8982 100644 --- a/kpops/components/streams_bootstrap/streams/model.py +++ b/kpops/components/streams_bootstrap/streams/model.py @@ -7,7 +7,6 @@ from kpops.api.exception import ValidationError from kpops.components.base_components.kafka_app import ( - KafkaAppValues, KafkaStreamsConfig, ) from kpops.components.common.streams_bootstrap import StreamsBootstrapValues @@ -240,7 +239,7 @@ def validate_mandatory_fields_are_set( return self -class StreamsAppValues(StreamsBootstrapValues, KafkaAppValues): +class StreamsAppValues(StreamsBootstrapValues): """streams-bootstrap app configurations. The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart. diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 6e55931d9..3d4997120 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -18,7 +18,7 @@ log = logging.getLogger("StreamsApp") -class StreamsAppCleaner(KafkaAppCleaner): +class StreamsAppCleaner(KafkaAppCleaner, StreamsBootstrap): from_: None = None to: None = None values: StreamsAppValues diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index 5e56c9402..10504605c 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -6,14 +6,11 @@ import pydantic from pydantic import AliasChoices, ConfigDict, Field -from typing_extensions import override -from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig -from kpops.components.base_components.cleaner import Cleaner +from kpops.components.base_components import KafkaApp from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.components.common.topic import KafkaTopic, KafkaTopicStr -from kpops.config import get_config from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( CamelCaseConfigModel, @@ -53,10 +50,8 @@ class StreamsBootstrapV3Values(HelmAppValues): description=describe_attr("image_tag", __doc__), ) - kafka: KafkaConfig = Field(default=..., description=describe_attr("kafka", __doc__)) - -class StreamsBootstrapV3(HelmApp, ABC): +class StreamsBootstrapV3(KafkaApp, HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. :param values: streams-bootstrap Helm values @@ -81,17 +76,6 @@ class StreamsBootstrapV3(HelmApp, ABC): description=describe_attr("version", __doc__), ) - @override - async def deploy(self, dry_run: bool) -> None: - if self.to: - for topic in self.to.kafka_topics: - await get_handlers().topic_handler.create_topic(topic, dry_run=dry_run) - - if schema_handler := get_handlers().schema_handler: - await schema_handler.submit_schemas(to_section=self.to, dry_run=dry_run) - - await super().deploy(dry_run) - @pydantic.model_validator(mode="after") def warning_for_latest_image_tag(self) -> Self: if self.validate_ and self.values.image_tag == "latest": @@ -159,31 +143,3 @@ def serialize_model( return exclude_defaults( self, exclude_by_value(default_serialize_handler(self), None) ) - - -class StreamsBootstrapV3Cleaner(Cleaner, StreamsBootstrapV3, ABC): - """Helm app for resetting and cleaning a streams-bootstrap app.""" - - from_: None = None - to: None = None - - @property - @override - def helm_chart(self) -> str: - raise NotImplementedError - - @override - async def clean(self, dry_run: bool) -> None: - """Clean an app using a cleanup job. - - :param dry_run: Dry run command - """ - log.info(f"Uninstall old cleanup job for {self.helm_release_name}") - await self.destroy(dry_run) - - log.info(f"Init cleanup job for {self.helm_release_name}") - await self.deploy(dry_run) - - if not get_config().retain_clean_jobs: - log.info(f"Uninstall cleanup job for {self.helm_release_name}") - await self.destroy(dry_run) diff --git a/kpops/components/streams_bootstrap_v3/producer/model.py b/kpops/components/streams_bootstrap_v3/producer/model.py index 220a39982..919843afb 100644 --- a/kpops/components/streams_bootstrap_v3/producer/model.py +++ b/kpops/components/streams_bootstrap_v3/producer/model.py @@ -18,7 +18,7 @@ class ProducerAppValues(StreamsBootstrapV3Values): """ kafka: ProducerConfig = Field( - default=..., description=describe_attr("streams", __doc__) + default=..., description=describe_attr("kafka", __doc__) ) model_config = ConfigDict(extra="allow") diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap_v3/producer/producer_app.py index 41e932997..26ca6eddd 100644 --- a/kpops/components/streams_bootstrap_v3/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v3/producer/producer_app.py @@ -4,9 +4,7 @@ from pydantic import Field, computed_field from typing_extensions import override -from kpops.components.base_components.kafka_app import ( - KafkaAppCleaner, -) +from kpops.components.base_components.kafka_app import KafkaAppCleaner from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, @@ -23,7 +21,7 @@ STREAMS_BOOTSTRAP_V3 = "3.0.0" -class ProducerAppCleaner(KafkaAppCleaner): +class ProducerAppCleaner(KafkaAppCleaner, StreamsBootstrapV3): values: ProducerAppValues @property diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index 9a5883592..343b49d49 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -249,7 +249,7 @@ class StreamsAppValues(StreamsBootstrapV3Values): kafka: StreamsConfig = Field( default=..., - description=describe_attr("streams", __doc__), + description=describe_attr("kafka", __doc__), ) autoscaling: StreamsAppAutoScaling | None = Field( default=None, diff --git a/kpops/components/streams_bootstrap_v3/streams/streams_app.py b/kpops/components/streams_bootstrap_v3/streams/streams_app.py index 325d3c726..88d380d42 100644 --- a/kpops/components/streams_bootstrap_v3/streams/streams_app.py +++ b/kpops/components/streams_bootstrap_v3/streams/streams_app.py @@ -6,11 +6,11 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.kafka_app import KafkaAppCleaner from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap_v3.app_type import AppType from kpops.components.streams_bootstrap_v3.base import ( StreamsBootstrapV3, - StreamsBootstrapV3Cleaner, ) from kpops.components.streams_bootstrap_v3.streams.model import ( StreamsAppValues, @@ -22,7 +22,7 @@ STREAMS_BOOTSTRAP_V3 = "3.0.0" -class StreamsAppCleaner(StreamsBootstrapV3Cleaner): +class StreamsAppCleaner(KafkaAppCleaner, StreamsBootstrapV3): from_: None = None to: None = None values: StreamsAppValues diff --git a/tests/pipeline/resources/defaults.yaml b/tests/pipeline/resources/defaults.yaml index 32aaf5aeb..b0785a3e3 100644 --- a/tests/pipeline/resources/defaults.yaml +++ b/tests/pipeline/resources/defaults.yaml @@ -4,7 +4,7 @@ pipeline-component: kubernetes-app: namespace: example-namespace -kafka-app: +streams-bootstrap: values: streams: brokers: ${config.kafka_brokers} diff --git a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml index b88979e76..a8202662e 100644 --- a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml +++ b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml @@ -1,9 +1,3 @@ -pipeline-component: - name: ${component.type} - -kubernetes-app: - namespace: example-namespace - streams-bootstrap-v3: values: kafka: @@ -11,9 +5,9 @@ streams-bootstrap-v3: schemaRegistryUrl: ${config.schema_registry.url} version: "3.0.0-SNAPSHOT" -producer-app-v3: {} # inherits from kafka-app +producer-app-v3: {} # inherits from streams-bootstrap-v3 -streams-app-v3: # inherits from kafka-app +streams-app-v3: # inherits from streams-bootstrap-v3 values: kafka: config: @@ -26,16 +20,3 @@ streams-app-v3: # inherits from kafka-app partitions_count: 1 configs: cleanup.policy: compact,delete - -kafka-sink-connector: - config: - batch.size: "2000" - behavior.on.malformed.documents: "warn" - behavior.on.null.values: "delete" - connection.compression: "true" - connector.class: "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector" - key.ignore: "false" - linger.ms: "5000" - max.buffered.records: "20000" - read.timeout.ms: "120000" - tasks.max: "1" diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 184534ad3..03764eeec 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -874,6 +874,7 @@ def test_streams_bootstrap_v3(self, snapshot: Snapshot): [ "generate", str(RESOURCE_PATH / "streams-bootstrap-v3" / PIPELINE_YAML), + "--verbose", ], catch_exceptions=False, ) From 831ac5deef65679289936df49d59f8c9be5c284a Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Fri, 2 Aug 2024 10:53:57 +0200 Subject: [PATCH 09/23] Update files --- tests/pipeline/test_generate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 03764eeec..184534ad3 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -874,7 +874,6 @@ def test_streams_bootstrap_v3(self, snapshot: Snapshot): [ "generate", str(RESOURCE_PATH / "streams-bootstrap-v3" / PIPELINE_YAML), - "--verbose", ], catch_exceptions=False, ) From 83e63e2f0a96bcf165731e11f679a395bb1028fa Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 5 Aug 2024 10:26:35 +0200 Subject: [PATCH 10/23] Address reviews and add version validator --- docs/docs/schema/defaults.json | 22 ++++++++++++-- docs/docs/schema/pipeline.json | 22 ++++++++++++-- kpops/components/streams_bootstrap_v3/base.py | 30 +++++++++++++++++-- .../streams_bootstrap_v3/streams/model.py | 15 ++++++++-- 4 files changed, 81 insertions(+), 8 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index cbfabb521..3ee266701 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1900,6 +1900,15 @@ "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", "properties": { + "additionalTriggers": { + "default": [], + "description": "List of additional KEDA triggers, see https://keda.sh/docs/latest/scalers/", + "items": { + "type": "string" + }, + "title": "Additionaltriggers", + "type": "array" + }, "cooldownPeriod": { "default": 300, "description": "The period to wait after the last trigger reported active before scaling the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod", @@ -1908,7 +1917,7 @@ }, "enabled": { "default": false, - "description": "", + "description": "Whether to enable auto-scaling using KEDA.", "title": "Enabled", "type": "boolean" }, @@ -1925,6 +1934,15 @@ "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", "title": "Idle replica count" }, + "internalTopics": { + "default": [], + "description": "List of auto-generated Kafka Streams topics used by the streams app", + "items": { + "type": "string" + }, + "title": "Internaltopics", + "type": "array" + }, "lagThreshold": { "anyOf": [ { @@ -1964,7 +1982,7 @@ }, "topics": { "default": [], - "description": "List of auto-generated Kafka Streams topics used by the streams app.", + "description": "List of topics used by the streams app", "items": { "type": "string" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 25b89d6c6..e20214713 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1538,6 +1538,15 @@ "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", "properties": { + "additionalTriggers": { + "default": [], + "description": "List of additional KEDA triggers, see https://keda.sh/docs/latest/scalers/", + "items": { + "type": "string" + }, + "title": "Additionaltriggers", + "type": "array" + }, "cooldownPeriod": { "default": 300, "description": "The period to wait after the last trigger reported active before scaling the resource back to 0. https://keda.sh/docs/2.9/concepts/scaling-deployments/#cooldownperiod", @@ -1546,7 +1555,7 @@ }, "enabled": { "default": false, - "description": "", + "description": "Whether to enable auto-scaling using KEDA.", "title": "Enabled", "type": "boolean" }, @@ -1563,6 +1572,15 @@ "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", "title": "Idle replica count" }, + "internalTopics": { + "default": [], + "description": "List of auto-generated Kafka Streams topics used by the streams app", + "items": { + "type": "string" + }, + "title": "Internaltopics", + "type": "array" + }, "lagThreshold": { "anyOf": [ { @@ -1602,7 +1620,7 @@ }, "topics": { "default": [], - "description": "List of auto-generated Kafka Streams topics used by the streams app.", + "description": "List of topics used by the streams app", "items": { "type": "string" }, diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index 10504605c..e6d714798 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import re from abc import ABC from typing import TYPE_CHECKING, Any @@ -31,6 +32,8 @@ url="https://bakdata.github.io/streams-bootstrap/", ) STREAMS_BOOTSTRAP_VERSION = "3.0.0" +STREAMS_BOOTSTRAP_VERSION_PATTERN = r"^(\d+)\.(\d+)\.(\d+)(-[a-zA-Z]+(\.[a-zA-Z]+)?)?$" +COMPILED_VERSION_PATTERN = re.compile(STREAMS_BOOTSTRAP_VERSION_PATTERN) # Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" @@ -42,6 +45,7 @@ class StreamsBootstrapV3Values(HelmAppValues): """Base value class for all streams bootstrap related components. :param image_tag: Docker image tag of the streams-bootstrap app. + :param kafka: Kafka configuration for the streams-bootstrap app. """ image_tag: str = Field( @@ -50,6 +54,11 @@ class StreamsBootstrapV3Values(HelmAppValues): description=describe_attr("image_tag", __doc__), ) + kafka: KafkaConfig = Field( + default=..., + description=describe_attr("kafka", __doc__), + ) + class StreamsBootstrapV3(KafkaApp, HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. @@ -70,12 +79,29 @@ class StreamsBootstrapV3(KafkaApp, HelmApp, ABC): description=describe_attr("repo_config", __doc__), ) - # TODO: validate that version is higher than 3.x.x - version: str | None = Field( + version: str = Field( default=STREAMS_BOOTSTRAP_VERSION, + pattern=STREAMS_BOOTSTRAP_VERSION_PATTERN, description=describe_attr("version", __doc__), ) + @pydantic.model_validator(mode="after") + def version_validator(self) -> Self: + pattern_match = COMPILED_VERSION_PATTERN.match(self.version) + + if not pattern_match: + msg = f"Invalid version format: {self.version}" + raise ValueError(msg) + + major, minor, patch, suffix, _ = pattern_match.groups() + major = int(major) + + if major < 3: + msg = f"The streams bootstrap version '{self.version}' must be at least 3.0.0." + raise ValueError(msg) + + return self + @pydantic.model_validator(mode="after") def warning_for_latest_image_tag(self) -> Self: if self.validate_ and self.values.image_tag == "latest": diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index 343b49d49..3524b10c4 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -142,13 +142,16 @@ class StreamsAppAutoScaling(CamelCaseConfigModel, DescConfigModel): down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount, defaults to None - :param topics: List of auto-generated Kafka Streams topics used by the streams app., + :param internal_topics: List of auto-generated Kafka Streams topics used by the streams app, defaults to [] + :param topics: List of topics used by the streams app, defaults to [] + :param additional_triggers: List of additional KEDA triggers, + see https://keda.sh/docs/latest/scalers/, defaults to [] """ enabled: bool = Field( default=False, - description=describe_attr("streams", __doc__), + description=describe_attr("enabled", __doc__), ) lag_threshold: int | None = Field( default=None, @@ -185,10 +188,18 @@ class StreamsAppAutoScaling(CamelCaseConfigModel, DescConfigModel): title="Idle replica count", description=describe_attr("idle_replicas", __doc__), ) + internal_topics: list[str] = Field( + default=[], + description=describe_attr("internal_topics", __doc__), + ) topics: list[str] = Field( default=[], description=describe_attr("topics", __doc__), ) + additional_triggers: list[str] = Field( + default=[], + description=describe_attr("additional_triggers", __doc__), + ) model_config = ConfigDict(extra="allow") @model_validator(mode="after") From e64b8ebeca04291b9f37e832083504434afb1f63 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 6 Aug 2024 11:31:17 +0200 Subject: [PATCH 11/23] add tests --- kpops/components/streams_bootstrap_v3/base.py | 98 +- .../components/streams_bootstrap_v3/model.py | 98 ++ .../streams_bootstrap_v3/producer/model.py | 2 +- .../streams_bootstrap_v3/streams/model.py | 33 +- .../streams_bootstrap_v3/__init__.py | 0 .../streams_bootstrap_v3/test_streams_app.py | 1019 +++++++++++++++++ .../test_streams_bootstrap.py | 124 ++ 7 files changed, 1267 insertions(+), 107 deletions(-) create mode 100644 kpops/components/streams_bootstrap_v3/model.py create mode 100644 tests/components/streams_bootstrap_v3/__init__.py create mode 100644 tests/components/streams_bootstrap_v3/test_streams_app.py create mode 100644 tests/components/streams_bootstrap_v3/test_streams_bootstrap.py diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index e6d714798..7e7f70fe5 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -3,22 +3,16 @@ import logging import re from abc import ABC -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import pydantic -from pydantic import AliasChoices, ConfigDict, Field +from pydantic import Field from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.components.base_components import KafkaApp -from kpops.components.base_components.helm_app import HelmApp, HelmAppValues -from kpops.components.common.topic import KafkaTopic, KafkaTopicStr +from kpops.components.base_components.helm_app import HelmApp +from kpops.components.streams_bootstrap_v3.model import StreamsBootstrapV3Values from kpops.utils.docstring import describe_attr -from kpops.utils.pydantic import ( - CamelCaseConfigModel, - DescConfigModel, - exclude_by_value, - exclude_defaults, -) if TYPE_CHECKING: try: @@ -35,31 +29,9 @@ STREAMS_BOOTSTRAP_VERSION_PATTERN = r"^(\d+)\.(\d+)\.(\d+)(-[a-zA-Z]+(\.[a-zA-Z]+)?)?$" COMPILED_VERSION_PATTERN = re.compile(STREAMS_BOOTSTRAP_VERSION_PATTERN) -# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names -IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" - log = logging.getLogger("StreamsBootstrapV3") -class StreamsBootstrapV3Values(HelmAppValues): - """Base value class for all streams bootstrap related components. - - :param image_tag: Docker image tag of the streams-bootstrap app. - :param kafka: Kafka configuration for the streams-bootstrap app. - """ - - image_tag: str = Field( - default="latest", - pattern=IMAGE_TAG_PATTERN, - description=describe_attr("image_tag", __doc__), - ) - - kafka: KafkaConfig = Field( - default=..., - description=describe_attr("kafka", __doc__), - ) - - class StreamsBootstrapV3(KafkaApp, HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. @@ -97,7 +69,7 @@ def version_validator(self) -> Self: major = int(major) if major < 3: - msg = f"The streams bootstrap version '{self.version}' must be at least 3.0.0." + msg = f"When using the streams bootstrap v3 component your version ('{self.version}') must be at least 3.0.0." raise ValueError(msg) return self @@ -109,63 +81,3 @@ def warning_for_latest_image_tag(self) -> Self: f"The image tag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable image tag." ) return self - - -class KafkaConfig(CamelCaseConfigModel, DescConfigModel): - """Kafka Streams config. - - :param bootstrap_servers: Brokers - :param schema_registry_url: URL of the schema registry, defaults to None - :param labeled_output_topics: Extra output topics - :param output_topic: Output topic, defaults to None - """ - - bootstrap_servers: str = Field( - default=..., description=describe_attr("bootstrap_servers", __doc__) - ) - schema_registry_url: str | None = Field( - default=None, - validation_alias=AliasChoices( - "schema_registry_url", "schemaRegistryUrl" - ), # TODO: same for other camelcase fields, avoids duplicates during enrichment - description=describe_attr("schema_registry_url", __doc__), - ) - labeled_output_topics: dict[str, KafkaTopicStr] = Field( - default={}, description=describe_attr("labeled_output_topics", __doc__) - ) - output_topic: KafkaTopicStr | None = Field( - default=None, - description=describe_attr("output_topic", __doc__), - json_schema_extra={}, - ) - - model_config = ConfigDict(extra="allow") - - @pydantic.field_validator("labeled_output_topics", mode="before") - @classmethod - def deserialize_labeled_output_topics( - cls, labeled_output_topics: dict[str, str] | Any - ) -> dict[str, KafkaTopic] | Any: - if isinstance(labeled_output_topics, dict): - return { - label: KafkaTopic(name=topic_name) - for label, topic_name in labeled_output_topics.items() - } - return labeled_output_topics - - @pydantic.field_serializer("labeled_output_topics") - def serialize_labeled_output_topics( - self, labeled_output_topics: dict[str, KafkaTopic] - ) -> dict[str, str]: - return {label: topic.name for label, topic in labeled_output_topics.items()} - - # TODO(Ivan Yordanov): Currently hacky and potentially unsafe. Find cleaner solution - @pydantic.model_serializer(mode="wrap", when_used="always") - def serialize_model( - self, - default_serialize_handler: pydantic.SerializerFunctionWrapHandler, - info: pydantic.SerializationInfo, - ) -> dict[str, Any]: - return exclude_defaults( - self, exclude_by_value(default_serialize_handler(self), None) - ) diff --git a/kpops/components/streams_bootstrap_v3/model.py b/kpops/components/streams_bootstrap_v3/model.py new file mode 100644 index 000000000..10911a2e0 --- /dev/null +++ b/kpops/components/streams_bootstrap_v3/model.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from typing import Any + +import pydantic +from pydantic import AliasChoices, ConfigDict, Field + +from kpops.components.base_components.helm_app import HelmAppValues +from kpops.components.common.topic import KafkaTopic, KafkaTopicStr +from kpops.utils.docstring import describe_attr +from kpops.utils.pydantic import ( + CamelCaseConfigModel, + DescConfigModel, + exclude_by_value, + exclude_defaults, +) + +# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names +IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" + + +class StreamsBootstrapV3Values(HelmAppValues): + """Base value class for all streams bootstrap related components. + + :param image_tag: Docker image tag of the streams-bootstrap app. + :param kafka: Kafka configuration for the streams-bootstrap app. + """ + + image_tag: str = Field( + default="latest", + pattern=IMAGE_TAG_PATTERN, + description=describe_attr("image_tag", __doc__), + ) + + kafka: KafkaConfig = Field( + default=..., + description=describe_attr("kafka", __doc__), + ) + + +class KafkaConfig(CamelCaseConfigModel, DescConfigModel): + """Kafka Streams config. + + :param bootstrap_servers: Brokers + :param schema_registry_url: URL of the schema registry, defaults to None + :param labeled_output_topics: Extra output topics + :param output_topic: Output topic, defaults to None + """ + + bootstrap_servers: str = Field( + default=..., description=describe_attr("bootstrap_servers", __doc__) + ) + schema_registry_url: str | None = Field( + default=None, + validation_alias=AliasChoices( + "schema_registry_url", "schemaRegistryUrl" + ), # TODO: same for other camelcase fields, avoids duplicates during enrichment + description=describe_attr("schema_registry_url", __doc__), + ) + labeled_output_topics: dict[str, KafkaTopicStr] = Field( + default={}, description=describe_attr("labeled_output_topics", __doc__) + ) + output_topic: KafkaTopicStr | None = Field( + default=None, + description=describe_attr("output_topic", __doc__), + json_schema_extra={}, + ) + + model_config = ConfigDict(extra="allow") + + @pydantic.field_validator("labeled_output_topics", mode="before") + @classmethod + def deserialize_labeled_output_topics( + cls, labeled_output_topics: dict[str, str] | Any + ) -> dict[str, KafkaTopic] | Any: + if isinstance(labeled_output_topics, dict): + return { + label: KafkaTopic(name=topic_name) + for label, topic_name in labeled_output_topics.items() + } + return labeled_output_topics + + @pydantic.field_serializer("labeled_output_topics") + def serialize_labeled_output_topics( + self, labeled_output_topics: dict[str, KafkaTopic] + ) -> dict[str, str]: + return {label: topic.name for label, topic in labeled_output_topics.items()} + + # TODO(Ivan Yordanov): Currently hacky and potentially unsafe. Find cleaner solution + @pydantic.model_serializer(mode="wrap", when_used="always") + def serialize_model( + self, + default_serialize_handler: pydantic.SerializerFunctionWrapHandler, + info: pydantic.SerializationInfo, + ) -> dict[str, Any]: + return exclude_defaults( + self, exclude_by_value(default_serialize_handler(self), None) + ) diff --git a/kpops/components/streams_bootstrap_v3/producer/model.py b/kpops/components/streams_bootstrap_v3/producer/model.py index 919843afb..d4111af1a 100644 --- a/kpops/components/streams_bootstrap_v3/producer/model.py +++ b/kpops/components/streams_bootstrap_v3/producer/model.py @@ -1,6 +1,6 @@ from pydantic import ConfigDict, Field -from kpops.components.streams_bootstrap_v3.base import ( +from kpops.components.streams_bootstrap_v3.model import ( KafkaConfig, StreamsBootstrapV3Values, ) diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index 3524b10c4..a20a68d93 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -7,7 +7,7 @@ from kpops.api.exception import ValidationError from kpops.components.common.topic import KafkaTopic, KafkaTopicStr -from kpops.components.streams_bootstrap_v3.base import ( +from kpops.components.streams_bootstrap_v3.model import ( KafkaConfig, StreamsBootstrapV3Values, ) @@ -202,18 +202,6 @@ class StreamsAppAutoScaling(CamelCaseConfigModel, DescConfigModel): ) model_config = ConfigDict(extra="allow") - @model_validator(mode="after") - def validate_mandatory_fields_are_set( - self: StreamsAppAutoScaling, - ) -> StreamsAppAutoScaling: # TODO: typing.Self for Python 3.11+ - if self.enabled and self.lag_threshold is None: - msg = ( - "If app.autoscaling.enabled is set to true, " - "the fields app.autoscaling.consumer_group and app.autoscaling.lag_threshold should be set." - ) - raise ValidationError(msg) - return self - class PersistenceConfig(BaseModel): """streams-bootstrap persistence configurations. @@ -275,3 +263,22 @@ class StreamsAppValues(StreamsBootstrapV3Values): description=describe_attr("persistence", __doc__), ) model_config = ConfigDict(extra="allow") + + @model_validator(mode="after") + def validate_mandatory_fields_are_set( + self: StreamsAppValues, + ) -> StreamsAppValues: # TODO: typing.Self for Python 3.11+ + if ( + self.autoscaling + and self.autoscaling.enabled + and ( + self.kafka.application_id is None + or self.autoscaling.lag_threshold is None + ) + ): + msg = ( + "If values.autoscaling.enabled is set to true, " + "the fields values.kafka.application_id and values.autoscaling.lag_threshold should be set." + ) + raise ValidationError(msg) + return self diff --git a/tests/components/streams_bootstrap_v3/__init__.py b/tests/components/streams_bootstrap_v3/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/components/streams_bootstrap_v3/test_streams_app.py b/tests/components/streams_bootstrap_v3/test_streams_app.py new file mode 100644 index 000000000..bd0ea0c07 --- /dev/null +++ b/tests/components/streams_bootstrap_v3/test_streams_app.py @@ -0,0 +1,1019 @@ +import logging +from pathlib import Path +from unittest.mock import ANY, AsyncMock, MagicMock + +import pytest +from pytest_mock import MockerFixture + +from kpops.api.exception import ValidationError +from kpops.component_handlers import get_handlers +from kpops.component_handlers.helm_wrapper.helm import Helm +from kpops.component_handlers.helm_wrapper.model import ( + HelmUpgradeInstallFlags, +) +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name +from kpops.components.base_components.models import TopicName +from kpops.components.base_components.models.to_section import ( + ToSection, +) +from kpops.components.common.topic import ( + KafkaTopic, + OutputTopicTypes, + TopicConfig, +) +from kpops.components.streams_bootstrap_v3 import StreamsAppV3 +from kpops.components.streams_bootstrap_v3.streams.model import ( + PersistenceConfig, + StreamsAppAutoScaling, +) +from kpops.components.streams_bootstrap_v3.streams.streams_app import ( + StreamsAppCleaner, +) + +RESOURCES_PATH = Path(__file__).parent / "resources" + +STREAMS_APP_NAME = "test-streams-app-with-long-name-0123456789abcdefghijklmnop" +STREAMS_APP_FULL_NAME = "${pipeline.name}-" + STREAMS_APP_NAME +STREAMS_APP_HELM_NAME_OVERRIDE = ( + "${pipeline.name}-" + "test-streams-app-with-long-name-01234567-a35c6" +) +STREAMS_APP_RELEASE_NAME = create_helm_release_name(STREAMS_APP_FULL_NAME) +STREAMS_APP_CLEAN_FULL_NAME = STREAMS_APP_FULL_NAME + "-clean" +STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE = ( + "${pipeline.name}-" + "test-streams-app-with-long-name-01-c98c5-clean" +) +STREAMS_APP_CLEAN_RELEASE_NAME = create_helm_release_name( + STREAMS_APP_CLEAN_FULL_NAME, "-clean" +) + +log = logging.getLogger("TestStreamsApp") + + +@pytest.mark.usefixtures("mock_env") +class TestStreamsApp: + def test_release_name(self): + assert STREAMS_APP_CLEAN_RELEASE_NAME.endswith("-clean") + + @pytest.fixture() + def streams_app(self) -> StreamsAppV3: + return StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "streams-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + } + }, + }, + ) + + @pytest.fixture() + def stateful_streams_app(self) -> StreamsAppV3: + return StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "statefulSet": True, + "persistence": {"enabled": True, "size": "5Gi"}, + "kafka": { + "bootstrapServers": "fake-broker:9092", + }, + }, + "to": { + "topics": { + "streams-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + } + }, + }, + ) + + @pytest.fixture() + def dry_run_handler_mock(self, mocker: MockerFixture) -> MagicMock: + return mocker.patch( + "kpops.components.base_components.helm_app.DryRunHandler" + ).return_value + + @pytest.fixture(autouse=True) + def empty_helm_get_values(self, mocker: MockerFixture) -> MagicMock: + return mocker.patch.object(Helm, "get_values", return_value=None) + + def test_cleaner(self, streams_app: StreamsAppV3): + cleaner = streams_app._cleaner + assert isinstance(cleaner, StreamsAppCleaner) + assert not hasattr(cleaner, "_cleaner") + + def test_cleaner_inheritance(self, streams_app: StreamsAppV3): + streams_app.values.kafka.application_id = "test-application-id" + streams_app.values.autoscaling = StreamsAppAutoScaling( + enabled=True, + lag_threshold=100, + idle_replicas=1, + ) + assert streams_app._cleaner.values == streams_app.values + + def test_raise_validation_error_when_autoscaling_enabled_and_mandatory_fields_not_set( + self, streams_app: StreamsAppV3 + ): + with pytest.raises(ValidationError) as error: + StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + "autoscaling": {"enabled": True}, + }, + "to": { + "topics": { + "streams-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + } + }, + }, + ) + msg = ( + "If values.autoscaling.enabled is set to true, " + "the fields values.kafka.application_id and values.autoscaling.lag_threshold should be set." + ) + assert str(error.value) == msg + + def test_raise_validation_error_when_autoscaling_enabled_and_only_consumer_group_set( + self, streams_app: StreamsAppV3 + ): + with pytest.raises(ValidationError) as error: + StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": { + "bootstrapServers": "fake-broker:9092", + "applicationId": "test-application-id", + }, + "autoscaling": {"enabled": True}, + }, + "to": { + "topics": { + "streams-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + } + }, + }, + ) + msg = ( + "If values.autoscaling.enabled is set to true, " + "the fields values.kafka.application_id and values.autoscaling.lag_threshold should be set." + ) + assert str(error.value) == msg + + def test_raise_validation_error_when_autoscaling_enabled_and_only_lag_threshold_is_set( + self, + ): + with pytest.raises(ValidationError) as error: + StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + "autoscaling": {"enabled": True, "lagThreshold": 100}, + }, + "to": { + "topics": { + "streams-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + } + }, + }, + ) + msg = ( + "If values.autoscaling.enabled is set to true, " + "the fields values.kafka.application_id and values.autoscaling.lag_threshold should be set." + ) + assert str(error.value) == msg + + def test_cleaner_helm_release_name(self, streams_app: StreamsAppV3): + assert ( + streams_app._cleaner.helm_release_name + == "${pipeline.name}-test-streams-app-with-lo-c98c5-clean" + ) + + def test_cleaner_helm_name_override(self, streams_app: StreamsAppV3): + assert ( + streams_app._cleaner.to_helm_values()["nameOverride"] + == STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE + ) + + def test_set_topics(self): + streams_app = StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "from": { + "topics": { + "example-input": {"type": "input"}, + "b": {"type": "input"}, + "a": {"type": "input"}, + "topic-extra2": {"role": "role2"}, + "topic-extra3": {"role": "role2"}, + "topic-extra": {"role": "role1"}, + ".*": {"type": "pattern"}, + "example.*": { + "type": "pattern", + "role": "another-pattern", + }, + } + }, + }, + ) + assert streams_app.values.kafka.input_topics == [ + KafkaTopic(name="example-input"), + KafkaTopic(name="b"), + KafkaTopic(name="a"), + ] + assert streams_app.values.kafka.labeled_input_topics == { + "role1": [KafkaTopic(name="topic-extra")], + "role2": [KafkaTopic(name="topic-extra2"), KafkaTopic(name="topic-extra3")], + } + assert streams_app.values.kafka.input_pattern == ".*" + assert streams_app.values.kafka.labeled_input_patterns == { + "another-pattern": "example.*" + } + + helm_values = streams_app.to_helm_values() + kafka_config = helm_values["kafka"] + assert kafka_config["inputTopics"] + assert "labeledInputTopics" in kafka_config + assert "inputPattern" in kafka_config + assert "labeledInputPatterns" in kafka_config + + def test_no_empty_input_topic(self): + streams_app = StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "from": { + "topics": { + ".*": {"type": "pattern"}, + } + }, + }, + ) + assert not streams_app.values.kafka.labeled_input_topics + assert not streams_app.values.kafka.input_topics + assert streams_app.values.kafka.input_pattern == ".*" + assert not streams_app.values.kafka.labeled_input_patterns + + helm_values = streams_app.to_helm_values() + streams_config = helm_values["kafka"] + assert "inputTopics" not in streams_config + assert "extraInputTopics" not in streams_config + assert "inputPattern" in streams_config + assert "extraInputPatterns" not in streams_config + + def test_should_validate(self): + # An exception should be raised when both role and type are defined and type is input + with pytest.raises( + ValueError, match="Define role only if `type` is `pattern` or `None`" + ): + StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "from": { + "topics": { + "topic-input": { + "type": "input", + "role": "role", + } + } + }, + }, + ) + + # An exception should be raised when both role and type are defined and type is error + with pytest.raises( + ValueError, match="Define `role` only if `type` is undefined" + ): + StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "topic-input": { + "type": "error", + "role": "role", + } + } + }, + }, + ) + + def test_set_streams_output_from_to(self): + streams_app = StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "streams-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + "streams-app-error-topic": TopicConfig( + type=OutputTopicTypes.ERROR, partitions_count=10 + ), + "extra-topic-1": TopicConfig( + role="first-extra-role", + partitions_count=10, + ), + "extra-topic-2": TopicConfig( + role="second-extra-role", + partitions_count=10, + ), + } + }, + }, + ) + assert streams_app.values.kafka.labeled_output_topics == { + "first-extra-role": KafkaTopic(name="extra-topic-1"), + "second-extra-role": KafkaTopic(name="extra-topic-2"), + } + assert streams_app.values.kafka.output_topic == KafkaTopic( + name="streams-app-output-topic" + ) + assert streams_app.values.kafka.error_topic == KafkaTopic( + name="streams-app-error-topic" + ) + + def test_weave_inputs_from_prev_component(self): + streams_app = StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + }, + ) + + streams_app.weave_from_topics( + ToSection( + topics={ + TopicName("prev-output-topic"): TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + TopicName("b"): TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + TopicName("a"): TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + TopicName("prev-error-topic"): TopicConfig( + type=OutputTopicTypes.ERROR, partitions_count=10 + ), + } + ) + ) + + assert streams_app.values.kafka.input_topics == [ + KafkaTopic(name="prev-output-topic"), + KafkaTopic(name="b"), + KafkaTopic(name="a"), + ] + + @pytest.mark.asyncio() + async def test_deploy_order_when_dry_run_is_false(self, mocker: MockerFixture): + streams_app = StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "streams-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + "streams-app-error-topic": TopicConfig( + type=OutputTopicTypes.ERROR, partitions_count=10 + ), + "extra-topic-1": TopicConfig( + role="first-extra-topic", + partitions_count=10, + ), + "extra-topic-2": TopicConfig( + role="second-extra-topic", + partitions_count=10, + ), + } + }, + }, + ) + mock_create_topic = mocker.patch.object( + get_handlers().topic_handler, "create_topic" + ) + mock_helm_upgrade_install = mocker.patch.object( + streams_app.helm, "upgrade_install" + ) + + mock = mocker.AsyncMock() + mock.attach_mock(mock_create_topic, "mock_create_topic") + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + + dry_run = False + await streams_app.deploy(dry_run=dry_run) + + assert streams_app.to + assert streams_app.to.kafka_topics == [ + KafkaTopic( + name="streams-app-output-topic", + config=TopicConfig( + type=OutputTopicTypes.OUTPUT, + partitions_count=10, + ), + ), + KafkaTopic( + name="streams-app-error-topic", + config=TopicConfig( + type=OutputTopicTypes.ERROR, + partitions_count=10, + ), + ), + KafkaTopic( + name="extra-topic-1", + config=TopicConfig( + partitions_count=10, + role="first-extra-topic", + ), + ), + KafkaTopic( + name="extra-topic-2", + config=TopicConfig( + partitions_count=10, + role="second-extra-topic", + ), + ), + ] + assert mock.mock_calls == [ + *( + mocker.call.mock_create_topic(topic, dry_run=dry_run) + for topic in streams_app.to.kafka_topics + ), + mocker.call.helm_upgrade_install( + STREAMS_APP_RELEASE_NAME, + "bakdata-streams-bootstrap/streams-app", + dry_run, + "test-namespace", + { + "nameOverride": STREAMS_APP_HELM_NAME_OVERRIDE, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "labeledOutputTopics": { + "first-extra-topic": "extra-topic-1", + "second-extra-topic": "extra-topic-2", + }, + "outputTopic": "streams-app-output-topic", + "errorTopic": "streams-app-error-topic", + }, + }, + HelmUpgradeInstallFlags( + create_namespace=False, + force=False, + username=None, + password=None, + ca_file=None, + insecure_skip_tls_verify=False, + timeout="5m0s", + version="3.0.0", + wait=True, + wait_for_jobs=False, + ), + ), + ] + + @pytest.mark.asyncio() + async def test_destroy( + self, + streams_app: StreamsAppV3, + mocker: MockerFixture, + ): + mock_helm_uninstall = mocker.patch.object(streams_app.helm, "uninstall") + + await streams_app.destroy(dry_run=True) + + mock_helm_uninstall.assert_called_once_with( + "test-namespace", STREAMS_APP_RELEASE_NAME, True + ) + + @pytest.mark.asyncio() + async def test_reset_when_dry_run_is_false( + self, + streams_app: StreamsAppV3, + empty_helm_get_values: MockerFixture, + mocker: MockerFixture, + ): + # actual component + mock_helm_uninstall_streams_app = mocker.patch.object( + streams_app.helm, "uninstall" + ) + + cleaner = streams_app._cleaner + assert isinstance(cleaner, StreamsAppCleaner) + + mock_helm_upgrade_install = mocker.patch.object(cleaner.helm, "upgrade_install") + mock_helm_uninstall = mocker.patch.object(cleaner.helm, "uninstall") + + mock = mocker.MagicMock() + mock.attach_mock( + mock_helm_uninstall_streams_app, "mock_helm_uninstall_streams_app" + ) + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + mock.attach_mock(mock_helm_uninstall, "helm_uninstall") + + dry_run = False + await streams_app.reset(dry_run=dry_run) + + mock.assert_has_calls( + [ + mocker.call.mock_helm_uninstall_streams_app( + "test-namespace", STREAMS_APP_RELEASE_NAME, dry_run + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_uninstall( + "test-namespace", + STREAMS_APP_CLEAN_RELEASE_NAME, + dry_run, + ), + ANY, # __bool__ # FIXME: why is this in the call stack? + ANY, # __str__ + mocker.call.helm_upgrade_install( + STREAMS_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/streams-app-cleanup-job", + dry_run, + "test-namespace", + { + "nameOverride": STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "streams-app-output-topic", + "deleteOutput": False, + }, + }, + HelmUpgradeInstallFlags( + version="3.0.0", wait=True, wait_for_jobs=True + ), + ), + mocker.call.helm_uninstall( + "test-namespace", + STREAMS_APP_CLEAN_RELEASE_NAME, + dry_run, + ), + ] + ) + + @pytest.mark.asyncio() + async def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean_up( + self, + streams_app: StreamsAppV3, + empty_helm_get_values: MockerFixture, + mocker: MockerFixture, + ): + # actual component + mock_helm_uninstall_streams_app = mocker.patch.object( + streams_app.helm, "uninstall" + ) + + mock_helm_upgrade_install = mocker.patch.object( + streams_app._cleaner.helm, "upgrade_install" + ) + mock_helm_uninstall = mocker.patch.object( + streams_app._cleaner.helm, "uninstall" + ) + + mock = mocker.MagicMock() + mock.attach_mock(mock_helm_uninstall_streams_app, "helm_uninstall_streams_app") + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + mock.attach_mock(mock_helm_uninstall, "helm_uninstall") + + dry_run = False + await streams_app.clean(dry_run=dry_run) + + mock.assert_has_calls( + [ + mocker.call.helm_uninstall_streams_app( + "test-namespace", STREAMS_APP_RELEASE_NAME, dry_run + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_uninstall( + "test-namespace", + STREAMS_APP_CLEAN_RELEASE_NAME, + dry_run, + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_upgrade_install( + STREAMS_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/streams-app-cleanup-job", + dry_run, + "test-namespace", + { + "nameOverride": STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "streams-app-output-topic", + "deleteOutput": True, + }, + }, + HelmUpgradeInstallFlags( + version="3.0.0", wait=True, wait_for_jobs=True + ), + ), + mocker.call.helm_uninstall( + "test-namespace", + STREAMS_APP_CLEAN_RELEASE_NAME, + dry_run, + ), + ] + ) + + @pytest.mark.asyncio() + async def test_should_deploy_clean_up_job_with_values_in_cluster_when_reset( + self, mocker: MockerFixture + ): + image_tag_in_cluster = "1.1.1" + mocker.patch.object( + Helm, + "get_values", + return_value={ + "image": "registry/streams-app", + "imageTag": image_tag_in_cluster, + "nameOverride": STREAMS_APP_NAME, + "replicaCount": 1, + "persistence": {"enabled": False, "size": "1Gi"}, + "statefulSet": False, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "inputTopics": ["test-input-topic"], + "outputTopic": "streams-app-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + ) + streams_app = StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "imageTag": "2.2.2", + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "from": { + "topics": { + "test-input-topic": {"type": "input"}, + } + }, + "to": { + "topics": { + "streams-app-output-topic": {"type": "output"}, + } + }, + }, + ) + + mocker.patch.object(streams_app.helm, "uninstall") + + mock_helm_upgrade_install = mocker.patch.object( + streams_app._cleaner.helm, "upgrade_install" + ) + mocker.patch.object(streams_app._cleaner.helm, "uninstall") + + mock = mocker.MagicMock() + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + + dry_run = False + await streams_app.reset(dry_run=dry_run) + + mock_helm_upgrade_install.assert_called_once_with( + STREAMS_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/streams-app-cleanup-job", + dry_run, + "test-namespace", + { + "image": "registry/streams-app", + "nameOverride": STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE, + "imageTag": image_tag_in_cluster, + "persistence": {"size": "1Gi"}, + "replicaCount": 1, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "inputTopics": ["test-input-topic"], + "outputTopic": "streams-app-output-topic", + "deleteOutput": False, + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + ) + + @pytest.mark.asyncio() + async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( + self, mocker: MockerFixture + ): + image_tag_in_cluster = "1.1.1" + mocker.patch.object( + Helm, + "get_values", + return_value={ + "image": "registry/streams-app", + "imageTag": image_tag_in_cluster, + "nameOverride": STREAMS_APP_NAME, + "replicaCount": 1, + "persistence": {"enabled": False, "size": "1Gi"}, + "statefulSet": False, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "inputTopics": ["test-input-topic"], + "outputTopic": "streams-app-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + ) + streams_app = StreamsAppV3( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "imageTag": "2.2.2", + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "from": { + "topics": { + "test-input-topic": {"type": "input"}, + } + }, + "to": { + "topics": { + "streams-app-output-topic": {"type": "output"}, + } + }, + }, + ) + + mocker.patch.object(streams_app.helm, "uninstall") + + mock_helm_upgrade_install = mocker.patch.object( + streams_app._cleaner.helm, "upgrade_install" + ) + mocker.patch.object(streams_app._cleaner.helm, "uninstall") + + mock = mocker.MagicMock() + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + + dry_run = False + await streams_app.clean(dry_run=dry_run) + + mock_helm_upgrade_install.assert_called_once_with( + STREAMS_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/streams-app-cleanup-job", + dry_run, + "test-namespace", + { + "image": "registry/streams-app", + "nameOverride": STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE, + "imageTag": image_tag_in_cluster, + "persistence": {"size": "1Gi"}, + "replicaCount": 1, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "inputTopics": ["test-input-topic"], + "outputTopic": "streams-app-output-topic", + "deleteOutput": True, + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + ) + + @pytest.mark.asyncio() + async def test_get_input_output_topics(self): + streams_app = StreamsAppV3( + name="my-app", + **{ + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "from": { + "topics": { + "example-input": {"type": "input"}, + "b": {"type": "input"}, + "a": {"type": "input"}, + "topic-extra2": {"role": "role2"}, + "topic-extra3": {"role": "role2"}, + "topic-extra": {"role": "role1"}, + ".*": {"type": "pattern"}, + "example.*": { + "type": "pattern", + "role": "another-pattern", + }, + } + }, + "to": { + "topics": { + "example-output": {"type": "output"}, + "extra-topic": {"role": "fake-role"}, + } + }, + }, + ) + + assert streams_app.values.kafka.input_topics == [ + KafkaTopic(name="example-input"), + KafkaTopic(name="b"), + KafkaTopic(name="a"), + ] + assert streams_app.values.kafka.labeled_input_topics == { + "role1": [KafkaTopic(name="topic-extra")], + "role2": [KafkaTopic(name="topic-extra2"), KafkaTopic(name="topic-extra3")], + } + assert streams_app.output_topic == KafkaTopic(name="example-output") + assert streams_app.extra_output_topics == { + "fake-role": KafkaTopic(name="extra-topic") + } + assert list(streams_app.outputs) == [ + KafkaTopic(name="example-output"), + KafkaTopic(name="extra-topic"), + ] + assert list(streams_app.inputs) == [ + KafkaTopic(name="example-input"), + KafkaTopic(name="b"), + KafkaTopic(name="a"), + KafkaTopic(name="topic-extra2"), + KafkaTopic(name="topic-extra3"), + KafkaTopic(name="topic-extra"), + ] + + def test_raise_validation_error_when_persistence_enabled_and_size_not_set( + self, stateful_streams_app: StreamsAppV3 + ): + with pytest.raises(ValidationError) as error: + stateful_streams_app.values.persistence = PersistenceConfig( + enabled=True, + ) + msg = ( + "If app.persistence.enabled is set to true, " + "the field app.persistence.size needs to be set." + ) + assert str(error.value) == msg + + @pytest.mark.asyncio() + async def test_stateful_clean_with_dry_run_false( + self, + stateful_streams_app: StreamsAppV3, + empty_helm_get_values: MockerFixture, + mocker: MockerFixture, + ): + # actual component + mock_helm_uninstall_streams_app = mocker.patch.object( + stateful_streams_app.helm, "uninstall" + ) + cleaner = stateful_streams_app._cleaner + assert isinstance(cleaner, StreamsAppCleaner) + + mock_helm_upgrade_install = mocker.patch.object(cleaner.helm, "upgrade_install") + mock_helm_uninstall = mocker.patch.object(cleaner.helm, "uninstall") + + module = StreamsAppCleaner.__module__ + mock_pvc_handler_instance = AsyncMock() + mock_delete_pvcs = mock_pvc_handler_instance.delete_pvcs + mock_delete_pvcs.return_value = AsyncMock() + + mocker.patch( + f"{module}.PVCHandler.create", return_value=mock_pvc_handler_instance + ) + + mock = MagicMock() + mock.attach_mock(mock_helm_uninstall_streams_app, "helm_uninstall_streams_app") + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + mock.attach_mock(mock_helm_uninstall, "helm_uninstall") + mock.attach_mock(mock_delete_pvcs, "delete_pvcs") + + dry_run = False + await stateful_streams_app.clean(dry_run=dry_run) + + mock.assert_has_calls( + [ + mocker.call.helm_uninstall_streams_app( + "test-namespace", STREAMS_APP_RELEASE_NAME, dry_run + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_uninstall( + "test-namespace", + STREAMS_APP_CLEAN_RELEASE_NAME, + dry_run, + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_upgrade_install( + STREAMS_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/streams-app-cleanup-job", + dry_run, + "test-namespace", + { + "nameOverride": STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE, + "statefulSet": True, + "persistence": {"enabled": True, "size": "5Gi"}, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "streams-app-output-topic", + "deleteOutput": True, + }, + }, + HelmUpgradeInstallFlags( + version="3.0.0", wait=True, wait_for_jobs=True + ), + ), + mocker.call.helm_uninstall( + "test-namespace", + STREAMS_APP_CLEAN_RELEASE_NAME, + dry_run, + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.delete_pvcs(), + ] + ) + + @pytest.mark.asyncio() + async def test_stateful_clean_with_dry_run_true( + self, + stateful_streams_app: StreamsAppV3, + empty_helm_get_values: MockerFixture, + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + ): + caplog.set_level(logging.INFO) + # actual component + mocker.patch.object(stateful_streams_app, "destroy") + + cleaner = stateful_streams_app._cleaner + assert isinstance(cleaner, StreamsAppCleaner) + + pvc_names = ["test-pvc1", "test-pvc2", "test-pvc3"] + + mock_pvc_handler_instance = AsyncMock() + mock_list_pvcs = mock_pvc_handler_instance.list_pvcs + mock_list_pvcs.return_value = pvc_names + + module = StreamsAppCleaner.__module__ + pvc_handler_create = mocker.patch( + f"{module}.PVCHandler.create", return_value=mock_pvc_handler_instance + ) + mocker.patch.object(cleaner, "destroy") + mocker.patch.object(cleaner, "deploy") + mocker.patch.object(mock_list_pvcs, "list_pvcs") + + dry_run = True + await stateful_streams_app.clean(dry_run=dry_run) + + pvc_handler_create.assert_called_once_with( + STREAMS_APP_FULL_NAME, "test-namespace" + ) + + mock_list_pvcs.assert_called_once() + assert ( + f"Deleting the PVCs {pvc_names} for StatefulSet '{STREAMS_APP_FULL_NAME}'" + in caplog.text + ) diff --git a/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py b/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py new file mode 100644 index 000000000..0672e6144 --- /dev/null +++ b/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py @@ -0,0 +1,124 @@ +import re + +import pytest +from pydantic import ValidationError +from pytest_mock import MockerFixture + +from kpops.component_handlers.helm_wrapper.model import ( + HelmRepoConfig, + HelmUpgradeInstallFlags, +) +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name +from kpops.components.streams_bootstrap_v3.base import StreamsBootstrapV3 +from kpops.components.streams_bootstrap_v3.model import StreamsBootstrapV3Values + + +@pytest.mark.usefixtures("mock_env") +class TestStreamsBootstrap: + def test_default_configs(self): + streams_bootstrap = StreamsBootstrapV3( + name="example-name", + **{ + "namespace": "test-namespace", + "values": { + "kafka": { + "bootstrapServers": "localhost:9092", + } + }, + }, + ) + assert streams_bootstrap.repo_config == HelmRepoConfig( + repository_name="bakdata-streams-bootstrap", + url="https://bakdata.github.io/streams-bootstrap/", + ) + assert streams_bootstrap.version == "3.0.0" + assert streams_bootstrap.namespace == "test-namespace" + assert streams_bootstrap.values.image_tag == "latest" + + @pytest.mark.asyncio() + async def test_should_deploy_streams_bootstrap_app(self, mocker: MockerFixture): + streams_bootstrap = StreamsBootstrapV3( + name="example-name", + **{ + "namespace": "test-namespace", + "values": { + "imageTag": "1.0.0", + "kafka": { + "outputTopic": "test", + "bootstrapServers": "fake-broker:9092", + }, + }, + "version": "3.2.1", + }, + ) + helm_upgrade_install = mocker.patch.object( + streams_bootstrap.helm, "upgrade_install" + ) + print_helm_diff = mocker.patch.object( + streams_bootstrap.dry_run_handler, "print_helm_diff" + ) + mocker.patch.object( + StreamsBootstrapV3, + "helm_chart", + return_value="test/test-chart", + new_callable=mocker.PropertyMock, + ) + + await streams_bootstrap.deploy(dry_run=True) + + print_helm_diff.assert_called_once() + helm_upgrade_install.assert_called_once_with( + create_helm_release_name("${pipeline.name}-example-name"), + "test/test-chart", + True, + "test-namespace", + { + "nameOverride": "${pipeline.name}-example-name", + "imageTag": "1.0.0", + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "test", + }, + }, + HelmUpgradeInstallFlags(version="3.2.1"), + ) + + @pytest.mark.asyncio() + async def test_should_raise_validation_error_for_invalid_image_tag(self): + with pytest.raises( + ValidationError, + match=re.escape( + "1 validation error for StreamsBootstrapV3Values\nimageTag\n String should match pattern '^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$'" + ), + ): + StreamsBootstrapV3Values( + **{ + "imageTag": "invalid image tag!", + "kafka": { + "bootstrapServers": "fake-broker:9092", + }, + } + ) + + @pytest.mark.asyncio() + async def test_should_raise_validation_error_for_invalid_helm_chart_version(self): + with pytest.raises( + ValueError, + match=re.escape( + "When using the streams bootstrap v3 component your version ('2.1.0') must be at least 3.0.0." + ), + ): + StreamsBootstrapV3( + name="example-name", + **{ + "namespace": "test-namespace", + "values": { + "imageTag": "1.0.0", + "kafka": { + "outputTopic": "test", + "bootstrapServers": "fake-broker:9092", + }, + }, + "version": "2.1.0", + }, + ) From 0bc2b13633f197ea668f796de9485eff1375e4b8 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 6 Aug 2024 11:38:13 +0200 Subject: [PATCH 12/23] Update files --- .../test_manifest/test_streams_bootstrap_v3/manifest.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml index 19eb16b15..4e69e4712 100644 --- a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml @@ -25,6 +25,8 @@ spec: value: APP_ - name: APP_BOOTSTRAP_SERVERS value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ - name: APP_OUTPUT_TOPIC value: my-producer-app-output-topic - name: APP_LABELED_OUTPUT_TOPICS @@ -101,6 +103,8 @@ spec: value: 'true' - name: APP_BOOTSTRAP_SERVERS value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ - name: APP_INPUT_TOPICS value: my-input-topic - name: APP_INPUT_PATTERN From 8c28a68594c6712acedcb3786f1ce6e7795bc6d0 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 7 Aug 2024 17:19:57 +0200 Subject: [PATCH 13/23] add producer app v3 tests --- kpops/component_handlers/helm_wrapper/helm.py | 3 +- .../test_producer_app_v3.py | 474 ++++++++++++++++++ tests/components/test_streams_app.py | 10 +- 3 files changed, 477 insertions(+), 10 deletions(-) create mode 100644 tests/components/streams_bootstrap_v3/test_producer_app_v3.py diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index 0381d552e..5ff76fb81 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -21,11 +21,12 @@ Version, ) from kpops.component_handlers.kubernetes.model import KubernetesManifest -from kpops.components.base_components.models.resource import Resource if TYPE_CHECKING: from collections.abc import Iterable, Iterator + from kpops.components.base_components.models.resource import Resource + log = logging.getLogger("Helm") diff --git a/tests/components/streams_bootstrap_v3/test_producer_app_v3.py b/tests/components/streams_bootstrap_v3/test_producer_app_v3.py new file mode 100644 index 000000000..78964033d --- /dev/null +++ b/tests/components/streams_bootstrap_v3/test_producer_app_v3.py @@ -0,0 +1,474 @@ +import logging +from unittest.mock import ANY, MagicMock + +import pytest +from pytest_mock import MockerFixture + +from kpops.component_handlers import get_handlers +from kpops.component_handlers.helm_wrapper.helm import Helm +from kpops.component_handlers.helm_wrapper.model import HelmUpgradeInstallFlags +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name +from kpops.components.common.topic import ( + KafkaTopic, + OutputTopicTypes, + TopicConfig, +) +from kpops.components.streams_bootstrap_v3.producer.producer_app import ( + ProducerAppCleaner, + ProducerAppV3, +) + +PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" +PRODUCER_APP_FULL_NAME = "${pipeline.name}-" + PRODUCER_APP_NAME +PRODUCER_APP_HELM_NAME_OVERRIDE = ( + "${pipeline.name}-" + "test-producer-app-with-long-name-0123456-c4c51" +) +PRODUCER_APP_RELEASE_NAME = create_helm_release_name(PRODUCER_APP_FULL_NAME) +PRODUCER_APP_CLEAN_FULL_NAME = PRODUCER_APP_FULL_NAME + "-clean" +PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE = ( + "${pipeline.name}-" + "test-producer-app-with-long-name-0-abc43-clean" +) +PRODUCER_APP_CLEAN_RELEASE_NAME = create_helm_release_name( + PRODUCER_APP_CLEAN_FULL_NAME, "-clean" +) + + +@pytest.mark.usefixtures("mock_env") +class TestProducerApp: + def test_release_name(self): + assert PRODUCER_APP_CLEAN_RELEASE_NAME.endswith("-clean") + + @pytest.fixture() + def producer_app(self) -> ProducerAppV3: + return ProducerAppV3( + name=PRODUCER_APP_NAME, + **{ + "version": "3.2.1", + "namespace": "test-namespace", + "values": { + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "clean_schemas": True, + "to": { + "topics": { + "producer-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + } + }, + }, + ) + + @pytest.fixture(autouse=True) + def empty_helm_get_values(self, mocker: MockerFixture) -> MagicMock: + return mocker.patch.object(Helm, "get_values", return_value=None) + + def test_cleaner(self, producer_app: ProducerAppV3): + cleaner = producer_app._cleaner + assert isinstance(cleaner, ProducerAppCleaner) + assert not hasattr(cleaner, "_cleaner") + + def test_cleaner_inheritance(self, producer_app: ProducerAppV3): + assert producer_app._cleaner.values == producer_app.values + + def test_cleaner_helm_release_name(self, producer_app: ProducerAppV3): + assert ( + producer_app._cleaner.helm_release_name + == "${pipeline.name}-test-producer-app-with-l-abc43-clean" + ) + + def test_cleaner_helm_name_override(self, producer_app: ProducerAppV3): + assert ( + producer_app._cleaner.to_helm_values()["nameOverride"] + == PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE + ) + + def test_output_topics(self): + producer_app = ProducerAppV3( + name=PRODUCER_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "namespace": "test-namespace", + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "producer-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + "extra-topic-1": TopicConfig( + role="first-extra-topic", + partitions_count=10, + ), + } + }, + }, + ) + + assert producer_app.values.kafka.output_topic == KafkaTopic( + name="producer-app-output-topic" + ) + assert producer_app.values.kafka.labeled_output_topics == { + "first-extra-topic": KafkaTopic(name="extra-topic-1") + } + + @pytest.mark.asyncio() + async def test_deploy_order_when_dry_run_is_false( + self, + producer_app: ProducerAppV3, + mocker: MockerFixture, + ): + mock_create_topic = mocker.patch.object( + get_handlers().topic_handler, "create_topic" + ) + + mock_helm_upgrade_install = mocker.patch.object( + producer_app.helm, "upgrade_install" + ) + + mock = mocker.AsyncMock() + mock.attach_mock(mock_create_topic, "mock_create_topic") + mock.attach_mock(mock_helm_upgrade_install, "mock_helm_upgrade_install") + + await producer_app.deploy(dry_run=False) + assert producer_app.to + assert mock.mock_calls == [ + *( + mocker.call.mock_create_topic(topic, dry_run=False) + for topic in producer_app.to.kafka_topics + ), + mocker.call.mock_helm_upgrade_install( + PRODUCER_APP_RELEASE_NAME, + "bakdata-streams-bootstrap/producer-app", + False, + "test-namespace", + { + "nameOverride": PRODUCER_APP_HELM_NAME_OVERRIDE, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "producer-app-output-topic", + }, + }, + HelmUpgradeInstallFlags( + force=False, + username=None, + password=None, + ca_file=None, + insecure_skip_tls_verify=False, + timeout="5m0s", + version="3.2.1", + wait=True, + wait_for_jobs=False, + ), + ), + ] + + @pytest.mark.asyncio() + async def test_destroy( + self, + producer_app: ProducerAppV3, + mocker: MockerFixture, + ): + mock_helm_uninstall = mocker.patch.object(producer_app.helm, "uninstall") + + await producer_app.destroy(dry_run=True) + + mock_helm_uninstall.assert_called_once_with( + "test-namespace", PRODUCER_APP_RELEASE_NAME, True + ) + + @pytest.mark.asyncio() + async def test_should_clean_producer_app( + self, + producer_app: ProducerAppV3, + empty_helm_get_values: MockerFixture, + mocker: MockerFixture, + ): + # actual component + mock_helm_uninstall_producer_app = mocker.patch.object( + producer_app.helm, "uninstall" + ) + + # cleaner + mock_helm_upgrade_install = mocker.patch.object( + producer_app._cleaner.helm, "upgrade_install" + ) + mock_helm_uninstall = mocker.patch.object( + producer_app._cleaner.helm, "uninstall" + ) + mock_helm_print_helm_diff = mocker.patch.object( + producer_app._cleaner.dry_run_handler, "print_helm_diff" + ) + + mock = mocker.MagicMock() + mock.attach_mock( + mock_helm_uninstall_producer_app, "helm_uninstall_producer_app" + ) + mock.attach_mock(mock_helm_uninstall, "helm_uninstall") + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + mock.attach_mock(mock_helm_print_helm_diff, "print_helm_diff") + + await producer_app.clean(dry_run=True) + + mock.assert_has_calls( + [ + mocker.call.helm_uninstall_producer_app( + "test-namespace", PRODUCER_APP_RELEASE_NAME, True + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_uninstall( + "test-namespace", + PRODUCER_APP_CLEAN_RELEASE_NAME, + True, + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_upgrade_install( + PRODUCER_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/producer-app-cleanup-job", + True, + "test-namespace", + { + "nameOverride": PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "producer-app-output-topic", + }, + }, + HelmUpgradeInstallFlags( + version="3.2.1", wait=True, wait_for_jobs=True + ), + ), + mocker.call.print_helm_diff( + ANY, + PRODUCER_APP_CLEAN_RELEASE_NAME, + logging.getLogger("HelmApp"), + ), + mocker.call.helm_uninstall( + "test-namespace", + PRODUCER_APP_CLEAN_RELEASE_NAME, + True, + ), + ANY, # __bool__ + ANY, # __str__ + ] + ) + + @pytest.mark.asyncio() + async def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clean_up_with_dry_run_false( + self, + mocker: MockerFixture, + producer_app: ProducerAppV3, + empty_helm_get_values: MockerFixture, + ): + # actual component + mock_helm_uninstall_producer_app = mocker.patch.object( + producer_app.helm, "uninstall" + ) + + # cleaner + mock_helm_upgrade_install = mocker.patch.object( + producer_app._cleaner.helm, "upgrade_install" + ) + mock_helm_uninstall = mocker.patch.object( + producer_app._cleaner.helm, "uninstall" + ) + + mock = mocker.MagicMock() + mock.attach_mock( + mock_helm_uninstall_producer_app, "helm_uninstall_producer_app" + ) + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + mock.attach_mock(mock_helm_uninstall, "helm_uninstall") + + await producer_app.clean(dry_run=False) + + mock.assert_has_calls( + [ + mocker.call.helm_uninstall_producer_app( + "test-namespace", PRODUCER_APP_RELEASE_NAME, False + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_uninstall( + "test-namespace", + PRODUCER_APP_CLEAN_RELEASE_NAME, + False, + ), + ANY, # __bool__ + ANY, # __str__ + mocker.call.helm_upgrade_install( + PRODUCER_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/producer-app-cleanup-job", + False, + "test-namespace", + { + "nameOverride": PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "producer-app-output-topic", + }, + }, + HelmUpgradeInstallFlags( + version="3.2.1", wait=True, wait_for_jobs=True + ), + ), + mocker.call.helm_uninstall( + "test-namespace", + PRODUCER_APP_CLEAN_RELEASE_NAME, + False, + ), + ANY, # __bool__ + ANY, # __str__ + ] + ) + + def test_get_output_topics(self): + producer_app = ProducerAppV3( + name="my-producer", + **{ + "namespace": "test-namespace", + "values": { + "namespace": "test-namespace", + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "producer-app-output-topic": TopicConfig( + type=OutputTopicTypes.OUTPUT, partitions_count=10 + ), + "extra-topic-1": TopicConfig( + role="first-extra-topic", + partitions_count=10, + ), + } + }, + }, + ) + assert producer_app.values.kafka.output_topic == KafkaTopic( + name="producer-app-output-topic" + ) + assert producer_app.values.kafka.labeled_output_topics == { + "first-extra-topic": KafkaTopic(name="extra-topic-1") + } + assert producer_app.input_topics == [] + assert list(producer_app.inputs) == [] + assert list(producer_app.outputs) == [ + KafkaTopic(name="producer-app-output-topic"), + KafkaTopic(name="extra-topic-1"), + ] + + @pytest.mark.asyncio() + async def test_should_not_deploy_clean_up_when_rest(self, mocker: MockerFixture): + image_tag_in_cluster = "1.1.1" + mocker.patch.object( + Helm, + "get_values", + return_value={ + "image": "registry/producer-app", + "imageTag": image_tag_in_cluster, + "nameOverride": PRODUCER_APP_NAME, + "replicaCount": 1, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "test-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + ) + producer_app = ProducerAppV3( + name=PRODUCER_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "imageTag": "2.2.2", + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "test-output-topic": {"type": "output"}, + } + }, + }, + ) + uninstall_producer_mock = mocker.patch.object(producer_app.helm, "uninstall") + mocker.patch.object(producer_app._cleaner.dry_run_handler, "print_helm_diff") + mocker.patch.object(producer_app._cleaner.helm, "uninstall") + + mock_helm_upgrade_install_clean_up = mocker.patch.object( + producer_app._cleaner.helm, "upgrade_install" + ) + + dry_run = True + await producer_app.reset(dry_run) + uninstall_producer_mock.assert_called_once_with( + "test-namespace", PRODUCER_APP_RELEASE_NAME, dry_run + ) + mock_helm_upgrade_install_clean_up.assert_not_called() + + @pytest.mark.asyncio() + async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( + self, mocker: MockerFixture + ): + image_tag_in_cluster = "1.1.1" + mocker.patch.object( + Helm, + "get_values", + return_value={ + "image": "registry/producer-app", + "imageTag": image_tag_in_cluster, + "nameOverride": PRODUCER_APP_NAME, + "replicaCount": 1, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "test-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + ) + producer_app = ProducerAppV3( + name=PRODUCER_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "imageTag": "2.2.2", + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "test-output-topic": {"type": "output"}, + } + }, + }, + ) + mocker.patch.object(producer_app.helm, "uninstall") + mocker.patch.object(producer_app._cleaner.dry_run_handler, "print_helm_diff") + mocker.patch.object(producer_app._cleaner.helm, "uninstall") + + mock_helm_upgrade_install = mocker.patch.object( + producer_app._cleaner.helm, "upgrade_install" + ) + + dry_run = True + await producer_app.clean(dry_run) + + mock_helm_upgrade_install.assert_called_once_with( + PRODUCER_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/producer-app-cleanup-job", + dry_run, + "test-namespace", + { + "image": "registry/producer-app", + "nameOverride": PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE, + "imageTag": image_tag_in_cluster, + "replicaCount": 1, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "test-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + ) diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 34afc0b21..1d3891016 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -8,9 +8,7 @@ from kpops.api.exception import ValidationError from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.helm import Helm -from kpops.component_handlers.helm_wrapper.model import ( - HelmUpgradeInstallFlags, -) +from kpops.component_handlers.helm_wrapper.model import HelmUpgradeInstallFlags from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.components.base_components.models import TopicName from kpops.components.base_components.models.to_section import ( @@ -96,12 +94,6 @@ def stateful_streams_app(self) -> StreamsApp: }, ) - @pytest.fixture() - def dry_run_handler_mock(self, mocker: MockerFixture) -> MagicMock: - return mocker.patch( - "kpops.components.base_components.helm_app.DryRunHandler" - ).return_value - @pytest.fixture(autouse=True) def empty_helm_get_values(self, mocker: MockerFixture) -> MagicMock: return mocker.patch.object(Helm, "get_values", return_value=None) From 4a93730beed07c808f3bea8112b90c00de3fdb6b Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 8 Aug 2024 10:31:30 +0200 Subject: [PATCH 14/23] change model validator to field validator --- kpops/components/streams_bootstrap_v3/base.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index 7e7f70fe5..d4a297c5a 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -57,22 +57,23 @@ class StreamsBootstrapV3(KafkaApp, HelmApp, ABC): description=describe_attr("version", __doc__), ) - @pydantic.model_validator(mode="after") - def version_validator(self) -> Self: - pattern_match = COMPILED_VERSION_PATTERN.match(self.version) + @pydantic.field_validator("version", mode="after") + @classmethod + def version_validator(cls, version: str) -> str: + pattern_match = COMPILED_VERSION_PATTERN.match(version) if not pattern_match: - msg = f"Invalid version format: {self.version}" + msg = f"Invalid version format: {version}" raise ValueError(msg) major, minor, patch, suffix, _ = pattern_match.groups() major = int(major) if major < 3: - msg = f"When using the streams bootstrap v3 component your version ('{self.version}') must be at least 3.0.0." + msg = f"When using the streams bootstrap v3 component your version ('{version}') must be at least 3.0.0." raise ValueError(msg) - return self + return version @pydantic.model_validator(mode="after") def warning_for_latest_image_tag(self) -> Self: From 7977c318e04551371274df672c8a344234ee50be Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 8 Aug 2024 11:32:44 +0200 Subject: [PATCH 15/23] remove Helm chart validations --- .../streams_bootstrap_v3/streams/model.py | 34 +------ .../streams_bootstrap_v3/test_streams_app.py | 99 ------------------- 2 files changed, 1 insertion(+), 132 deletions(-) diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index a20a68d93..ab195df87 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -3,9 +3,8 @@ from typing import Any import pydantic -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field -from kpops.api.exception import ValidationError from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.components.streams_bootstrap_v3.model import ( KafkaConfig, @@ -224,18 +223,6 @@ class PersistenceConfig(BaseModel): description="Storage class to use for the persistent volume.", ) - @model_validator(mode="after") - def validate_mandatory_fields_are_set( - self: PersistenceConfig, - ) -> PersistenceConfig: # TODO: typing.Self for Python 3.11+ - if self.enabled and self.size is None: - msg = ( - "If app.persistence.enabled is set to true, " - "the field app.persistence.size needs to be set." - ) - raise ValidationError(msg) - return self - class StreamsAppValues(StreamsBootstrapV3Values): """streams-bootstrap app configurations. @@ -263,22 +250,3 @@ class StreamsAppValues(StreamsBootstrapV3Values): description=describe_attr("persistence", __doc__), ) model_config = ConfigDict(extra="allow") - - @model_validator(mode="after") - def validate_mandatory_fields_are_set( - self: StreamsAppValues, - ) -> StreamsAppValues: # TODO: typing.Self for Python 3.11+ - if ( - self.autoscaling - and self.autoscaling.enabled - and ( - self.kafka.application_id is None - or self.autoscaling.lag_threshold is None - ) - ): - msg = ( - "If values.autoscaling.enabled is set to true, " - "the fields values.kafka.application_id and values.autoscaling.lag_threshold should be set." - ) - raise ValidationError(msg) - return self diff --git a/tests/components/streams_bootstrap_v3/test_streams_app.py b/tests/components/streams_bootstrap_v3/test_streams_app.py index bd0ea0c07..17b3521ea 100644 --- a/tests/components/streams_bootstrap_v3/test_streams_app.py +++ b/tests/components/streams_bootstrap_v3/test_streams_app.py @@ -5,7 +5,6 @@ import pytest from pytest_mock import MockerFixture -from kpops.api.exception import ValidationError from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.model import ( @@ -23,7 +22,6 @@ ) from kpops.components.streams_bootstrap_v3 import StreamsAppV3 from kpops.components.streams_bootstrap_v3.streams.model import ( - PersistenceConfig, StreamsAppAutoScaling, ) from kpops.components.streams_bootstrap_v3.streams.streams_app import ( @@ -120,90 +118,6 @@ def test_cleaner_inheritance(self, streams_app: StreamsAppV3): ) assert streams_app._cleaner.values == streams_app.values - def test_raise_validation_error_when_autoscaling_enabled_and_mandatory_fields_not_set( - self, streams_app: StreamsAppV3 - ): - with pytest.raises(ValidationError) as error: - StreamsAppV3( - name=STREAMS_APP_NAME, - **{ - "namespace": "test-namespace", - "values": { - "kafka": {"bootstrapServers": "fake-broker:9092"}, - "autoscaling": {"enabled": True}, - }, - "to": { - "topics": { - "streams-app-output-topic": TopicConfig( - type=OutputTopicTypes.OUTPUT, partitions_count=10 - ), - } - }, - }, - ) - msg = ( - "If values.autoscaling.enabled is set to true, " - "the fields values.kafka.application_id and values.autoscaling.lag_threshold should be set." - ) - assert str(error.value) == msg - - def test_raise_validation_error_when_autoscaling_enabled_and_only_consumer_group_set( - self, streams_app: StreamsAppV3 - ): - with pytest.raises(ValidationError) as error: - StreamsAppV3( - name=STREAMS_APP_NAME, - **{ - "namespace": "test-namespace", - "values": { - "kafka": { - "bootstrapServers": "fake-broker:9092", - "applicationId": "test-application-id", - }, - "autoscaling": {"enabled": True}, - }, - "to": { - "topics": { - "streams-app-output-topic": TopicConfig( - type=OutputTopicTypes.OUTPUT, partitions_count=10 - ), - } - }, - }, - ) - msg = ( - "If values.autoscaling.enabled is set to true, " - "the fields values.kafka.application_id and values.autoscaling.lag_threshold should be set." - ) - assert str(error.value) == msg - - def test_raise_validation_error_when_autoscaling_enabled_and_only_lag_threshold_is_set( - self, - ): - with pytest.raises(ValidationError) as error: - StreamsAppV3( - name=STREAMS_APP_NAME, - **{ - "namespace": "test-namespace", - "values": { - "kafka": {"bootstrapServers": "fake-broker:9092"}, - "autoscaling": {"enabled": True, "lagThreshold": 100}, - }, - "to": { - "topics": { - "streams-app-output-topic": TopicConfig( - type=OutputTopicTypes.OUTPUT, partitions_count=10 - ), - } - }, - }, - ) - msg = ( - "If values.autoscaling.enabled is set to true, " - "the fields values.kafka.application_id and values.autoscaling.lag_threshold should be set." - ) - assert str(error.value) == msg - def test_cleaner_helm_release_name(self, streams_app: StreamsAppV3): assert ( streams_app._cleaner.helm_release_name @@ -884,19 +798,6 @@ async def test_get_input_output_topics(self): KafkaTopic(name="topic-extra"), ] - def test_raise_validation_error_when_persistence_enabled_and_size_not_set( - self, stateful_streams_app: StreamsAppV3 - ): - with pytest.raises(ValidationError) as error: - stateful_streams_app.values.persistence = PersistenceConfig( - enabled=True, - ) - msg = ( - "If app.persistence.enabled is set to true, " - "the field app.persistence.size needs to be set." - ) - assert str(error.value) == msg - @pytest.mark.asyncio() async def test_stateful_clean_with_dry_run_false( self, From a3c3d147e5360a37e8a3ef063d49aec2ec6d78af Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 8 Aug 2024 12:02:02 +0200 Subject: [PATCH 16/23] add todos --- tests/api/test_registry.py | 1 + .../streams-bootstrap-v3/defaults.yaml | 1 + .../streams-bootstrap-v3/pipeline.yaml | 7 +- .../test_streams_bootstrap_v3/pipeline.yaml | 12 --- .../test_streams_bootstrap_v3/manifest.yaml | 79 +++++++++---------- 5 files changed, 39 insertions(+), 61 deletions(-) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 609b41702..3507c95e1 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -101,6 +101,7 @@ def test_registry(): "kafka-source-connector": KafkaSourceConnector, "kubernetes-app": KubernetesApp, "pipeline-component": PipelineComponent, + # TODO: change the old sterams bootstrap to -v2 and remove -v3 "producer-app": ProducerApp, "producer-app-v3": ProducerAppV3, "streams-app": StreamsApp, diff --git a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml index a8202662e..7dc1f296a 100644 --- a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml +++ b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml @@ -3,6 +3,7 @@ streams-bootstrap-v3: kafka: bootstrapServers: ${config.kafka_brokers} schemaRegistryUrl: ${config.schema_registry.url} + # TODO: change to stable version after the PR is merged version: "3.0.0-SNAPSHOT" producer-app-v3: {} # inherits from streams-bootstrap-v3 diff --git a/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml b/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml index ae03216b3..a22bafd85 100644 --- a/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml +++ b/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml @@ -4,7 +4,6 @@ imageTag: "1.0.0" commandLine: FAKE_ARG: "fake-arg-value" - schedule: "30 3/8 * * *" to: topics: @@ -22,11 +21,7 @@ applicationId: "my-streams-app-id" commandLine: CONVERT_XML: true - resources: - limits: - memory: 2G - requests: - memory: 2G + from: topics: my-input-topic: diff --git a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml index 503bacae3..3d5a7bf9d 100644 --- a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml @@ -20,7 +20,6 @@ my-producer-app-output-topic-label: my-labeled-producer-app-topic-output outputTopic: my-producer-app-output-topic schemaRegistryUrl: http://localhost:8081/ - schedule: 30 3/8 * * * version: 3.0.0-SNAPSHOT name: my-producer-app namespace: example-namespace @@ -51,7 +50,6 @@ my-producer-app-output-topic-label: my-labeled-producer-app-topic-output outputTopic: my-producer-app-output-topic schemaRegistryUrl: http://localhost:8081/ - schedule: 30 3/8 * * * version: 3.0.0-SNAPSHOT - _cleaner: name: my-streams-app @@ -89,11 +87,6 @@ schemaRegistryUrl: http://localhost:8081/ persistence: enabled: false - resources: - limits: - memory: 2G - requests: - memory: 2G statefulSet: false version: 3.0.0-SNAPSHOT from: @@ -160,11 +153,6 @@ schemaRegistryUrl: http://localhost:8081/ persistence: enabled: false - resources: - limits: - memory: 2G - requests: - memory: 2G statefulSet: false version: 3.0.0-SNAPSHOT diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml index 4e69e4712..89ce55efb 100644 --- a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml @@ -1,6 +1,6 @@ --- -apiVersion: batch/v1beta1 -kind: CronJob +apiVersion: batch/v1 +kind: Job metadata: labels: app: resources-streams-bootstrap-v3-my-producer-app @@ -8,47 +8,40 @@ metadata: release: resources-streams-bootstrap-v3-my-producer-app name: resources-streams-bootstrap-v3-my-producer-app spec: - concurrencyPolicy: Replace - failedJobsHistoryLimit: 1 - jobTemplate: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-streams-bootstrap-v3-my-producer-app + release: resources-streams-bootstrap-v3-my-producer-app spec: - backoffLimit: 6 - template: - metadata: - labels: - app: resources-streams-bootstrap-v3-my-producer-app - release: resources-streams-bootstrap-v3-my-producer-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_OUTPUT_TOPIC - value: my-producer-app-output-topic - - name: APP_LABELED_OUTPUT_TOPICS - value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, - - name: APP_FAKE_ARG - value: fake-arg-value - - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-producer-image:1.0.0 - imagePullPolicy: Always - name: resources-streams-bootstrap-v3-my-producer-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - restartPolicy: OnFailure - schedule: 30 3/8 * * * - successfulJobsHistoryLimit: 1 - suspend: false + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: APP_FAKE_ARG + value: fake-arg-value + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-v3-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure --- apiVersion: v1 @@ -138,7 +131,7 @@ spec: memory: 2G requests: cpu: 200m - memory: 2G + memory: 300Mi - command: - java - -XX:+UnlockExperimentalVMOptions From 116c94ceccf63d53b01ca2b03c638ff90063d9d9 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 12 Aug 2024 09:34:23 +0200 Subject: [PATCH 17/23] move app type to common --- .../components/{streams_bootstrap => common}/app_type.py | 0 .../components/streams_bootstrap/producer/producer_app.py | 2 +- kpops/components/streams_bootstrap/streams/streams_app.py | 2 +- kpops/components/streams_bootstrap_v3/app_type.py | 8 -------- .../streams_bootstrap_v3/producer/producer_app.py | 2 +- .../streams_bootstrap_v3/streams/streams_app.py | 2 +- .../component_handlers/helm_wrapper/test_helm_wrapper.py | 2 +- 7 files changed, 5 insertions(+), 13 deletions(-) rename kpops/components/{streams_bootstrap => common}/app_type.py (100%) delete mode 100644 kpops/components/streams_bootstrap_v3/app_type.py diff --git a/kpops/components/streams_bootstrap/app_type.py b/kpops/components/common/app_type.py similarity index 100% rename from kpops/components/streams_bootstrap/app_type.py rename to kpops/components/common/app_type.py diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index b2373b7d8..6822c2841 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -8,13 +8,13 @@ KafkaApp, KafkaAppCleaner, ) +from kpops.components.common.app_type import AppType from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 3d4997120..77481e0ce 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -7,9 +7,9 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp from kpops.components.base_components.kafka_app import KafkaApp, KafkaAppCleaner +from kpops.components.common.app_type import AppType from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.common.topic import KafkaTopic -from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.streams.model import ( StreamsAppValues, ) diff --git a/kpops/components/streams_bootstrap_v3/app_type.py b/kpops/components/streams_bootstrap_v3/app_type.py deleted file mode 100644 index 982ad07fa..000000000 --- a/kpops/components/streams_bootstrap_v3/app_type.py +++ /dev/null @@ -1,8 +0,0 @@ -from enum import Enum - - -class AppType(Enum): - STREAMS_APP = "streams-app" - PRODUCER_APP = "producer-app" - CLEANUP_STREAMS_APP = "streams-app-cleanup-job" - CLEANUP_PRODUCER_APP = "producer-app-cleanup-job" diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap_v3/producer/producer_app.py index 26ca6eddd..3c037a32b 100644 --- a/kpops/components/streams_bootstrap_v3/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v3/producer/producer_app.py @@ -5,12 +5,12 @@ from typing_extensions import override from kpops.components.base_components.kafka_app import KafkaAppCleaner +from kpops.components.common.app_type import AppType from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap_v3.app_type import AppType from kpops.components.streams_bootstrap_v3.base import ( StreamsBootstrapV3, ) diff --git a/kpops/components/streams_bootstrap_v3/streams/streams_app.py b/kpops/components/streams_bootstrap_v3/streams/streams_app.py index 88d380d42..286cc8938 100644 --- a/kpops/components/streams_bootstrap_v3/streams/streams_app.py +++ b/kpops/components/streams_bootstrap_v3/streams/streams_app.py @@ -7,8 +7,8 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp from kpops.components.base_components.kafka_app import KafkaAppCleaner +from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic -from kpops.components.streams_bootstrap_v3.app_type import AppType from kpops.components.streams_bootstrap_v3.base import ( StreamsBootstrapV3, ) diff --git a/tests/component_handlers/helm_wrapper/test_helm_wrapper.py b/tests/component_handlers/helm_wrapper/test_helm_wrapper.py index aa4040ef2..23c855251 100644 --- a/tests/component_handlers/helm_wrapper/test_helm_wrapper.py +++ b/tests/component_handlers/helm_wrapper/test_helm_wrapper.py @@ -18,7 +18,7 @@ RepoAuthFlags, Version, ) -from kpops.components.streams_bootstrap.app_type import AppType +from kpops.components.common.app_type import AppType class TestHelmWrapper: From 4af7a94251f1bde175b026d06102c451f870da7c Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 12 Aug 2024 09:38:02 +0200 Subject: [PATCH 18/23] Fallback to user defined model when the validation of cluster model fails (#521) --- kpops/component_handlers/helm_wrapper/helm.py | 3 +- .../producer/producer_app.py | 15 +++- .../streams_bootstrap/streams/streams_app.py | 15 +++- tests/components/test_producer_app.py | 71 +++++++++++++++ tests/components/test_streams_app.py | 90 +++++++++++++++++-- 5 files changed, 181 insertions(+), 13 deletions(-) diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index 0381d552e..5ff76fb81 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -21,11 +21,12 @@ Version, ) from kpops.component_handlers.kubernetes.model import KubernetesManifest -from kpops.components.base_components.models.resource import Resource if TYPE_CHECKING: from collections.abc import Iterable, Iterator + from kpops.components.base_components.models.resource import Resource + log = logging.getLogger("Helm") diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 564487616..1c26ab974 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,7 +1,7 @@ import logging from functools import cached_property -from pydantic import Field, computed_field +from pydantic import Field, ValidationError, computed_field from typing_extensions import override from kpops.components.base_components.kafka_app import ( @@ -16,6 +16,7 @@ ) from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.producer.model import ProducerAppValues +from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.utils.docstring import describe_attr log = logging.getLogger("ProducerApp") @@ -106,8 +107,16 @@ async def destroy(self, dry_run: bool) -> None: if cluster_values: log.debug("Fetched Helm chart values from cluster") name_override = self._cleaner.helm_name_override - self._cleaner.values = self.values.model_validate(cluster_values) - self._cleaner.values.name_override = name_override + try: + self._cleaner.values = self.values.model_validate(cluster_values) + self._cleaner.values.name_override = name_override + except ValidationError as validation_error: + warning_msg = f"The values in the cluster are invalid with the current model. Falling back to the enriched values of {PIPELINE_YAML} and {DEFAULTS_YAML}" + log.warning(warning_msg) + debug_msg = f"Cluster values: {cluster_values}" + log.debug(debug_msg) + debug_msg = f"Validation error: {validation_error}" + log.debug(debug_msg) await super().destroy(dry_run) diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 6e55931d9..5dc1352a6 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,7 +1,7 @@ import logging from functools import cached_property -from pydantic import Field, computed_field +from pydantic import Field, ValidationError, computed_field from typing_extensions import override from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler @@ -13,6 +13,7 @@ from kpops.components.streams_bootstrap.streams.model import ( StreamsAppValues, ) +from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.utils.docstring import describe_attr log = logging.getLogger("StreamsApp") @@ -131,8 +132,16 @@ async def destroy(self, dry_run: bool) -> None: if cluster_values: log.debug("Fetched Helm chart values from cluster") name_override = self._cleaner.helm_name_override - self._cleaner.values = self.values.model_validate(cluster_values) - self._cleaner.values.name_override = name_override + try: + self._cleaner.values = self.values.model_validate(cluster_values) + self._cleaner.values.name_override = name_override + except ValidationError as validation_error: + warning_msg = f"The values in the cluster are invalid with the current model. Falling back to the enriched values of {PIPELINE_YAML} and {DEFAULTS_YAML}" + log.warning(warning_msg) + debug_msg = f"Cluster values: {cluster_values}" + log.debug(debug_msg) + debug_msg = f"Validation error: {validation_error}" + log.debug(debug_msg) await super().destroy(dry_run) diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 00b0e70d3..fc49debf8 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -472,3 +472,74 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( }, HelmUpgradeInstallFlags(version="2.9.0", wait=True, wait_for_jobs=True), ) + + @pytest.mark.asyncio() + async def test_clean_should_fall_back_to_local_values_when_validation_of_cluster_values_fails( + self, mocker: MockerFixture, caplog: pytest.LogCaptureFixture + ): + caplog.set_level(logging.WARNING) + + # invalid model + mocker.patch.object( + Helm, + "get_values", + return_value={ + "image": "registry/producer-app", + "imageTag": "1.1.1", + "nameOverride": PRODUCER_APP_NAME, + "kafka": { + "boostrapServers": "fake-broker:9092", + "outputTopic": "test-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + ) + + # user defined model + producer_app = ProducerApp( + name=PRODUCER_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "image": "registry/producer-app", + "imageTag": "2.2.2", + "streams": {"brokers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "test-output-topic": {"type": "output"}, + } + }, + }, + ) + mocker.patch.object(producer_app.helm, "uninstall") + mocker.patch.object(producer_app._cleaner.dry_run_handler, "print_helm_diff") + mocker.patch.object(producer_app._cleaner.helm, "uninstall") + + mock_helm_upgrade_install = mocker.patch.object( + producer_app._cleaner.helm, "upgrade_install" + ) + + dry_run = True + await producer_app.clean(dry_run) + assert ( + "The values in the cluster are invalid with the current model. Falling back to the enriched values of pipeline.yaml and defaults.yaml" + in caplog.text + ) + + mock_helm_upgrade_install.assert_called_once_with( + PRODUCER_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/producer-app-cleanup-job", + dry_run, + "test-namespace", + { + "image": "registry/producer-app", + "imageTag": "2.2.2", + "nameOverride": PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE, + "streams": { + "brokers": "fake-broker:9092", + "outputTopic": "test-output-topic", + }, + }, + HelmUpgradeInstallFlags(version="2.9.0", wait=True, wait_for_jobs=True), + ) diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 34afc0b21..f0e523a3d 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -96,12 +96,6 @@ def stateful_streams_app(self) -> StreamsApp: }, ) - @pytest.fixture() - def dry_run_handler_mock(self, mocker: MockerFixture) -> MagicMock: - return mocker.patch( - "kpops.components.base_components.helm_app.DryRunHandler" - ).return_value - @pytest.fixture(autouse=True) def empty_helm_get_values(self, mocker: MockerFixture) -> MagicMock: return mocker.patch.object(Helm, "get_values", return_value=None) @@ -972,3 +966,87 @@ async def test_stateful_clean_with_dry_run_true( f"Deleting the PVCs {pvc_names} for StatefulSet '{STREAMS_APP_FULL_NAME}'" in caplog.text ) + + @pytest.mark.asyncio() + async def test_clean_should_fall_back_to_local_values_when_validation_of_cluster_values_fails( + self, + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + ): + caplog.set_level(logging.WARNING) + + # invalid model + mocker.patch.object( + Helm, + "get_values", + return_value={ + "image": "registry/producer-app", + "imageTag": "1.1.1", + "nameOverride": STREAMS_APP_NAME, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "inputTopics": ["test-input-topic"], + "outputTopic": "streams-app-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + ) + + streams_app = StreamsApp( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "image": "registry/streams-app", + "imageTag": "2.2.2", + "streams": {"brokers": "fake-broker:9092"}, + }, + "from": { + "topics": { + "test-input-topic": {"type": "input"}, + } + }, + "to": { + "topics": { + "streams-app-output-topic": {"type": "output"}, + } + }, + }, + ) + + mocker.patch.object(streams_app.helm, "uninstall") + + mock_helm_upgrade_install = mocker.patch.object( + streams_app._cleaner.helm, "upgrade_install" + ) + mocker.patch.object(streams_app._cleaner.helm, "uninstall") + + mock = mocker.MagicMock() + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + + dry_run = False + await streams_app.clean(dry_run=dry_run) + + assert ( + "The values in the cluster are invalid with the current model. Falling back to the enriched values of pipeline.yaml and defaults.yaml" + in caplog.text + ) + + mock_helm_upgrade_install.assert_called_once_with( + STREAMS_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/streams-app-cleanup-job", + dry_run, + "test-namespace", + { + "image": "registry/streams-app", + "nameOverride": STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE, + "imageTag": "2.2.2", + "streams": { + "brokers": "fake-broker:9092", + "inputTopics": ["test-input-topic"], + "outputTopic": "streams-app-output-topic", + "deleteOutput": True, + }, + }, + HelmUpgradeInstallFlags(version="2.9.0", wait=True, wait_for_jobs=True), + ) From fa876434c05ccfaa795e460a3962da27fb3afae7 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 12 Aug 2024 16:07:51 +0200 Subject: [PATCH 19/23] Update files --- .../dependencies/kpops_structure.yaml | 19 ++ docs/docs/schema/defaults.json | 183 +++++++++++++++++- .../streams_bootstrap_v3/__init__.py | 4 +- kpops/components/streams_bootstrap_v3/base.py | 3 +- ...roducer_app_v3.py => test_producer_app.py} | 0 5 files changed, 205 insertions(+), 4 deletions(-) rename tests/components/streams_bootstrap_v3/{test_producer_app_v3.py => test_producer_app.py} (100%) diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 830445552..85cd0a175 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -95,6 +95,15 @@ kpops_components_fields: - values - repo_config - version + streams-bootstrap-v3: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version kpops_components_inheritance_ref: helm-app: bases: @@ -189,3 +198,13 @@ kpops_components_inheritance_ref: - kubernetes-app - pipeline-component - base-defaults-component + streams-bootstrap-v3: + bases: + - kafka-app + - helm-app + parents: + - kafka-app + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index e223e0eb8..2210e0876 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -265,6 +265,56 @@ "title": "KafkaApp", "type": "object" }, + "KafkaConfig": { + "additionalProperties": true, + "description": "Kafka Streams config.", + "properties": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "KafkaConfig", + "type": "object" + }, "KafkaConnector": { "additionalProperties": true, "description": "Base class for all Kafka connectors.\nShould only be used to set defaults", @@ -1422,6 +1472,133 @@ "title": "StreamsBootstrap", "type": "object" }, + "StreamsBootstrapV3": { + "additionalProperties": true, + "description": "Base for components with a streams-bootstrap Helm chart.", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsBootstrapV3Values" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.0", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "title": "StreamsBootstrapV3", + "type": "object" + }, + "StreamsBootstrapV3Values": { + "additionalProperties": true, + "description": "Base value class for all streams bootstrap related components.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/KafkaConfig" + } + ], + "description": "Kafka configuration for the streams-bootstrap app." + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + } + }, + "required": [ + "kafka" + ], + "title": "StreamsBootstrapV3Values", + "type": "object" + }, "StreamsBootstrapValues": { "additionalProperties": true, "description": "Base value class for all streams bootstrap related components.", @@ -2299,6 +2476,9 @@ }, "streams-bootstrap": { "$ref": "#/$defs/StreamsBootstrap" + }, + "streams-bootstrap-v3": { + "$ref": "#/$defs/StreamsBootstrapV3" } }, "required": [ @@ -2313,7 +2493,8 @@ "streams-app", "streams-bootstrap", "producer-app-v3", - "streams-app-v3" + "streams-app-v3", + "streams-bootstrap-v3" ], "title": "DefaultsSchema", "type": "object" diff --git a/kpops/components/streams_bootstrap_v3/__init__.py b/kpops/components/streams_bootstrap_v3/__init__.py index adb6a8c92..07b11bf21 100644 --- a/kpops/components/streams_bootstrap_v3/__init__.py +++ b/kpops/components/streams_bootstrap_v3/__init__.py @@ -1,6 +1,6 @@ -from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.streams_bootstrap_v3.base import StreamsBootstrapV3 from .producer.producer_app import ProducerAppV3 from .streams.streams_app import StreamsAppV3 -__all__ = ("StreamsBootstrap", "StreamsAppV3", "ProducerAppV3") +__all__ = ("StreamsBootstrapV3", "StreamsAppV3", "ProducerAppV3") diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index d4a297c5a..6f9a26357 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -20,11 +20,12 @@ except ImportError: from typing_extensions import Self - STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( repository_name="bakdata-streams-bootstrap", url="https://bakdata.github.io/streams-bootstrap/", ) + +# TODO: Update this with the latest stable version release STREAMS_BOOTSTRAP_VERSION = "3.0.0" STREAMS_BOOTSTRAP_VERSION_PATTERN = r"^(\d+)\.(\d+)\.(\d+)(-[a-zA-Z]+(\.[a-zA-Z]+)?)?$" COMPILED_VERSION_PATTERN = re.compile(STREAMS_BOOTSTRAP_VERSION_PATTERN) diff --git a/tests/components/streams_bootstrap_v3/test_producer_app_v3.py b/tests/components/streams_bootstrap_v3/test_producer_app.py similarity index 100% rename from tests/components/streams_bootstrap_v3/test_producer_app_v3.py rename to tests/components/streams_bootstrap_v3/test_producer_app.py From cc12d61ecaca927a2b784f2a824a7532212a276a Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 12 Aug 2024 16:55:53 +0200 Subject: [PATCH 20/23] Update files --- kpops/components/streams_bootstrap_v3/base.py | 4 ++-- .../streams_bootstrap_v3/producer/producer_app.py | 6 +++--- kpops/components/streams_bootstrap_v3/streams/model.py | 2 +- .../components/streams_bootstrap_v3/streams/streams_app.py | 7 +++---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index 6f9a26357..72e51fc99 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -70,8 +70,8 @@ def version_validator(cls, version: str) -> str: major, minor, patch, suffix, _ = pattern_match.groups() major = int(major) - if major < 3: - msg = f"When using the streams bootstrap v3 component your version ('{version}') must be at least 3.0.0." + if major == 3: + msg = f"When using the streams-bootstrap v3 component your version ('{version}') must be at least 3.0.0." raise ValueError(msg) return version diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap_v3/producer/producer_app.py index 3c037a32b..3b70132e0 100644 --- a/kpops/components/streams_bootstrap_v3/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v3/producer/producer_app.py @@ -12,13 +12,13 @@ TopicConfig, ) from kpops.components.streams_bootstrap_v3.base import ( + STREAMS_BOOTSTRAP_VERSION, StreamsBootstrapV3, ) from kpops.components.streams_bootstrap_v3.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr -log = logging.getLogger("ProducerApp") -STREAMS_BOOTSTRAP_V3 = "3.0.0" +log = logging.getLogger("ProducerAppV3") class ProducerAppCleaner(KafkaAppCleaner, StreamsBootstrapV3): @@ -55,7 +55,7 @@ class ProducerAppV3(StreamsBootstrapV3): description=describe_attr("from_", __doc__), ) - version: str | None = STREAMS_BOOTSTRAP_V3 + version: str | None = STREAMS_BOOTSTRAP_VERSION @computed_field @cached_property diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index ab195df87..2d641660b 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -69,7 +69,7 @@ def deserialize_input_topics( @pydantic.field_validator("labeled_input_topics", mode="before") @classmethod def deserialize_labeled_input_topics( - cls, labeled_input_topics: Any + cls, labeled_input_topics: dict[str, list[str]] | Any ) -> dict[str, list[KafkaTopic]] | Any: if isinstance(labeled_input_topics, dict): return { diff --git a/kpops/components/streams_bootstrap_v3/streams/streams_app.py b/kpops/components/streams_bootstrap_v3/streams/streams_app.py index 286cc8938..964c2acd0 100644 --- a/kpops/components/streams_bootstrap_v3/streams/streams_app.py +++ b/kpops/components/streams_bootstrap_v3/streams/streams_app.py @@ -10,6 +10,7 @@ from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap_v3.base import ( + STREAMS_BOOTSTRAP_VERSION, StreamsBootstrapV3, ) from kpops.components.streams_bootstrap_v3.streams.model import ( @@ -17,9 +18,7 @@ ) from kpops.utils.docstring import describe_attr -log = logging.getLogger("StreamsApp") - -STREAMS_BOOTSTRAP_V3 = "3.0.0" +log = logging.getLogger("StreamsAppV3") class StreamsAppCleaner(KafkaAppCleaner, StreamsBootstrapV3): @@ -67,7 +66,7 @@ class StreamsAppV3(StreamsBootstrapV3): description=describe_attr("values", __doc__), ) - version: str | None = STREAMS_BOOTSTRAP_V3 + version: str | None = STREAMS_BOOTSTRAP_VERSION @computed_field @cached_property From 2ed9db4c58133302fcdc488b2c72f2e3f68a3bd3 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 12 Aug 2024 16:58:51 +0200 Subject: [PATCH 21/23] Update files --- kpops/components/streams_bootstrap_v3/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index 72e51fc99..26ee45d24 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -70,7 +70,7 @@ def version_validator(cls, version: str) -> str: major, minor, patch, suffix, _ = pattern_match.groups() major = int(major) - if major == 3: + if major != 3: msg = f"When using the streams-bootstrap v3 component your version ('{version}') must be at least 3.0.0." raise ValueError(msg) From a90af1b176ed3b2c739fae7db4c5089094f0cc2b Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 12 Aug 2024 17:00:31 +0200 Subject: [PATCH 22/23] Update files --- tests/components/streams_bootstrap_v3/test_streams_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py b/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py index 0672e6144..cdb892119 100644 --- a/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py +++ b/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py @@ -105,7 +105,7 @@ async def test_should_raise_validation_error_for_invalid_helm_chart_version(self with pytest.raises( ValueError, match=re.escape( - "When using the streams bootstrap v3 component your version ('2.1.0') must be at least 3.0.0." + "When using the streams-bootstrap v3 component your version ('2.1.0') must be at least 3.0.0." ), ): StreamsBootstrapV3( From eb6f49d49eee1a797a1f060a4df3e924f2f81eec Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 12 Aug 2024 17:03:24 +0200 Subject: [PATCH 23/23] Update files --- docs/docs/schema/defaults.json | 26 ++++++------------- docs/docs/schema/pipeline.json | 26 ++++++------------- .../producer/producer_app.py | 3 --- .../streams/streams_app.py | 3 --- tests/api/test_registry.py | 7 ++++- 5 files changed, 22 insertions(+), 43 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 2210e0876..d3dbc7905 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -994,16 +994,11 @@ "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], "default": "3.0.0", - "title": "Version" + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ @@ -1358,16 +1353,11 @@ "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], "default": "3.0.0", - "title": "Version" + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index e20214713..86edc0b8e 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -654,16 +654,11 @@ "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], "default": "3.0.0", - "title": "Version" + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ @@ -1018,16 +1013,11 @@ "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], "default": "3.0.0", - "title": "Version" + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap_v3/producer/producer_app.py index 3b70132e0..97df4a632 100644 --- a/kpops/components/streams_bootstrap_v3/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v3/producer/producer_app.py @@ -12,7 +12,6 @@ TopicConfig, ) from kpops.components.streams_bootstrap_v3.base import ( - STREAMS_BOOTSTRAP_VERSION, StreamsBootstrapV3, ) from kpops.components.streams_bootstrap_v3.producer.model import ProducerAppValues @@ -55,8 +54,6 @@ class ProducerAppV3(StreamsBootstrapV3): description=describe_attr("from_", __doc__), ) - version: str | None = STREAMS_BOOTSTRAP_VERSION - @computed_field @cached_property def _cleaner(self) -> ProducerAppCleaner: diff --git a/kpops/components/streams_bootstrap_v3/streams/streams_app.py b/kpops/components/streams_bootstrap_v3/streams/streams_app.py index 964c2acd0..56fde5aac 100644 --- a/kpops/components/streams_bootstrap_v3/streams/streams_app.py +++ b/kpops/components/streams_bootstrap_v3/streams/streams_app.py @@ -10,7 +10,6 @@ from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap_v3.base import ( - STREAMS_BOOTSTRAP_VERSION, StreamsBootstrapV3, ) from kpops.components.streams_bootstrap_v3.streams.model import ( @@ -66,8 +65,6 @@ class StreamsAppV3(StreamsBootstrapV3): description=describe_attr("values", __doc__), ) - version: str | None = STREAMS_BOOTSTRAP_VERSION - @computed_field @cached_property def _cleaner(self) -> StreamsAppCleaner: diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 3507c95e1..9429dab83 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -21,7 +21,11 @@ from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp -from kpops.components.streams_bootstrap_v3 import ProducerAppV3, StreamsAppV3 +from kpops.components.streams_bootstrap_v3 import ( + ProducerAppV3, + StreamsAppV3, + StreamsBootstrapV3, +) from tests.cli.resources.custom_module import CustomSchemaProvider @@ -107,6 +111,7 @@ def test_registry(): "streams-app": StreamsApp, "streams-app-v3": StreamsAppV3, "streams-bootstrap": StreamsBootstrap, + "streams-bootstrap-v3": StreamsBootstrapV3, } for _type, _class in registry._classes.items(): assert registry[_type] is _class