Introduce KPOps operation and manifest resources for deployment (#541)

Closes #539, #538

docs/docs/resources/variables/cli_env_vars.env

@@ -14,6 +14,8 @@ KPOPS_DOTENV_PATH # No default value, not required
 # Suffix your environment files with this value (e.g.
 # defaults_development.yaml for environment=development).
 KPOPS_ENVIRONMENT # No default value, not required
+# How KPOps should operate.
+KPOPS_OPERATION_MODE=managed
 # Paths to dir containing 'pipeline.yaml' or files named
 # 'pipeline.yaml'.
 KPOPS_PIPELINE_PATHS # No default value, required

docs/docs/resources/variables/cli_env_vars.md

@@ -5,5 +5,6 @@ These variables take precedence over the commands' flags. If a variable is set,
 |KPOPS_CONFIG_PATH   |.            |False   |Path to the dir containing config.yaml files                                                                                                                                |
 |KPOPS_DOTENV_PATH   |             |False   |Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.                                         |
 |KPOPS_ENVIRONMENT   |             |False   |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).|
+|KPOPS_OPERATION_MODE|managed      |False   |How KPOps should operate.                                                                                                                                                   |
 |KPOPS_PIPELINE_PATHS|             |True    |Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.                                                                                                     |
 |KPOPS_PIPELINE_STEPS|             |False   |Comma separated list of steps to apply the command on                                                                                                                       |

docs/docs/resources/variables/config_env_vars.env

@@ -58,3 +58,6 @@ KPOPS_HELM_DIFF_CONFIG__IGNORE # No default value, required
 # Whether to retain clean up jobs in the cluster or uninstall the,
 # after completion.
 KPOPS_RETAIN_CLEAN_JOBS=False
+# operation_mode
+# The operation mode of KPOps (managed, manifest, argo).
+KPOPS_OPERATION_MODE=managed

docs/docs/resources/variables/config_env_vars.md

@@ -19,3 +19,4 @@ These variables take precedence over the settings in `config.yaml`. Variables ma
 |KPOPS_HELM_CONFIG__API_VERSION                    |                                        |False   |Kubernetes API version used for `Capabilities.APIVersions`                        |helm_config.api_version                    |
 |KPOPS_HELM_DIFF_CONFIG__IGNORE                    |                                        |True    |Set of keys that should not be checked.                                           |helm_diff_config.ignore                    |
 |KPOPS_RETAIN_CLEAN_JOBS                           |False                                   |False   |Whether to retain clean up jobs in the cluster or uninstall the, after completion.|retain_clean_jobs                          |
+|KPOPS_OPERATION_MODE                              |managed                                 |False   |The operation mode of KPOps (managed, manifest, argo).                            |operation_mode                             |

docs/docs/schema/config.json

@@ -118,6 +118,15 @@
             "title": "KafkaRestConfig",
             "type": "object"
         },
+        "OperationMode": {
+            "enum": [
+                "argo",
+                "manifest",
+                "managed"
+            ],
+            "title": "OperationMode",
+            "type": "string"
+        },
         "SchemaRegistryConfig": {
             "additionalProperties": false,
             "description": "Configuration for Schema Registry.",
@@ -239,6 +248,15 @@
             },
             "description": "Configuration for Kafka REST Proxy."
         },
+        "operation_mode": {
+            "allOf": [
+                {
+                    "$ref": "#/$defs/OperationMode"
+                }
+            ],
+            "default": "managed",
+            "description": "The operation mode of KPOps (managed, manifest, argo)."
+        },
         "pipeline_base_dir": {
             "default": ".",
             "description": "Base directory to the pipelines (default is current working directory)",

docs/docs/user/references/cli-commands.md

@@ -20,7 +20,6 @@ $ kpops [OPTIONS] COMMAND [ARGS]...
 * `destroy`: Destroy pipeline steps
 * `generate`: Generate enriched pipeline representation
 * `init`: Initialize a new KPOps project.
-* `manifest`: Render final resource representation
 * `reset`: Reset pipeline steps
 * `schema`: Generate JSON schema.
 
@@ -74,6 +73,7 @@ $ kpops deploy [OPTIONS] PIPELINE_PATHS...
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
 * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially.  [default: no-parallel]
+* `--operation-mode [argo|manifest|managed]`: How KPOps should operate.  [env var: KPOPS_OPERATION_MODE; default: managed]
 * `--help`: Show this message and exit.
 
 ## `kpops destroy`
@@ -142,31 +142,7 @@ $ kpops init [OPTIONS] PATH
 
 **Options**:
 
-* `--config-include-opt / --no-config-include-opt`: Whether to include non-required settings in the generated 'config.yaml'  [default: no-config-include-opt]
-* `--help`: Show this message and exit.
-
-## `kpops manifest`
-
-In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests.
-
-**Usage**:
-
-```console
-$ kpops manifest [OPTIONS] PIPELINE_PATHS...
-```
-
-**Arguments**:
-
-* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
-
-**Options**:
-
-* `--dotenv FILE`: Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.  [env var: KPOPS_DOTENV_PATH]
-* `--config DIRECTORY`: Path to the dir containing config.yaml files  [env var: KPOPS_CONFIG_PATH; default: .]
-* `--steps TEXT`: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]
-* `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: include]
-* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).   [env var: KPOPS_ENVIRONMENT]
-* `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
+* `--config-include-optional / --no-config-include-optional`: Whether to include non-required settings in the generated 'config.yaml'  [default: no-config-include-optional]
 * `--help`: Show this message and exit.
 
 ## `kpops reset`

kpops/api/__init__.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
 import asyncio
+from collections.abc import Iterator
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from kpops.api.logs import log, log_action
+from kpops.api.operation import OperationMode
 from kpops.api.options import FilterType
 from kpops.api.registry import Registry
 from kpops.component_handlers import ComponentHandlers
@@ -14,6 +16,7 @@
 from kpops.component_handlers.schema_handler.schema_handler import SchemaHandler
 from kpops.component_handlers.topic.handler import TopicHandler
 from kpops.component_handlers.topic.proxy_wrapper import ProxyWrapper
+from kpops.components.base_components.models.resource import Resource
 from kpops.config import KpopsConfig
 from kpops.pipeline import (
     Pipeline,
@@ -22,7 +25,6 @@
 from kpops.utils.cli_commands import init_project
 
 if TYPE_CHECKING:
-    from kpops.components.base_components.models.resource import Resource
     from kpops.components.base_components.pipeline_component import PipelineComponent
     from kpops.config import KpopsConfig
 
@@ -35,6 +37,7 @@ def generate(
     filter_type: FilterType = FilterType.INCLUDE,
     environment: str | None = None,
     verbose: bool = False,
+    operation_mode: OperationMode = OperationMode.MANAGED,
 ) -> Pipeline:
     """Generate enriched pipeline representation.
 
@@ -45,13 +48,11 @@ def generate(
     :param filter_type: Whether `steps` should include/exclude the steps.
     :param environment: The environment to generate and deploy the pipeline to.
     :param verbose: Enable verbose printing.
+    :param operation_mode: How KPOps should operate.
     :return: Generated `Pipeline` object.
     """
     kpops_config = KpopsConfig.create(
-        config,
-        dotenv,
-        environment,
-        verbose,
+        config, dotenv, environment, verbose, operation_mode
     )
     pipeline = _create_pipeline(pipeline_path, kpops_config, environment)
     log.info(f"Picked up pipeline '{pipeline_path.parent.name}'")
@@ -69,26 +70,16 @@ def generate(
     return pipeline
 
 
-def manifest(
+def manifest_deploy(
     pipeline_path: Path,
     dotenv: list[Path] | None = None,
     config: Path = Path(),
     steps: set[str] | None = None,
     filter_type: FilterType = FilterType.INCLUDE,
     environment: str | None = None,
-    verbose: bool = False,
-) -> list[Resource]:
-    """Generate pipeline, return final resource representations for each step.
-
-    :param pipeline_path: Path to pipeline definition yaml file.
-    :param dotenv: Paths to dotenv files.
-    :param config: Path to the dir containing config.yaml files.
-    :param steps: Set of steps (components) to apply the command on.
-    :param filter_type: Whether `steps` should include/exclude the steps.
-    :param environment: The environment to generate and deploy the pipeline to.
-    :param verbose: Enable verbose printing.
-    :return: Resources.
-    """
+    verbose: bool = True,
+    operation_mode: OperationMode = OperationMode.MANIFEST,
+) -> Iterator[Resource]:
     pipeline = generate(
         pipeline_path=pipeline_path,
         dotenv=dotenv,
@@ -97,12 +88,11 @@ def manifest(
         filter_type=filter_type,
         environment=environment,
         verbose=verbose,
+        operation_mode=operation_mode,
     )
-    resources: list[Resource] = []
     for component in pipeline.components:
-        resource = component.manifest()
-        resources.append(resource)
-    return resources
+        resource = component.manifest_deploy()
+        yield resource
 
 
 def deploy(
@@ -301,20 +291,20 @@ async def async_clean():
 
 def init(
     path: Path,
-    config_include_opt: bool = False,
+    config_include_optional: bool = False,
 ):
     """Initiate a default empty project.
 
     :param path: Directory in which the project should be initiated.
-    :param conf_incl_opt: Whether to include non-required settings
+    :param config_include_optional: Whether to include non-required settings
         in the generated config file.
     """
     if not path.exists():
         path.mkdir(parents=False)
     elif next(path.iterdir(), False):
         log.warning("Please provide a path to an empty directory.")
         return
-    init_project(path, config_include_opt)
+    init_project(path, config_include_optional)
 
 
 def _create_pipeline(

kpops/api/operation.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import enum
+
+
+class OperationMode(str, enum.Enum):
+    ARGO = "argo"
+    MANIFEST = "manifest"
+    MANAGED = "managed"

kpops/cli/main.py

@@ -5,6 +5,7 @@
 import typer
 
 import kpops.api as kpops
+from kpops.api.operation import OperationMode
 from kpops.api.options import FilterType
 from kpops.cli.utils import (
     collect_pipeline_paths,
@@ -110,6 +111,11 @@
         "Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). "
     ),
 )
+OPERATION_MODE_OPTION: OperationMode = typer.Option(
+    default=OperationMode.MANAGED,
+    envvar=f"{ENV_PREFIX}OPERATION_MODE",
+    help="How KPOps should operate.",
+)
 
 
 def parse_steps(steps: str | None) -> set[str] | None:
@@ -119,9 +125,9 @@ def parse_steps(steps: str | None) -> set[str] | None:
 @app.command(help="Initialize a new KPOps project.")
 def init(
     path: Path = PROJECT_PATH,
-    config_include_opt: bool = CONFIG_INCLUDE_OPTIONAL,
+    config_include_optional: bool = CONFIG_INCLUDE_OPTIONAL,
 ):
-    kpops.init(path, config_include_opt=config_include_opt)
+    kpops.init(path, config_include_optional=config_include_optional)
 
 
 @app.command(
@@ -180,34 +186,6 @@ def generate(
         print_yaml(pipeline.to_yaml())
 
 
-@app.command(
-    short_help="Render final resource representation",
-    help="In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests.",
-)
-def manifest(
-    pipeline_paths: list[Path] = PIPELINE_PATHS_ARG,
-    dotenv: list[Path] | None = DOTENV_PATH_OPTION,
-    config: Path = CONFIG_PATH_OPTION,
-    steps: str | None = PIPELINE_STEPS,
-    filter_type: FilterType = FILTER_TYPE,
-    environment: str | None = ENVIRONMENT,
-    verbose: bool = VERBOSE_OPTION,
-):
-    for pipeline_file_path in collect_pipeline_paths(pipeline_paths):
-        resources = kpops.manifest(
-            pipeline_path=pipeline_file_path,
-            dotenv=dotenv,
-            config=config,
-            steps=parse_steps(steps),
-            filter_type=filter_type,
-            environment=environment,
-            verbose=verbose,
-        )
-        for resource in resources:
-            for rendered_manifest in resource:
-                print_yaml(rendered_manifest)
-
-
 @app.command(help="Deploy pipeline steps")
 def deploy(
     pipeline_paths: list[Path] = PIPELINE_PATHS_ARG,
@@ -219,19 +197,37 @@ def deploy(
     dry_run: bool = DRY_RUN,
     verbose: bool = VERBOSE_OPTION,
     parallel: bool = PARALLEL,
+    operation_mode: OperationMode = OPERATION_MODE_OPTION,
 ):
-    for pipeline_file_path in collect_pipeline_paths(pipeline_paths):
-        kpops.deploy(
-            pipeline_path=pipeline_file_path,
-            dotenv=dotenv,
-            config=config,
-            steps=parse_steps(steps),
-            filter_type=filter_type,
-            environment=environment,
-            dry_run=dry_run,
-            verbose=verbose,
-            parallel=parallel,
-        )
+    match operation_mode:
+        case OperationMode.MANAGED:
+            for pipeline_file_path in collect_pipeline_paths(pipeline_paths):
+                kpops.deploy(
+                    pipeline_path=pipeline_file_path,
+                    dotenv=dotenv,
+                    config=config,
+                    steps=parse_steps(steps),
+                    filter_type=filter_type,
+                    environment=environment,
+                    dry_run=dry_run,
+                    verbose=verbose,
+                    parallel=parallel,
+                )
+        case _:
+            for pipeline_file_path in collect_pipeline_paths(pipeline_paths):
+                resources = kpops.manifest_deploy(
+                    pipeline_file_path,
+                    dotenv,
+                    config,
+                    parse_steps(steps),
+                    filter_type,
+                    environment,
+                    verbose,
+                    operation_mode,
+                )
+                for resource in resources:
+                    for rendered_manifest in resource:
+                        print_yaml(rendered_manifest)
 
 
 @app.command(help="Destroy pipeline steps")

kpops/component_handlers/helm_wrapper/helm.py

@@ -27,6 +27,7 @@
 
     from kpops.components.base_components.models.resource import Resource
 
+
 log = logging.getLogger("Helm")