diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index feb6d9161..3c46cb11b 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -19,6 +19,7 @@ $ kpops [OPTIONS] COMMAND [ARGS]... * `deploy`: Deploy pipeline steps * `destroy`: Destroy pipeline steps * `generate`: Generate enriched pipeline representation +* `init`: Initialize a new KPOps project. * `manifest`: Render final resource representation * `reset`: Reset pipeline steps * `schema`: Generate JSON schema. @@ -126,6 +127,25 @@ $ kpops generate [OPTIONS] PIPELINE_PATH * `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--help`: Show this message and exit. +## `kpops init` + +Initialize a new KPOps project. + +**Usage**: + +```console +$ kpops init [OPTIONS] PATH +``` + +**Arguments**: + +* `PATH`: Path for a new KPOps project. It should lead to an empty (or non-existent) directory. The part of the path that doesn't exist will be created. [required] + +**Options**: + +* `--config-include-opt / --no-config-include-opt`: Whether to include non-required settings in the generated 'config.yaml' [default: no-config-include-opt] +* `--help`: Show this message and exit. + ## `kpops manifest` In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests. diff --git a/hooks/gen_docs/gen_docs_env_vars.py b/hooks/gen_docs/gen_docs_env_vars.py index 909c4021f..f0cd71a74 100644 --- a/hooks/gen_docs/gen_docs_env_vars.py +++ b/hooks/gen_docs/gen_docs_env_vars.py @@ -228,7 +228,6 @@ def write_csv_to_md_file( :param source: path to csv file to read from :param target: path to md file to overwrite or create :param title: Title for the table, optional - """ if heading: heading += " " diff --git a/kpops/__init__.py b/kpops/__init__.py index 4422c15e6..08daf2920 100644 --- a/kpops/__init__.py +++ b/kpops/__init__.py @@ -1,7 +1,7 @@ __version__ = "4.0.2" # export public API functions -from kpops.cli.main import clean, deploy, destroy, generate, manifest, reset +from kpops.cli.main import clean, deploy, destroy, generate, init, manifest, reset __all__ = ( "generate", @@ -10,4 +10,5 @@ "destroy", "reset", "clean", + "init", ) diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 05c7f41b2..6a55e27eb 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -22,6 +22,7 @@ from kpops.components.base_components.models.resource import Resource from kpops.config import ENV_PREFIX, KpopsConfig from kpops.pipeline import ComponentFilterPredicate, Pipeline, PipelineGenerator +from kpops.utils.cli_commands import init_project from kpops.utils.gen_schema import ( SchemaScope, gen_config_schema, @@ -71,7 +72,22 @@ help="Path to YAML with pipeline definition", ) -PIPELINE_STEPS: str | None = typer.Option( +PROJECT_PATH: Path = typer.Argument( + default=..., + exists=False, + file_okay=False, + dir_okay=True, + readable=True, + resolve_path=True, + help="Path for a new KPOps project. It should lead to an empty (or non-existent) directory. The part of the path that doesn't exist will be created.", +) + +CONFIG_INCLUDE_OPTIONAL: bool = typer.Option( + default=False, + help="Whether to include non-required settings in the generated 'config.yaml'", +) + +PIPELINE_STEPS: Optional[str] = typer.Option( default=None, envvar=f"{ENV_PREFIX}PIPELINE_STEPS", help="Comma separated list of steps to apply the command on", @@ -109,6 +125,7 @@ ), ) + logger = logging.getLogger() logging.getLogger("httpx").setLevel(logging.WARNING) stream_handler = logging.StreamHandler() @@ -189,6 +206,21 @@ def create_kpops_config( ) +@app.command( # pyright: ignore[reportCallIssue] https://github.com/rec/dtyper/issues/8 + help="Initialize a new KPOps project." +) +def init( + path: Path = PROJECT_PATH, + config_include_opt: bool = CONFIG_INCLUDE_OPTIONAL, +): + if not path.exists(): + path.mkdir(parents=False) + elif next(path.iterdir(), False): + log.warning("Please provide a path to an empty directory.") + return + init_project(path, config_include_opt) + + @app.command( # pyright: ignore[reportCallIssue] https://github.com/rec/dtyper/issues/8 help=""" Generate JSON schema. diff --git a/kpops/utils/cli_commands.py b/kpops/utils/cli_commands.py new file mode 100644 index 000000000..27cfc6b6d --- /dev/null +++ b/kpops/utils/cli_commands.py @@ -0,0 +1,78 @@ +from pathlib import Path +from typing import Any + +import yaml +from pydantic import BaseModel +from pydantic.fields import FieldInfo +from pydantic_core import PydanticUndefined + +from hooks.gen_docs.gen_docs_env_vars import collect_fields +from kpops.config import KpopsConfig +from kpops.utils.docstring import describe_object +from kpops.utils.json import is_jsonable +from kpops.utils.pydantic import issubclass_patched + + +def extract_config_fields_for_yaml( + fields: dict[str, Any], required: bool +) -> dict[str, Any]: + """Return only (non-)required fields and their respective default values. + + :param fields: Dict containing the fields to be categorized. The key of a + record is the name of the field, the value is the field's type. + :param required: Whether to extract only the required fields or only the + non-required ones. + """ + extracted_fields = {} + for key, value in fields.items(): + if issubclass(type(value), FieldInfo): + if required and value.default in [PydanticUndefined, Ellipsis]: + extracted_fields[key] = None + elif not (required or value.default in [PydanticUndefined, Ellipsis]): + if is_jsonable(value.default): + extracted_fields[key] = value.default + elif issubclass_patched(value.default, BaseModel): + extracted_fields[key] = value.default.model_dump(mode="json") + else: + extracted_fields[key] = str(value.default) + else: + extracted_fields[key] = extract_config_fields_for_yaml( + fields[key], required + ) + return extracted_fields + + +def create_config(file_name: str, dir_path: Path, include_optional: bool) -> None: + """Create a KPOps config yaml. + + :param file_name: Name for the file + :param dir_path: Directory in which the file should be created + :param include_optional: Whether to include non-required settings + """ + file_path = Path(dir_path / (file_name + ".yaml")) + file_path.touch(exist_ok=False) + with file_path.open(mode="w") as conf: + conf.write("# " + describe_object(KpopsConfig.__doc__)) # Write title + non_required = extract_config_fields_for_yaml( + collect_fields(KpopsConfig), False + ) + required = extract_config_fields_for_yaml(collect_fields(KpopsConfig), True) + for k in non_required: + required.pop(k, None) + conf.write("\n\n# Required fields\n") + conf.write(yaml.dump(required)) + if include_optional: + conf.write("\n# Non-required fields\n") + conf.write(yaml.dump(non_required)) + + +def init_project(path: Path, conf_incl_opt: bool): + """Initiate a default empty project. + + :param path: Directory in which the project should be initiated + :param conf_incl_opt: Whether to include non-required settings + in the generated config file + """ + create_config("config", path, conf_incl_opt) + Path(path / "pipeline.yaml").touch(exist_ok=False) + Path(path / "defaults.yaml").touch(exist_ok=False) diff --git a/kpops/utils/docstring.py b/kpops/utils/docstring.py index d5ca287d3..66c8572bf 100644 --- a/kpops/utils/docstring.py +++ b/kpops/utils/docstring.py @@ -59,6 +59,7 @@ def _trim_description_end(desc: str) -> str: desc_enders = [ ":param ", ":returns:", + ":raises:", "defaults to ", ] end_index = len(desc) diff --git a/kpops/utils/json.py b/kpops/utils/json.py new file mode 100644 index 000000000..9e0a58d77 --- /dev/null +++ b/kpops/utils/json.py @@ -0,0 +1,15 @@ +import json +from typing import Any + + +def is_jsonable(input: Any) -> bool: + """Check whether a value is json-serializable. + + :param input: Value to be checked. + """ + try: + json.dumps(input) + except (TypeError, OverflowError): + return False + else: + return True diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index fb5d715d3..1cb972492 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -109,8 +109,8 @@ def issubclass_patched( issubclass(BaseSettings, BaseModel) # True issubclass(set[str], BaseModel) # raises Exception - :param cls: class to check - :base: class(es) to check against, defaults to ``BaseModel`` + :param __cls: class to check + :param __class_or_tuple: class(es) to check against, defaults to ``BaseModel`` :return: Whether 'cls' is derived from another class or is the same class. """ try: diff --git a/poetry.lock b/poetry.lock index c347a8446..878e27212 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1353,6 +1353,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1360,8 +1361,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1378,6 +1386,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1385,6 +1394,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, diff --git a/tests/cli/snapshots/test_init/test_init_project/config_exclude_opt.yaml b/tests/cli/snapshots/test_init/test_init_project/config_exclude_opt.yaml new file mode 100644 index 000000000..230f8f025 --- /dev/null +++ b/tests/cli/snapshots/test_init/test_init_project/config_exclude_opt.yaml @@ -0,0 +1,4 @@ +# Global configuration for KPOps project. + +# Required fields +kafka_brokers: null diff --git a/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml b/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml new file mode 100644 index 000000000..337de0d96 --- /dev/null +++ b/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml @@ -0,0 +1,27 @@ +# Global configuration for KPOps project. + +# Required fields +kafka_brokers: null + +# Non-required fields +components_module: null +create_namespace: false +defaults_filename_prefix: defaults +helm_config: + api_version: null + context: null + debug: false +helm_diff_config: {} +kafka_connect: + url: http://localhost:8083/ +kafka_rest: + url: http://localhost:8082/ +pipeline_base_dir: . +retain_clean_jobs: false +schema_registry: + enabled: false + url: http://localhost:8081/ +timeout: 300 +topic_name_config: + default_error_topic_name: ${pipeline.name}-${component.name}-error + default_output_topic_name: ${pipeline.name}-${component.name} diff --git a/tests/cli/snapshots/test_init/test_init_project/defaults.yaml b/tests/cli/snapshots/test_init/test_init_project/defaults.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cli/snapshots/test_init/test_init_project/pipeline.yaml b/tests/cli/snapshots/test_init/test_init_project/pipeline.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py new file mode 100644 index 000000000..f3245f814 --- /dev/null +++ b/tests/cli/test_init.py @@ -0,0 +1,52 @@ +from pathlib import Path + +from pytest_snapshot.plugin import Snapshot +from typer.testing import CliRunner + +import kpops +from kpops.cli.main import app +from kpops.utils.cli_commands import create_config + +runner = CliRunner() + + +def test_create_config(tmp_path: Path): + opt_conf_name = "config_with_non_required" + req_conf_name = "config_with_only_required" + create_config(opt_conf_name, tmp_path, True) + create_config(req_conf_name, tmp_path, False) + assert (opt_conf := Path(tmp_path / (opt_conf_name + ".yaml"))).exists() + assert (req_conf := Path(tmp_path / (req_conf_name + ".yaml"))).exists() + assert len(opt_conf.read_text()) > len(req_conf.read_text()) + + +def test_init_project(tmp_path: Path, snapshot: Snapshot): + opt_path = tmp_path / "opt" + opt_path.mkdir() + kpops.init(opt_path, config_include_opt=False) + snapshot.assert_match( + Path(opt_path / "config.yaml").read_text(), "config_exclude_opt.yaml" + ) + snapshot.assert_match(Path(opt_path / "pipeline.yaml").read_text(), "pipeline.yaml") + snapshot.assert_match(Path(opt_path / "defaults.yaml").read_text(), "defaults.yaml") + + req_path = tmp_path / "req" + req_path.mkdir() + kpops.init(req_path, config_include_opt=True) + snapshot.assert_match( + Path(req_path / "config.yaml").read_text(), "config_include_opt.yaml" + ) + + +def test_init_project_from_cli_with_bad_path(tmp_path: Path): + bad_path = Path(tmp_path / "random_file.yaml") + bad_path.touch() + result = runner.invoke( + app, + [ + "init", + str(bad_path), + ], + catch_exceptions=False, + ) + assert result.exit_code == 2