From 2552334dd6f369207df056fd0b25f359fef740c9 Mon Sep 17 00:00:00 2001 From: Michel El Nacouzi Date: Wed, 28 Aug 2024 13:02:09 -0400 Subject: [PATCH] Add templating processor for native apps (#1503) --- RELEASE-NOTES.md | 1 + .../_plugins/nativeapp/codegen/compiler.py | 6 + .../setup/native_app_setup_processor.py | 2 +- .../codegen/templates/templates_processor.py | 93 ++++++++ .../cli/_plugins/nativeapp/exceptions.py | 6 +- src/snowflake/cli/api/commands/flags.py | 2 +- src/snowflake/cli/api/entities/utils.py | 6 +- .../rendering/project_definition_templates.py | 2 +- .../cli/api/rendering/sql_templates.py | 11 +- .../cli/api/utils/definition_rendering.py | 10 +- tests/__snapshots__/test_help_messages.ambr | 30 +-- tests/__snapshots__/test_sql.ambr | 2 +- tests/api/utils/test_definition_rendering.py | 4 +- .../templating/test_templates_processor.py | 215 ++++++++++++++++++ tests/nativeapp/test_package_scripts.py | 6 +- .../nativeapp/test_project_templating.py | 86 +++++++ .../app/README.md | 4 + .../app/another_script.sql | 6 + .../app/manifest.yml | 9 + .../app/nesteddir/testfile.txt | 1 + .../app/setup_script.sql | 10 + .../snowflake.yml | 13 ++ .../app/README.md | 4 + .../app/another_script.sql | 6 + .../app/manifest.yml | 9 + .../app/nesteddir/testfile.txt | 1 + .../app/setup_script.sql | 10 + .../snowflake.yml | 20 ++ 28 files changed, 535 insertions(+), 40 deletions(-) create mode 100644 src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py create mode 100644 tests/nativeapp/codegen/templating/test_templates_processor.py create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v1/app/README.md create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v1/app/another_script.sql create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v1/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v1/app/nesteddir/testfile.txt create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v1/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v1/snowflake.yml create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v2/app/README.md create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v2/app/another_script.sql create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v2/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v2/app/nesteddir/testfile.txt create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v2/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_templates_processors_v2/snowflake.yml diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 6b812e54d9..ab06308bca 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -20,6 +20,7 @@ ## Deprecations ## New additions +* Added templates expansion of arbitrary files for Native Apps through `templates` processor. ## Fixes and improvements diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py b/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py index 72e677ca3a..98018b6b78 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py @@ -18,6 +18,7 @@ import re from typing import Dict, Optional +from click import ClickException from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import ( ArtifactProcessor, @@ -29,6 +30,9 @@ from snowflake.cli._plugins.nativeapp.codegen.snowpark.python_processor import ( SnowparkAnnotationProcessor, ) +from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import ( + TemplatesProcessor, +) from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.project.schemas.native_app.path_mapping import ( @@ -37,10 +41,12 @@ SNOWPARK_PROCESSOR = "snowpark" NA_SETUP_PROCESSOR = "native app setup" +TEMPLATES_PROCESSOR = "templates" _REGISTERED_PROCESSORS_BY_NAME = { SNOWPARK_PROCESSOR: SnowparkAnnotationProcessor, NA_SETUP_PROCESSOR: NativeAppSetupProcessor, + TEMPLATES_PROCESSOR: TemplatesProcessor, } diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py index fc80246622..2ef3ef4964 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py @@ -94,7 +94,7 @@ def process( self._create_or_update_sandbox() - cc.phase("Processing Python setup files") + cc.step("Processing Python setup files") files_to_process = [] for src_file, dest_file in bundle_map.all_mappings( diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py new file mode 100644 index 0000000000..3078ee77fd --- /dev/null +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -0,0 +1,93 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Optional + +import jinja2 +from snowflake.cli._plugins.nativeapp.artifacts import BundleMap +from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import ( + ArtifactProcessor, +) +from snowflake.cli._plugins.nativeapp.exceptions import InvalidTemplateInFileError +from snowflake.cli.api.cli_global_context import get_cli_context +from snowflake.cli.api.console import cli_console as cc +from snowflake.cli.api.project.schemas.native_app.path_mapping import ( + PathMapping, + ProcessorMapping, +) +from snowflake.cli.api.rendering.project_definition_templates import ( + get_client_side_jinja_env, +) +from snowflake.cli.api.rendering.sql_templates import ( + choose_sql_jinja_env_based_on_template_syntax, +) + + +class TemplatesProcessor(ArtifactProcessor): + """ + Processor class to perform template expansion on all relevant artifacts (specified in the project definition file). + """ + + def process( + self, + artifact_to_process: PathMapping, + processor_mapping: Optional[ProcessorMapping], + **kwargs, + ): + """ + Process the artifact by executing the template expansion logic on it. + """ + cc.step(f"Processing artifact {artifact_to_process} with templates processor") + + bundle_map = BundleMap( + project_root=self._bundle_ctx.project_root, + deploy_root=self._bundle_ctx.deploy_root, + ) + bundle_map.add(artifact_to_process) + + for src, dest in bundle_map.all_mappings( + absolute=True, + expand_directories=True, + ): + if src.is_dir(): + continue + with self.edit_file(dest) as f: + file_name = src.relative_to(self._bundle_ctx.project_root) + + jinja_env = ( + choose_sql_jinja_env_based_on_template_syntax( + f.contents, reference_name=file_name + ) + if dest.name.lower().endswith(".sql") + else get_client_side_jinja_env() + ) + + try: + expanded_template = jinja_env.from_string(f.contents).render( + get_cli_context().template_context + ) + + # For now, we are printing the source file path in the error message + # instead of the destination file path to make it easier for the user + # to identify the file that has the error, and edit the correct file. + except jinja2.TemplateSyntaxError as e: + raise InvalidTemplateInFileError(file_name, e, e.lineno) from e + + except jinja2.UndefinedError as e: + raise InvalidTemplateInFileError(file_name, e) from e + + if expanded_template != f.contents: + f.edited_contents = expanded_template diff --git a/src/snowflake/cli/_plugins/nativeapp/exceptions.py b/src/snowflake/cli/_plugins/nativeapp/exceptions.py index 108ff69d15..167ce415ac 100644 --- a/src/snowflake/cli/_plugins/nativeapp/exceptions.py +++ b/src/snowflake/cli/_plugins/nativeapp/exceptions.py @@ -64,15 +64,15 @@ def __init__(self, relpath: str): super().__init__(f'Script "{relpath}" does not exist') -class InvalidScriptError(ClickException): - """A referenced script had syntax error(s).""" +class InvalidTemplateInFileError(ClickException): + """A referenced templated file had syntax error(s).""" def __init__( self, relpath: str, err: jinja2.TemplateError, lineno: Optional[int] = None ): lineno_str = f":{lineno}" if lineno is not None else "" super().__init__( - f'Script "{relpath}{lineno_str}" does not contain a valid template: {err.message}' + f'File "{relpath}{lineno_str}" does not contain a valid template: {err.message}' ) self.err = err diff --git a/src/snowflake/cli/api/commands/flags.py b/src/snowflake/cli/api/commands/flags.py index 40f1ff08c6..d4677395cc 100644 --- a/src/snowflake/cli/api/commands/flags.py +++ b/src/snowflake/cli/api/commands/flags.py @@ -506,7 +506,7 @@ def project_env_overrides_callback(env_overrides_args_list: list[str]) -> None: return typer.Option( [], "--env", - help="String in format of key=value. Overrides variables from env section used for templating.", + help="String in format of key=value. Overrides variables from env section used for templates.", callback=_callback(lambda: project_env_overrides_callback), show_default=False, ) diff --git a/src/snowflake/cli/api/entities/utils.py b/src/snowflake/cli/api/entities/utils.py index 19e8ca7b17..a2d6797a9d 100644 --- a/src/snowflake/cli/api/entities/utils.py +++ b/src/snowflake/cli/api/entities/utils.py @@ -11,7 +11,7 @@ ) from snowflake.cli._plugins.nativeapp.constants import OWNER_COL from snowflake.cli._plugins.nativeapp.exceptions import ( - InvalidScriptError, + InvalidTemplateInFileError, MissingScriptError, UnexpectedOwnerError, ) @@ -313,9 +313,9 @@ def render_script_templates( raise MissingScriptError(relpath) from e except jinja2.TemplateSyntaxError as e: - raise InvalidScriptError(relpath, e, e.lineno) from e + raise InvalidTemplateInFileError(relpath, e, e.lineno) from e except jinja2.UndefinedError as e: - raise InvalidScriptError(relpath, e) from e + raise InvalidTemplateInFileError(relpath, e) from e return scripts_contents diff --git a/src/snowflake/cli/api/rendering/project_definition_templates.py b/src/snowflake/cli/api/rendering/project_definition_templates.py index 3e5774b27e..90aa4c87a2 100644 --- a/src/snowflake/cli/api/rendering/project_definition_templates.py +++ b/src/snowflake/cli/api/rendering/project_definition_templates.py @@ -24,7 +24,7 @@ _YML_TEMPLATE_END = "%>" -def get_project_definition_cli_jinja_env() -> Environment: +def get_client_side_jinja_env() -> Environment: _random_block = "___very___unique___block___to___disable___logic___blocks___" return env_bootstrap( IgnoreAttrEnvironment( diff --git a/src/snowflake/cli/api/rendering/sql_templates.py b/src/snowflake/cli/api/rendering/sql_templates.py index f832417670..bb260353bf 100644 --- a/src/snowflake/cli/api/rendering/sql_templates.py +++ b/src/snowflake/cli/api/rendering/sql_templates.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Dict +from typing import Dict, Optional from click import ClickException from jinja2 import Environment, StrictUndefined, loaders, meta @@ -55,19 +55,22 @@ def _does_template_have_env_syntax(env: Environment, template_content: str) -> b return bool(meta.find_undeclared_variables(template)) -def choose_sql_jinja_env_based_on_template_syntax(template_content: str) -> Environment: +def choose_sql_jinja_env_based_on_template_syntax( + template_content: str, reference_name: Optional[str] = None +) -> Environment: old_syntax_env = _get_sql_jinja_env(_OLD_SQL_TEMPLATE_START, _OLD_SQL_TEMPLATE_END) new_syntax_env = _get_sql_jinja_env(_SQL_TEMPLATE_START, _SQL_TEMPLATE_END) has_old_syntax = _does_template_have_env_syntax(old_syntax_env, template_content) has_new_syntax = _does_template_have_env_syntax(new_syntax_env, template_content) + reference_name_str = f" in {reference_name}" if reference_name else "" if has_old_syntax and has_new_syntax: raise InvalidTemplate( - f"The SQL query mixes {_OLD_SQL_TEMPLATE_START} ... {_OLD_SQL_TEMPLATE_END} syntax" + f"The SQL query{reference_name_str} mixes {_OLD_SQL_TEMPLATE_START} ... {_OLD_SQL_TEMPLATE_END} syntax" f" and {_SQL_TEMPLATE_START} ... {_SQL_TEMPLATE_END} syntax." ) if has_old_syntax: cli_console.warning( - f"Warning: {_OLD_SQL_TEMPLATE_START} ... {_OLD_SQL_TEMPLATE_END} syntax is deprecated." + f"Warning: {_OLD_SQL_TEMPLATE_START} ... {_OLD_SQL_TEMPLATE_END} syntax{reference_name_str} is deprecated." f" Use {_SQL_TEMPLATE_START} ... {_SQL_TEMPLATE_END} syntax instead." ) return old_syntax_env diff --git a/src/snowflake/cli/api/utils/definition_rendering.py b/src/snowflake/cli/api/utils/definition_rendering.py index c5023e3029..9716e4186c 100644 --- a/src/snowflake/cli/api/utils/definition_rendering.py +++ b/src/snowflake/cli/api/utils/definition_rendering.py @@ -28,7 +28,7 @@ from snowflake.cli.api.project.schemas.updatable_model import context from snowflake.cli.api.rendering.jinja import CONTEXT_KEY, FUNCTION_KEY from snowflake.cli.api.rendering.project_definition_templates import ( - get_project_definition_cli_jinja_env, + get_client_side_jinja_env, ) from snowflake.cli.api.utils.dict_utils import deep_merge_dicts, traverse from snowflake.cli.api.utils.graph import Graph, Node @@ -96,7 +96,7 @@ def _get_referenced_vars( ) or current_attr_chain is not None ): - raise InvalidTemplate(f"Unexpected templating syntax in {template_value}") + raise InvalidTemplate(f"Unexpected template syntax in {template_value}") for child_node in ast_node.iter_child_nodes(): all_referenced_vars.update( @@ -318,7 +318,7 @@ def render_definition_template( if definition is None: return ProjectProperties(None, {CONTEXT_KEY: {"env": environment_overrides}}) - template_env = TemplatedEnvironment(get_project_definition_cli_jinja_env()) + template_env = TemplatedEnvironment(get_client_side_jinja_env()) if "definition_version" not in definition or Version( definition["definition_version"] @@ -353,9 +353,7 @@ def render_definition_template( ) def on_cycle_action(node: Node[TemplateVar]): - raise CycleDetectedError( - f"Cycle detected in templating variable {node.data.key}" - ) + raise CycleDetectedError(f"Cycle detected in template variable {node.data.key}") dependencies_graph.dfs( visit_action=lambda node: _render_graph_node(template_env, node), diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 2d15cb2525..0ead0779b1 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -51,7 +51,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -113,7 +113,7 @@ | working directory. | | --env TEXT String in format of key=value. | | Overrides variables from env | - | section used for templating. | + | section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -185,7 +185,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -308,7 +308,7 @@ | key=value. Overrides | | variables from env | | section used for | - | templating. | + | templates. | | --help -h Show this message and | | exit. | +------------------------------------------------------------------------------+ @@ -467,7 +467,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -619,7 +619,7 @@ | key=value. Overrides | | variables from env | | section used for | - | templating. | + | templates. | | --help -h Show this message and | | exit. | +------------------------------------------------------------------------------+ @@ -715,7 +715,7 @@ | working directory. | | --env TEXT String in format of key=value. | | Overrides variables from env | - | section used for templating. | + | section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -787,7 +787,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -907,7 +907,7 @@ | key=value. Overrides | | variables from env | | section used for | - | templating. | + | templates. | | --help -h Show this message and | | exit. | +------------------------------------------------------------------------------+ @@ -1006,7 +1006,7 @@ | working directory. | | --env TEXT String in format of key=value. | | Overrides variables from env | - | section used for templating. | + | section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -1078,7 +1078,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -2952,7 +2952,7 @@ | directory. | | --env TEXT String in format of key=value. | | Overrides variables from env section | - | used for templating. | + | used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -3030,7 +3030,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -6400,7 +6400,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ @@ -7217,7 +7217,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ diff --git a/tests/__snapshots__/test_sql.ambr b/tests/__snapshots__/test_sql.ambr index 82a475d3b1..0fd7ae1bfd 100644 --- a/tests/__snapshots__/test_sql.ambr +++ b/tests/__snapshots__/test_sql.ambr @@ -22,7 +22,7 @@ | --project -p TEXT Path where Snowflake project resides. Defaults to | | current working directory. | | --env TEXT String in format of key=value. Overrides variables | - | from env section used for templating. | + | from env section used for templates. | | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ diff --git a/tests/api/utils/test_definition_rendering.py b/tests/api/utils/test_definition_rendering.py index f09c314471..12747051ad 100644 --- a/tests/api/utils/test_definition_rendering.py +++ b/tests/api/utils/test_definition_rendering.py @@ -336,7 +336,7 @@ def test_resolve_variables_error_on_cycle(definition): with pytest.raises(CycleDetectedError) as err: render_definition_template(definition, {}) - assert err.value.message.startswith("Cycle detected in templating variable ") + assert err.value.message.startswith("Cycle detected in template variable ") @pytest.mark.parametrize( @@ -534,7 +534,7 @@ def test_invalid_templating_syntax(template_value): with pytest.raises(InvalidTemplate) as err: render_definition_template(definition, {}) - assert err.value.message == f"Unexpected templating syntax in {template_value}" + assert err.value.message == f"Unexpected template syntax in {template_value}" def test_invalid_type_for_env_section(): diff --git a/tests/nativeapp/codegen/templating/test_templates_processor.py b/tests/nativeapp/codegen/templating/test_templates_processor.py new file mode 100644 index 0000000000..e4998b20f0 --- /dev/null +++ b/tests/nativeapp/codegen/templating/test_templates_processor.py @@ -0,0 +1,215 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest import mock + +import pytest +from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext +from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import ( + TemplatesProcessor, +) +from snowflake.cli._plugins.nativeapp.exceptions import InvalidTemplateInFileError +from snowflake.cli.api.exceptions import InvalidTemplate +from snowflake.cli.api.project.schemas.native_app.path_mapping import PathMapping + +from tests.nativeapp.utils import CLI_GLOBAL_TEMPLATE_CONTEXT + + +@dataclass +class BundleResult: + """ + Dataclass to hold the test setup result + """ + + artifact_to_process: PathMapping + bundle_ctx: BundleContext + output_files: list[Path] + + +def bundle_files( + tmp_dir: str, file_names: list[str], file_contents: list[str] +) -> BundleResult: + project_root = Path(tmp_dir) + + deploy_root = Path(tmp_dir) / "output" / "deploy" + deploy_root.mkdir(parents=True, exist_ok=True) + + src_root = project_root / "src" + src_root.mkdir(parents=True, exist_ok=True) + + for index, file_name in enumerate(file_names): + test_file = src_root / file_name + test_file.write_text(file_contents[index]) + + # create a symlink to the test file from the deploy directory: + output_files = [] + for file_name in file_names: + deploy_file = deploy_root / file_name + output_files.append(deploy_file) + deploy_file.symlink_to(src_root / file_name) + + artifact_to_process = PathMapping(src="src/*", dest="./", processors=["templates"]) + + bundle_context = BundleContext( + package_name="test_package_name", + project_root=project_root, + artifacts=[artifact_to_process], + bundle_root=deploy_root / "bundle", + generated_root=deploy_root / "generated", + deploy_root=deploy_root, + ) + + return BundleResult(artifact_to_process, bundle_context, output_files) + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {}) +def test_templates_processor_valid_files_no_templates(): + file_names = ["test_file.txt"] + file_contents = ["This is a test file\n with some content"] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_names, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + templates_processor.process(bundle_result.artifact_to_process, None) + + assert bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0] + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {"ctx": {"env": {"TEST_VAR": "test_value"}}}) +def test_one_file_with_template_and_one_without(): + file_names = ["test_file.txt", "test_file_with_template.txt"] + file_contents = [ + "This is a test file\n with some content", + "This is a test file\n with some <% ctx.env.TEST_VAR %>", + ] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_names, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + templates_processor.process(bundle_result.artifact_to_process, None) + + assert bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0] + + assert not bundle_result.output_files[1].is_symlink() + assert bundle_result.output_files[1].read_text() == file_contents[1].replace( + "<% ctx.env.TEST_VAR %>", "test_value" + ) + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {"ctx": {"native_app": {"name": "test_app"}}}) +def test_templates_with_sql_and_non_sql_files_and_mix_syntax(): + file_names = ["test_sql.sql", "test_non_sql.txt"] + file_contents = [ + "This is a sql file with &{ ctx.native_app.name }", + "This is a non sql file with <% ctx.native_app.name %>", + ] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_names, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + templates_processor.process(bundle_result.artifact_to_process, None) + + assert not bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0].replace( + "&{ ctx.native_app.name }", "test_app" + ) + + assert not bundle_result.output_files[1].is_symlink() + assert bundle_result.output_files[1].read_text() == file_contents[1].replace( + "<% ctx.native_app.name %>", "test_app" + ) + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {"ctx": {"env": {"name": "test_name"}}}) +def test_templates_with_sql_new_syntax(): + file_names = ["test_sql.sql"] + file_contents = ["This is a sql file with <% ctx.env.name %>"] + + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_names, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + templates_processor.process(bundle_result.artifact_to_process, None) + + assert not bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0].replace( + "<% ctx.env.name %>", "test_name" + ) + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {"ctx": {"env": {"name": "test_name"}}}) +def test_templates_with_sql_old_syntax(): + file_names = ["test_sql.sql"] + file_contents = ["This is a sql file with &{ ctx.env.name }"] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_names, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + templates_processor.process(bundle_result.artifact_to_process, None) + + assert not bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0].replace( + "&{ ctx.env.name }", "test_name" + ) + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {"ctx": {"env": {"name": "test_name"}}}) +def test_templates_with_sql_both_old_and_new_syntax(): + file_names = ["test_sql.sql"] + file_contents = ["This is a sql file with &{ ctx.env.name } and <% ctx.env.name %>"] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_names, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + + with pytest.raises(InvalidTemplate) as e: + templates_processor.process(bundle_result.artifact_to_process, None) + + assert "mixes &{ ... } syntax and <% ... %> syntax." in str(e.value) + assert bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0] + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {}) +def test_file_with_syntax_error(): + file_name = ["test_file.txt"] + file_contents = ["This is a test file with invalid <% ctx.env.TEST_VAR"] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_name, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + + with pytest.raises(InvalidTemplateInFileError) as e: + templates_processor.process(bundle_result.artifact_to_process, None) + + assert "does not contain a valid template" in str(e.value) + assert bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0] + + +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, {}) +def test_file_with_undefined_variable(): + file_name = ["test_file.txt"] + file_contents = ["This is a test file with invalid <% ctx.env.TEST_VAR %>"] + with TemporaryDirectory() as tmp_dir: + bundle_result = bundle_files(tmp_dir, file_name, file_contents) + templates_processor = TemplatesProcessor(bundle_ctx=bundle_result.bundle_ctx) + + with pytest.raises(InvalidTemplateInFileError) as e: + templates_processor.process(bundle_result.artifact_to_process, None) + + assert "'ctx' is undefined" in str(e.value) + assert "does not contain a valid template" in str(e.value) + assert bundle_result.output_files[0].is_symlink() + assert bundle_result.output_files[0].read_text() == file_contents[0] diff --git a/tests/nativeapp/test_package_scripts.py b/tests/nativeapp/test_package_scripts.py index ed18f0708e..2ead5ad686 100644 --- a/tests/nativeapp/test_package_scripts.py +++ b/tests/nativeapp/test_package_scripts.py @@ -19,7 +19,7 @@ import pytest from click import ClickException from snowflake.cli._plugins.nativeapp.exceptions import ( - InvalidScriptError, + InvalidTemplateInFileError, MissingScriptError, ) from snowflake.cli._plugins.nativeapp.run_processor import NativeAppRunProcessor @@ -217,7 +217,7 @@ def test_invalid_package_script(mock_conn, mock_execute, project_definition_file mock_conn.return_value = MockConnectionCtx() working_dir: Path = project_definition_files[0].parent native_app_manager = _get_na_manager(str(working_dir)) - with pytest.raises(InvalidScriptError): + with pytest.raises(InvalidTemplateInFileError): second_file = working_dir / "002-shared.sql" second_file.unlink() second_file.write_text("select * from {{ package_name") @@ -236,7 +236,7 @@ def test_undefined_var_package_script( mock_conn.return_value = MockConnectionCtx() working_dir: Path = project_definition_files[0].parent native_app_manager = _get_na_manager(str(working_dir)) - with pytest.raises(InvalidScriptError): + with pytest.raises(InvalidTemplateInFileError): second_file = working_dir / "001-shared.sql" second_file.unlink() second_file.write_text("select * from {{ abc }}") diff --git a/tests_integration/nativeapp/test_project_templating.py b/tests_integration/nativeapp/test_project_templating.py index f698842906..ad8753edda 100644 --- a/tests_integration/nativeapp/test_project_templating.py +++ b/tests_integration/nativeapp/test_project_templating.py @@ -18,6 +18,9 @@ contains_row_with, row_from_snowflake_session, ) +from tests_integration.testing_utils.working_directory_utils import ( + WorkingDirectoryChanger, +) # Tests a simple flow of native app with template reading env variables from OS @@ -335,3 +338,86 @@ def test_nativeapp_project_templating_bundle_deploy_successful( env=local_test_env, ) assert result.exit_code == 0 + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_project", ["napp_templates_processors_v1", "napp_templates_processors_v2"] +) +@pytest.mark.parametrize("with_project_flag", [True, False]) +def test_nativeapp_templates_processor_with_run( + runner, + snowflake_session, + default_username, + resource_suffix, + nativeapp_project_directory, + test_project, + with_project_flag, +): + project_name = "myapp" + app_name = f"{project_name}_{default_username}{resource_suffix}" + + with nativeapp_project_directory(test_project) as tmp_dir: + project_args = ["--project", f"{tmp_dir}"] if with_project_flag else [] + + if with_project_flag: + working_directory_changer = WorkingDirectoryChanger() + working_directory_changer.change_working_directory_to("app") + try: + result = runner.invoke_with_connection_json( + ["app", "run"] + project_args, + env={ + "schema_name": "test_schema", + "table_name": "test_table", + "value": "test_value", + }, + ) + assert result.exit_code == 0 + + result = row_from_snowflake_session( + snowflake_session.execute_string( + f"select * from {app_name}.test_schema.test_table", + ) + ) + assert result == [{"NAME": "test_value"}] + + finally: + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force"] + project_args + ) + assert result.exit_code == 0 + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_project", ["napp_templates_processors_v1", "napp_templates_processors_v2"] +) +@pytest.mark.parametrize("with_project_flag", [True, False]) +def test_nativeapp_templates_processor_with_deploy( + runner, + nativeapp_project_directory, + test_project, + with_project_flag, +): + + with nativeapp_project_directory(test_project) as tmp_dir: + project_args = ["--project", f"{tmp_dir}"] if with_project_flag else [] + + if with_project_flag: + working_directory_changer = WorkingDirectoryChanger() + working_directory_changer.change_working_directory_to("app") + + result = runner.invoke_with_connection_json( + ["app", "deploy"] + project_args, + env={ + "schema_name": "test_schema", + "table_name": "test_table", + "value": "test_value", + }, + ) + assert result.exit_code == 0 + + with open( + tmp_dir / "output" / "deploy" / "another_script.sql", "r", encoding="utf-8" + ) as f: + assert "test_value" in f.read() diff --git a/tests_integration/test_data/projects/napp_templates_processors_v1/app/README.md b/tests_integration/test_data/projects/napp_templates_processors_v1/app/README.md new file mode 100644 index 0000000000..f66bf75c9b --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v1/app/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains an extremely simple application that is used for +integration testing SnowCLI. diff --git a/tests_integration/test_data/projects/napp_templates_processors_v1/app/another_script.sql b/tests_integration/test_data/projects/napp_templates_processors_v1/app/another_script.sql new file mode 100644 index 0000000000..9111a30ba4 --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v1/app/another_script.sql @@ -0,0 +1,6 @@ +-- This file uses old templates syntax +CREATE OR REPLACE TABLE &{ ctx.env.schema_name }.&{ ctx.env.table_name } ( + name STRING +); + +insert into &{ ctx.env.schema_name }.&{ ctx.env.table_name } values ('&{ ctx.env.value }'); diff --git a/tests_integration/test_data/projects/napp_templates_processors_v1/app/manifest.yml b/tests_integration/test_data/projects/napp_templates_processors_v1/app/manifest.yml new file mode 100644 index 0000000000..5b8ef74e8a --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v1/app/manifest.yml @@ -0,0 +1,9 @@ +# This is a manifest.yml file, a required component of creating a Snowflake Native App. +# This file defines properties required by the application package, including the location of the setup script and version definitions. +# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file. + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md diff --git a/tests_integration/test_data/projects/napp_templates_processors_v1/app/nesteddir/testfile.txt b/tests_integration/test_data/projects/napp_templates_processors_v1/app/nesteddir/testfile.txt new file mode 100644 index 0000000000..d9d045443f --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v1/app/nesteddir/testfile.txt @@ -0,0 +1 @@ +Test file to test templates diff --git a/tests_integration/test_data/projects/napp_templates_processors_v1/app/setup_script.sql b/tests_integration/test_data/projects/napp_templates_processors_v1/app/setup_script.sql new file mode 100644 index 0000000000..2aae4f834e --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v1/app/setup_script.sql @@ -0,0 +1,10 @@ +-- This is the setup script that runs while installing a Snowflake Native App in a consumer account. +-- To write this script, you can familiarize yourself with some of the following concepts: +-- Application Roles +-- Versioned Schemas +-- UDFs/Procs +-- Extension Code +-- Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script for a detailed understanding of this file. + +CREATE OR ALTER VERSIONED SCHEMA <% ctx.env.schema_name %>; +EXECUTE IMMEDIATE from '/another_script.sql'; diff --git a/tests_integration/test_data/projects/napp_templates_processors_v1/snowflake.yml b/tests_integration/test_data/projects/napp_templates_processors_v1/snowflake.yml new file mode 100644 index 0000000000..ef3a17bd51 --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v1/snowflake.yml @@ -0,0 +1,13 @@ +definition_version: 1.1 +native_app: + name: myapp + + artifacts: + - src: app/* + dest: ./ + processors: + - templates + +env: + schema: app_schema + pkg_schema: pkg_schema diff --git a/tests_integration/test_data/projects/napp_templates_processors_v2/app/README.md b/tests_integration/test_data/projects/napp_templates_processors_v2/app/README.md new file mode 100644 index 0000000000..f66bf75c9b --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v2/app/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains an extremely simple application that is used for +integration testing SnowCLI. diff --git a/tests_integration/test_data/projects/napp_templates_processors_v2/app/another_script.sql b/tests_integration/test_data/projects/napp_templates_processors_v2/app/another_script.sql new file mode 100644 index 0000000000..9111a30ba4 --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v2/app/another_script.sql @@ -0,0 +1,6 @@ +-- This file uses old templates syntax +CREATE OR REPLACE TABLE &{ ctx.env.schema_name }.&{ ctx.env.table_name } ( + name STRING +); + +insert into &{ ctx.env.schema_name }.&{ ctx.env.table_name } values ('&{ ctx.env.value }'); diff --git a/tests_integration/test_data/projects/napp_templates_processors_v2/app/manifest.yml b/tests_integration/test_data/projects/napp_templates_processors_v2/app/manifest.yml new file mode 100644 index 0000000000..5b8ef74e8a --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v2/app/manifest.yml @@ -0,0 +1,9 @@ +# This is a manifest.yml file, a required component of creating a Snowflake Native App. +# This file defines properties required by the application package, including the location of the setup script and version definitions. +# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file. + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md diff --git a/tests_integration/test_data/projects/napp_templates_processors_v2/app/nesteddir/testfile.txt b/tests_integration/test_data/projects/napp_templates_processors_v2/app/nesteddir/testfile.txt new file mode 100644 index 0000000000..d9d045443f --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v2/app/nesteddir/testfile.txt @@ -0,0 +1 @@ +Test file to test templates diff --git a/tests_integration/test_data/projects/napp_templates_processors_v2/app/setup_script.sql b/tests_integration/test_data/projects/napp_templates_processors_v2/app/setup_script.sql new file mode 100644 index 0000000000..2aae4f834e --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v2/app/setup_script.sql @@ -0,0 +1,10 @@ +-- This is the setup script that runs while installing a Snowflake Native App in a consumer account. +-- To write this script, you can familiarize yourself with some of the following concepts: +-- Application Roles +-- Versioned Schemas +-- UDFs/Procs +-- Extension Code +-- Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script for a detailed understanding of this file. + +CREATE OR ALTER VERSIONED SCHEMA <% ctx.env.schema_name %>; +EXECUTE IMMEDIATE from '/another_script.sql'; diff --git a/tests_integration/test_data/projects/napp_templates_processors_v2/snowflake.yml b/tests_integration/test_data/projects/napp_templates_processors_v2/snowflake.yml new file mode 100644 index 0000000000..8f74891531 --- /dev/null +++ b/tests_integration/test_data/projects/napp_templates_processors_v2/snowflake.yml @@ -0,0 +1,20 @@ +# This is the v2 version of the "napp_application_post_deploy_v1" project definition +definition_version: 2 +entities: + pkg: + type: application package + identifier: myapp_pkg_<% ctx.env.USER %> + artifacts: + - src: app/* + dest: ./ + processors: + - templates + manifest: app/manifest.yml + app: + type: application + identifier: myapp_<% ctx.env.USER %> + from: + target: pkg +env: + schema: app_schema + pkg_schema: pkg_schema