Add templating processor for native apps (#1503)

RELEASE-NOTES.md

@@ -20,6 +20,7 @@
 ## Deprecations
 
 ## New additions
+* Added templates expansion of arbitrary files for Native Apps through `templates` processor.
 
 ## Fixes and improvements
 

src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py

@@ -18,6 +18,7 @@
 import re
 from typing import Dict, Optional
 
+from click import ClickException
 from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext
 from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import (
     ArtifactProcessor,
@@ -29,6 +30,9 @@
 from snowflake.cli._plugins.nativeapp.codegen.snowpark.python_processor import (
     SnowparkAnnotationProcessor,
 )
+from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import (
+    TemplatesProcessor,
+)
 from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag
 from snowflake.cli.api.console import cli_console as cc
 from snowflake.cli.api.project.schemas.native_app.path_mapping import (
@@ -37,10 +41,12 @@
 
 SNOWPARK_PROCESSOR = "snowpark"
 NA_SETUP_PROCESSOR = "native app setup"
+TEMPLATES_PROCESSOR = "templates"
 
 _REGISTERED_PROCESSORS_BY_NAME = {
     SNOWPARK_PROCESSOR: SnowparkAnnotationProcessor,
     NA_SETUP_PROCESSOR: NativeAppSetupProcessor,
+    TEMPLATES_PROCESSOR: TemplatesProcessor,
 }
 
 

src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py

@@ -94,7 +94,7 @@ def process(
 
         self._create_or_update_sandbox()
 
-        cc.phase("Processing Python setup files")
+        cc.step("Processing Python setup files")
 
         files_to_process = []
         for src_file, dest_file in bundle_map.all_mappings(

src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Optional
+
+import jinja2
+from snowflake.cli._plugins.nativeapp.artifacts import BundleMap
+from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import (
+    ArtifactProcessor,
+)
+from snowflake.cli._plugins.nativeapp.exceptions import InvalidTemplateInFileError
+from snowflake.cli.api.cli_global_context import get_cli_context
+from snowflake.cli.api.console import cli_console as cc
+from snowflake.cli.api.project.schemas.native_app.path_mapping import (
+    PathMapping,
+    ProcessorMapping,
+)
+from snowflake.cli.api.rendering.project_definition_templates import (
+    get_client_side_jinja_env,
+)
+from snowflake.cli.api.rendering.sql_templates import (
+    choose_sql_jinja_env_based_on_template_syntax,
+)
+
+
+class TemplatesProcessor(ArtifactProcessor):
+    """
+    Processor class to perform template expansion on all relevant artifacts (specified in the project definition file).
+    """
+
+    def process(
+        self,
+        artifact_to_process: PathMapping,
+        processor_mapping: Optional[ProcessorMapping],
+        **kwargs,
+    ):
+        """
+        Process the artifact by executing the template expansion logic on it.
+        """
+        cc.step(f"Processing artifact {artifact_to_process} with templates processor")
+
+        bundle_map = BundleMap(
+            project_root=self._bundle_ctx.project_root,
+            deploy_root=self._bundle_ctx.deploy_root,
+        )
+        bundle_map.add(artifact_to_process)
+
+        for src, dest in bundle_map.all_mappings(
+            absolute=True,
+            expand_directories=True,
+        ):
+            if src.is_dir():
+                continue
+            with self.edit_file(dest) as f:
+                file_name = src.relative_to(self._bundle_ctx.project_root)
+
+                jinja_env = (
+                    choose_sql_jinja_env_based_on_template_syntax(
+                        f.contents, reference_name=file_name
+                    )
+                    if dest.name.lower().endswith(".sql")
+                    else get_client_side_jinja_env()
+                )
+
+                try:
+                    expanded_template = jinja_env.from_string(f.contents).render(
+                        get_cli_context().template_context
+                    )
+
+                # For now, we are printing the source file path in the error message
+                # instead of the destination file path to make it easier for the user
+                # to identify the file that has the error, and edit the correct file.
+                except jinja2.TemplateSyntaxError as e:
+                    raise InvalidTemplateInFileError(file_name, e, e.lineno) from e
+
+                except jinja2.UndefinedError as e:
+                    raise InvalidTemplateInFileError(file_name, e) from e
+
+                if expanded_template != f.contents:
+                    f.edited_contents = expanded_template

src/snowflake/cli/_plugins/nativeapp/exceptions.py

@@ -64,15 +64,15 @@ def __init__(self, relpath: str):
         super().__init__(f'Script "{relpath}" does not exist')
 
 
-class InvalidScriptError(ClickException):
-    """A referenced script had syntax error(s)."""
+class InvalidTemplateInFileError(ClickException):
+    """A referenced templated file had syntax error(s)."""
 
     def __init__(
         self, relpath: str, err: jinja2.TemplateError, lineno: Optional[int] = None
     ):
         lineno_str = f":{lineno}" if lineno is not None else ""
         super().__init__(
-            f'Script "{relpath}{lineno_str}" does not contain a valid template: {err.message}'
+            f'File "{relpath}{lineno_str}" does not contain a valid template: {err.message}'
         )
         self.err = err
 

src/snowflake/cli/api/commands/flags.py

@@ -506,7 +506,7 @@ def project_env_overrides_callback(env_overrides_args_list: list[str]) -> None:
     return typer.Option(
         [],
         "--env",
-        help="String in format of key=value. Overrides variables from env section used for templating.",
+        help="String in format of key=value. Overrides variables from env section used for templates.",
         callback=_callback(lambda: project_env_overrides_callback),
         show_default=False,
     )

src/snowflake/cli/api/entities/utils.py

@@ -11,7 +11,7 @@
 )
 from snowflake.cli._plugins.nativeapp.constants import OWNER_COL
 from snowflake.cli._plugins.nativeapp.exceptions import (
-    InvalidScriptError,
+    InvalidTemplateInFileError,
     MissingScriptError,
     UnexpectedOwnerError,
 )
@@ -313,9 +313,9 @@ def render_script_templates(
             raise MissingScriptError(relpath) from e
 
         except jinja2.TemplateSyntaxError as e:
-            raise InvalidScriptError(relpath, e, e.lineno) from e
+            raise InvalidTemplateInFileError(relpath, e, e.lineno) from e
 
         except jinja2.UndefinedError as e:
-            raise InvalidScriptError(relpath, e) from e
+            raise InvalidTemplateInFileError(relpath, e) from e
 
     return scripts_contents

src/snowflake/cli/api/rendering/project_definition_templates.py

@@ -24,7 +24,7 @@
 _YML_TEMPLATE_END = "%>"
 
 
-def get_project_definition_cli_jinja_env() -> Environment:
+def get_client_side_jinja_env() -> Environment:
     _random_block = "___very___unique___block___to___disable___logic___blocks___"
     return env_bootstrap(
         IgnoreAttrEnvironment(

src/snowflake/cli/api/rendering/sql_templates.py

@@ -14,7 +14,7 @@
 
 from __future__ import annotations
 
-from typing import Dict
+from typing import Dict, Optional
 
 from click import ClickException
 from jinja2 import Environment, StrictUndefined, loaders, meta
@@ -55,19 +55,22 @@ def _does_template_have_env_syntax(env: Environment, template_content: str) -> b
     return bool(meta.find_undeclared_variables(template))
 
 
-def choose_sql_jinja_env_based_on_template_syntax(template_content: str) -> Environment:
+def choose_sql_jinja_env_based_on_template_syntax(
+    template_content: str, reference_name: Optional[str] = None
+) -> Environment:
     old_syntax_env = _get_sql_jinja_env(_OLD_SQL_TEMPLATE_START, _OLD_SQL_TEMPLATE_END)
     new_syntax_env = _get_sql_jinja_env(_SQL_TEMPLATE_START, _SQL_TEMPLATE_END)
     has_old_syntax = _does_template_have_env_syntax(old_syntax_env, template_content)
     has_new_syntax = _does_template_have_env_syntax(new_syntax_env, template_content)
+    reference_name_str = f" in {reference_name}" if reference_name else ""
     if has_old_syntax and has_new_syntax:
         raise InvalidTemplate(
-            f"The SQL query mixes {_OLD_SQL_TEMPLATE_START} ... {_OLD_SQL_TEMPLATE_END} syntax"
+            f"The SQL query{reference_name_str} mixes {_OLD_SQL_TEMPLATE_START} ... {_OLD_SQL_TEMPLATE_END} syntax"
             f" and {_SQL_TEMPLATE_START} ... {_SQL_TEMPLATE_END} syntax."
         )
     if has_old_syntax:
         cli_console.warning(
-            f"Warning: {_OLD_SQL_TEMPLATE_START} ... {_OLD_SQL_TEMPLATE_END} syntax is deprecated."
+            f"Warning: {_OLD_SQL_TEMPLATE_START} ... {_OLD_SQL_TEMPLATE_END} syntax{reference_name_str} is deprecated."
             f" Use {_SQL_TEMPLATE_START} ... {_SQL_TEMPLATE_END} syntax instead."
         )
         return old_syntax_env

src/snowflake/cli/api/utils/definition_rendering.py

@@ -28,7 +28,7 @@
 from snowflake.cli.api.project.schemas.updatable_model import context
 from snowflake.cli.api.rendering.jinja import CONTEXT_KEY, FUNCTION_KEY
 from snowflake.cli.api.rendering.project_definition_templates import (
-    get_project_definition_cli_jinja_env,
+    get_client_side_jinja_env,
 )
 from snowflake.cli.api.utils.dict_utils import deep_merge_dicts, traverse
 from snowflake.cli.api.utils.graph import Graph, Node
@@ -96,7 +96,7 @@ def _get_referenced_vars(
             )
             or current_attr_chain is not None
         ):
-            raise InvalidTemplate(f"Unexpected templating syntax in {template_value}")
+            raise InvalidTemplate(f"Unexpected template syntax in {template_value}")
 
         for child_node in ast_node.iter_child_nodes():
             all_referenced_vars.update(
@@ -318,7 +318,7 @@ def render_definition_template(
     if definition is None:
         return ProjectProperties(None, {CONTEXT_KEY: {"env": environment_overrides}})
 
-    template_env = TemplatedEnvironment(get_project_definition_cli_jinja_env())
+    template_env = TemplatedEnvironment(get_client_side_jinja_env())
 
     if "definition_version" not in definition or Version(
         definition["definition_version"]
@@ -353,9 +353,7 @@ def render_definition_template(
     )
 
     def on_cycle_action(node: Node[TemplateVar]):
-        raise CycleDetectedError(
-            f"Cycle detected in templating variable {node.data.key}"
-        )
+        raise CycleDetectedError(f"Cycle detected in template variable {node.data.key}")
 
     dependencies_graph.dfs(
         visit_action=lambda node: _render_graph_node(template_env, node),