Refactor and connect sql templating to the common logic

RELEASE-NOTES.md

@@ -8,6 +8,9 @@
 * Added `snow notebook` commands:
   * `snow notebook execute` enabling head-less execution of a notebook.
   * `snow notebook create` proving an option to create a Snowflake Notebook from a file on stage.
+* Added templating support for project definition file.
+  * Templates can now be used within the main section of the project definition file.
+  * Resolved values of the project definition file are available to all modules.
 
 ## Fixes and improvements
 

src/snowflake/cli/api/cli_global_context.py

@@ -212,6 +212,7 @@ def __init__(self):
         self._experimental = False
         self._project_definition = None
         self._project_root = None
+        self._template_context = None
         self._silent: bool = False
 
     def reset(self):
@@ -259,6 +260,13 @@ def project_root(self):
     def set_project_root(self, project_root: Path):
         self._project_root = project_root
 
+    @property
+    def template_context(self):
+        return self._template_context
+
+    def set_template_context(self, template_context: dict):
+        self._template_context = template_context
+
     @property
     def connection_context(self) -> _ConnectionContext:
         return self._connection_context
@@ -311,6 +319,10 @@ def project_definition(self) -> ProjectDefinition | None:
     def project_root(self):
         return self._manager.project_root
 
+    @property
+    def template_context(self):
+        return self._manager.template_context
+
     @property
     def silent(self) -> bool:
         if self._should_force_mute_intermediate_output:

src/snowflake/cli/api/commands/flags.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 import tempfile
 from dataclasses import dataclass
 from enum import Enum
@@ -14,6 +15,7 @@
 from snowflake.cli.api.console import cli_console
 from snowflake.cli.api.exceptions import MissingConfiguration
 from snowflake.cli.api.output.formats import OutputFormat
+from snowflake.cli.api.utils.rendering import CONTEXT_KEY
 
 DEFAULT_CONTEXT_SETTINGS = {"help_option_names": ["--help", "-h"]}
 
@@ -499,6 +501,7 @@ def _callback(project_path: Optional[str]):
 
         cli_context_manager.set_project_definition(project_definition)
         cli_context_manager.set_project_root(project_root)
+        cli_context_manager.set_template_context(dm.template_context)
         return project_definition
 
     if project_name == "native_app":
@@ -527,14 +530,17 @@ def _callback(project_path: Optional[str]):
             dm = DefinitionManager(project_path)
             project_definition = dm.project_definition
             project_root = dm.project_root
+            template_context = dm.template_context
         except MissingConfiguration:
             if optional:
                 project_definition = None
                 project_root = None
+                template_context = {CONTEXT_KEY: {"env": os.environ}}
             else:
                 raise
         cli_context_manager.set_project_definition(project_definition)
         cli_context_manager.set_project_root(project_root)
+        cli_context_manager.set_template_context(template_context)
         return project_definition
 
     return typer.Option(

src/snowflake/cli/api/project/definition.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import List
 
 import yaml.loader
 from snowflake.cli.api.cli_global_context import cli_context
@@ -15,33 +16,19 @@
 )
 from snowflake.cli.api.secure_path import SecurePath
 from snowflake.cli.api.utils.definition_rendering import render_definition_template
+from snowflake.cli.api.utils.dict_utils import deep_merge_dicts
 from yaml import load
 
 DEFAULT_USERNAME = "unknown_user"
 
 
-def merge_two_dicts(
-    original_values: Dict[str, Any], update_values: Dict[str, Any]
-):  # TODO update name of function
-    if not isinstance(update_values, dict) or not isinstance(original_values, dict):
-        return
+@dataclass
+class ProjectProperties:
+    project_definition: ProjectDefinition
+    raw_project_definition: dict
 
-    for field, value in update_values.items():
-        if (
-            field in original_values
-            and isinstance(original_values[field], dict)
-            and isinstance(value, dict)
-        ):
-            merge_two_dicts(original_values[field], value)
-        else:
-            original_values[field] = value
 
-
-def load_project_definition(paths: List[Path]) -> ProjectDefinition:
-    """
-    Loads project definition, optionally overriding values. Definition values
-    are merged in left-to-right order (increasing precedence).
-    """
+def _get_merged_project_files(paths: List[Path]) -> dict:
     spaths: List[SecurePath] = [SecurePath(p) for p in paths]
     if len(spaths) == 0:
         raise ValueError("Need at least one definition file.")
@@ -54,11 +41,21 @@ def load_project_definition(paths: List[Path]) -> ProjectDefinition:
             "r", read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB
         ) as override_yml:
             overrides = load(override_yml.read(), Loader=yaml.loader.BaseLoader)
-            merge_two_dicts(definition, overrides)
+            deep_merge_dicts(definition, overrides)
+
+    return definition
 
-    rendered_definition = render_definition_template(definition)
-    rendered_project = ProjectDefinition(**rendered_definition)
-    return rendered_project
+
+def load_project(paths: List[Path]) -> ProjectProperties:
+    """
+    Loads project definition, optionally overriding values. Definition values
+    are merged in left-to-right order (increasing precedence).
+    """
+    merged_files = _get_merged_project_files(paths)
+    rendered_definition = render_definition_template(merged_files)
+    return ProjectProperties(
+        ProjectDefinition(**rendered_definition), rendered_definition
+    )
 
 
 def generate_local_override_yml(

src/snowflake/cli/api/project/definition_manager.py

@@ -6,8 +6,9 @@
 from typing import List, Optional
 
 from snowflake.cli.api.exceptions import MissingConfiguration
-from snowflake.cli.api.project.definition import load_project_definition
+from snowflake.cli.api.project.definition import ProjectProperties, load_project
 from snowflake.cli.api.project.schemas.project_definition import ProjectDefinition
+from snowflake.cli.api.utils.rendering import CONTEXT_KEY
 
 
 def _compat_is_mount(path: Path):
@@ -100,6 +101,16 @@ def _user_definition_file_if_available(project_path: Path) -> Optional[Path]:
             DefinitionManager.USER_DEFINITION_FILENAME, project_path
         )
 
+    @functools.cached_property
+    def _project_properties(self) -> ProjectProperties:
+        return load_project(self._project_config_paths)
+
     @functools.cached_property
     def project_definition(self) -> ProjectDefinition:
-        return load_project_definition(self._project_config_paths)
+        return self._project_properties.project_definition
+
+    @functools.cached_property
+    def template_context(self) -> dict:
+        definition = self._project_properties.raw_project_definition
+
+        return {CONTEXT_KEY: definition}