Merge branch 'main' into AddScriptToTestInFollowingPR

RELEASE-NOTES.md

@@ -16,6 +16,16 @@
 
 # Unreleased version
 ## Backward incompatibility
+
+## Deprecations
+
+## New additions
+
+## Fixes and improvements
+
+
+# v3.0.0
+## Backward incompatibility
 * Dropped support for Python below 3.10 version.
 * `snow object stage` commands are removed in favour of `snow stage`.
 * `snow snowpark init` and `snow streamlit init` commands are removed in favor of `snow init` command.
@@ -27,6 +37,7 @@
   * `snow snowpark deploy` uploads all artifacts created during build step. Dependencies zip is upload once to
      every Snowpark stage specified in project definition.
   * The changes are compatible with V1 projects definition though the result state (file layout) is different.
+  * `snow snowpark package` commands no longer fallback to Anaconda Channel metadata when fetching available packages info fails.
 
 ## Deprecations
 
@@ -36,6 +47,7 @@
 * Added support for external access (api integrations and secrets) in Streamlit.
 * Added support for `<% ... %>` syntax in SQL templating.
 * Support multiple Streamlit application in single snowflake.yml project definition file.
+* Added `snow ws migrate` command to migrate `snowflake.yml` file from V1 to V2.
 
 ## Fixes and improvements
 * Fixed problem with whitespaces in `snow connection add` command.

src/snowflake/cli/_app/dev/docs/commands_docs_generator.py

@@ -15,7 +15,7 @@
 from __future__ import annotations
 
 import logging
-from typing import List, Optional
+from typing import Any, List, Optional
 
 from click import Command
 from snowflake.cli._app.dev.docs.template_utils import get_template_environment
@@ -86,15 +86,33 @@ def _render_command_usage(
     # Included files, which these are, need to use the .txt extension.
     file_path = root / f"usage-{command_name}.txt"
     log.info("Creating %s", file_path)
+    command_help_params = _split_docstring(command.help)
+    template_params = {
+        "name": command_name,
+        "options": options,
+        "arguments": arguments,
+        "path": path,
+    }
     with file_path.open("w+") as fh:
-        fh.write(
-            template.render(
-                {
-                    "help": command.help,
-                    "name": command_name,
-                    "options": options,
-                    "arguments": arguments,
-                    "path": path,
-                }
-            )
+        fh.write(template.render(command_help_params | template_params))
+
+
+def _split_docstring(command_help: Optional[str]) -> dict[str, Any]:
+    if command_help is None:
+        return {}
+
+    split_command_help = command_help.split("## ")
+
+    if len(split_command_help) == 1:
+        return {"help": split_command_help[0]}
+
+    additional_sections = []
+    for section in split_command_help[1:]:
+        lines = section.split("\n")
+        additional_sections.append(
+            {"title": lines[0], "content": "\n".join(lines[1:]).strip()}
         )
+    return {
+        "help": split_command_help[0],
+        "additional_sections": additional_sections,
+    }

src/snowflake/cli/_app/dev/docs/templates/usage.rst.jinja2

@@ -1,4 +1,4 @@
-{{ help | replace("`", "``") }}
+{{ help | replace("`", "``") | replace("\n", " ") }}
 
 Syntax
 ===============================================================================
@@ -26,7 +26,7 @@ Arguments
         {{ param.make_metavar().replace("[", "").replace("]", "").lower() }}
     {%- endif -%}
     {{ '}' }}`
-{% if param.help %}{{ "  " + param.help | replace("`", "``") }}{% if param.help[-1] != '.' %}.{% endif %}{% if param.default %} Default: {{ param.default }}.{% endif %}{% else %}  TBD{% endif %}
+{% if param.help %}{{ "  " + param.help | replace("`", "``") | replace("\n", " ") }}{% if param.help[-1] != '.' %}.{% endif %}{% if param.default %} Default: {{ param.default }}.{% endif %}{% else %}  TBD{% endif %}
 {% endfor %}
 {% else %}
 
@@ -48,10 +48,20 @@ Options
         {%- if param.type.name != "choice" %}{{ ' {' }}{% else %} {% endif %}{{ param.make_metavar() }}{% if param.type.name != "choice" %}{{ '}' }}
         {%- endif %}
     {%- endif %}`
-{% if param.help %}{{ "  " + param.help | replace("`", "``") }}{% if param.help[-1] != '.' %}.{% endif %}{% if param.default is not none %} Default: {{ param.default }}.{% endif %}{% else %}  TBD{% endif %}
-{% endfor %}
+{% if param.help %}{{ "  " + param.help | replace("`", "``") | replace("\n", " ") }}{% if param.help[-1] != '.' %}.{% endif %}{% if param.default is not none %} Default: {{ param.default }}.{% endif %}{% else %}  TBD{% endif %}
+{% endfor -%}
 {% else %}
 
 None
 
 {% endif -%}
+
+{%- if additional_sections %}
+..
+{%- for section in additional_sections %}
+  {{ section.title }}
+  ===============================================================================
+
+  {{ section.content | indent(2) }}
+{% endfor %}
+{% endif %}

src/snowflake/cli/_plugins/git/commands.py

@@ -106,6 +106,8 @@ def setup(
     """
     Sets up a git repository object.
 
+    ## Usage notes
+
     You will be prompted for:
 
     * url - address of repository to be used for git clone operation

src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py

@@ -14,6 +14,8 @@
 
 from __future__ import annotations
 
+import copy
+import re
 from typing import Dict, Optional
 
 from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext
@@ -34,7 +36,7 @@
 )
 
 SNOWPARK_PROCESSOR = "snowpark"
-NA_SETUP_PROCESSOR = "native-app-setup"
+NA_SETUP_PROCESSOR = "native app setup"
 
 _REGISTERED_PROCESSORS_BY_NAME = {
     SNOWPARK_PROCESSOR: SnowparkAnnotationProcessor,
@@ -110,7 +112,15 @@ def _try_create_processor(
             # No registered processor with the specified name
             return None
 
-        current_processor = processor_factory(self._bundle_ctx)
+        processor_ctx = copy.copy(self._bundle_ctx)
+        processor_subdirectory = re.sub(r"[^a-zA-Z0-9_$]", "_", processor_name)
+        processor_ctx.bundle_root = (
+            self._bundle_ctx.bundle_root / processor_subdirectory
+        )
+        processor_ctx.generated_root = (
+            self._bundle_ctx.generated_root / processor_subdirectory
+        )
+        current_processor = processor_factory(processor_ctx)
         self.cached_processors[processor_name] = current_processor
 
         return current_processor

src/snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py

@@ -15,12 +15,18 @@
 from __future__ import annotations
 
 import json
+import logging
 import os.path
 from pathlib import Path
 from typing import List, Optional
 
+import yaml
 from click import ClickException
-from snowflake.cli._plugins.nativeapp.artifacts import BundleMap, find_setup_script_file
+from snowflake.cli._plugins.nativeapp.artifacts import (
+    BundleMap,
+    find_manifest_file,
+    find_setup_script_file,
+)
 from snowflake.cli._plugins.nativeapp.codegen.artifact_processor import (
     ArtifactProcessor,
     is_python_file_artifact,
@@ -40,6 +46,32 @@
 DEFAULT_TIMEOUT = 30
 DRIVER_PATH = Path(__file__).parent / "setup_driver.py.source"
 
+log = logging.getLogger(__name__)
+
+
+def safe_set(d: dict, *keys: str, **kwargs) -> None:
+    """
+    Sets a value in a nested dictionary structure, creating intermediate dictionaries as needed.
+    Sample usage:
+
+      d = {}
+      safe_set(d, "a", "b", "c", value=42)
+
+    d is now:
+      {
+        "a": {
+          "b": {
+            "c": 42
+          }
+        }
+      }
+    """
+    curr = d
+    for k in keys[:-1]:
+        curr = curr.setdefault(k, {})
+
+    curr[keys[-1]] = kwargs.get("value")
+
 
 class NativeAppSetupProcessor(ArtifactProcessor):
     def __init__(self, *args, **kwargs):
@@ -73,18 +105,55 @@ def process(
             )
             files_to_process.append(src_file)
 
-        sql_files_mapping = self._execute_in_sandbox(files_to_process)
-        self._generate_setup_sql(sql_files_mapping)
+        result = self._execute_in_sandbox(files_to_process)
+        if not result:
+            return  # nothing to do
+
+        logs = result.get("logs", [])
+        for msg in logs:
+            log.debug(msg)
+
+        warnings = result.get("warnings", [])
+        for msg in warnings:
+            cc.warning(msg)
+
+        schema_version = result.get("schema_version")
+        if schema_version != "1":
+            raise ClickException(
+                f"Unsupported schema version returned from snowflake-app-python library: {schema_version}"
+            )
+
+        setup_script_mods = [
+            mod
+            for mod in result.get("modifications", [])
+            if mod.get("target") == "native_app:setup_script"
+        ]
+        if setup_script_mods:
+            self._edit_setup_sql(setup_script_mods)
+
+        manifest_mods = [
+            mod
+            for mod in result.get("modifications", [])
+            if mod.get("target") == "native_app:manifest"
+        ]
+        if manifest_mods:
+            self._edit_manifest(manifest_mods)
 
     def _execute_in_sandbox(self, py_files: List[Path]) -> dict:
         file_count = len(py_files)
         cc.step(f"Processing {file_count} setup file{'s' if file_count > 1 else ''}")
 
+        manifest_path = find_manifest_file(deploy_root=self._bundle_ctx.deploy_root)
+
+        generated_root = self._bundle_ctx.generated_root
+        generated_root.mkdir(exist_ok=True, parents=True)
+
         env_vars = {
             "_SNOWFLAKE_CLI_PROJECT_PATH": str(self._bundle_ctx.project_root),
             "_SNOWFLAKE_CLI_SETUP_FILES": os.pathsep.join(map(str, py_files)),
             "_SNOWFLAKE_CLI_APP_NAME": str(self._bundle_ctx.package_name),
-            "_SNOWFLAKE_CLI_SQL_DEST_DIR": str(self.generated_root),
+            "_SNOWFLAKE_CLI_SQL_DEST_DIR": str(generated_root),
+            "_SNOWFLAKE_CLI_MANIFEST_PATH": str(manifest_path),
         }
 
         try:
@@ -102,56 +171,68 @@ def _execute_in_sandbox(self, py_files: List[Path]) -> dict:
             )
 
         if result.returncode == 0:
-            sql_file_mappings = json.loads(result.stdout)
-            return sql_file_mappings
+            return json.loads(result.stdout)
         else:
             raise ClickException(
                 f"Failed to execute python setup script logic: {result.stderr}"
             )
 
-    def _generate_setup_sql(self, sql_file_mappings: dict) -> None:
-        if not sql_file_mappings:
-            # Nothing to generate
-            return
-
-        generated_root = self.generated_root
-        generated_root.mkdir(exist_ok=True, parents=True)
-
+    def _edit_setup_sql(self, modifications: List[dict]) -> None:
         cc.step("Patching setup script")
         setup_file_path = find_setup_script_file(
             deploy_root=self._bundle_ctx.deploy_root
         )
-        with self.edit_file(setup_file_path) as f:
-            new_contents = [f.contents]
 
-            if sql_file_mappings["schemas"]:
-                schemas_file = generated_root / sql_file_mappings["schemas"]
-                new_contents.insert(
-                    0,
-                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(schemas_file.relative_to(self._bundle_ctx.deploy_root))}';",
-                )
-
-            if sql_file_mappings["compute_pools"]:
-                compute_pools_file = generated_root / sql_file_mappings["compute_pools"]
-                new_contents.append(
-                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(compute_pools_file.relative_to(self._bundle_ctx.deploy_root))}';"
-                )
-
-            if sql_file_mappings["services"]:
-                services_file = generated_root / sql_file_mappings["services"]
-                new_contents.append(
-                    f"EXECUTE IMMEDIATE FROM '/{to_stage_path(services_file.relative_to(self._bundle_ctx.deploy_root))}';"
-                )
-
-            f.edited_contents = "\n".join(new_contents)
+        with self.edit_file(setup_file_path) as f:
+            prepended = []
+            appended = []
+
+            for mod in modifications:
+                for inst in mod.get("instructions", []):
+                    if inst.get("type") == "insert":
+                        default_loc = inst.get("default_location")
+                        if default_loc == "end":
+                            appended.append(self._setup_mod_instruction_to_sql(inst))
+                        elif default_loc == "start":
+                            prepended.append(self._setup_mod_instruction_to_sql(inst))
+
+            if prepended or appended:
+                f.edited_contents = "\n".join(prepended + [f.contents] + appended)
+
+    def _edit_manifest(self, modifications: List[dict]) -> None:
+        cc.step("Patching manifest")
+        manifest_path = find_manifest_file(deploy_root=self._bundle_ctx.deploy_root)
+
+        with self.edit_file(manifest_path) as f:
+            manifest = yaml.safe_load(f.contents)
+
+            for mod in modifications:
+                for inst in mod.get("instructions", []):
+                    if inst.get("type") == "set":
+                        payload = inst.get("payload")
+                        if payload:
+                            key = payload.get("key")
+                            value = payload.get("value")
+                            safe_set(manifest, *key.split("."), value=value)
+            f.edited_contents = yaml.safe_dump(manifest, sort_keys=False)
+
+    def _setup_mod_instruction_to_sql(self, mod_inst: dict) -> str:
+        payload = mod_inst.get("payload")
+        if not payload:
+            raise ClickException("Unsupported instruction received: no payload found")
+
+        payload_type = payload.get("type")
+        if payload_type == "execute immediate":
+            file_path = payload.get("file_path")
+            if file_path:
+                sql_file_path = self._bundle_ctx.generated_root / file_path
+                return f"EXECUTE IMMEDIATE FROM '/{to_stage_path(sql_file_path.relative_to(self._bundle_ctx.deploy_root))}';"
+
+        raise ClickException(f"Unsupported instruction type received: {payload_type}")
 
     @property
     def sandbox_root(self):
-        return self._bundle_ctx.bundle_root / "setup_py_venv"
-
-    @property
-    def generated_root(self):
-        return self._bundle_ctx.generated_root / "setup_py"
+        return self._bundle_ctx.bundle_root / "venv"
 
     def _create_or_update_sandbox(self):
         sandbox_root = self.sandbox_root

src/snowflake/cli/_plugins/nativeapp/codegen/setup/setup_driver.py.source

@@ -20,8 +20,11 @@ from pathlib import Path
 import snowflake.app.context as ctx
 from snowflake.app.sql import SQLGenerator
 
-ctx._project_path = os.environ["_SNOWFLAKE_CLI_PROJECT_PATH"]
-ctx._current_app_name = os.environ["_SNOWFLAKE_CLI_APP_NAME"]
+ctx.configure("project_path", os.environ.get("_SNOWFLAKE_CLI_PROJECT_PATH", None))
+ctx.configure("manifest_path", os.environ.get("_SNOWFLAKE_CLI_MANIFEST_PATH", None))
+ctx.configure("current_app_name", os.environ.get("_SNOWFLAKE_CLI_APP_NAME", None))
+ctx.configure("enable_sql_generation", True)
+
 __snowflake_internal_py_files = os.environ["_SNOWFLAKE_CLI_SETUP_FILES"].split(
     os.pathsep
 )