Add optional deploy files argument (#933)

* add deploy files argument

* add --prune flag

* add --recursive flag

RELEASE-NOTES.md

@@ -6,6 +6,10 @@
 
 ## New additions
 * `snow sql` command supports now client-side templating of queries.
+* New `snow app deploy` functionality:
+  * Passing files and directories as arguments syncs these only: `snow app deploy some-file some-dir`.
+  * `--recursive` syncs all files and subdirectories recursively.
+  * `--prune` deletes specified files from the stage if they don't exist locally.
 
 ## Fixes and improvements
 * More human-friendly errors in case of corrupted `config.toml` file.

src/snowflake/cli/plugins/nativeapp/artifacts.py

@@ -1,14 +1,17 @@
 import os
 from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
-from click import ClickException
+from click.exceptions import ClickException
 from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
 from snowflake.cli.api.project.schemas.native_app.path_mapping import PathMapping
 from snowflake.cli.api.secure_path import SecurePath
 from yaml import safe_load
 
+# Map from source directories and files in the project directory to their path in the deploy directory. Both paths are absolute.
+ArtifactDeploymentMap = Dict[Path, Path]
+
 
 class DeployRootError(ClickException):
     """
@@ -195,11 +198,14 @@ def resolve_without_follow(path: Path) -> Path:
 
 
 def build_bundle(
-    project_root: Path, deploy_root: Path, artifacts: List[ArtifactMapping]
-):
+    project_root: Path,
+    deploy_root: Path,
+    artifacts: List[ArtifactMapping],
+) -> ArtifactDeploymentMap:
     """
     Prepares a local folder (deploy_root) with configured app artifacts.
     This folder can then be uploaded to a stage.
+    Returns a map of the copied source files, pointing to where they were copied.
     """
     resolved_root = deploy_root.resolve()
     if resolved_root.exists() and not resolved_root.is_dir():
@@ -217,6 +223,7 @@ def build_bundle(
     if resolved_root.exists():
         delete(resolved_root)
 
+    mapped_files: ArtifactDeploymentMap = {}
     for artifact in artifacts:
         dest_path = resolve_without_follow(Path(resolved_root, artifact.dest))
         source_paths = get_source_paths(artifact, project_root)
@@ -228,7 +235,9 @@ def build_bundle(
 
             # copy all files as children of the given destination path
             for source_path in source_paths:
-                symlink_or_copy(source_path, dest_path / source_path.name)
+                dest_child_path = dest_path / source_path.name
+                symlink_or_copy(source_path, dest_child_path)
+                mapped_files[source_path.resolve()] = dest_child_path
         else:
             # ensure we are copying into the deploy root, not replacing it!
             if resolved_root not in dest_path.parents:
@@ -237,9 +246,11 @@ def build_bundle(
             if len(source_paths) == 1:
                 # copy a single file as the given destination path
                 symlink_or_copy(source_paths[0], dest_path)
+                mapped_files[source_paths[0].resolve()] = dest_path
             else:
                 # refuse to map multiple source files to one destination (undefined behaviour)
                 raise TooManyFilesError(dest_path)
+    return mapped_files
 
 
 def find_manifest_file(deploy_root: Path) -> Path:
@@ -283,3 +294,31 @@ def find_version_info_in_manifest_file(
             patch_name = version_info[patch_field]
 
     return version_name, patch_name
+
+
+def source_path_to_deploy_path(
+    source_path: Path, mapped_files: ArtifactDeploymentMap
+) -> Path:
+    """Returns the absolute path where the specified source path was copied to during bundle."""
+
+    source_path = source_path.resolve()
+
+    if source_path in mapped_files:
+        return mapped_files[source_path]
+
+    # Find the first parent directory that exists in mapped_files
+    common_root = source_path
+    while common_root:
+        if common_root in mapped_files:
+            break
+        elif common_root.parent != common_root:
+            common_root = common_root.parent
+        else:
+            raise ClickException(f"Could not find the deploy path of {source_path}")
+
+    # Construct the target deploy path
+    path_to_symlink = mapped_files[common_root]
+    relative_path_to_target = Path(source_path).relative_to(common_root)
+    result = Path(path_to_symlink, relative_path_to_target)
+
+    return result

src/snowflake/cli/plugins/nativeapp/commands.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Optional
+from pathlib import Path
+from typing import List, Optional
 
 import typer
 from snowflake.cli.api.cli_global_context import cli_context
@@ -229,17 +230,44 @@ def app_teardown(
 @app.command("deploy", requires_connection=True)
 @with_project_definition("native_app")
 def app_deploy(
+    prune: Optional[bool] = typer.Option(
+        default=None,
+        help=f"""Whether to delete specified files from the stage if they don't exist locally. If set, the command deletes files that exist in the stage, but not in the local filesystem.""",
+    ),
+    recursive: Optional[bool] = typer.Option(
+        None,
+        "--recursive",
+        "-r",
+        help=f"""Whether to traverse and deploy files from subdirectories. If set, the command deploys all files and subdirectories; otherwise, only files in the current directory are deployed.""",
+    ),
+    files: Optional[List[Path]] = typer.Argument(
+        default=None,
+        show_default=False,
+        help=f"""Paths, relative to the the project root, of files you want to upload to a stage. The paths must match one of the artifacts src pattern entries in snowflake.yml. If unspecified, the command syncs all local changes to the stage.""",
+    ),
     **options,
 ) -> CommandResult:
     """
     Creates an application package in your Snowflake account and syncs the local changes to the stage without creating or updating the application.
+    Running this command with no arguments at all, as in `snow app deploy`, is a shorthand for `snow app deploy --prune --recursive`.
     """
+    if files is None:
+        files = []
+    if prune is None and recursive is None and len(files) == 0:
+        prune = True
+        recursive = True
+    else:
+        if prune is None:
+            prune = False
+        if recursive is None:
+            recursive = False
+
     manager = NativeAppManager(
         project_definition=cli_context.project_definition,
         project_root=cli_context.project_root,
     )
 
-    manager.build_bundle()
-    manager.deploy()
+    mapped_files = manager.build_bundle()
+    manager.deploy(prune, recursive, files, mapped_files)
 
     return MessageResult(f"Deployed successfully.")

src/snowflake/cli/plugins/nativeapp/manager.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from abc import ABC, abstractmethod
 from functools import cached_property
 from pathlib import Path
@@ -23,8 +24,11 @@
 from snowflake.cli.api.sql_execution import SqlExecutionMixin
 from snowflake.cli.plugins.connection.util import make_snowsight_url
 from snowflake.cli.plugins.nativeapp.artifacts import (
+    ArtifactDeploymentMap,
     ArtifactMapping,
     build_bundle,
+    resolve_without_follow,
+    source_path_to_deploy_path,
     translate_artifact,
 )
 from snowflake.cli.plugins.nativeapp.constants import (
@@ -43,8 +47,10 @@
     MissingPackageScriptError,
     UnexpectedOwnerError,
 )
+from snowflake.cli.plugins.nativeapp.utils import verify_exists, verify_no_directories
 from snowflake.cli.plugins.stage.diff import (
     DiffResult,
+    filter_from_diff,
     stage_diff,
     sync_local_diff_with_stage,
 )
@@ -100,6 +106,20 @@ def ensure_correct_owner(row: dict, role: str, obj_name: str) -> None:
         raise UnexpectedOwnerError(obj_name, role, actual_owner)
 
 
+def _get_paths_to_sync(paths_to_sync: List[Path], deploy_root: Path) -> List[str]:
+    """Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root."""
+    paths = []
+    for path in paths_to_sync:
+        if path.is_dir():
+            for current_dir, _dirs, files in os.walk(path):
+                for file in files:
+                    deploy_path = Path(current_dir, file).relative_to(deploy_root)
+                    paths.append(str(deploy_path))
+        else:
+            paths.append(str(path.relative_to(deploy_root)))
+    return paths
+
+
 class NativeAppCommandProcessor(ABC):
     @abstractmethod
     def process(self, *args, **kwargs):
@@ -278,13 +298,20 @@ def verify_project_distribution(
             return False
         return True
 
-    def build_bundle(self) -> None:
+    def build_bundle(self) -> ArtifactDeploymentMap:
         """
         Populates the local deploy root from artifact sources.
         """
-        build_bundle(self.project_root, self.deploy_root, self.artifacts)
-
-    def sync_deploy_root_with_stage(self, role: str) -> DiffResult:
+        return build_bundle(self.project_root, self.deploy_root, self.artifacts)
+
+    def sync_deploy_root_with_stage(
+        self,
+        role: str,
+        prune: bool,
+        recursive: bool,
+        paths_to_sync: List[Path] = [],  # relative to project root
+        mapped_files: Optional[ArtifactDeploymentMap] = None,
+    ) -> DiffResult:
         """
         Ensures that the files on our remote stage match the artifacts we have in
         the local filesystem. Returns the DiffResult used to make changes.
@@ -310,6 +337,37 @@ def sync_deploy_root_with_stage(self, role: str) -> DiffResult:
             % self.deploy_root
         )
         diff: DiffResult = stage_diff(self.deploy_root, self.stage_fqn)
+
+        files_not_removed = []
+        if len(paths_to_sync) > 0:
+            # Deploying specific files/directories
+            resolved_paths_to_sync = [resolve_without_follow(p) for p in paths_to_sync]
+            if not recursive:
+                verify_no_directories(resolved_paths_to_sync)
+            deploy_paths_to_sync = [
+                source_path_to_deploy_path(p, mapped_files)
+                for p in resolved_paths_to_sync
+            ]
+            verify_exists(deploy_paths_to_sync)
+            paths_to_sync_set = set(
+                _get_paths_to_sync(deploy_paths_to_sync, self.deploy_root.resolve())
+            )
+            files_not_removed = filter_from_diff(diff, paths_to_sync_set, prune)
+        else:
+            # Full deploy
+            if not recursive:
+                deploy_files = os.listdir(str(self.deploy_root.resolve()))
+                verify_no_directories([Path(path_str) for path_str in deploy_files])
+            if not prune:
+                files_not_removed = diff.only_on_stage
+                diff.only_on_stage = []
+
+        if len(files_not_removed) > 0:
+            files_not_removed_str = "\n".join(files_not_removed)
+            cc.warning(
+                f"The following files exist only on the stage:\n{files_not_removed_str}\n\nUse the --prune flag to delete them from the stage."
+            )
+
         cc.message(str(diff))
 
         # Upload diff-ed files to application package stage
@@ -435,7 +493,13 @@ def _apply_package_scripts(self) -> None:
                 err, role=self.package_role, warehouse=self.package_warehouse
             )
 
-    def deploy(self) -> DiffResult:
+    def deploy(
+        self,
+        prune: bool,
+        recursive: bool,
+        paths_to_sync: List[Path] = [],
+        mapped_files: Optional[ArtifactDeploymentMap] = None,
+    ) -> DiffResult:
         """app deploy process"""
 
         # 1. Create an empty application package, if none exists
@@ -446,6 +510,8 @@ def deploy(self) -> DiffResult:
             self._apply_package_scripts()
 
             # 3. Upload files from deploy root local folder to the above stage
-            diff = self.sync_deploy_root_with_stage(self.package_role)
+            diff = self.sync_deploy_root_with_stage(
+                self.package_role, prune, recursive, paths_to_sync, mapped_files
+            )
 
         return diff

src/snowflake/cli/plugins/nativeapp/run_processor.py

@@ -297,5 +297,5 @@ def process(
             )
             return
 
-        diff = self.deploy()
+        diff = self.deploy(prune=True, recursive=True)
         self._create_dev_app(diff)

src/snowflake/cli/plugins/nativeapp/utils.py

@@ -1,7 +1,9 @@
 from os import PathLike
 from pathlib import Path
 from sys import stdin, stdout
-from typing import Optional, Union
+from typing import List, Optional, Union
+
+from click import ClickException
 
 
 def needs_confirmation(needs_confirm: bool, auto_yes: bool) -> bool:
@@ -65,3 +67,17 @@ def shallow_git_clone(url: Union[str, PathLike], to_path: Union[str, PathLike]):
     repo.close()
 
     return repo
+
+
+def verify_no_directories(paths_to_sync: List[Path]):
+    for path in paths_to_sync:
+        if path.is_dir():
+            raise ClickException(
+                f"{path} is a directory. Add the -r flag to deploy directories."  #
+            )
+
+
+def verify_exists(paths_to_sync: List[Path]):
+    for path in paths_to_sync:
+        if not path.exists():
+            raise ClickException(f"The following path does not exist: {path}")

src/snowflake/cli/plugins/nativeapp/version/version_processor.py

@@ -199,7 +199,9 @@ def process(
             self._apply_package_scripts()
 
             # Upload files from deploy root local folder to the above stage
-            self.sync_deploy_root_with_stage(self.package_role)
+            self.sync_deploy_root_with_stage(
+                self.package_role, prune=True, recursive=True
+            )
 
         # Warn if the version exists in a release directive(s)
         existing_release_directives = (

src/snowflake/cli/plugins/stage/diff.py

@@ -3,9 +3,11 @@
 import re
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Set
 
-from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError
+from snowflake.cli.api.exceptions import (
+    SnowflakeSQLExecutionError,
+)
 from snowflake.cli.api.secure_path import UNLIMITED, SecurePath
 from snowflake.connector.cursor import DictCursor
 
@@ -154,7 +156,26 @@ def build_md5_map(list_stage_cursor: DictCursor) -> Dict[str, str]:
     }
 
 
-def stage_diff(local_path: Path, stage_fqn: str) -> DiffResult:
+def filter_from_diff(
+    diff: DiffResult, paths_to_sync: Set[str], prune: bool
+) -> List[str]:
+    """Modifies the given diff, keeping only the provided paths. If prune is false, remote-only paths will be empty and the non-removed paths will be returned."""
+    diff.different = [i for i in diff.different if i in paths_to_sync]
+    diff.only_local = [i for i in diff.only_local if i in paths_to_sync]
+    only_on_stage = [i for i in diff.only_on_stage if i in paths_to_sync]
+    files_not_removed = []
+    if prune:
+        diff.only_on_stage = only_on_stage
+    else:
+        files_not_removed = only_on_stage
+        diff.only_on_stage = []
+    return files_not_removed
+
+
+def stage_diff(
+    local_path: Path,
+    stage_fqn: str,
+) -> DiffResult:
     """
     Diffs the files in a stage with a local folder.
     """