diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 853a78c381..bb8e947690 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -6,6 +6,10 @@ ## New additions * `snow sql` command supports now client-side templating of queries. +* New `snow app deploy` functionality: + * Passing files and directories as arguments syncs these only: `snow app deploy some-file some-dir`. + * `--recursive` syncs all files and subdirectories recursively. + * `--prune` deletes specified files from the stage if they don't exist locally. ## Fixes and improvements * More human-friendly errors in case of corrupted `config.toml` file. diff --git a/src/snowflake/cli/plugins/nativeapp/artifacts.py b/src/snowflake/cli/plugins/nativeapp/artifacts.py index 01fe864255..87cd0278ab 100644 --- a/src/snowflake/cli/plugins/nativeapp/artifacts.py +++ b/src/snowflake/cli/plugins/nativeapp/artifacts.py @@ -1,14 +1,17 @@ import os from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union -from click import ClickException +from click.exceptions import ClickException from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB from snowflake.cli.api.project.schemas.native_app.path_mapping import PathMapping from snowflake.cli.api.secure_path import SecurePath from yaml import safe_load +# Map from source directories and files in the project directory to their path in the deploy directory. Both paths are absolute. +ArtifactDeploymentMap = Dict[Path, Path] + class DeployRootError(ClickException): """ @@ -195,11 +198,14 @@ def resolve_without_follow(path: Path) -> Path: def build_bundle( - project_root: Path, deploy_root: Path, artifacts: List[ArtifactMapping] -): + project_root: Path, + deploy_root: Path, + artifacts: List[ArtifactMapping], +) -> ArtifactDeploymentMap: """ Prepares a local folder (deploy_root) with configured app artifacts. This folder can then be uploaded to a stage. + Returns a map of the copied source files, pointing to where they were copied. """ resolved_root = deploy_root.resolve() if resolved_root.exists() and not resolved_root.is_dir(): @@ -217,6 +223,7 @@ def build_bundle( if resolved_root.exists(): delete(resolved_root) + mapped_files: ArtifactDeploymentMap = {} for artifact in artifacts: dest_path = resolve_without_follow(Path(resolved_root, artifact.dest)) source_paths = get_source_paths(artifact, project_root) @@ -228,7 +235,9 @@ def build_bundle( # copy all files as children of the given destination path for source_path in source_paths: - symlink_or_copy(source_path, dest_path / source_path.name) + dest_child_path = dest_path / source_path.name + symlink_or_copy(source_path, dest_child_path) + mapped_files[source_path.resolve()] = dest_child_path else: # ensure we are copying into the deploy root, not replacing it! if resolved_root not in dest_path.parents: @@ -237,9 +246,11 @@ def build_bundle( if len(source_paths) == 1: # copy a single file as the given destination path symlink_or_copy(source_paths[0], dest_path) + mapped_files[source_paths[0].resolve()] = dest_path else: # refuse to map multiple source files to one destination (undefined behaviour) raise TooManyFilesError(dest_path) + return mapped_files def find_manifest_file(deploy_root: Path) -> Path: @@ -283,3 +294,31 @@ def find_version_info_in_manifest_file( patch_name = version_info[patch_field] return version_name, patch_name + + +def source_path_to_deploy_path( + source_path: Path, mapped_files: ArtifactDeploymentMap +) -> Path: + """Returns the absolute path where the specified source path was copied to during bundle.""" + + source_path = source_path.resolve() + + if source_path in mapped_files: + return mapped_files[source_path] + + # Find the first parent directory that exists in mapped_files + common_root = source_path + while common_root: + if common_root in mapped_files: + break + elif common_root.parent != common_root: + common_root = common_root.parent + else: + raise ClickException(f"Could not find the deploy path of {source_path}") + + # Construct the target deploy path + path_to_symlink = mapped_files[common_root] + relative_path_to_target = Path(source_path).relative_to(common_root) + result = Path(path_to_symlink, relative_path_to_target) + + return result diff --git a/src/snowflake/cli/plugins/nativeapp/commands.py b/src/snowflake/cli/plugins/nativeapp/commands.py index 5b7398cf50..01a00822e8 100644 --- a/src/snowflake/cli/plugins/nativeapp/commands.py +++ b/src/snowflake/cli/plugins/nativeapp/commands.py @@ -1,5 +1,6 @@ import logging -from typing import Optional +from pathlib import Path +from typing import List, Optional import typer from snowflake.cli.api.cli_global_context import cli_context @@ -229,17 +230,44 @@ def app_teardown( @app.command("deploy", requires_connection=True) @with_project_definition("native_app") def app_deploy( + prune: Optional[bool] = typer.Option( + default=None, + help=f"""Whether to delete specified files from the stage if they don't exist locally. If set, the command deletes files that exist in the stage, but not in the local filesystem.""", + ), + recursive: Optional[bool] = typer.Option( + None, + "--recursive", + "-r", + help=f"""Whether to traverse and deploy files from subdirectories. If set, the command deploys all files and subdirectories; otherwise, only files in the current directory are deployed.""", + ), + files: Optional[List[Path]] = typer.Argument( + default=None, + show_default=False, + help=f"""Paths, relative to the the project root, of files you want to upload to a stage. The paths must match one of the artifacts src pattern entries in snowflake.yml. If unspecified, the command syncs all local changes to the stage.""", + ), **options, ) -> CommandResult: """ Creates an application package in your Snowflake account and syncs the local changes to the stage without creating or updating the application. + Running this command with no arguments at all, as in `snow app deploy`, is a shorthand for `snow app deploy --prune --recursive`. """ + if files is None: + files = [] + if prune is None and recursive is None and len(files) == 0: + prune = True + recursive = True + else: + if prune is None: + prune = False + if recursive is None: + recursive = False + manager = NativeAppManager( project_definition=cli_context.project_definition, project_root=cli_context.project_root, ) - manager.build_bundle() - manager.deploy() + mapped_files = manager.build_bundle() + manager.deploy(prune, recursive, files, mapped_files) return MessageResult(f"Deployed successfully.") diff --git a/src/snowflake/cli/plugins/nativeapp/manager.py b/src/snowflake/cli/plugins/nativeapp/manager.py index c583c47eed..e0f1ce7009 100644 --- a/src/snowflake/cli/plugins/nativeapp/manager.py +++ b/src/snowflake/cli/plugins/nativeapp/manager.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from abc import ABC, abstractmethod from functools import cached_property from pathlib import Path @@ -23,8 +24,11 @@ from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.cli.plugins.connection.util import make_snowsight_url from snowflake.cli.plugins.nativeapp.artifacts import ( + ArtifactDeploymentMap, ArtifactMapping, build_bundle, + resolve_without_follow, + source_path_to_deploy_path, translate_artifact, ) from snowflake.cli.plugins.nativeapp.constants import ( @@ -43,8 +47,10 @@ MissingPackageScriptError, UnexpectedOwnerError, ) +from snowflake.cli.plugins.nativeapp.utils import verify_exists, verify_no_directories from snowflake.cli.plugins.stage.diff import ( DiffResult, + filter_from_diff, stage_diff, sync_local_diff_with_stage, ) @@ -100,6 +106,20 @@ def ensure_correct_owner(row: dict, role: str, obj_name: str) -> None: raise UnexpectedOwnerError(obj_name, role, actual_owner) +def _get_paths_to_sync(paths_to_sync: List[Path], deploy_root: Path) -> List[str]: + """Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root.""" + paths = [] + for path in paths_to_sync: + if path.is_dir(): + for current_dir, _dirs, files in os.walk(path): + for file in files: + deploy_path = Path(current_dir, file).relative_to(deploy_root) + paths.append(str(deploy_path)) + else: + paths.append(str(path.relative_to(deploy_root))) + return paths + + class NativeAppCommandProcessor(ABC): @abstractmethod def process(self, *args, **kwargs): @@ -278,13 +298,20 @@ def verify_project_distribution( return False return True - def build_bundle(self) -> None: + def build_bundle(self) -> ArtifactDeploymentMap: """ Populates the local deploy root from artifact sources. """ - build_bundle(self.project_root, self.deploy_root, self.artifacts) - - def sync_deploy_root_with_stage(self, role: str) -> DiffResult: + return build_bundle(self.project_root, self.deploy_root, self.artifacts) + + def sync_deploy_root_with_stage( + self, + role: str, + prune: bool, + recursive: bool, + paths_to_sync: List[Path] = [], # relative to project root + mapped_files: Optional[ArtifactDeploymentMap] = None, + ) -> DiffResult: """ Ensures that the files on our remote stage match the artifacts we have in the local filesystem. Returns the DiffResult used to make changes. @@ -310,6 +337,37 @@ def sync_deploy_root_with_stage(self, role: str) -> DiffResult: % self.deploy_root ) diff: DiffResult = stage_diff(self.deploy_root, self.stage_fqn) + + files_not_removed = [] + if len(paths_to_sync) > 0: + # Deploying specific files/directories + resolved_paths_to_sync = [resolve_without_follow(p) for p in paths_to_sync] + if not recursive: + verify_no_directories(resolved_paths_to_sync) + deploy_paths_to_sync = [ + source_path_to_deploy_path(p, mapped_files) + for p in resolved_paths_to_sync + ] + verify_exists(deploy_paths_to_sync) + paths_to_sync_set = set( + _get_paths_to_sync(deploy_paths_to_sync, self.deploy_root.resolve()) + ) + files_not_removed = filter_from_diff(diff, paths_to_sync_set, prune) + else: + # Full deploy + if not recursive: + deploy_files = os.listdir(str(self.deploy_root.resolve())) + verify_no_directories([Path(path_str) for path_str in deploy_files]) + if not prune: + files_not_removed = diff.only_on_stage + diff.only_on_stage = [] + + if len(files_not_removed) > 0: + files_not_removed_str = "\n".join(files_not_removed) + cc.warning( + f"The following files exist only on the stage:\n{files_not_removed_str}\n\nUse the --prune flag to delete them from the stage." + ) + cc.message(str(diff)) # Upload diff-ed files to application package stage @@ -435,7 +493,13 @@ def _apply_package_scripts(self) -> None: err, role=self.package_role, warehouse=self.package_warehouse ) - def deploy(self) -> DiffResult: + def deploy( + self, + prune: bool, + recursive: bool, + paths_to_sync: List[Path] = [], + mapped_files: Optional[ArtifactDeploymentMap] = None, + ) -> DiffResult: """app deploy process""" # 1. Create an empty application package, if none exists @@ -446,6 +510,8 @@ def deploy(self) -> DiffResult: self._apply_package_scripts() # 3. Upload files from deploy root local folder to the above stage - diff = self.sync_deploy_root_with_stage(self.package_role) + diff = self.sync_deploy_root_with_stage( + self.package_role, prune, recursive, paths_to_sync, mapped_files + ) return diff diff --git a/src/snowflake/cli/plugins/nativeapp/run_processor.py b/src/snowflake/cli/plugins/nativeapp/run_processor.py index 05dcfe4d9d..828d640c5c 100644 --- a/src/snowflake/cli/plugins/nativeapp/run_processor.py +++ b/src/snowflake/cli/plugins/nativeapp/run_processor.py @@ -297,5 +297,5 @@ def process( ) return - diff = self.deploy() + diff = self.deploy(prune=True, recursive=True) self._create_dev_app(diff) diff --git a/src/snowflake/cli/plugins/nativeapp/utils.py b/src/snowflake/cli/plugins/nativeapp/utils.py index 9091142a6c..0a6249765e 100644 --- a/src/snowflake/cli/plugins/nativeapp/utils.py +++ b/src/snowflake/cli/plugins/nativeapp/utils.py @@ -1,7 +1,9 @@ from os import PathLike from pathlib import Path from sys import stdin, stdout -from typing import Optional, Union +from typing import List, Optional, Union + +from click import ClickException def needs_confirmation(needs_confirm: bool, auto_yes: bool) -> bool: @@ -65,3 +67,17 @@ def shallow_git_clone(url: Union[str, PathLike], to_path: Union[str, PathLike]): repo.close() return repo + + +def verify_no_directories(paths_to_sync: List[Path]): + for path in paths_to_sync: + if path.is_dir(): + raise ClickException( + f"{path} is a directory. Add the -r flag to deploy directories." # + ) + + +def verify_exists(paths_to_sync: List[Path]): + for path in paths_to_sync: + if not path.exists(): + raise ClickException(f"The following path does not exist: {path}") diff --git a/src/snowflake/cli/plugins/nativeapp/version/version_processor.py b/src/snowflake/cli/plugins/nativeapp/version/version_processor.py index 574fa8e7f3..1763687180 100644 --- a/src/snowflake/cli/plugins/nativeapp/version/version_processor.py +++ b/src/snowflake/cli/plugins/nativeapp/version/version_processor.py @@ -199,7 +199,9 @@ def process( self._apply_package_scripts() # Upload files from deploy root local folder to the above stage - self.sync_deploy_root_with_stage(self.package_role) + self.sync_deploy_root_with_stage( + self.package_role, prune=True, recursive=True + ) # Warn if the version exists in a release directive(s) existing_release_directives = ( diff --git a/src/snowflake/cli/plugins/stage/diff.py b/src/snowflake/cli/plugins/stage/diff.py index af6c42e913..a9b2037763 100644 --- a/src/snowflake/cli/plugins/stage/diff.py +++ b/src/snowflake/cli/plugins/stage/diff.py @@ -3,9 +3,11 @@ import re from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Set -from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError +from snowflake.cli.api.exceptions import ( + SnowflakeSQLExecutionError, +) from snowflake.cli.api.secure_path import UNLIMITED, SecurePath from snowflake.connector.cursor import DictCursor @@ -154,7 +156,26 @@ def build_md5_map(list_stage_cursor: DictCursor) -> Dict[str, str]: } -def stage_diff(local_path: Path, stage_fqn: str) -> DiffResult: +def filter_from_diff( + diff: DiffResult, paths_to_sync: Set[str], prune: bool +) -> List[str]: + """Modifies the given diff, keeping only the provided paths. If prune is false, remote-only paths will be empty and the non-removed paths will be returned.""" + diff.different = [i for i in diff.different if i in paths_to_sync] + diff.only_local = [i for i in diff.only_local if i in paths_to_sync] + only_on_stage = [i for i in diff.only_on_stage if i in paths_to_sync] + files_not_removed = [] + if prune: + diff.only_on_stage = only_on_stage + else: + files_not_removed = only_on_stage + diff.only_on_stage = [] + return files_not_removed + + +def stage_diff( + local_path: Path, + stage_fqn: str, +) -> DiffResult: """ Diffs the files in a stage with a local folder. """ diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index f85460b4b7..ec92dc207b 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -59,15 +59,36 @@ # name: test_help_messages[app.deploy] ''' - Usage: default app deploy [OPTIONS] + Usage: default app deploy [OPTIONS] [FILES]... Creates an application package in your Snowflake account and syncs the local - changes to the stage without creating or updating the application. + changes to the stage without creating or updating the application. Running + this command with no arguments at all, as in `snow app deploy`, is a shorthand + for `snow app deploy --prune --recursive`. + ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ + │ files [FILES]... Paths, relative to the the project root, of files │ + │ you want to upload to a stage. The paths must match │ + │ one of the artifacts src pattern entries in │ + │ snowflake.yml. If unspecified, the command syncs │ + │ all local changes to the stage. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ ╭─ Options ────────────────────────────────────────────────────────────────────╮ - │ --project -p TEXT Path where the Snowflake Native App project │ - │ resides. Defaults to current working directory. │ - │ --help -h Show this message and exit. │ + │ --prune --no-prune Whether to delete specified files from │ + │ the stage if they don't exist locally. │ + │ If set, the command deletes files that │ + │ exist in the stage, but not in the │ + │ local filesystem. │ + │ [default: no-prune] │ + │ --recursive -r Whether to traverse and deploy files │ + │ from subdirectories. If set, the │ + │ command deploys all files and │ + │ subdirectories; otherwise, only files │ + │ in the current directory are deployed. │ + │ --project -p TEXT Path where the Snowflake Native App │ + │ project resides. Defaults to current │ + │ working directory. │ + │ --help -h Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────╯ ╭─ Connection configuration ───────────────────────────────────────────────────╮ │ --connection,--environment -c TEXT Name of the connection, as defined │ @@ -741,7 +762,9 @@ ╭─ Commands ───────────────────────────────────────────────────────────────────╮ │ deploy Creates an application package in your Snowflake account and │ │ syncs the local changes to the stage without creating or updating │ - │ the application. │ + │ the application. Running this command with no arguments at all, │ + │ as in `snow app deploy`, is a shorthand for `snow app deploy │ + │ --prune --recursive`. │ │ init Initializes a Snowflake Native App project. │ │ open Opens the Snowflake Native App inside of your browser, once it │ │ has been installed in your account. │ diff --git a/tests/nativeapp/test_artifacts.py b/tests/nativeapp/test_artifacts.py index 51b311d8d6..41b56e6754 100644 --- a/tests/nativeapp/test_artifacts.py +++ b/tests/nativeapp/test_artifacts.py @@ -1,7 +1,9 @@ +import os from pathlib import Path from typing import List, Optional import pytest +from click.exceptions import ClickException from snowflake.cli.api.project.definition import load_project_definition from snowflake.cli.plugins.nativeapp.artifacts import ( ArtifactMapping, @@ -11,9 +13,13 @@ SourceNotFoundError, TooManyFilesError, build_bundle, + resolve_without_follow, + source_path_to_deploy_path, translate_artifact, ) +from tests.nativeapp.utils import touch + def trimmed_contents(path: Path) -> Optional[str]: if not path.is_file(): @@ -163,3 +169,60 @@ def test_too_many_files(project_definition_files): ArtifactMapping("app/streamlit/*.py", "somehow_combined_streamlits.py") ], ) + + +@pytest.mark.parametrize( + "project_path,expected_path", + [ + [ + "srcfile", + "deploy/file", + ], + [ + "srcdir", + "deploy/dir", + ], + [ + "srcdir/nested_file1", + "deploy/dir/nested_file1", + ], + [ + "srcdir/nested_dir/nested_file2", + "deploy/dir/nested_dir/nested_file2", + ], + [ + "srcdir/nested_dir", + "deploy/dir/nested_dir", + ], + [ + "not-in-deploy", + None, + ], + ], +) +def test_source_path_to_deploy_path( + temp_dir, + project_path, + expected_path, +): + # Source files + touch("srcfile") + touch("srcdir/nested_file1") + touch("srcdir/nested_dir/nested_file2") + touch("not-in-deploy") + # Build + os.mkdir("deploy") + os.symlink("srcfile", "deploy/file") + os.symlink(Path("srcdir").resolve(), Path("deploy/dir")) + + files_mapping = { + Path("srcdir").resolve(): resolve_without_follow(Path("deploy/dir")), + Path("srcfile").resolve(): resolve_without_follow(Path("deploy/file")), + } + + if expected_path: + result = source_path_to_deploy_path(Path(project_path).resolve(), files_mapping) + assert result == resolve_without_follow(Path(expected_path)) + else: + with pytest.raises(ClickException): + source_path_to_deploy_path(Path(project_path).resolve(), files_mapping) diff --git a/tests/nativeapp/test_manager.py b/tests/nativeapp/test_manager.py index 49816cfbbb..49d8bba489 100644 --- a/tests/nativeapp/test_manager.py +++ b/tests/nativeapp/test_manager.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from textwrap import dedent from unittest import mock @@ -17,9 +18,12 @@ from snowflake.cli.plugins.nativeapp.manager import ( NativeAppManager, SnowflakeSQLExecutionError, + _get_paths_to_sync, ensure_correct_owner, ) -from snowflake.cli.plugins.stage.diff import DiffResult +from snowflake.cli.plugins.stage.diff import ( + DiffResult, +) from snowflake.connector import ProgrammingError from snowflake.connector.cursor import DictCursor @@ -34,6 +38,7 @@ NATIVEAPP_MODULE, mock_execute_helper, mock_snowflake_yml_file, + touch, ) from tests.testing_utils.files_and_dirs import create_named_file @@ -78,7 +83,7 @@ def test_sync_deploy_root_with_stage( native_app_manager = _get_na_manager() assert mock_diff_result.has_changes() - native_app_manager.sync_deploy_root_with_stage("new_role") + native_app_manager.sync_deploy_root_with_stage("new_role", True, True) expected = [ mock.call("select current_role()", cursor_class=DictCursor), @@ -104,6 +109,56 @@ def test_sync_deploy_root_with_stage( ) +@mock.patch(NATIVEAPP_MANAGER_EXECUTE) +@mock.patch(f"{NATIVEAPP_MODULE}.sync_local_diff_with_stage") +@mock.patch(f"{NATIVEAPP_MODULE}.stage_diff") +@mock.patch(f"{NATIVEAPP_MODULE}.cc.warning") +@pytest.mark.parametrize( + "prune,only_on_stage_files,expected_warn", + [ + [ + True, + ["only-stage.txt"], + False, + ], + [ + False, + ["only-stage-1.txt", "only-stage-2.txt"], + True, + ], + ], +) +def test_sync_deploy_root_with_stage_prune( + mock_warning, + mock_stage_diff, + mock_local_diff_with_stage, + mock_execute, + prune, + only_on_stage_files, + expected_warn, + temp_dir, +): + mock_stage_diff.return_value = DiffResult(only_on_stage=only_on_stage_files) + create_named_file( + file_name="snowflake.yml", + dir_name=os.getcwd(), + contents=[mock_snowflake_yml_file], + ) + native_app_manager = _get_na_manager() + + native_app_manager.sync_deploy_root_with_stage("role", prune, True) + + if expected_warn: + files_str = "\n".join(only_on_stage_files) + warn_message = f"""The following files exist only on the stage: +{files_str} + +Use the --prune flag to delete them from the stage.""" + mock_warning.assert_called_once_with(warn_message) + else: + mock_warning.assert_not_called() + + @mock.patch(NATIVEAPP_MANAGER_EXECUTE) def test_get_app_pkg_distribution_in_snowflake(mock_execute, temp_dir, mock_cursor): @@ -716,3 +771,32 @@ def test_create_app_pkg_internal_distribution_no_special_comment( mock_warning.assert_called_once_with( "Continuing to execute `snow app run` on application package app_pkg with distribution 'internal'." ) + + +@pytest.mark.parametrize( + "paths_to_sync,expected_result", + [ + [ + ["deploy/dir"], + ["dir/nested_file1", "dir/nested_file2", "dir/nested_dir/nested_file3"], + ], + [["deploy/dir/nested_dir"], ["dir/nested_dir/nested_file3"]], + [ + ["deploy/file", "deploy/dir/nested_dir/nested_file3"], + ["file", "dir/nested_dir/nested_file3"], + ], + ], +) +def test_get_paths_to_sync( + temp_dir, + paths_to_sync, + expected_result, +): + touch("deploy/file") + touch("deploy/dir/nested_file1") + touch("deploy/dir/nested_file2") + touch("deploy/dir/nested_dir/nested_file3") + + paths_to_sync = [Path(p) for p in paths_to_sync] + result = _get_paths_to_sync(paths_to_sync, Path("deploy/")) + assert result.sort() == expected_result.sort() diff --git a/tests/nativeapp/utils.py b/tests/nativeapp/utils.py index 12be464ff0..2c915ccd15 100644 --- a/tests/nativeapp/utils.py +++ b/tests/nativeapp/utils.py @@ -1,3 +1,4 @@ +from pathlib import Path from textwrap import dedent NATIVEAPP_MODULE = "snowflake.cli.plugins.nativeapp.manager" @@ -78,3 +79,9 @@ def mock_execute_helper(mock_input: list): side_effects, expected = map(list, zip(*mock_input)) return side_effects, expected + + +def touch(path: str): + file = Path(path) + file.parent.mkdir(exist_ok=True, parents=True) + file.write_text("") diff --git a/tests/stage/test_diff.py b/tests/stage/test_diff.py index 09bb52a8d3..ab36f617d9 100644 --- a/tests/stage/test_diff.py +++ b/tests/stage/test_diff.py @@ -4,12 +4,15 @@ from unittest import mock import pytest -from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError +from snowflake.cli.api.exceptions import ( + SnowflakeSQLExecutionError, +) from snowflake.cli.plugins.stage.diff import ( DiffResult, build_md5_map, delete_only_on_stage_files, enumerate_files, + filter_from_diff, get_stage_path_from_file, put_files_on_stage, stage_diff, @@ -20,6 +23,7 @@ from tests.testing_utils.files_and_dirs import temp_local_dir STAGE_MANAGER = "snowflake.cli.plugins.stage.manager.StageManager" +STAGE_DIFF = "snowflake.cli.plugins.object.stage.diff" FILE_CONTENTS = { "README.md": "This is a README\n", @@ -227,7 +231,7 @@ def test_sync_local_diff_with_stage(mock_remove, other_directory): temp_dir = Path(other_directory) mock_remove.side_effect = Exception("Mock Exception") mock_remove.return_value = None - diff: DiffResult = DiffResult() + diff = DiffResult() diff.only_on_stage = ["some_file_on_stage"] stage_name = "some_stage_name" @@ -238,3 +242,66 @@ def test_sync_local_diff_with_stage(mock_remove, other_directory): diff_result=diff, stage_path=stage_name, ) + + +def test_filter_from_diff(): + diff = DiffResult() + diff.different = [ + "different", + "different-2", + "dir/different", + "dir/different-2", + ] + diff.only_local = [ + "only_local", + "only_local-2", + "dir/only_local", + "dir/only_local-2", + ] + diff.only_on_stage = [ + "only_on_stage", + "only_on_stage-2", + "dir/only_on_stage", + "dir/only_on_stage-2", + ] + + paths_to_sync = set( + [ + "different", + "only-local", + "only-stage", + "dir/different", + "dir/only-local", + "dir/only-stage", + ] + ) + filter_from_diff(diff, paths_to_sync, True) + + for path in diff.different: + assert path in paths_to_sync + for path in diff.only_local: + assert path in paths_to_sync + for path in diff.only_on_stage: + assert path in paths_to_sync + + +# When prune flag is off, remote-only files are filtered out +def test_filter_from_diff_no_prune(): + diff = DiffResult() + diff.only_on_stage = [ + "only-stage-1.txt", + "only-stage-2.txt", + "only-stage-3.txt", + ] + paths_to_sync = set( + [ + "on-both.txt", + "only-stage-1.txt", + "only-stage-2.txt", + "only-local.txt", + ] + ) + + filter_from_diff(diff, paths_to_sync, False) + + assert len(diff.only_on_stage) == 0 diff --git a/tests_integration/test_nativeapp.py b/tests_integration/test_nativeapp.py index dda1967793..b7139150c4 100644 --- a/tests_integration/test_nativeapp.py +++ b/tests_integration/test_nativeapp.py @@ -4,6 +4,8 @@ from snowflake.cli.api.project.util import generate_user_env from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.plugins.nativeapp.init import OFFICIAL_TEMPLATES_GITHUB_URL +from tests.nativeapp.utils import touch +from click import ClickException from tests.project.fixtures import * from tests_integration.test_utils import ( @@ -500,7 +502,7 @@ def test_nativeapp_init_from_repo_with_single_template( # Tests a simple flow of executing "snow app deploy", verifying that an application package was created, and an application was not @pytest.mark.integration -def test_nativeapp_init_deploy( +def test_nativeapp_deploy( runner, snowflake_session, temporary_working_directory, @@ -572,3 +574,261 @@ def test_nativeapp_init_deploy( env=TEST_ENV, ) assert result.exit_code == 0 + + +@pytest.mark.integration +@pytest.mark.parametrize( + "command,contains,not_contains", + [ + # deploy --prune removes remote-only files + ["app deploy --prune", ["stage/manifest.yml"], ["stage/README.md"]], + # deploy removes remote-only files (--prune is the default value) + ["app deploy", ["stage/manifest.yml"], ["stage/README.md"]], + # deploy --no-prune does not delete remote-only files + ["app deploy --no-prune", ["stage/README.md"], []], + ], +) +def test_nativeapp_deploy_prune( + command, + contains, + not_contains, + runner, + snowflake_session, + temporary_working_directory, +): + project_name = "myapp" + result = runner.invoke_json( + ["app", "init", project_name], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + with pushd(Path(os.getcwd(), project_name)): + result = runner.invoke_with_connection_json( + ["app", "deploy"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + try: + # delete a file locally + os.remove(os.path.join("app", "README.md")) + + # deploy + result = runner.invoke_with_connection_json( + command.split(), + env=TEST_ENV, + ) + assert result.exit_code == 0 + + # verify the file does not exist on the stage + package_name = f"{project_name}_pkg_{USER_NAME}".upper() + stage_name = "app_src.stage" # as defined in native-apps-templates/basic + stage_files = runner.invoke_with_connection_json( + ["stage", "list-files", f"{package_name}.{stage_name}"], + env=TEST_ENV, + ) + for name in contains: + assert contains_row_with(stage_files.json, {"name": name}) + for name in not_contains: + assert not_contains_row_with(stage_files.json, {"name": name}) + + # make sure we always delete the app + result = runner.invoke_with_connection_json( + ["app", "teardown"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + finally: + # teardown is idempotent, so we can execute it again with no ill effects + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + +# Tests a simple flow of executing "snow app deploy [files]", verifying that only the specified files are synced to the stage +@pytest.mark.integration +def test_nativeapp_deploy_files( + runner, + temporary_working_directory, +): + project_name = "myapp" + result = runner.invoke_json( + ["app", "init", project_name], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + with pushd(Path(os.getcwd(), project_name)): + # sync only two specific files to stage + result = runner.invoke_with_connection_json( + ["app", "deploy", "app/manifest.yml", "app/setup_script.sql"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + try: + # manifest and script files exist, readme doesn't exist + package_name = f"{project_name}_pkg_{USER_NAME}".upper() + stage_name = "app_src.stage" # as defined in native-apps-templates/basic + stage_files = runner.invoke_with_connection_json( + ["stage", "list-files", f"{package_name}.{stage_name}"], + env=TEST_ENV, + ) + assert contains_row_with(stage_files.json, {"name": "stage/manifest.yml"}) + assert contains_row_with( + stage_files.json, {"name": "stage/setup_script.sql"} + ) + assert not_contains_row_with(stage_files.json, {"name": "stage/README.md"}) + + # make sure we always delete the app + result = runner.invoke_with_connection_json( + ["app", "teardown"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + finally: + # teardown is idempotent, so we can execute it again with no ill effects + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + +# Tests that files inside of a symlinked directory are deployed +@pytest.mark.integration +def test_nativeapp_deploy_nested_directories( + runner, + temporary_working_directory, +): + project_name = "myapp" + project_dir = "app root" + result = runner.invoke_json( + ["app", "init", project_dir, "--name", project_name], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + with pushd(Path(os.getcwd(), project_dir)): + # create nested file under app/ + touch("app/nested/dir/file.txt") + + result = runner.invoke_with_connection_json( + ["app", "deploy", "app/nested/dir/file.txt"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + try: + package_name = f"{project_name}_pkg_{USER_NAME}".upper() + stage_name = "app_src.stage" # as defined in native-apps-templates/basic + stage_files = runner.invoke_with_connection_json( + ["stage", "list-files", f"{package_name}.{stage_name}"], + env=TEST_ENV, + ) + assert contains_row_with( + stage_files.json, {"name": "stage/nested/dir/file.txt"} + ) or contains_row_with( + # Windows path + stage_files.json, + {"name": "stage/nested\\dir/file.txt"}, + ) + + # make sure we always delete the app + result = runner.invoke_with_connection_json( + ["app", "teardown"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + finally: + # teardown is idempotent, so we can execute it again with no ill effects + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + +# Tests that deploying a directory recursively syncs all of its contents +@pytest.mark.integration +def test_nativeapp_deploy_directory( + runner, + temporary_working_directory, +): + project_name = "myapp" + project_dir = "app root" + result = runner.invoke_json( + ["app", "init", project_dir, "--name", project_name], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + with pushd(Path(os.getcwd(), project_dir)): + touch("app/dir/file.txt") + result = runner.invoke_with_connection_json( + ["app", "deploy", "app/dir", "-r"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + try: + package_name = f"{project_name}_pkg_{USER_NAME}".upper() + stage_name = "app_src.stage" # as defined in native-apps-templates/basic + stage_files = runner.invoke_with_connection_json( + ["stage", "list-files", f"{package_name}.{stage_name}"], + env=TEST_ENV, + ) + assert contains_row_with(stage_files.json, {"name": "stage/dir/file.txt"}) + + # make sure we always delete the app + result = runner.invoke_with_connection_json( + ["app", "teardown"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + finally: + # teardown is idempotent, so we can execute it again with no ill effects + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + +# Tests that deploying a directory without specifying -r returns an error +@pytest.mark.integration +def test_nativeapp_deploy_directory_no_recursive( + runner, + temporary_working_directory, +): + project_name = "myapp" + project_dir = "app root" + result = runner.invoke_json( + ["app", "init", project_dir, "--name", project_name], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + with pushd(Path(os.getcwd(), project_dir)): + try: + touch("app/nested/dir/file.txt") + result = runner.invoke_with_connection_json( + ["app", "deploy", "app/nested"], + env=TEST_ENV, + ) + assert result.exit_code == 1, result.output + + finally: + # teardown is idempotent, so we can execute it again with no ill effects + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force"], + env=TEST_ENV, + ) + assert result.exit_code == 0