From 0f6296be26a7ca95abfacc669da10c099c6787f9 Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Wed, 17 Jul 2024 15:34:44 +0200 Subject: [PATCH 1/7] Fixed logs init with disabled save_logs and debug flag (#1324) --- src/snowflake/cli/app/loggers.py | 4 +++- tests/test_logs.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/snowflake/cli/app/loggers.py b/src/snowflake/cli/app/loggers.py index bf63930242..bc2b46838e 100644 --- a/src/snowflake/cli/app/loggers.py +++ b/src/snowflake/cli/app/loggers.py @@ -183,7 +183,9 @@ def create_loggers(verbose: bool, debug: bool): else: # We need to remove handler definition - otherwise it creates file even if `save_logs` is False del config.handlers["file"] - config.loggers["snowflake.cli"].handlers.remove("file") + for logger in config.loggers.values(): + if "file" in logger.handlers: + logger.handlers.remove("file") config.loggers["snowflake.cli"].level = global_log_level config.loggers["snowflake"].level = global_log_level diff --git a/tests/test_logs.py b/tests/test_logs.py index 47329d13d2..8af4c477ae 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -213,3 +213,9 @@ def test_log_files_permissions(setup_config_and_logs): with setup_config_and_logs(save_logs=True) as logs_path: print_log_messages() assert_file_permissions_are_strict(get_logs_file(logs_path)) + + +def test_disabled_logs_with_debug_flag(setup_config_and_logs): + with setup_config_and_logs(save_logs=False, debug=True): + print_log_messages() + # Should not raise exception From 58e85e13a6b0d83e4edd277c0418829bed2fd6c6 Mon Sep 17 00:00:00 2001 From: Guy Bloom Date: Wed, 17 Jul 2024 10:28:43 -0400 Subject: [PATCH 2/7] Support project definition v2 in "snow app open" (#1307) * create decorator * unit tests * add "app open" integration tests * teardown with v1 * add tests --- .../cli/plugins/nativeapp/commands.py | 4 + .../v2_conversions/v2_to_v1_decorator.py | 99 +++++++++++ tests/nativeapp/test_v2_to_v1.py | 156 ++++++++++++++++++ tests_integration/nativeapp/test_open.py | 117 +++++++++++++ .../projects/napp_init_v1/app/README.md | 4 + .../projects/napp_init_v1/app/manifest.yml | 9 + .../napp_init_v1/app/setup_script.sql | 11 ++ .../projects/napp_init_v1/snowflake.yml | 7 + .../projects/napp_init_v2/app/README.md | 4 + .../projects/napp_init_v2/app/manifest.yml | 9 + .../napp_init_v2/app/setup_script.sql | 11 ++ .../projects/napp_init_v2/snowflake.yml | 16 ++ 12 files changed, 447 insertions(+) create mode 100644 src/snowflake/cli/plugins/nativeapp/v2_conversions/v2_to_v1_decorator.py create mode 100644 tests/nativeapp/test_v2_to_v1.py create mode 100644 tests_integration/nativeapp/test_open.py create mode 100644 tests_integration/test_data/projects/napp_init_v1/app/README.md create mode 100644 tests_integration/test_data/projects/napp_init_v1/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_init_v1/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_init_v1/snowflake.yml create mode 100644 tests_integration/test_data/projects/napp_init_v2/app/README.md create mode 100644 tests_integration/test_data/projects/napp_init_v2/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_init_v2/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_init_v2/snowflake.yml diff --git a/src/snowflake/cli/plugins/nativeapp/commands.py b/src/snowflake/cli/plugins/nativeapp/commands.py index f0822f8d98..ecfa4210e7 100644 --- a/src/snowflake/cli/plugins/nativeapp/commands.py +++ b/src/snowflake/cli/plugins/nativeapp/commands.py @@ -58,6 +58,9 @@ get_first_paragraph_from_markdown_file, shallow_git_clone, ) +from snowflake.cli.plugins.nativeapp.v2_conversions.v2_to_v1_decorator import ( + nativeapp_definition_v2_to_v1, +) from snowflake.cli.plugins.nativeapp.version.commands import app as versions_app app = SnowTyperFactory( @@ -228,6 +231,7 @@ def app_run( @app.command("open", requires_connection=True) @with_project_definition() +@nativeapp_definition_v2_to_v1 def app_open( **options, ) -> CommandResult: diff --git a/src/snowflake/cli/plugins/nativeapp/v2_conversions/v2_to_v1_decorator.py b/src/snowflake/cli/plugins/nativeapp/v2_conversions/v2_to_v1_decorator.py new file mode 100644 index 0000000000..b71d003407 --- /dev/null +++ b/src/snowflake/cli/plugins/nativeapp/v2_conversions/v2_to_v1_decorator.py @@ -0,0 +1,99 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from functools import wraps +from typing import Any, Dict, Optional + +from click import ClickException +from snowflake.cli.api.cli_global_context import cli_context, cli_context_manager +from snowflake.cli.api.project.schemas.entities.application_entity import ( + ApplicationEntity, +) +from snowflake.cli.api.project.schemas.entities.application_package_entity import ( + ApplicationPackageEntity, +) +from snowflake.cli.api.project.schemas.project_definition import ( + DefinitionV11, + DefinitionV20, +) + + +def _pdf_v2_to_v1(v2_definition: DefinitionV20) -> DefinitionV11: + pdfv1: Dict[str, Any] = {"definition_version": "1.1", "native_app": {}} + + app_package_definition: ApplicationPackageEntity = None + app_definition: Optional[ApplicationEntity] = None + + for key, entity in v2_definition.entities.items(): + if entity.get_type() == ApplicationPackageEntity.get_type(): + if app_package_definition: + raise ClickException( + "More than one application package entity exists in the project definition file." + ) + app_package_definition = entity + elif entity.get_type() == ApplicationEntity.get_type(): + if app_definition: + raise ClickException( + "More than one application entity exists in the project definition file." + ) + app_definition = entity + if not app_package_definition: + raise ClickException( + "Could not find an application package entity in the project definition file." + ) + + # NativeApp + pdfv1["native_app"]["name"] = "Auto converted NativeApp project from V2" + pdfv1["native_app"]["artifacts"] = app_package_definition.artifacts + pdfv1["native_app"]["source_stage"] = app_package_definition.stage + + # Package + pdfv1["native_app"]["package"] = {} + pdfv1["native_app"]["package"]["name"] = app_package_definition.name + + # Application + if app_definition: + pdfv1["native_app"]["application"] = {} + pdfv1["native_app"]["application"]["name"] = app_definition.name + if app_definition.meta and app_definition.meta.role: + pdfv1["native_app"]["application"]["role"] = app_definition.meta.role + + # Override the definition object in global context + return DefinitionV11(**pdfv1) + + +def nativeapp_definition_v2_to_v1(func): + """ + A command decorator that attempts to automatically convert a native app project from + definition v2 to v1.1. Assumes with_project_definition() has already been called. + The definition object in CliGlobalContext will be replaced with the converted object. + Exactly one application package entity type is expected, and up to one application + entity type is expected. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + original_pdf: DefinitionV20 = cli_context.project_definition + if not original_pdf: + raise ValueError( + "Project definition could not be found. The nativeapp_definition_v2_to_v1 command decorator assumes with_project_definition() was called before it." + ) + if original_pdf.definition_version == "2": + pdfv1 = _pdf_v2_to_v1(original_pdf) + cli_context_manager.set_project_definition(pdfv1) + return func(*args, **kwargs) + + return wrapper diff --git a/tests/nativeapp/test_v2_to_v1.py b/tests/nativeapp/test_v2_to_v1.py new file mode 100644 index 0000000000..99beb19705 --- /dev/null +++ b/tests/nativeapp/test_v2_to_v1.py @@ -0,0 +1,156 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest +from click import ClickException +from snowflake.cli.api.cli_global_context import cli_context, cli_context_manager +from snowflake.cli.api.project.schemas.project_definition import ( + DefinitionV11, + DefinitionV20, +) +from snowflake.cli.plugins.nativeapp.v2_conversions.v2_to_v1_decorator import ( + _pdf_v2_to_v1, + nativeapp_definition_v2_to_v1, +) + +from tests.testing_utils.mock_config import mock_config_key + + +@pytest.mark.parametrize( + "pdfv2_input, expected_pdfv1, expected_error", + [ + [ + { + "definition_version": "2", + "entities": { + "pkg1": { + "type": "application package", + "name": "pkg", + "artifacts": [], + "manifest": "", + }, + "pkg2": { + "type": "application package", + "name": "pkg", + "artifacts": [], + "manifest": "", + }, + }, + }, + None, + "More than one application package entity exists", + ], + [ + { + "definition_version": "2", + "entities": { + "pkg": { + "type": "application package", + "name": "pkg", + "artifacts": [], + "manifest": "", + }, + "app1": { + "type": "application", + "name": "pkg", + "from": {"target": "pkg"}, + }, + "app2": { + "type": "application", + "name": "pkg", + "from": {"target": "pkg"}, + }, + }, + }, + None, + "More than one application entity exists", + ], + [ + { + "definition_version": "2", + "entities": { + "pkg": { + "type": "application package", + "name": "pkg_name", + "artifacts": [{"src": "app/*", "dest": "./"}], + "manifest": "", + "stage": "app.stage", + }, + "app": { + "type": "application", + "name": "app_name", + "from": {"target": "pkg"}, + "meta": {"role": "app_role"}, + }, + }, + }, + { + "definition_version": "1.1", + "native_app": { + "name": "Auto converted NativeApp project from V2", + "artifacts": [{"src": "app/*", "dest": "./"}], + "source_stage": "app.stage", + "package": { + "name": "pkg_name", + }, + "application": { + "name": "app_name", + "role": "app_role", + }, + }, + }, + None, + ], + ], +) +def test_v2_to_v1_conversions(pdfv2_input, expected_pdfv1, expected_error): + with mock_config_key("enable_project_definition_v2", True): + pdfv2 = DefinitionV20(**pdfv2_input) + if expected_error: + with pytest.raises(ClickException, match=expected_error) as err: + _pdf_v2_to_v1(pdfv2) + else: + pdfv1_actual = vars(_pdf_v2_to_v1(pdfv2)) + pdfv1_expected = vars(DefinitionV11(**expected_pdfv1)) + + # Assert that the expected dict is a subset of the actual dict + assert {**pdfv1_actual, **pdfv1_expected} == pdfv1_actual + + +def test_decorator_error_when_no_project_exists(): + with pytest.raises(ValueError, match="Project definition could not be found"): + nativeapp_definition_v2_to_v1(lambda *args: None)() + + +@mock.patch( + "snowflake.cli.plugins.nativeapp.v2_conversions.v2_to_v1_decorator._pdf_v2_to_v1" +) +def test_decorator_skips_when_project_is_not_v2(mock_pdf_v2_to_v1): + pdfv1 = DefinitionV11( + **{ + "definition_version": "1.1", + "native_app": { + "name": "test", + "artifacts": [{"src": "*", "dest": "./"}], + }, + }, + ) + cli_context_manager.set_project_definition(pdfv1) + + nativeapp_definition_v2_to_v1(lambda *args: None)() + + mock_pdf_v2_to_v1.launch.assert_not_called() + assert cli_context.project_definition == pdfv1 diff --git a/tests_integration/nativeapp/test_open.py b/tests_integration/nativeapp/test_open.py new file mode 100644 index 0000000000..dbdaae9bb3 --- /dev/null +++ b/tests_integration/nativeapp/test_open.py @@ -0,0 +1,117 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid +from unittest import mock +import re +import os + +from snowflake.cli.api.project.util import generate_user_env + +from tests.project.fixtures import * + +USER_NAME = f"user_{uuid.uuid4().hex}" +TEST_ENV = generate_user_env(USER_NAME) + + +@pytest.mark.integration +@mock.patch("typer.launch") +def test_nativeapp_open( + mock_typer_launch, + runner, + snowflake_session, + project_directory, +): + project_name = "myapp" + app_name = f"{project_name}_{USER_NAME}" + + with project_directory("napp_init_v1"): + try: + result = runner.invoke_with_connection_json( + ["app", "run"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + result = runner.invoke_with_connection_json( + ["app", "open"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + assert "Snowflake Native App opened in browser." in result.output + + mock_call = mock_typer_launch.call_args_list[0].args[0] + assert re.match( + rf"https://app.snowflake.com/.*#/apps/application/{app_name}", + mock_call, + re.IGNORECASE, + ) + + finally: + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force", "--cascade"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + +@pytest.mark.integration +@mock.patch.dict( + os.environ, + { + "SNOWFLAKE_CLI_FEATURES_ENABLE_PROJECT_DEFINITION_V2": "true", + }, +) +@mock.patch("typer.launch") +def test_nativeapp_open_v2( + mock_typer_launch, + runner, + snowflake_session, + project_directory, +): + project_name = "myapp" + app_name = f"{project_name}_{USER_NAME}" + + # TODO Move to napp_init_v2 block once "snow app run" supports definition v2 + with project_directory("napp_init_v1"): + result = runner.invoke_with_connection_json( + ["app", "run"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + with project_directory("napp_init_v2"): + try: + result = runner.invoke_with_connection_json( + ["app", "open"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + assert "Snowflake Native App opened in browser." in result.output + + mock_call = mock_typer_launch.call_args_list[0].args[0] + assert re.match( + rf"https://app.snowflake.com/.*#/apps/application/{app_name}", + mock_call, + re.IGNORECASE, + ) + + finally: + # TODO Move to napp_init_v2 block once "snow app run" supports definition v2 + with project_directory("napp_init_v1"): + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force", "--cascade"], + env=TEST_ENV, + ) + assert result.exit_code == 0 diff --git a/tests_integration/test_data/projects/napp_init_v1/app/README.md b/tests_integration/test_data/projects/napp_init_v1/app/README.md new file mode 100644 index 0000000000..f66bf75c9b --- /dev/null +++ b/tests_integration/test_data/projects/napp_init_v1/app/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains an extremely simple application that is used for +integration testing SnowCLI. diff --git a/tests_integration/test_data/projects/napp_init_v1/app/manifest.yml b/tests_integration/test_data/projects/napp_init_v1/app/manifest.yml new file mode 100644 index 0000000000..5b8ef74e8a --- /dev/null +++ b/tests_integration/test_data/projects/napp_init_v1/app/manifest.yml @@ -0,0 +1,9 @@ +# This is a manifest.yml file, a required component of creating a Snowflake Native App. +# This file defines properties required by the application package, including the location of the setup script and version definitions. +# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file. + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md diff --git a/tests_integration/test_data/projects/napp_init_v1/app/setup_script.sql b/tests_integration/test_data/projects/napp_init_v1/app/setup_script.sql new file mode 100644 index 0000000000..7fc3682b6e --- /dev/null +++ b/tests_integration/test_data/projects/napp_init_v1/app/setup_script.sql @@ -0,0 +1,11 @@ +-- This is the setup script that runs while installing a Snowflake Native App in a consumer account. +-- To write this script, you can familiarize yourself with some of the following concepts: +-- Application Roles +-- Versioned Schemas +-- UDFs/Procs +-- Extension Code +-- Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script for a detailed understanding of this file. + +CREATE OR ALTER VERSIONED SCHEMA core; + +-- The rest of this script is left blank for purposes of your learning and exploration. diff --git a/tests_integration/test_data/projects/napp_init_v1/snowflake.yml b/tests_integration/test_data/projects/napp_init_v1/snowflake.yml new file mode 100644 index 0000000000..bb42e3a5ee --- /dev/null +++ b/tests_integration/test_data/projects/napp_init_v1/snowflake.yml @@ -0,0 +1,7 @@ +definition_version: 1 +native_app: + name: myapp + source_stage: app_src.stage + artifacts: + - src: app/* + dest: ./ diff --git a/tests_integration/test_data/projects/napp_init_v2/app/README.md b/tests_integration/test_data/projects/napp_init_v2/app/README.md new file mode 100644 index 0000000000..f66bf75c9b --- /dev/null +++ b/tests_integration/test_data/projects/napp_init_v2/app/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains an extremely simple application that is used for +integration testing SnowCLI. diff --git a/tests_integration/test_data/projects/napp_init_v2/app/manifest.yml b/tests_integration/test_data/projects/napp_init_v2/app/manifest.yml new file mode 100644 index 0000000000..5b8ef74e8a --- /dev/null +++ b/tests_integration/test_data/projects/napp_init_v2/app/manifest.yml @@ -0,0 +1,9 @@ +# This is a manifest.yml file, a required component of creating a Snowflake Native App. +# This file defines properties required by the application package, including the location of the setup script and version definitions. +# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file. + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md diff --git a/tests_integration/test_data/projects/napp_init_v2/app/setup_script.sql b/tests_integration/test_data/projects/napp_init_v2/app/setup_script.sql new file mode 100644 index 0000000000..7fc3682b6e --- /dev/null +++ b/tests_integration/test_data/projects/napp_init_v2/app/setup_script.sql @@ -0,0 +1,11 @@ +-- This is the setup script that runs while installing a Snowflake Native App in a consumer account. +-- To write this script, you can familiarize yourself with some of the following concepts: +-- Application Roles +-- Versioned Schemas +-- UDFs/Procs +-- Extension Code +-- Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script for a detailed understanding of this file. + +CREATE OR ALTER VERSIONED SCHEMA core; + +-- The rest of this script is left blank for purposes of your learning and exploration. diff --git a/tests_integration/test_data/projects/napp_init_v2/snowflake.yml b/tests_integration/test_data/projects/napp_init_v2/snowflake.yml new file mode 100644 index 0000000000..76134ccff2 --- /dev/null +++ b/tests_integration/test_data/projects/napp_init_v2/snowflake.yml @@ -0,0 +1,16 @@ +# This is the v2 version of the napp_init_v1 project definition + +definition_version: 2 +entities: + pkg: + type: application package + name: myapp_pkg_<% ctx.env.USER %> + artifacts: + - src: app/* + dest: ./ + manifest: app/manifest.yml + app: + type: application + name: myapp_<% ctx.env.USER %> + from: + target: pkg From 7bb9b28193e3c753bff354cbe0a6240b7fc75bb5 Mon Sep 17 00:00:00 2001 From: Jorge Arturo Vasquez Rojas Date: Wed, 17 Jul 2024 09:01:22 -0600 Subject: [PATCH 3/7] Add command error to telemetry (#1266) Add error events to telemetry --- .../cli/api/commands/execution_metadata.py | 40 +++++++++ src/snowflake/cli/api/commands/snow_typer.py | 29 +++++-- src/snowflake/cli/app/telemetry.py | 43 ++++++++- tests/api/commands/test_snow_typer.py | 16 ++-- tests/app/test_telemetry.py | 87 ++++++++++++++++--- 5 files changed, 186 insertions(+), 29 deletions(-) create mode 100644 src/snowflake/cli/api/commands/execution_metadata.py diff --git a/src/snowflake/cli/api/commands/execution_metadata.py b/src/snowflake/cli/api/commands/execution_metadata.py new file mode 100644 index 0000000000..b53ab72943 --- /dev/null +++ b/src/snowflake/cli/api/commands/execution_metadata.py @@ -0,0 +1,40 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time +import uuid +from dataclasses import dataclass, field +from enum import Enum + + +class ExecutionStatus(Enum): + SUCCESS = "success" + FAILURE = "failure" + + +@dataclass +class ExecutionMetadata: + start_time: float = 0.0 + end_time: float = 0.0 + status: ExecutionStatus = ExecutionStatus.SUCCESS + execution_id: str = field(default_factory=lambda: uuid.uuid4().hex) + + def __post_init__(self): + self.start_time = time.monotonic() + + def complete(self, status: ExecutionStatus): + self.end_time = time.monotonic() + self.status = status + + def get_duration(self): + return self.end_time - self.start_time diff --git a/src/snowflake/cli/api/commands/snow_typer.py b/src/snowflake/cli/api/commands/snow_typer.py index 2cbbf959b4..64a4e2b4b2 100644 --- a/src/snowflake/cli/api/commands/snow_typer.py +++ b/src/snowflake/cli/api/commands/snow_typer.py @@ -24,6 +24,10 @@ global_options, global_options_with_connection, ) +from snowflake.cli.api.commands.execution_metadata import ( + ExecutionMetadata, + ExecutionStatus, +) from snowflake.cli.api.commands.flags import DEFAULT_CONTEXT_SETTINGS from snowflake.cli.api.commands.typer_pre_execute import run_pre_execute_commands from snowflake.cli.api.exceptions import CommandReturnTypeError @@ -91,15 +95,18 @@ def custom_command(command_callable): @wraps(command_callable) def command_callable_decorator(*args, **kw): """Wrapper around command callable. This is what happens at "runtime".""" - self.pre_execute() + execution = ExecutionMetadata() + self.pre_execute(execution) try: result = command_callable(*args, **kw) - return self.process_result(result) + self.process_result(result) + execution.complete(ExecutionStatus.SUCCESS) except Exception as err: - self.exception_handler(err) + execution.complete(ExecutionStatus.FAILURE) + self.exception_handler(err, execution) raise finally: - self.post_execute() + self.post_execute(execution) return super(SnowTyper, self).command(name=name, **kwargs)( command_callable_decorator @@ -108,7 +115,7 @@ def command_callable_decorator(*args, **kw): return custom_command @staticmethod - def pre_execute(): + def pre_execute(execution: ExecutionMetadata): """ Callback executed before running any command callable (after context execution). Pay attention to make this method safe to use if performed operations are not necessary @@ -118,7 +125,7 @@ def pre_execute(): log.debug("Executing command pre execution callback") run_pre_execute_commands() - log_command_usage() + log_command_usage(execution) @staticmethod def process_result(result): @@ -134,21 +141,25 @@ def process_result(result): print_result(result) @staticmethod - def exception_handler(exception: Exception): + def exception_handler(exception: Exception, execution: ExecutionMetadata): """ Callback executed on command execution error. """ + from snowflake.cli.app.telemetry import log_command_execution_error + log.debug("Executing command exception callback") + log_command_execution_error(exception, execution) @staticmethod - def post_execute(): + def post_execute(execution: ExecutionMetadata): """ Callback executed after running any command callable. Pay attention to make this method safe to use if performed operations are not necessary for executing the command in proper way. """ - from snowflake.cli.app.telemetry import flush_telemetry + from snowflake.cli.app.telemetry import flush_telemetry, log_command_result log.debug("Executing command post execution callback") + log_command_result(execution) flush_telemetry() diff --git a/src/snowflake/cli/app/telemetry.py b/src/snowflake/cli/app/telemetry.py index bc0a307f04..95cd9fb041 100644 --- a/src/snowflake/cli/app/telemetry.py +++ b/src/snowflake/cli/app/telemetry.py @@ -22,6 +22,7 @@ import click from snowflake.cli.__about__ import VERSION from snowflake.cli.api.cli_global_context import cli_context +from snowflake.cli.api.commands.execution_metadata import ExecutionMetadata from snowflake.cli.api.config import get_feature_flags_section from snowflake.cli.api.output.formats import OutputFormat from snowflake.cli.api.utils.error_handling import ignore_exceptions @@ -44,19 +45,25 @@ class CLITelemetryField(Enum): COMMAND = "command" COMMAND_GROUP = "command_group" COMMAND_FLAGS = "command_flags" + COMMAND_EXECUTION_ID = "command_execution_id" + COMMAND_RESULT_STATUS = "command_result_status" COMMAND_OUTPUT_TYPE = "command_output_type" + COMMAND_EXECUTION_TIME = "command_execution_time" # Configuration CONFIG_FEATURE_FLAGS = "config_feature_flags" # Information EVENT = "event" ERROR_MSG = "error_msg" ERROR_TYPE = "error_type" + IS_CLI_EXCEPTION = "is_cli_exception" # Project context PROJECT_DEFINITION_VERSION = "project_definition_version" class TelemetryEvent(Enum): CMD_EXECUTION = "executing_command" + CMD_EXECUTION_ERROR = "error_executing_command" + CMD_EXECUTION_RESULT = "result_executing_command" TelemetryDict = Dict[Union[CLITelemetryField, TelemetryField], Any] @@ -141,8 +148,40 @@ def flush(self): @ignore_exceptions() -def log_command_usage(): - _telemetry.send({TelemetryField.KEY_TYPE: TelemetryEvent.CMD_EXECUTION.value}) +def log_command_usage(execution: ExecutionMetadata): + _telemetry.send( + { + TelemetryField.KEY_TYPE: TelemetryEvent.CMD_EXECUTION.value, + CLITelemetryField.COMMAND_EXECUTION_ID: execution.execution_id, + } + ) + + +@ignore_exceptions() +def log_command_result(execution: ExecutionMetadata): + _telemetry.send( + { + TelemetryField.KEY_TYPE: TelemetryEvent.CMD_EXECUTION_RESULT.value, + CLITelemetryField.COMMAND_EXECUTION_ID: execution.execution_id, + CLITelemetryField.COMMAND_RESULT_STATUS: execution.status.value, + CLITelemetryField.COMMAND_EXECUTION_TIME: execution.get_duration(), + } + ) + + +@ignore_exceptions() +def log_command_execution_error(exception: Exception, execution: ExecutionMetadata): + exception_type: str = type(exception).__name__ + is_cli_exception: bool = issubclass(exception.__class__, click.ClickException) + _telemetry.send( + { + TelemetryField.KEY_TYPE: TelemetryEvent.CMD_EXECUTION_ERROR.value, + CLITelemetryField.COMMAND_EXECUTION_ID: execution.execution_id, + CLITelemetryField.ERROR_TYPE: exception_type, + CLITelemetryField.IS_CLI_EXCEPTION: is_cli_exception, + CLITelemetryField.COMMAND_EXECUTION_TIME: execution.get_duration(), + } + ) @ignore_exceptions() diff --git a/tests/api/commands/test_snow_typer.py b/tests/api/commands/test_snow_typer.py index 9225280728..2a005df45a 100644 --- a/tests/api/commands/test_snow_typer.py +++ b/tests/api/commands/test_snow_typer.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from functools import partial from unittest import mock from unittest.mock import MagicMock @@ -31,14 +30,14 @@ def class_factory( ): class _CustomTyper(SnowTyper): @staticmethod - def pre_execute(): + def pre_execute(execution): if pre_execute: - pre_execute() + pre_execute(execution) @staticmethod - def post_execute(): + def post_execute(execution): if post_execute: - post_execute() + post_execute(execution) @staticmethod def process_result(result): @@ -46,9 +45,9 @@ def process_result(result): result_handler(result) @staticmethod - def exception_handler(err): + def exception_handler(err, execution): if exception_handler: - exception_handler(err) + exception_handler(err, execution) def create_instance(self): return self @@ -192,8 +191,9 @@ def test_enabled_command_is_not_visible(cli, os_agnostic_snapshot): @mock.patch("snowflake.cli.app.telemetry.log_command_usage") def test_snow_typer_pre_execute_sends_telemetry(mock_log_command_usage, cli): result = cli(app_factory(SnowTyperFactory))(["simple_cmd", "Norma"]) + assert result.exit_code == 0 - mock_log_command_usage.assert_called_once_with() + mock_log_command_usage.assert_called_once_with(mock.ANY) @mock.patch("snowflake.cli.app.telemetry.flush_telemetry") diff --git a/tests/app/test_telemetry.py b/tests/app/test_telemetry.py index ce68d5c836..26965a2322 100644 --- a/tests/app/test_telemetry.py +++ b/tests/app/test_telemetry.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import uuid from unittest import mock from snowflake.connector.version import VERSION as DRIVER_VERSION @@ -22,25 +23,30 @@ "snowflake.cli.app.telemetry.python_version", ) @mock.patch("snowflake.cli.app.telemetry.platform.platform") +@mock.patch("uuid.uuid4") @mock.patch("snowflake.cli.app.telemetry.get_time_millis") @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli.plugins.connection.commands.ObjectManager") @mock.patch.dict(os.environ, {"SNOWFLAKE_CLI_FEATURES_FOO": "False"}) -def test_executing_command_sends_telemetry_data( - _, mock_conn, mock_time, mock_platform, mock_version, runner +def test_executing_command_sends_telemetry_usage_data( + _, mock_conn, mock_time, mock_uuid4, mock_platform, mock_version, runner ): mock_time.return_value = "123" mock_platform.return_value = "FancyOS" mock_version.return_value = "2.3.4" - + mock_uuid4.return_value = uuid.UUID("8a2225b3800c4017a4a9eab941db58fa") result = runner.invoke(["connection", "test"], catch_exceptions=False) assert result.exit_code == 0, result.output - # The method is called with a TelemetryData type, so we cast it to dict for simpler comparison - actual_call = mock_conn.return_value._telemetry.try_add_log_to_batch.call_args.args[ # noqa: SLF001 - 0 - ].to_dict() - assert actual_call == { + usage_command_event = ( + mock_conn.return_value._telemetry.try_add_log_to_batch.call_args_list[ # noqa: SLF001 + 0 + ] + .args[0] + .to_dict() + ) + + assert usage_command_event == { "message": { "driver_type": "PythonConnector", "driver_version": ".".join(str(s) for s in DRIVER_VERSION[:3]), @@ -50,6 +56,7 @@ def test_executing_command_sends_telemetry_data( "version_python": "2.3.4", "command": ["connection", "test"], "command_group": "connection", + "command_execution_id": "8a2225b3800c4017a4a9eab941db58fa", "command_flags": {"diag_log_path": "DEFAULT", "format": "DEFAULT"}, "command_output_type": "TABLE", "type": "executing_command", @@ -64,14 +71,47 @@ def test_executing_command_sends_telemetry_data( } +@mock.patch( + "snowflake.cli.app.telemetry.python_version", +) +@mock.patch("snowflake.cli.app.telemetry.platform.platform") +@mock.patch("uuid.uuid4") +@mock.patch("snowflake.connector.time_util.get_time_millis") +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli.plugins.connection.commands.ObjectManager") +@mock.patch.dict(os.environ, {"SNOWFLAKE_CLI_FEATURES_FOO": "False"}) +def test_executing_command_sends_telemetry_result_data( + _, mock_conn, mock_time, mock_uuid4, mock_platform, mock_version, runner +): + mock_time.return_value = "123" + mock_platform.return_value = "FancyOS" + mock_version.return_value = "2.3.4" + mock_uuid4.return_value = uuid.UUID("8a2225b3800c4017a4a9eab941db58fa") + result = runner.invoke(["connection", "test"], catch_exceptions=False) + assert result.exit_code == 0, result.output + + # The method is called with a TelemetryData type, so we cast it to dict for simpler comparison + result_command_event = ( + mock_conn.return_value._telemetry.try_add_log_to_batch.call_args_list[ # noqa: SLF001 + 1 + ] + .args[0] + .to_dict() + ) + assert ( + result_command_event["message"]["type"] == "result_executing_command" + and result_command_event["message"]["command_result_status"] == "success" + and result_command_event["message"]["command_execution_time"] + ) + + @mock.patch("snowflake.connector.connect") @mock.patch("snowflake.cli.plugins.streamlit.commands.StreamlitManager") def test_executing_command_sends_project_definition_in_telemetry_data( _, mock_conn, project_directory, runner ): - with project_directory("streamlit_full_definition"): - result = runner.invoke(["streamlit", "deploy"], catch_exceptions=False) + result = runner.invoke(["streamlit", "deploy"]) assert result.exit_code == 0, result.output # The method is called with a TelemetryData type, so we cast it to dict for simpler comparison @@ -79,3 +119,30 @@ def test_executing_command_sends_project_definition_in_telemetry_data( 0 ].to_dict() assert actual_call["message"]["project_definition_version"] == "1" + + +@mock.patch("snowflake.connector.connect") +@mock.patch("uuid.uuid4") +@mock.patch("snowflake.cli.plugins.streamlit.commands.StreamlitManager") +def test_failing_executing_command_sends_telemetry_data( + _, mock_uuid4, mock_conn, project_directory, runner +): + mock_uuid4.return_value = uuid.UUID("8a2225b3800c4017a4a9eab941db58fa") + with project_directory("napp_post_deploy_missing_file"): + runner.invoke(["app", "run"], catch_exceptions=False) + + # The method is called with a TelemetryData type, so we cast it to dict for simpler comparison + result_command_event = ( + mock_conn.return_value._telemetry.try_add_log_to_batch.call_args_list[ # noqa: SLF001 + 1 + ] + .args[0] + .to_dict() + ) + assert ( + result_command_event["message"]["type"] == "error_executing_command" + and result_command_event["message"]["error_type"] == "SourceNotFoundError" + and result_command_event["message"]["is_cli_exception"] == True + and result_command_event["message"]["command_execution_id"] + == "8a2225b3800c4017a4a9eab941db58fa" + ) From 170fa1a965b1762e249e55c6f486af079d869692 Mon Sep 17 00:00:00 2001 From: Bruno Dufour Date: Wed, 17 Jul 2024 11:19:43 -0400 Subject: [PATCH 4/7] [Alpha] Add support for snowflake.app-generated SQL setup (#1318) --- .../nativeapp/codegen/artifact_processor.py | 40 ++++ .../cli/plugins/nativeapp/codegen/compiler.py | 84 ++++++--- .../cli/plugins/nativeapp/codegen/sandbox.py | 109 ++++++++++- .../setup/native_app_setup_processor.py | 172 ++++++++++++++++++ .../codegen/setup/setup_driver.py.source | 56 ++++++ .../codegen/snowpark/python_processor.py | 30 ++- .../__snapshots__/test_python_processor.ambr | 58 +++--- .../codegen/snowpark/test_python_processor.py | 18 +- .../codegen/test_artifact_processor.py | 58 ++++++ tests/nativeapp/codegen/test_sandbox.py | 93 +++++++--- 10 files changed, 600 insertions(+), 118 deletions(-) create mode 100644 src/snowflake/cli/plugins/nativeapp/codegen/setup/native_app_setup_processor.py create mode 100644 src/snowflake/cli/plugins/nativeapp/codegen/setup/setup_driver.py.source create mode 100644 tests/nativeapp/codegen/test_artifact_processor.py diff --git a/src/snowflake/cli/plugins/nativeapp/codegen/artifact_processor.py b/src/snowflake/cli/plugins/nativeapp/codegen/artifact_processor.py index 67bcf1f7dc..da20ad9608 100644 --- a/src/snowflake/cli/plugins/nativeapp/codegen/artifact_processor.py +++ b/src/snowflake/cli/plugins/nativeapp/codegen/artifact_processor.py @@ -15,6 +15,7 @@ from __future__ import annotations from abc import ABC, abstractmethod +from pathlib import Path from typing import Optional from click import ClickException @@ -34,6 +35,42 @@ def __init__(self, processor_name: str): ) +def is_python_file_artifact(src: Path, _: Path): + """Determines whether the provided source path is an existing python file.""" + return src.is_file() and src.suffix == ".py" + + +class ProjectFileContextManager: + """ + A context manager that encapsulates the logic required to update a project file + in processor logic. The processor can use this manager to gain access to the contents + of a file, and optionally provide replacement contents. If it does, the file is + correctly modified in the deploy root directory to reflect the new contents. + """ + + def __init__(self, path: Path): + self.path = path + self._contents = None + self.edited_contents = None + + @property + def contents(self): + return self._contents + + def __enter__(self): + self._contents = self.path.read_text(encoding="utf-8") + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.edited_contents is not None: + if self.path.is_symlink(): + # if the file is a symlink, make sure we don't overwrite the original + self.path.unlink() + + self.path.write_text(self.edited_contents, encoding="utf-8") + + class ArtifactProcessor(ABC): def __init__( self, @@ -49,3 +86,6 @@ def process( **kwargs, ) -> None: pass + + def edit_file(self, path: Path): + return ProjectFileContextManager(path) diff --git a/src/snowflake/cli/plugins/nativeapp/codegen/compiler.py b/src/snowflake/cli/plugins/nativeapp/codegen/compiler.py index 8815d008ed..2bdd869c12 100644 --- a/src/snowflake/cli/plugins/nativeapp/codegen/compiler.py +++ b/src/snowflake/cli/plugins/nativeapp/codegen/compiler.py @@ -24,12 +24,22 @@ ArtifactProcessor, UnsupportedArtifactProcessorError, ) +from snowflake.cli.plugins.nativeapp.codegen.setup.native_app_setup_processor import ( + NativeAppSetupProcessor, +) from snowflake.cli.plugins.nativeapp.codegen.snowpark.python_processor import ( SnowparkAnnotationProcessor, ) +from snowflake.cli.plugins.nativeapp.feature_flags import FeatureFlag from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel SNOWPARK_PROCESSOR = "snowpark" +NA_SETUP_PROCESSOR = "native-app-setup" + +_REGISTERED_PROCESSORS_BY_NAME = { + SNOWPARK_PROCESSOR: SnowparkAnnotationProcessor, + NA_SETUP_PROCESSOR: NativeAppSetupProcessor, +} class NativeAppCompiler: @@ -54,28 +64,31 @@ def compile_artifacts(self): Go through every artifact object in the project definition of a native app, and execute processors in order of specification for each of the artifact object. May have side-effects on the filesystem by either directly editing source files or the deploy root. """ - should_proceed = False - for artifact in self._na_project.artifacts: - if artifact.processors: - should_proceed = True - break - if not should_proceed: + + if not self._should_invoke_processors(): return with cc.phase("Invoking artifact processors"): + if self._na_project.generated_root.exists(): + raise ClickException( + f"Path {self._na_project.generated_root} already exists. Please choose a different name for your generated directory in the project definition file." + ) + for artifact in self._na_project.artifacts: for processor in artifact.processors: - artifact_processor = self._try_create_processor( - processor_mapping=processor, - ) - if artifact_processor is None: - raise UnsupportedArtifactProcessorError( - processor_name=processor.name - ) - else: - artifact_processor.process( - artifact_to_process=artifact, processor_mapping=processor + if self._is_enabled(processor): + artifact_processor = self._try_create_processor( + processor_mapping=processor, ) + if artifact_processor is None: + raise UnsupportedArtifactProcessorError( + processor_name=processor.name + ) + else: + artifact_processor.process( + artifact_to_process=artifact, + processor_mapping=processor, + ) def _try_create_processor( self, @@ -86,15 +99,32 @@ def _try_create_processor( Fetch processor object if one already exists in the cached_processors dictionary. Else, initialize a new object to return, and add it to the cached_processors dictionary. """ - if processor_mapping.name.lower() == SNOWPARK_PROCESSOR: - curr_processor = self.cached_processors.get(SNOWPARK_PROCESSOR, None) - if curr_processor is not None: - return curr_processor - else: - curr_processor = SnowparkAnnotationProcessor( - na_project=self._na_project, - ) - self.cached_processors[SNOWPARK_PROCESSOR] = curr_processor - return curr_processor - else: + processor_name = processor_mapping.name.lower() + current_processor = self.cached_processors.get(processor_name) + + if current_processor is not None: + return current_processor + + processor_factory = _REGISTERED_PROCESSORS_BY_NAME.get(processor_name) + if processor_factory is None: + # No registered processor with the specified name return None + + current_processor = processor_factory( + na_project=self._na_project, + ) + self.cached_processors[processor_name] = current_processor + + return current_processor + + def _should_invoke_processors(self): + for artifact in self._na_project.artifacts: + for processor in artifact.processors: + if self._is_enabled(processor): + return True + return False + + def _is_enabled(self, processor: ProcessorMapping) -> bool: + if processor.name.lower() == NA_SETUP_PROCESSOR: + return FeatureFlag.ENABLE_NATIVE_APP_PYTHON_SETUP.is_enabled() + return True diff --git a/src/snowflake/cli/plugins/nativeapp/codegen/sandbox.py b/src/snowflake/cli/plugins/nativeapp/codegen/sandbox.py index 824a15863d..be8cbe5c69 100644 --- a/src/snowflake/cli/plugins/nativeapp/codegen/sandbox.py +++ b/src/snowflake/cli/plugins/nativeapp/codegen/sandbox.py @@ -20,10 +20,13 @@ import sys from enum import Enum from pathlib import Path -from typing import Optional, Sequence, Union +from typing import Any, Mapping, Optional, Sequence, Union +from venv import EnvBuilder from click.exceptions import ClickException +EnvVars = Mapping[str, str] # Only support str -> str for cross-platform compatibility + class SandboxExecutionError(ClickException): """An error occurred while executing a python script.""" @@ -57,6 +60,7 @@ def _execute_python_interpreter( script_source: str, cwd: Optional[Union[str, Path]], timeout: Optional[int], + env_vars: Optional[EnvVars], ) -> subprocess.CompletedProcess: if not python_executable: raise SandboxExecutionError("No python executable found") @@ -73,6 +77,7 @@ def _execute_python_interpreter( input=script_source, timeout=timeout, cwd=cwd, + env=env_vars, ) @@ -81,6 +86,7 @@ def _execute_in_venv( venv_path: Optional[Union[str, Path]] = None, cwd: Optional[Union[str, Path]] = None, timeout: Optional[int] = None, + env_vars: Optional[EnvVars] = None, ) -> subprocess.CompletedProcess: resolved_venv_path = None if venv_path is None: @@ -114,7 +120,7 @@ def _execute_in_venv( ) return _execute_python_interpreter( - python_executable, script_source, timeout=timeout, cwd=cwd + python_executable, script_source, timeout=timeout, cwd=cwd, env_vars=env_vars ) @@ -123,6 +129,7 @@ def _execute_in_conda_env( env_name: Optional[str] = None, cwd: Optional[Union[str, Path]] = None, timeout: Optional[int] = None, + env_vars: Optional[EnvVars] = None, ) -> subprocess.CompletedProcess: conda_env = env_name if conda_env is None: @@ -142,6 +149,7 @@ def _execute_in_conda_env( script_source, timeout=timeout, cwd=cwd, + env_vars=env_vars, ) @@ -149,13 +157,18 @@ def _execute_with_system_path_python( script_source: str, cwd: Optional[Union[str, Path]] = None, timeout: Optional[int] = None, + env_vars: Optional[EnvVars] = None, ) -> subprocess.CompletedProcess: python_executable = ( shutil.which("python3") or shutil.which("python") or sys.executable ) return _execute_python_interpreter( - python_executable, script_source, timeout=timeout, cwd=cwd + python_executable, + script_source, + timeout=timeout, + cwd=cwd, + env_vars=env_vars, ) @@ -172,6 +185,7 @@ def execute_script_in_sandbox( env_type: ExecutionEnvironmentType = ExecutionEnvironmentType.AUTO_DETECT, cwd: Optional[Union[str, Path]] = None, timeout: Optional[int] = None, + env_vars: Optional[EnvVars] = None, **kwargs, ) -> subprocess.CompletedProcess: """ @@ -194,24 +208,99 @@ def execute_script_in_sandbox( """ if env_type == ExecutionEnvironmentType.AUTO_DETECT: if _is_venv_active(): - return _execute_in_venv(script_source, cwd=cwd, timeout=timeout) + return _execute_in_venv( + script_source, cwd=cwd, timeout=timeout, env_vars=env_vars + ) elif _is_conda_active(): - return _execute_in_conda_env(script_source, cwd=cwd, timeout=timeout) + return _execute_in_conda_env( + script_source, cwd=cwd, timeout=timeout, env_vars=env_vars + ) else: return _execute_with_system_path_python( - script_source, cwd=cwd, timeout=timeout + script_source, cwd=cwd, timeout=timeout, env_vars=env_vars ) elif env_type == ExecutionEnvironmentType.VENV: return _execute_in_venv( - script_source, kwargs.get("path"), cwd=cwd, timeout=timeout + script_source, + kwargs.get("path"), + cwd=cwd, + timeout=timeout, + env_vars=env_vars, ) elif env_type == ExecutionEnvironmentType.CONDA: return _execute_in_conda_env( - script_source, kwargs.get("name"), cwd=cwd, timeout=timeout + script_source, + kwargs.get("name"), + cwd=cwd, + timeout=timeout, + env_vars=env_vars, ) elif env_type == ExecutionEnvironmentType.SYSTEM_PATH: - return _execute_with_system_path_python(script_source, cwd=cwd, timeout=timeout) + return _execute_with_system_path_python( + script_source, cwd=cwd, timeout=timeout, env_vars=env_vars + ) else: # ExecutionEnvironmentType.CURRENT return _execute_python_interpreter( - sys.executable, script_source, cwd=cwd, timeout=timeout + sys.executable, script_source, cwd=cwd, timeout=timeout, env_vars=env_vars ) + + +class SandboxEnvBuilder(EnvBuilder): + """ + A virtual environment builder that can be used to build an environment suitable for + executing user-provided python scripts in an isolated sandbox. + """ + + def __init__(self, path: Path, **kwargs) -> None: + """ + Creates a new builder with the specified destination path. The path need not + exist, it will be created when needed (recursively if necessary). + + Parameters: + path (Path): The directory in which the sandbox environment will be created. + """ + super().__init__(**kwargs) + self.path = path + self._context: Any = None # cached context + + def post_setup(self, context) -> None: + self._context = context + + def ensure_created(self) -> None: + """ + Ensures that the sandbox environment has been created and correctly initialized. + """ + if self.path.exists(): + self._context = self.ensure_directories(self.path) + else: + self.path.mkdir(parents=True, exist_ok=True) + self.create( + self.path + ) # will set self._context through the post_setup callback + + def run_python(self, *args) -> str: + """ + Executes the python interpreter in the sandboxed environment with the provided arguments. + This raises a CalledProcessError if the python interpreter was not executed successfully. + + Returns: + The output of running the command. + """ + positional_args = [ + self._context.env_exe, + "-E", # passing -E ignores all PYTHON* env vars + *args, + ] + kwargs = { + "cwd": self._context.env_dir, + "stderr": subprocess.STDOUT, + } + env = dict(os.environ) + env["VIRTUAL_ENV"] = self._context.env_dir + return subprocess.check_output(positional_args, **kwargs) + + def pip_install(self, *args: Any) -> None: + """ + Invokes pip install with the provided arguments. + """ + self.run_python("-m", "pip", "install", *[str(arg) for arg in args]) diff --git a/src/snowflake/cli/plugins/nativeapp/codegen/setup/native_app_setup_processor.py b/src/snowflake/cli/plugins/nativeapp/codegen/setup/native_app_setup_processor.py new file mode 100644 index 0000000000..b1cfca12d2 --- /dev/null +++ b/src/snowflake/cli/plugins/nativeapp/codegen/setup/native_app_setup_processor.py @@ -0,0 +1,172 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import os.path +from pathlib import Path +from typing import List, Optional + +from click import ClickException +from snowflake.cli.api.console import cli_console as cc +from snowflake.cli.api.project.schemas.native_app.path_mapping import ( + PathMapping, + ProcessorMapping, +) +from snowflake.cli.plugins.nativeapp.artifacts import BundleMap, find_setup_script_file +from snowflake.cli.plugins.nativeapp.codegen.artifact_processor import ( + ArtifactProcessor, + is_python_file_artifact, +) +from snowflake.cli.plugins.nativeapp.codegen.sandbox import ( + ExecutionEnvironmentType, + SandboxEnvBuilder, + execute_script_in_sandbox, +) +from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel +from snowflake.cli.plugins.stage.diff import to_stage_path + +DEFAULT_TIMEOUT = 30 +DRIVER_PATH = Path(__file__).parent / "setup_driver.py.source" + + +class NativeAppSetupProcessor(ArtifactProcessor): + def __init__( + self, + na_project: NativeAppProjectModel, + ): + super().__init__(na_project=na_project) + + def process( + self, + artifact_to_process: PathMapping, + processor_mapping: Optional[ProcessorMapping], + **kwargs, + ) -> None: + """ + Processes a Python setup script and generates the corresponding SQL commands. + """ + bundle_map = BundleMap( + project_root=self._na_project.project_root, + deploy_root=self._na_project.deploy_root, + ) + bundle_map.add(artifact_to_process) + + self._create_or_update_sandbox() + + cc.phase("Processing Python setup files") + + files_to_process = [] + for src_file, dest_file in bundle_map.all_mappings( + absolute=True, expand_directories=True, predicate=is_python_file_artifact + ): + cc.message( + f"Found Python setup file: {src_file.relative_to(self._na_project.project_root)}" + ) + files_to_process.append(src_file) + + sql_files_mapping = self._execute_in_sandbox(files_to_process) + self._generate_setup_sql(sql_files_mapping) + + def _execute_in_sandbox(self, py_files: List[Path]) -> dict: + file_count = len(py_files) + cc.step(f"Processing {file_count} setup file{'s' if file_count > 1 else ''}") + + env_vars = { + "_SNOWFLAKE_CLI_PROJECT_PATH": str(self._na_project.project_root), + "_SNOWFLAKE_CLI_SETUP_FILES": os.pathsep.join(map(str, py_files)), + "_SNOWFLAKE_CLI_APP_NAME": str(self._na_project.package_name), + "_SNOWFLAKE_CLI_SQL_DEST_DIR": str(self.generated_root), + } + + try: + result = execute_script_in_sandbox( + script_source=DRIVER_PATH.read_text(), + env_type=ExecutionEnvironmentType.VENV, + cwd=self._na_project.bundle_root, + timeout=DEFAULT_TIMEOUT, + path=self.sandbox_root, + env_vars=env_vars, + ) + except Exception as e: + raise ClickException( + f"Exception while executing python setup script logic: {e}" + ) + + if result.returncode == 0: + sql_file_mappings = json.loads(result.stdout) + return sql_file_mappings + else: + raise ClickException( + f"Failed to execute python setup script logic: {result.stderr}" + ) + + def _generate_setup_sql(self, sql_file_mappings: dict) -> None: + if not sql_file_mappings: + # Nothing to generate + return + + generated_root = self.generated_root + generated_root.mkdir(exist_ok=True, parents=True) + + cc.step("Patching setup script") + setup_file_path = find_setup_script_file( + deploy_root=self._na_project.deploy_root + ) + with self.edit_file(setup_file_path) as f: + new_contents = [f.contents] + + if sql_file_mappings["schemas"]: + schemas_file = generated_root / sql_file_mappings["schemas"] + new_contents.insert( + 0, + f"EXECUTE IMMEDIATE FROM '/{to_stage_path(schemas_file.relative_to(self._na_project.deploy_root))}';", + ) + + if sql_file_mappings["compute_pools"]: + compute_pools_file = generated_root / sql_file_mappings["compute_pools"] + new_contents.append( + f"EXECUTE IMMEDIATE FROM '/{to_stage_path(compute_pools_file.relative_to(self._na_project.deploy_root))}';" + ) + + if sql_file_mappings["services"]: + services_file = generated_root / sql_file_mappings["services"] + new_contents.append( + f"EXECUTE IMMEDIATE FROM '/{to_stage_path(services_file.relative_to(self._na_project.deploy_root))}';" + ) + + f.edited_contents = "\n".join(new_contents) + + @property + def sandbox_root(self): + return self._na_project.bundle_root / "setup_py_venv" + + @property + def generated_root(self): + return self._na_project.generated_root / "setup_py" + + def _create_or_update_sandbox(self): + sandbox_root = self.sandbox_root + env_builder = SandboxEnvBuilder(sandbox_root, with_pip=True) + if sandbox_root.exists(): + cc.step("Virtual environment found") + else: + cc.step( + f"Creating virtual environment in {sandbox_root.relative_to(self._na_project.project_root)}" + ) + env_builder.ensure_created() + + # Temporarily fetch the library from a location specified via env vars + env_builder.pip_install(os.environ["SNOWFLAKE_APP_PYTHON_LOC"]) diff --git a/src/snowflake/cli/plugins/nativeapp/codegen/setup/setup_driver.py.source b/src/snowflake/cli/plugins/nativeapp/codegen/setup/setup_driver.py.source new file mode 100644 index 0000000000..4a261cbf1e --- /dev/null +++ b/src/snowflake/cli/plugins/nativeapp/codegen/setup/setup_driver.py.source @@ -0,0 +1,56 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import os +import sys +from pathlib import Path + +import snowflake.app.context as ctx +from snowflake.app.sql import SQLGenerator + +ctx._project_path = os.environ["_SNOWFLAKE_CLI_PROJECT_PATH"] +ctx._current_app_name = os.environ["_SNOWFLAKE_CLI_APP_NAME"] +__snowflake_internal_py_files = os.environ["_SNOWFLAKE_CLI_SETUP_FILES"].split( + os.pathsep +) +__snowflake_internal_sql_dest_dir = os.environ["_SNOWFLAKE_CLI_SQL_DEST_DIR"] + +try: + import importlib + + with contextlib.redirect_stdout(None): + with contextlib.redirect_stderr(None): + for __snowflake_internal_py_file in __snowflake_internal_py_files: + __snowflake_internal_spec = importlib.util.spec_from_file_location( + "", __snowflake_internal_py_file + ) + __snowflake_internal_module = importlib.util.module_from_spec( + __snowflake_internal_spec + ) + __snowflake_internal_spec.loader.exec_module( + __snowflake_internal_module + ) +except Exception as exc: # Catch any error + print("An exception occurred while executing file: ", exc, file=sys.stderr) + sys.exit(1) + + +import json + +output_dir = Path(__snowflake_internal_sql_dest_dir) +output_dir.mkdir(exist_ok=True, parents=True) +path_mappings = SQLGenerator(dest_dir=output_dir).generate() + +print(json.dumps(path_mappings, default=str)) diff --git a/src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py b/src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py index e28a401e6d..c25e3715e7 100644 --- a/src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py +++ b/src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py @@ -20,7 +20,6 @@ from textwrap import dedent from typing import Any, Dict, List, Optional, Set -from click import ClickException from pydantic import ValidationError from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.project.schemas.native_app.path_mapping import ( @@ -32,7 +31,10 @@ BundleMap, find_setup_script_file, ) -from snowflake.cli.plugins.nativeapp.codegen.artifact_processor import ArtifactProcessor +from snowflake.cli.plugins.nativeapp.codegen.artifact_processor import ( + ArtifactProcessor, + is_python_file_artifact, +) from snowflake.cli.plugins.nativeapp.codegen.sandbox import ( ExecutionEnvironmentType, SandboxExecutionError, @@ -167,16 +169,6 @@ def __init__( ): super().__init__(na_project=na_project) - assert self._na_project.bundle_root.is_absolute() - assert self._na_project.deploy_root.is_absolute() - assert self._na_project.generated_root.is_absolute() - assert self._na_project.project_root.is_absolute() - - if self._na_project.generated_root.exists(): - raise ClickException( - f"Path {self._na_project.generated_root} already exists. Please choose a different name for your generated directory in the project definition file." - ) - def process( self, artifact_to_process: PathMapping, @@ -238,9 +230,13 @@ def process( edit_setup_script_with_exec_imm_sql( collected_sql_files=collected_sql_files, deploy_root=bundle_map.deploy_root(), - generated_root=self._na_project.generated_root, + generated_root=self._generated_root, ) + @property + def _generated_root(self): + return self._na_project.generated_root / "snowpark" + def _normalize_imports( self, extension_fn: NativeAppExtensionFunction, @@ -332,7 +328,7 @@ def collect_extension_functions( bundle_map.all_mappings( absolute=True, expand_directories=True, - predicate=_is_python_file_artifact, + predicate=is_python_file_artifact, ) ): cc.step( @@ -374,9 +370,7 @@ def generate_new_sql_file_name(self, py_file: Path) -> Path: Generates a SQL filename for the generated root from the python file, and creates its parent directories. """ relative_py_file = py_file.relative_to(self._na_project.deploy_root) - sql_file = Path( - self._na_project.generated_root, relative_py_file.with_suffix(".sql") - ) + sql_file = Path(self._generated_root, relative_py_file.with_suffix(".sql")) if sql_file.exists(): cc.warning( f"""\ @@ -494,7 +488,7 @@ def edit_setup_script_with_exec_imm_sql( Adds an 'execute immediate' to setup script for every SQL file in the map """ # Create a __generated.sql in the __generated folder - generated_file_path = Path(generated_root, f"{generated_root.stem}.sql") + generated_file_path = Path(generated_root, f"__generated.sql") generated_file_path.parent.mkdir(exist_ok=True, parents=True) if generated_file_path.exists(): diff --git a/tests/nativeapp/codegen/snowpark/__snapshots__/test_python_processor.ambr b/tests/nativeapp/codegen/snowpark/__snapshots__/test_python_processor.ambr index e0434f998a..1342c540f1 100644 --- a/tests/nativeapp/codegen/snowpark/__snapshots__/test_python_processor.ambr +++ b/tests/nativeapp/codegen/snowpark/__snapshots__/test_python_processor.ambr @@ -3,10 +3,11 @@ ''' d output/deploy d output/deploy/__generated - f output/deploy/__generated/__generated.sql - f output/deploy/__generated/dummy.sql - d output/deploy/__generated/moduleB - f output/deploy/__generated/moduleB/dummy.sql + d output/deploy/__generated/snowpark + f output/deploy/__generated/snowpark/__generated.sql + f output/deploy/__generated/snowpark/dummy.sql + d output/deploy/__generated/snowpark/moduleB + f output/deploy/__generated/snowpark/moduleB/dummy.sql f output/deploy/manifest.yml d output/deploy/moduleA d output/deploy/moduleA/moduleC @@ -15,21 +16,21 @@ # --- # name: test_edit_setup_script_with_exec_imm_sql.1 ''' - ===== Contents of: output/deploy/__generated/__generated.sql ===== - EXECUTE IMMEDIATE FROM '/__generated/moduleB/dummy.sql'; - EXECUTE IMMEDIATE FROM '/__generated/dummy.sql'; + ===== Contents of: output/deploy/__generated/snowpark/__generated.sql ===== + EXECUTE IMMEDIATE FROM '/__generated/snowpark/moduleB/dummy.sql'; + EXECUTE IMMEDIATE FROM '/__generated/snowpark/dummy.sql'; ''' # --- # name: test_edit_setup_script_with_exec_imm_sql.2 ''' - ===== Contents of: output/deploy/__generated/dummy.sql ===== + ===== Contents of: output/deploy/__generated/snowpark/dummy.sql ===== #this is a file ''' # --- # name: test_edit_setup_script_with_exec_imm_sql.3 ''' - ===== Contents of: output/deploy/__generated/moduleB/dummy.sql ===== + ===== Contents of: output/deploy/__generated/snowpark/moduleB/dummy.sql ===== #this is a file ''' # --- @@ -47,7 +48,7 @@ ''' ===== Contents of: output/deploy/moduleA/moduleC/setup.sql ===== create application role app_public; - EXECUTE IMMEDIATE FROM '/__generated/__generated.sql'; + EXECUTE IMMEDIATE FROM '/__generated/snowpark/__generated.sql'; ''' # --- @@ -55,7 +56,8 @@ ''' d output/deploy d output/deploy/__generated - f output/deploy/__generated/__generated.sql + d output/deploy/__generated/snowpark + f output/deploy/__generated/snowpark/__generated.sql f output/deploy/manifest.yml d output/deploy/moduleA d output/deploy/moduleA/moduleC @@ -64,7 +66,7 @@ # --- # name: test_edit_setup_script_with_exec_imm_sql_noop.1 ''' - ===== Contents of: output/deploy/__generated/__generated.sql ===== + ===== Contents of: output/deploy/__generated/snowpark/__generated.sql ===== #some text ''' # --- @@ -82,16 +84,17 @@ ''' d output/deploy d output/deploy/__generated - f output/deploy/__generated/__generated.sql + d output/deploy/__generated/snowpark + f output/deploy/__generated/snowpark/__generated.sql f output/deploy/manifest.yml f output/deploy/setup.sql ''' # --- # name: test_edit_setup_script_with_exec_imm_sql_symlink.1 ''' - ===== Contents of: output/deploy/__generated/__generated.sql ===== - EXECUTE IMMEDIATE FROM '/__generated/moduleB/dummy.sql'; - EXECUTE IMMEDIATE FROM '/__generated/dummy.sql'; + ===== Contents of: output/deploy/__generated/snowpark/__generated.sql ===== + EXECUTE IMMEDIATE FROM '/__generated/snowpark/moduleB/dummy.sql'; + EXECUTE IMMEDIATE FROM '/__generated/snowpark/dummy.sql'; ''' # --- @@ -109,7 +112,7 @@ ''' ===== Contents of: output/deploy/setup.sql ===== create application role admin; - EXECUTE IMMEDIATE FROM '/__generated/__generated.sql'; + EXECUTE IMMEDIATE FROM '/__generated/snowpark/__generated.sql'; ''' # --- @@ -527,10 +530,11 @@ ''' d output/deploy d output/deploy/__generated - f output/deploy/__generated/__generated.sql - d output/deploy/__generated/stagepath - f output/deploy/__generated/stagepath/data.sql - f output/deploy/__generated/stagepath/main.sql + d output/deploy/__generated/snowpark + f output/deploy/__generated/snowpark/__generated.sql + d output/deploy/__generated/snowpark/stagepath + f output/deploy/__generated/snowpark/stagepath/data.sql + f output/deploy/__generated/snowpark/stagepath/main.sql f output/deploy/manifest.yml d output/deploy/moduleA d output/deploy/moduleA/moduleC @@ -544,15 +548,15 @@ # --- # name: test_process_with_collected_functions.1 ''' - ===== Contents of: output/deploy/__generated/__generated.sql ===== - EXECUTE IMMEDIATE FROM '/__generated/stagepath/data.sql'; - EXECUTE IMMEDIATE FROM '/__generated/stagepath/main.sql'; + ===== Contents of: output/deploy/__generated/snowpark/__generated.sql ===== + EXECUTE IMMEDIATE FROM '/__generated/snowpark/stagepath/data.sql'; + EXECUTE IMMEDIATE FROM '/__generated/snowpark/stagepath/main.sql'; ''' # --- # name: test_process_with_collected_functions.2 ''' - ===== Contents of: output/deploy/__generated/stagepath/data.sql ===== + ===== Contents of: output/deploy/__generated/snowpark/stagepath/data.sql ===== -- Generated by the Snowflake CLI from stagepath/data.py -- DO NOT EDIT CREATE OR REPLACE @@ -575,7 +579,7 @@ # --- # name: test_process_with_collected_functions.3 ''' - ===== Contents of: output/deploy/__generated/stagepath/main.sql ===== + ===== Contents of: output/deploy/__generated/snowpark/stagepath/main.sql ===== -- Generated by the Snowflake CLI from stagepath/main.py -- DO NOT EDIT CREATE OR REPLACE @@ -610,7 +614,7 @@ ''' ===== Contents of: output/deploy/moduleA/moduleC/setup.sql ===== create application role app_public; - EXECUTE IMMEDIATE FROM '/__generated/__generated.sql'; + EXECUTE IMMEDIATE FROM '/__generated/snowpark/__generated.sql'; ''' # --- diff --git a/tests/nativeapp/codegen/snowpark/test_python_processor.py b/tests/nativeapp/codegen/snowpark/test_python_processor.py index cea459f059..1d3c93619d 100644 --- a/tests/nativeapp/codegen/snowpark/test_python_processor.py +++ b/tests/nativeapp/codegen/snowpark/test_python_processor.py @@ -216,14 +216,14 @@ def test_edit_setup_script_with_exec_imm_sql(os_agnostic_snapshot): dir_structure = { "output/deploy/manifest.yml": manifest_contents, "output/deploy/moduleA/moduleC/setup.sql": "create application role app_public;", - "output/deploy/__generated/moduleB/dummy.sql": "#this is a file", - "output/deploy/__generated/dummy.sql": "#this is a file", + "output/deploy/__generated/snowpark/moduleB/dummy.sql": "#this is a file", + "output/deploy/__generated/snowpark/dummy.sql": "#this is a file", } with temp_local_dir(dir_structure=dir_structure) as local_path: with pushd(local_path): deploy_root = Path(local_path, "output", "deploy") - generated_root = Path(deploy_root, "__generated") + generated_root = Path(deploy_root, "__generated", "snowpark") collected_sql_files = [ Path(generated_root, "moduleB", "dummy.sql"), Path(generated_root, "dummy.sql"), @@ -252,19 +252,19 @@ def test_edit_setup_script_with_exec_imm_sql_noop(os_agnostic_snapshot): dir_structure = { "output/deploy/manifest.yml": manifest_contents, "output/deploy/moduleA/moduleC/setup.sql": None, - "output/deploy/__generated/__generated.sql": "#some text", + "output/deploy/__generated/snowpark/__generated.sql": "#some text", } with temp_local_dir(dir_structure=dir_structure) as local_path: with pushd(local_path): deploy_root = Path(local_path, "output", "deploy") collected_sql_files = [ - Path(deploy_root, "__generated", "dummy.sql"), + Path(deploy_root, "__generated", "snowpark", "dummy.sql"), ] edit_setup_script_with_exec_imm_sql( collected_sql_files=collected_sql_files, deploy_root=deploy_root, - generated_root=Path(deploy_root, "__generated"), + generated_root=Path(deploy_root, "__generated", "snowpark"), ) assert_dir_snapshot( @@ -296,7 +296,7 @@ def test_edit_setup_script_with_exec_imm_sql_symlink(os_agnostic_snapshot): deploy_root_setup_script = Path(deploy_root, "setup.sql") deploy_root_setup_script.symlink_to(Path(local_path, "setup.sql")) - generated_root = Path(deploy_root, "__generated") + generated_root = Path(deploy_root, "__generated", "snowpark") collected_sql_files = [ Path(generated_root, "moduleB", "dummy.sql"), Path(generated_root, "dummy.sql"), @@ -304,7 +304,7 @@ def test_edit_setup_script_with_exec_imm_sql_symlink(os_agnostic_snapshot): edit_setup_script_with_exec_imm_sql( collected_sql_files=collected_sql_files, deploy_root=deploy_root, - generated_root=Path(deploy_root, "__generated"), + generated_root=Path(deploy_root, "__generated", "snowpark"), ) assert_dir_snapshot( @@ -468,6 +468,6 @@ def test_package_normalization( processor_mapping=processor_mapping, ) - dest_file = project.generated_root / "stagepath" / "main.sql" + dest_file = project.generated_root / "snowpark" / "stagepath" / "main.sql" assert dest_file.is_file() assert dest_file.read_text(encoding="utf-8") == os_agnostic_snapshot diff --git a/tests/nativeapp/codegen/test_artifact_processor.py b/tests/nativeapp/codegen/test_artifact_processor.py new file mode 100644 index 0000000000..bde9320742 --- /dev/null +++ b/tests/nativeapp/codegen/test_artifact_processor.py @@ -0,0 +1,58 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from snowflake.cli.plugins.nativeapp.codegen.artifact_processor import ( + ProjectFileContextManager, +) + +from tests.testing_utils.files_and_dirs import temp_local_dir +from tests_common import IS_WINDOWS + +ORIGINAL_CONTENTS = "# This is the original contents" +EDITED_CONTENTS = "# This is the edited contents" + + +def test_project_file_context_manager(): + dir_contents = {"foo.txt": ORIGINAL_CONTENTS} + with temp_local_dir(dir_contents) as root: + foo_path = root / "foo.txt" + with ProjectFileContextManager(foo_path) as cm: + assert cm.contents == ORIGINAL_CONTENTS + + cm.edited_contents = EDITED_CONTENTS + + assert foo_path.read_text(encoding="utf-8") == EDITED_CONTENTS + + +@pytest.mark.skipif( + IS_WINDOWS, reason="Symlinks on Windows are restricted to Developer mode or admins" +) +def test_project_file_context_manager_unlinks_symlinks(): + dir_contents = {"foo.txt": ORIGINAL_CONTENTS} + with temp_local_dir(dir_contents) as root: + foo_path = root / "foo.txt" + foo_link_path = root / "foo_link.txt" + foo_link_path.symlink_to(foo_path) + + assert foo_link_path.is_symlink() + + with ProjectFileContextManager(foo_link_path) as cm: + assert cm.contents == ORIGINAL_CONTENTS + + cm.edited_contents = EDITED_CONTENTS + + assert foo_path.read_text(encoding="utf-8") == ORIGINAL_CONTENTS + assert foo_link_path.read_text(encoding="utf-8") == EDITED_CONTENTS + assert not foo_link_path.is_symlink() diff --git a/tests/nativeapp/codegen/test_sandbox.py b/tests/nativeapp/codegen/test_sandbox.py index ff657e8d16..796744dbd1 100644 --- a/tests/nativeapp/codegen/test_sandbox.py +++ b/tests/nativeapp/codegen/test_sandbox.py @@ -36,6 +36,7 @@ VENV_ONLY_ENVIRON = {"VIRTUAL_ENV": VIRTUAL_ENV_ROOT_FROM_ENVIRON} TIMEOUT = 60 NEW_CWD = "/path/to/cwd" +ENV_VARS = {"TEST_VAR": "foo"} @pytest.fixture @@ -98,19 +99,20 @@ def fake_venv_interpreter_win32(fake_venv_root_win32): @pytest.mark.parametrize( - "expected_timeout, expected_cwd", + "expected_timeout, expected_cwd, expected_env", [ - (None, None), - (TIMEOUT, None), - (None, NEW_CWD), - (None, Path(NEW_CWD)), - (TIMEOUT, NEW_CWD), + (None, None, None), + (TIMEOUT, None, None), + (None, NEW_CWD, None), + (None, Path(NEW_CWD), None), + (None, None, ENV_VARS), + (TIMEOUT, NEW_CWD, ENV_VARS), ], ) @mock.patch("subprocess.run") @mock.patch("shutil.which") def test_execute_in_named_conda_env( - mock_which, mock_run, mock_environ, expected_timeout, expected_cwd + mock_which, mock_run, mock_environ, expected_timeout, expected_cwd, expected_env ): mock_which.side_effect = ( lambda executable: "/path/to/conda" if executable == "conda" else None @@ -128,6 +130,7 @@ def test_execute_in_named_conda_env( name="foo", timeout=expected_timeout, cwd=expected_cwd, + env_vars=expected_env, ) mock_run.assert_called_once_with( @@ -137,6 +140,7 @@ def test_execute_in_named_conda_env( input=PYTHON_SCRIPT, cwd=expected_cwd, timeout=expected_timeout, + env=expected_env, ) assert actual.args == SCRIPT_ARGS @@ -179,6 +183,7 @@ def test_execute_in_conda_env_falls_back_to_activated_one( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -218,13 +223,14 @@ def test_execute_in_conda_env_fails_when_conda_env_cannot_be_determined( @pytest.mark.parametrize( - "expected_timeout, expected_cwd", + "expected_timeout, expected_cwd, expected_env", [ - (None, None), - (TIMEOUT, None), - (None, NEW_CWD), - (None, Path(NEW_CWD)), - (TIMEOUT, NEW_CWD), + (None, None, None), + (TIMEOUT, None, None), + (None, NEW_CWD, None), + (None, Path(NEW_CWD), None), + (None, None, ENV_VARS), + (TIMEOUT, NEW_CWD, ENV_VARS), ], ) @mock.patch("sys.platform", "darwin") @@ -236,6 +242,7 @@ def test_execute_in_specified_venv_root_unix( fake_venv_interpreter_unix, expected_timeout, expected_cwd, + expected_env, ): mock_environ.side_effect = VENV_ONLY_ENVIRON.get @@ -250,6 +257,7 @@ def test_execute_in_specified_venv_root_unix( path=fake_venv_root_unix, timeout=expected_timeout, cwd=expected_cwd, + env_vars=expected_env, ) mock_run.assert_called_once_with( @@ -259,6 +267,7 @@ def test_execute_in_specified_venv_root_unix( input=PYTHON_SCRIPT, cwd=expected_cwd, timeout=expected_timeout, + env=expected_env, ) assert actual.args == SCRIPT_ARGS @@ -292,6 +301,7 @@ def test_execute_in_specified_venv_root_as_string( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -323,6 +333,7 @@ def test_execute_in_specified_venv_root_windows( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -354,6 +365,7 @@ def test_execute_in_venv_falls_back_to_activated_one( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -408,19 +420,20 @@ def test_execute_in_venv_fails_when_interpreter_not_found( @pytest.mark.parametrize( - "expected_timeout, expected_cwd", + "expected_timeout, expected_cwd, expected_env", [ - (None, None), - (TIMEOUT, None), - (None, NEW_CWD), - (None, Path(NEW_CWD)), - (TIMEOUT, NEW_CWD), + (None, None, None), + (TIMEOUT, None, None), + (None, NEW_CWD, None), + (None, Path(NEW_CWD), None), + (None, None, ENV_VARS), + (TIMEOUT, NEW_CWD, ENV_VARS), ], ) @mock.patch("subprocess.run") @mock.patch("shutil.which") def test_execute_system_python_looks_for_python3( - mock_which, mock_run, mock_environ, expected_timeout, expected_cwd + mock_which, mock_run, mock_environ, expected_timeout, expected_cwd, expected_env ): expected_interpreter = Path("/path/to/python3") mock_which.side_effect = ( @@ -437,6 +450,7 @@ def test_execute_system_python_looks_for_python3( sandbox.ExecutionEnvironmentType.SYSTEM_PATH, cwd=expected_cwd, timeout=expected_timeout, + env_vars=expected_env, ) mock_run.assert_called_once_with( @@ -446,6 +460,7 @@ def test_execute_system_python_looks_for_python3( input=PYTHON_SCRIPT, cwd=expected_cwd, timeout=expected_timeout, + env=expected_env, ) assert actual.args == SCRIPT_ARGS @@ -478,6 +493,7 @@ def test_execute_system_python_falls_back_to_python(mock_which, mock_run, mock_e input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -514,6 +530,7 @@ def test_execute_system_python_falls_back_to_current_interpreter( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -544,20 +561,21 @@ def test_execute_system_python_fails_when_no_interpreter_available( @pytest.mark.parametrize( - "expected_timeout, expected_cwd", + "expected_timeout, expected_cwd, expected_env", [ - (None, None), - (TIMEOUT, None), - (None, NEW_CWD), - (None, Path(NEW_CWD)), - (TIMEOUT, NEW_CWD), + (None, None, None), + (TIMEOUT, None, None), + (None, NEW_CWD, None), + (None, Path(NEW_CWD), None), + (None, None, ENV_VARS), + (TIMEOUT, NEW_CWD, ENV_VARS), ], ) @mock.patch("subprocess.run") @mock.patch("shutil.which") @mock.patch("sys.executable", "/path/to/python") def test_execute_in_current_interpreter( - mock_which, mock_run, mock_environ, expected_timeout, expected_cwd + mock_which, mock_run, mock_environ, expected_timeout, expected_cwd, expected_env ): expected_interpreter = "/path/to/python" mock_which.return_value = "/path/to/ignored/python" @@ -572,6 +590,7 @@ def test_execute_in_current_interpreter( sandbox.ExecutionEnvironmentType.CURRENT, timeout=expected_timeout, cwd=expected_cwd, + env_vars=expected_env, ) mock_run.assert_called_once_with( @@ -581,6 +600,7 @@ def test_execute_in_current_interpreter( input=PYTHON_SCRIPT, cwd=expected_cwd, timeout=expected_timeout, + env=expected_env, ) assert actual.args == SCRIPT_ARGS @@ -613,6 +633,7 @@ def test_execute_auto_detects_venv( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -653,6 +674,7 @@ def test_execute_auto_detects_conda(mock_which, mock_run, mock_environ): input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -687,6 +709,7 @@ def test_execute_auto_detect_falls_back_to_system_python( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -721,6 +744,7 @@ def test_execute_auto_detect_chooses_venv_over_conda( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -755,6 +779,7 @@ def test_execute_auto_detect_is_default( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -789,6 +814,7 @@ def test_execute_does_not_interpret_return_codes( input=PYTHON_SCRIPT, cwd=None, timeout=None, + env=None, ) assert actual.args == SCRIPT_ARGS @@ -797,3 +823,16 @@ def test_execute_does_not_interpret_return_codes( assert actual.stderr == SCRIPT_ERR assert not mock_which.called + + +def test_sandbox_env_builder(temp_dir): + env_path = Path(temp_dir) / "venv" + builder = sandbox.SandboxEnvBuilder(env_path) + builder.ensure_created() # exercise the creation path + + builder.run_python("--version") # should not raise an exception + + # verify that a builder works correctly when the virtual env already exists + builder = sandbox.SandboxEnvBuilder(env_path) + builder.ensure_created() + builder.run_python("--help") # should not raise an exception From 2696516c8d6f8b00e8795f19596846768a7d7193 Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Thu, 18 Jul 2024 10:03:46 +0200 Subject: [PATCH 5/7] bump version to 2.6.1 (#1328) --- src/snowflake/cli/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index 5537ed282d..3b1988c250 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -14,4 +14,4 @@ from __future__ import annotations -VERSION = "2.6.0.dev0" +VERSION = "2.6.1.dev0" From 474469601f37c7e8a5c181aa6131aa35540dbca2 Mon Sep 17 00:00:00 2001 From: Guy Bloom Date: Thu, 18 Jul 2024 10:30:44 -0400 Subject: [PATCH 6/7] Support project definition v2 in "snow app teardown" (#1326) * add v2 decorator to "app teardown" command * util to enable v2 feature flag * v2 coverage for teardown cascade integration test --- .../cli/plugins/nativeapp/commands.py | 1 + tests_integration/nativeapp/test_open.py | 28 ++--- tests_integration/nativeapp/test_teardown.py | 108 +++++++----------- .../projects/napp_create_db_v1/app/README.md | 1 + .../napp_create_db_v1/app/manifest.yml | 8 ++ .../napp_create_db_v1/app/setup_script.sql | 11 ++ .../projects/napp_create_db_v1/snowflake.yml | 7 ++ .../projects/napp_create_db_v2/app/README.md | 3 + .../napp_create_db_v2/app/manifest.yml | 10 ++ .../napp_create_db_v2/app/setup_script.sql | 13 +++ .../projects/napp_create_db_v2/snowflake.yml | 17 +++ .../projects/napp_init_v2/app/README.md | 3 +- .../projects/napp_init_v2/app/manifest.yml | 4 +- .../napp_init_v2/app/setup_script.sql | 10 +- tests_integration/test_utils.py | 10 ++ .../workspaces/test_validate_schema.py | 15 +-- 16 files changed, 143 insertions(+), 106 deletions(-) create mode 100644 tests_integration/test_data/projects/napp_create_db_v1/app/README.md create mode 100644 tests_integration/test_data/projects/napp_create_db_v1/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_create_db_v1/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_create_db_v1/snowflake.yml create mode 100644 tests_integration/test_data/projects/napp_create_db_v2/app/README.md create mode 100644 tests_integration/test_data/projects/napp_create_db_v2/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_create_db_v2/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_create_db_v2/snowflake.yml diff --git a/src/snowflake/cli/plugins/nativeapp/commands.py b/src/snowflake/cli/plugins/nativeapp/commands.py index ecfa4210e7..2759197eb3 100644 --- a/src/snowflake/cli/plugins/nativeapp/commands.py +++ b/src/snowflake/cli/plugins/nativeapp/commands.py @@ -257,6 +257,7 @@ def app_open( @app.command("teardown", requires_connection=True) @with_project_definition() +@nativeapp_definition_v2_to_v1 def app_teardown( force: Optional[bool] = ForceOption, cascade: Optional[bool] = typer.Option( diff --git a/tests_integration/nativeapp/test_open.py b/tests_integration/nativeapp/test_open.py index dbdaae9bb3..6e2dd15792 100644 --- a/tests_integration/nativeapp/test_open.py +++ b/tests_integration/nativeapp/test_open.py @@ -15,9 +15,9 @@ import uuid from unittest import mock import re -import os from snowflake.cli.api.project.util import generate_user_env +from tests_integration.test_utils import enable_definition_v2_feature_flag from tests.project.fixtures import * @@ -67,23 +67,19 @@ def test_nativeapp_open( @pytest.mark.integration -@mock.patch.dict( - os.environ, - { - "SNOWFLAKE_CLI_FEATURES_ENABLE_PROJECT_DEFINITION_V2": "true", - }, -) +@enable_definition_v2_feature_flag @mock.patch("typer.launch") +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_open_v2( mock_typer_launch, runner, - snowflake_session, + definition_version, project_directory, ): project_name = "myapp" app_name = f"{project_name}_{USER_NAME}" - # TODO Move to napp_init_v2 block once "snow app run" supports definition v2 + # TODO Use the main project_directory block once "snow app run" supports definition v2 with project_directory("napp_init_v1"): result = runner.invoke_with_connection_json( ["app", "run"], @@ -91,7 +87,7 @@ def test_nativeapp_open_v2( ) assert result.exit_code == 0 - with project_directory("napp_init_v2"): + with project_directory(f"napp_init_{definition_version}"): try: result = runner.invoke_with_connection_json( ["app", "open"], @@ -108,10 +104,8 @@ def test_nativeapp_open_v2( ) finally: - # TODO Move to napp_init_v2 block once "snow app run" supports definition v2 - with project_directory("napp_init_v1"): - result = runner.invoke_with_connection_json( - ["app", "teardown", "--force", "--cascade"], - env=TEST_ENV, - ) - assert result.exit_code == 0 + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force", "--cascade"], + env=TEST_ENV, + ) + assert result.exit_code == 0 diff --git a/tests_integration/nativeapp/test_teardown.py b/tests_integration/nativeapp/test_teardown.py index 6463e59f43..9ab2883050 100644 --- a/tests_integration/nativeapp/test_teardown.py +++ b/tests_integration/nativeapp/test_teardown.py @@ -15,6 +15,7 @@ import os import uuid from textwrap import dedent +from unittest import mock from snowflake.cli.api.project.util import generate_user_env @@ -24,6 +25,7 @@ contains_row_with, not_contains_row_with, row_from_snowflake_session, + enable_definition_v2_feature_flag, ) USER_NAME = f"user_{uuid.uuid4().hex}" @@ -31,6 +33,7 @@ @pytest.mark.integration +@enable_definition_v2_feature_flag @pytest.mark.parametrize( "command,expected_error", [ @@ -46,52 +49,30 @@ ], ) @pytest.mark.parametrize("orphan_app", [True, False]) +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_teardown_cascade( command, expected_error, orphan_app, + definition_version, + project_directory, runner, snowflake_session, - temporary_working_directory, ): project_name = "myapp" app_name = f"{project_name}_{USER_NAME}".upper() db_name = f"{project_name}_db_{USER_NAME}".upper() - result = runner.invoke_json( - ["app", "init", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_name)): - # Add a procedure to the setup script that creates an app-owned database - with open("app/setup_script.sql", "a") as file: - file.write( - dedent( - f""" - create or replace procedure core.create_db() - returns boolean - language sql - as $$ - begin - create or replace database {db_name}; - return true; - end; - $$; - """ - ) - ) - with open("app/manifest.yml", "a") as file: - file.write( - dedent( - f""" - privileges: - - CREATE DATABASE: - description: "Permission to create databases" - """ - ) - ) + # TODO Use the main project_directory block once "snow app run" supports definition v2 + with project_directory(f"napp_create_db_v1"): + # Replacing the static DB name with a unique one to avoid collisions between tests + with open("app/setup_script.sql", "r") as file: + setup_script_content = file.read() + setup_script_content = setup_script_content.replace( + "DB_NAME_PLACEHOLDER", db_name + ) + with open("app/setup_script.sql", "w") as file: + file.write(setup_script_content) result = runner.invoke_with_connection_json( ["app", "run"], @@ -99,6 +80,7 @@ def test_nativeapp_teardown_cascade( ) assert result.exit_code == 0 + with project_directory(f"napp_create_db_{definition_version}"): try: # Grant permission to create databases snowflake_session.execute_string( @@ -176,29 +158,27 @@ def test_nativeapp_teardown_cascade( @pytest.mark.integration +@enable_definition_v2_feature_flag @pytest.mark.parametrize("force", [True, False]) +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_teardown_unowned_app( runner, - snowflake_session, - temporary_working_directory, force, + definition_version, + project_directory, ): project_name = "myapp" app_name = f"{project_name}_{USER_NAME}" - result = runner.invoke_json( - ["app", "init", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_name)): + # TODO Use the main project_directory block once "snow app run" supports definition v2 + with project_directory("napp_init_v1"): result = runner.invoke_with_connection_json( ["app", "run"], env=TEST_ENV, ) assert result.exit_code == 0 + with project_directory(f"napp_init_{definition_version}"): try: result = runner.invoke_with_connection_json( ["sql", "-q", f"alter application {app_name} set comment = 'foo'"], @@ -228,28 +208,26 @@ def test_nativeapp_teardown_unowned_app( @pytest.mark.integration +@enable_definition_v2_feature_flag @pytest.mark.parametrize("default_release_directive", [True, False]) +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_teardown_pkg_versions( runner, - snowflake_session, - temporary_working_directory, default_release_directive, + definition_version, + project_directory, ): project_name = "myapp" pkg_name = f"{project_name}_pkg_{USER_NAME}" - result = runner.invoke_json( - ["app", "init", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_name)): - result = runner.invoke_with_connection( - ["app", "version", "create", "v1"], - env=TEST_ENV, - ) - assert result.exit_code == 0 + with project_directory(f"napp_init_{definition_version}"): + # TODO Use the main project_directory block once "snow app version" supports definition v2 + with project_directory("napp_init_v1"): + result = runner.invoke_with_connection( + ["app", "version", "create", "v1"], + env=TEST_ENV, + ) + assert result.exit_code == 0 try: # when setting a release directive, we will not have the ability to drop the version later @@ -274,12 +252,14 @@ def test_nativeapp_teardown_pkg_versions( teardown_args = [] if not default_release_directive: - # if we didn't set a release directive, we can drop the version and try again - result = runner.invoke_with_connection( - ["app", "version", "drop", "v1", "--force"], - env=TEST_ENV, - ) - assert result.exit_code == 0 + # TODO Use the main project_directory block once "snow app version" supports definition v2 + with project_directory("napp_init_v1"): + # if we didn't set a release directive, we can drop the version and try again + result = runner.invoke_with_connection( + ["app", "version", "drop", "v1", "--force"], + env=TEST_ENV, + ) + assert result.exit_code == 0 else: # if we did set a release directive, we need --force for teardown to work teardown_args = ["--force"] diff --git a/tests_integration/test_data/projects/napp_create_db_v1/app/README.md b/tests_integration/test_data/projects/napp_create_db_v1/app/README.md new file mode 100644 index 0000000000..7e59600739 --- /dev/null +++ b/tests_integration/test_data/projects/napp_create_db_v1/app/README.md @@ -0,0 +1 @@ +# README diff --git a/tests_integration/test_data/projects/napp_create_db_v1/app/manifest.yml b/tests_integration/test_data/projects/napp_create_db_v1/app/manifest.yml new file mode 100644 index 0000000000..9e1dda96f5 --- /dev/null +++ b/tests_integration/test_data/projects/napp_create_db_v1/app/manifest.yml @@ -0,0 +1,8 @@ +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md +privileges: + - CREATE DATABASE: + description: "Permission to create databases" diff --git a/tests_integration/test_data/projects/napp_create_db_v1/app/setup_script.sql b/tests_integration/test_data/projects/napp_create_db_v1/app/setup_script.sql new file mode 100644 index 0000000000..7ab44e0d96 --- /dev/null +++ b/tests_integration/test_data/projects/napp_create_db_v1/app/setup_script.sql @@ -0,0 +1,11 @@ +CREATE OR ALTER VERSIONED SCHEMA core; + +create or replace procedure core.create_db() + returns boolean + language sql + as $$ + begin + create or replace database DB_NAME_PLACEHOLDER; + return true; + end; + $$; diff --git a/tests_integration/test_data/projects/napp_create_db_v1/snowflake.yml b/tests_integration/test_data/projects/napp_create_db_v1/snowflake.yml new file mode 100644 index 0000000000..bb42e3a5ee --- /dev/null +++ b/tests_integration/test_data/projects/napp_create_db_v1/snowflake.yml @@ -0,0 +1,7 @@ +definition_version: 1 +native_app: + name: myapp + source_stage: app_src.stage + artifacts: + - src: app/* + dest: ./ diff --git a/tests_integration/test_data/projects/napp_create_db_v2/app/README.md b/tests_integration/test_data/projects/napp_create_db_v2/app/README.md new file mode 100644 index 0000000000..031bf34098 --- /dev/null +++ b/tests_integration/test_data/projects/napp_create_db_v2/app/README.md @@ -0,0 +1,3 @@ +# README + +This is the v2 version of the napp_create_db_v1 project diff --git a/tests_integration/test_data/projects/napp_create_db_v2/app/manifest.yml b/tests_integration/test_data/projects/napp_create_db_v2/app/manifest.yml new file mode 100644 index 0000000000..8f1cc46319 --- /dev/null +++ b/tests_integration/test_data/projects/napp_create_db_v2/app/manifest.yml @@ -0,0 +1,10 @@ +# This is the v2 version of the napp_create_db_v1 project + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md +privileges: + - CREATE DATABASE: + description: "Permission to create databases" diff --git a/tests_integration/test_data/projects/napp_create_db_v2/app/setup_script.sql b/tests_integration/test_data/projects/napp_create_db_v2/app/setup_script.sql new file mode 100644 index 0000000000..d00f61535e --- /dev/null +++ b/tests_integration/test_data/projects/napp_create_db_v2/app/setup_script.sql @@ -0,0 +1,13 @@ +-- This is the v2 version of the napp_create_db_v1 project + +CREATE OR ALTER VERSIONED SCHEMA core; + +create or replace procedure core.create_db() + returns boolean + language sql + as $$ + begin + create or replace database DB_NAME_PLACEHOLDER; + return true; + end; + $$; diff --git a/tests_integration/test_data/projects/napp_create_db_v2/snowflake.yml b/tests_integration/test_data/projects/napp_create_db_v2/snowflake.yml new file mode 100644 index 0000000000..22724d559e --- /dev/null +++ b/tests_integration/test_data/projects/napp_create_db_v2/snowflake.yml @@ -0,0 +1,17 @@ +# This is the v2 version of the napp_create_db_v1 project definition + +definition_version: 2 +entities: + pkg: + type: application package + name: myapp_pkg_<% ctx.env.USER %> + stage: app_src.stage + artifacts: + - src: app/* + dest: ./ + manifest: app/manifest.yml + app: + type: application + name: myapp_<% ctx.env.USER %> + from: + target: pkg diff --git a/tests_integration/test_data/projects/napp_init_v2/app/README.md b/tests_integration/test_data/projects/napp_init_v2/app/README.md index f66bf75c9b..7df9b2f35c 100644 --- a/tests_integration/test_data/projects/napp_init_v2/app/README.md +++ b/tests_integration/test_data/projects/napp_init_v2/app/README.md @@ -1,4 +1,3 @@ # README -This directory contains an extremely simple application that is used for -integration testing SnowCLI. +This is the v2 version of the napp_init_v1 project diff --git a/tests_integration/test_data/projects/napp_init_v2/app/manifest.yml b/tests_integration/test_data/projects/napp_init_v2/app/manifest.yml index 5b8ef74e8a..0b8b9b892c 100644 --- a/tests_integration/test_data/projects/napp_init_v2/app/manifest.yml +++ b/tests_integration/test_data/projects/napp_init_v2/app/manifest.yml @@ -1,6 +1,4 @@ -# This is a manifest.yml file, a required component of creating a Snowflake Native App. -# This file defines properties required by the application package, including the location of the setup script and version definitions. -# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file. +# This is the v2 version of the napp_init_v1 project manifest_version: 1 diff --git a/tests_integration/test_data/projects/napp_init_v2/app/setup_script.sql b/tests_integration/test_data/projects/napp_init_v2/app/setup_script.sql index 7fc3682b6e..3b3562c461 100644 --- a/tests_integration/test_data/projects/napp_init_v2/app/setup_script.sql +++ b/tests_integration/test_data/projects/napp_init_v2/app/setup_script.sql @@ -1,11 +1,3 @@ --- This is the setup script that runs while installing a Snowflake Native App in a consumer account. --- To write this script, you can familiarize yourself with some of the following concepts: --- Application Roles --- Versioned Schemas --- UDFs/Procs --- Extension Code --- Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script for a detailed understanding of this file. +-- This is the v2 version of the napp_init_v1 project CREATE OR ALTER VERSIONED SCHEMA core; - --- The rest of this script is left blank for purposes of your learning and exploration. diff --git a/tests_integration/test_utils.py b/tests_integration/test_utils.py index 81dec980ec..ff66689e7e 100644 --- a/tests_integration/test_utils.py +++ b/tests_integration/test_utils.py @@ -14,6 +14,8 @@ import os import datetime +from functools import wraps +from unittest import mock from typing import Any, Dict, List from contextlib import contextmanager from pathlib import Path @@ -78,3 +80,11 @@ def not_contains_row_with(rows: List[Dict[str, Any]], values: Dict[str, Any]) -> if row.items() >= values_items: return False return True + + +enable_definition_v2_feature_flag = mock.patch.dict( + os.environ, + { + "SNOWFLAKE_CLI_FEATURES_ENABLE_PROJECT_DEFINITION_V2": "true", + }, +) diff --git a/tests_integration/workspaces/test_validate_schema.py b/tests_integration/workspaces/test_validate_schema.py index 3c4220ce77..ffe7d07bde 100644 --- a/tests_integration/workspaces/test_validate_schema.py +++ b/tests_integration/workspaces/test_validate_schema.py @@ -13,20 +13,13 @@ # limitations under the License. import pytest -import os -from unittest import mock +from tests_integration.test_utils import enable_definition_v2_feature_flag @pytest.mark.integration -@mock.patch.dict( - os.environ, - { - "SNOWFLAKE_CLI_FEATURES_ENABLE_PROJECT_DEFINITION_V2": "true", - }, - clear=True, -) -def test_validate_project_definition_v2(runner, snowflake_session, project_directory): - with project_directory("project_definition_v2") as tmp_dir: +@enable_definition_v2_feature_flag +def test_validate_project_definition_v2(runner, project_directory): + with project_directory("project_definition_v2"): result = runner.invoke_with_connection_json(["ws", "validate"]) assert result.exit_code == 0 From 59905dcb18673faddfbae112422f001707bbce5f Mon Sep 17 00:00:00 2001 From: Guy Bloom Date: Thu, 18 Jul 2024 16:34:26 -0400 Subject: [PATCH 7/7] Support project definition v2 in "snow app" commands: bundle, validate, deploy (#1332) * support v2 in "snow app bundle" * refactor deploy_root * update unit test * Support project definition v2 in "app validate" and "app deploy" (#1339) support v2 in validate and deploy * remove pushd --- .../cli/plugins/nativeapp/commands.py | 3 + .../v2_conversions/v2_to_v1_decorator.py | 23 +- tests/nativeapp/test_v2_to_v1.py | 6 + .../nativeapp/__snapshots__/test_deploy.ambr | 122 ++++- tests_integration/nativeapp/test_bundle.py | 460 ++++++++---------- tests_integration/nativeapp/test_deploy.py | 173 +++---- tests_integration/nativeapp/test_teardown.py | 4 - tests_integration/nativeapp/test_validate.py | 29 +- .../app/README.md | 4 + .../app/manifest.yml | 9 + .../app/setup_script.sql | 11 + .../lib/parent/child/a.py | 0 .../lib/parent/child/b.py | 0 .../lib/parent/child/c/c.py | 0 .../snowflake.yml | 10 + .../napp_deploy_prefix_matches_v1/src/main.py | 0 .../app/README.md | 3 + .../app/manifest.yml | 7 + .../app/setup_script.sql | 3 + .../lib/parent/child/a.py | 0 .../lib/parent/child/b.py | 0 .../lib/parent/child/c/c.py | 0 .../snowflake.yml | 19 + .../napp_deploy_prefix_matches_v2/src/main.py | 0 24 files changed, 486 insertions(+), 400 deletions(-) create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/README.md create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/a.py create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/b.py create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/c/c.py create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/snowflake.yml create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/src/main.py create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/README.md create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/a.py create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/b.py create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/c/c.py create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/snowflake.yml create mode 100644 tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/src/main.py diff --git a/src/snowflake/cli/plugins/nativeapp/commands.py b/src/snowflake/cli/plugins/nativeapp/commands.py index 2759197eb3..ae94c93501 100644 --- a/src/snowflake/cli/plugins/nativeapp/commands.py +++ b/src/snowflake/cli/plugins/nativeapp/commands.py @@ -150,6 +150,7 @@ def app_list_templates(**options) -> CommandResult: @app.command("bundle") @with_project_definition() +@nativeapp_definition_v2_to_v1 def app_bundle( **options, ) -> CommandResult: @@ -284,6 +285,7 @@ def app_teardown( @app.command("deploy", requires_connection=True) @with_project_definition() +@nativeapp_definition_v2_to_v1 def app_deploy( prune: Optional[bool] = typer.Option( default=None, @@ -350,6 +352,7 @@ def app_deploy( @app.command("validate", requires_connection=True) @with_project_definition() +@nativeapp_definition_v2_to_v1 def app_validate(**options): """ Validates a deployed Snowflake Native App's setup script. diff --git a/src/snowflake/cli/plugins/nativeapp/v2_conversions/v2_to_v1_decorator.py b/src/snowflake/cli/plugins/nativeapp/v2_conversions/v2_to_v1_decorator.py index b71d003407..a886e46116 100644 --- a/src/snowflake/cli/plugins/nativeapp/v2_conversions/v2_to_v1_decorator.py +++ b/src/snowflake/cli/plugins/nativeapp/v2_conversions/v2_to_v1_decorator.py @@ -15,7 +15,8 @@ from __future__ import annotations from functools import wraps -from typing import Any, Dict, Optional +from pathlib import Path +from typing import Any, Dict, Optional, Union from click import ClickException from snowflake.cli.api.cli_global_context import cli_context, cli_context_manager @@ -25,12 +26,25 @@ from snowflake.cli.api.project.schemas.entities.application_package_entity import ( ApplicationPackageEntity, ) +from snowflake.cli.api.project.schemas.native_app.path_mapping import PathMapping from snowflake.cli.api.project.schemas.project_definition import ( DefinitionV11, DefinitionV20, ) +def _convert_v2_artifact_to_v1_dict( + v2_artifact: Union[PathMapping, Path] +) -> Union[Dict, str]: + if isinstance(v2_artifact, PathMapping): + return { + "src": v2_artifact.src, + "dest": v2_artifact.dest, + "processors": v2_artifact.processors, + } + return str(v2_artifact) + + def _pdf_v2_to_v1(v2_definition: DefinitionV20) -> DefinitionV11: pdfv1: Dict[str, Any] = {"definition_version": "1.1", "native_app": {}} @@ -57,8 +71,13 @@ def _pdf_v2_to_v1(v2_definition: DefinitionV20) -> DefinitionV11: # NativeApp pdfv1["native_app"]["name"] = "Auto converted NativeApp project from V2" - pdfv1["native_app"]["artifacts"] = app_package_definition.artifacts + pdfv1["native_app"]["artifacts"] = [ + _convert_v2_artifact_to_v1_dict(a) for a in app_package_definition.artifacts + ] pdfv1["native_app"]["source_stage"] = app_package_definition.stage + pdfv1["native_app"]["bundle_root"] = str(app_package_definition.bundle_root) + pdfv1["native_app"]["generated_root"] = str(app_package_definition.generated_root) + pdfv1["native_app"]["deploy_root"] = str(app_package_definition.deploy_root) # Package pdfv1["native_app"]["package"] = {} diff --git a/tests/nativeapp/test_v2_to_v1.py b/tests/nativeapp/test_v2_to_v1.py index 99beb19705..66110e25a0 100644 --- a/tests/nativeapp/test_v2_to_v1.py +++ b/tests/nativeapp/test_v2_to_v1.py @@ -88,6 +88,9 @@ "artifacts": [{"src": "app/*", "dest": "./"}], "manifest": "", "stage": "app.stage", + "bundle_root": "bundle_root", + "generated_root": "generated_root", + "deploy_root": "deploy_root", }, "app": { "type": "application", @@ -103,6 +106,9 @@ "name": "Auto converted NativeApp project from V2", "artifacts": [{"src": "app/*", "dest": "./"}], "source_stage": "app.stage", + "bundle_root": "bundle_root", + "generated_root": "generated_root", + "deploy_root": "deploy_root", "package": { "name": "pkg_name", }, diff --git a/tests_integration/nativeapp/__snapshots__/test_deploy.ambr b/tests_integration/nativeapp/__snapshots__/test_deploy.ambr index f72c71c40c..002cfe8f03 100644 --- a/tests_integration/nativeapp/__snapshots__/test_deploy.ambr +++ b/tests_integration/nativeapp/__snapshots__/test_deploy.ambr @@ -1,5 +1,5 @@ # serializer version: 1 -# name: test_nativeapp_deploy +# name: test_nativeapp_deploy[v1] ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. @@ -14,7 +14,7 @@ ''' # --- -# name: test_nativeapp_deploy_dot +# name: test_nativeapp_deploy[v2] ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. @@ -29,20 +29,78 @@ ''' # --- -# name: test_nativeapp_deploy_files +# name: test_nativeapp_deploy_dot[v1] ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. Local changes to be deployed: + added: app/README.md -> README.md + added: app/manifest.yml -> manifest.yml + added: app/setup_script.sql -> setup_script.sql + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. + Validating Snowflake Native App setup script. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_dot[v2] + ''' + Creating new application package myapp_pkg_@@USER@@ in account. + Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Local changes to be deployed: + added: app/README.md -> README.md + added: app/manifest.yml -> manifest.yml + added: app/setup_script.sql -> setup_script.sql + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. + Validating Snowflake Native App setup script. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_files[v1] + ''' + Creating new application package myapp_pkg_@@USER@@ in account. + Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Local changes to be deployed: + added: app/manifest.yml -> manifest.yml + added: app/setup_script.sql -> setup_script.sql + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_files[v2] + ''' + Creating new application package myapp_pkg_@@USER@@ in account. + Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Local changes to be deployed: + added: app/manifest.yml -> manifest.yml + added: app/setup_script.sql -> setup_script.sql + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_looks_for_prefix_matches[v1] + ''' + Creating new application package myapp_pkg_@@USER@@ in account. + Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Local changes to be deployed: + added: app/README.md -> README.md added: app/manifest.yml -> manifest.yml added: app/setup_script.sql -> setup_script.sql Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. + Validating Snowflake Native App setup script. Deployed successfully. Application package and stage are up-to-date. ''' # --- -# name: test_nativeapp_deploy_looks_for_prefix_matches +# name: test_nativeapp_deploy_looks_for_prefix_matches[v2] ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. @@ -57,7 +115,7 @@ ''' # --- -# name: test_nativeapp_deploy_nested_directories +# name: test_nativeapp_deploy_nested_directories[v1] ''' Creating new application package myapp_pkg_@@USER@@ in account. Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. @@ -69,7 +127,55 @@ ''' # --- -# name: test_nativeapp_deploy_prune[app deploy --no-prune-contains2-not_contains2] +# name: test_nativeapp_deploy_nested_directories[v2] + ''' + Creating new application package myapp_pkg_@@USER@@ in account. + Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Local changes to be deployed: + added: app/nested/dir/file.txt -> nested/dir/file.txt + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_prune[v1-app deploy --no-prune-contains2-not_contains2] + ''' + Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + The following files exist only on the stage: + README.md + + Use the --prune flag to delete them from the stage. + Your stage is up-to-date with your local deploy root. + Validating Snowflake Native App setup script. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_prune[v1-app deploy --no-validate-contains1-not_contains1] + ''' + Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Deleted paths to be removed from your stage: + deleted: README.md + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_prune[v1-app deploy --prune --no-validate-contains0-not_contains0] + ''' + Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. + Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. + Deleted paths to be removed from your stage: + deleted: README.md + Updating the Snowflake stage from your local @@DEPLOY_ROOT@@ directory. + Deployed successfully. Application package and stage are up-to-date. + + ''' +# --- +# name: test_nativeapp_deploy_prune[v2-app deploy --no-prune-contains2-not_contains2] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. @@ -83,7 +189,7 @@ ''' # --- -# name: test_nativeapp_deploy_prune[app deploy --no-validate-contains1-not_contains1] +# name: test_nativeapp_deploy_prune[v2-app deploy --no-validate-contains1-not_contains1] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. @@ -94,7 +200,7 @@ ''' # --- -# name: test_nativeapp_deploy_prune[app deploy --prune --no-validate-contains0-not_contains0] +# name: test_nativeapp_deploy_prune[v2-app deploy --prune --no-validate-contains0-not_contains0] ''' Checking if stage myapp_pkg_@@USER@@.app_src.stage exists, or creating a new one if none exists. Performing a diff between the Snowflake stage and your local deploy_root ('@@DEPLOY_ROOT@@') directory. diff --git a/tests_integration/nativeapp/test_bundle.py b/tests_integration/nativeapp/test_bundle.py index 762c085e48..e4e4fb00f0 100644 --- a/tests_integration/nativeapp/test_bundle.py +++ b/tests_integration/nativeapp/test_bundle.py @@ -14,13 +14,13 @@ import os import os.path +import yaml import uuid -from textwrap import dedent from snowflake.cli.api.project.util import generate_user_env from tests.project.fixtures import * -from tests_integration.test_utils import pushd +from tests_integration.test_utils import enable_definition_v2_feature_flag from tests_integration.testing_utils import ( assert_that_result_failed_with_message_containing, ) @@ -29,30 +29,59 @@ TEST_ENV = generate_user_env(USER_NAME) -@pytest.fixture -def template_setup(runner, temporary_working_directory): - project_name = "myapp" - result = runner.invoke_json( - ["app", "init", project_name], # Uses default template - env=TEST_ENV, - ) - assert result.exit_code == 0 +@pytest.fixture(scope="function", params=["v1", "v2"]) +def template_setup(runner, project_directory, request): + definition_version = request.param + with enable_definition_v2_feature_flag: + with project_directory(f"napp_init_{definition_version}") as project_root: + # Vanilla bundle on the unmodified template + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) + assert result.exit_code == 0 - # Vanilla bundle on the unmodified template - result = runner.invoke_json( - ["app", "bundle", "--project", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - # The newly created deploy_root is explicitly deleted here, as bundle should take care of it. + # The newly created deploy_root is explicitly deleted here, as bundle should take care of it. + + deploy_root = Path(project_root, "output", "deploy") + assert Path(deploy_root, "manifest.yml").is_file() + assert Path(deploy_root, "setup_script.sql").is_file() + assert Path(deploy_root, "README.md").is_file() - project_root = Path(temporary_working_directory, project_name) - deploy_root = Path(project_root, "output", "deploy") - assert Path(deploy_root, "manifest.yml").is_file() - assert Path(deploy_root, "setup_script.sql").is_file() - assert Path(deploy_root, "README.md").is_file() + yield project_root, runner, definition_version - return project_root, runner + +def override_snowflake_yml_artifacts( + definition_version, artifacts_section, deploy_root=Path("output", "deploy") +): + with open("snowflake.yml", "w") as f: + if definition_version == "v2": + file_content = yaml.dump( + { + "definition_version": "2", + "entities": { + "pkg": { + "type": "application package", + "name": "myapp_pkg_<% ctx.env.USER %>", + "artifacts": artifacts_section, + "manifest": "app/manifest.yml", + "deploy_root": str(deploy_root), + } + }, + } + ) + else: + file_content = yaml.dump( + { + "definition_version": "1", + "native_app": { + "name": "myapp", + "artifacts": artifacts_section, + "deploy_root": str(deploy_root), + }, + } + ) + f.write(file_content) # Tests that we copy files/directories directly to the deploy root instead of creating symlinks. @@ -60,50 +89,40 @@ def template_setup(runner, temporary_working_directory): def test_nativeapp_bundle_does_explicit_copy( template_setup, ): - project_root, runner = template_setup - - with pushd(project_root): - # overwrite the snowflake.yml rules - with open("snowflake.yml", "w") as f: - f.write( - dedent( - f""" - definition_version: 1 - native_app: - name: myapp - artifacts: - - src: app - dest: ./ - - src: snowflake.yml - dest: ./app/ - """ - ) - ) + project_root, runner, definition_version = template_setup + + override_snowflake_yml_artifacts( + definition_version, + artifacts_section=[ + {"src": "app", "dest": "./"}, + {"src": "snowflake.yml", "dest": "./app/"}, + ], + ) - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) - assert result.exit_code == 0 - assert not os.path.exists("app/snowflake.yml") - app_path = Path("output", "deploy", "app") - assert app_path.exists() and not app_path.is_symlink() - assert ( - Path(app_path, "manifest.yml").exists() - and Path(app_path, "manifest.yml").is_symlink() - ) - assert ( - Path(app_path, "setup_script.sql").exists() - and Path(app_path, "setup_script.sql").is_symlink() - ) - assert ( - Path(app_path, "README.md").exists() - and Path(app_path, "README.md").is_symlink() - ) - assert ( - Path(app_path, "snowflake.yml").exists() - and Path(app_path, "snowflake.yml").is_symlink() - ) + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + assert not os.path.exists("app/snowflake.yml") + app_path = Path("output", "deploy", "app") + assert app_path.exists() and not app_path.is_symlink() + assert ( + Path(app_path, "manifest.yml").exists() + and Path(app_path, "manifest.yml").is_symlink() + ) + assert ( + Path(app_path, "setup_script.sql").exists() + and Path(app_path, "setup_script.sql").is_symlink() + ) + assert ( + Path(app_path, "README.md").exists() + and Path(app_path, "README.md").is_symlink() + ) + assert ( + Path(app_path, "snowflake.yml").exists() + and Path(app_path, "snowflake.yml").is_symlink() + ) # Tests restrictions on the deploy root: It must be a sub-directory within the project directory @@ -112,227 +131,170 @@ def test_nativeapp_bundle_throws_error_due_to_project_root_deploy_root_mismatch( template_setup, ): - project_root, runner = template_setup + project_root, runner, definition_version = template_setup # Delete deploy_root since we test requirement of deploy_root being a directory shutil.rmtree(Path(project_root, "output", "deploy")) - with pushd(project_root) as project_dir: - deploy_root = Path(project_dir, "output") - # Make deploy root a file instead of directory - deploy_root_as_file = Path(deploy_root, "deploy") - deploy_root_as_file.touch(exist_ok=False) - - assert deploy_root_as_file.is_file() + deploy_root = Path(project_root, "output") + # Make deploy root a file instead of directory + deploy_root_as_file = Path(deploy_root, "deploy") + deploy_root_as_file.touch(exist_ok=False) - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) + assert deploy_root_as_file.is_file() - assert result.exit_code == 1 - assert_that_result_failed_with_message_containing( - result, "exists, but is not a directory!" - ) + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) - os.remove(deploy_root_as_file) - deploy_root.rmdir() + assert result.exit_code == 1 + assert_that_result_failed_with_message_containing( + result, "exists, but is not a directory!" + ) - original_cwd = os.getcwd() - assert not Path(original_cwd, "output").exists() + os.remove(deploy_root_as_file) + deploy_root.rmdir() # Make deploy root outside the project directory - deploy_root = Path(original_cwd, "output", "deploy") - deploy_root.mkdir(parents=True, exist_ok=False) - - with pushd(project_root): - with open("snowflake.yml", "w") as f: - f.write( - dedent( - f""" - definition_version: 1 - native_app: - name: myapp - deploy_root: {deploy_root} - artifacts: - - src: app - dest: ./ - - src: snowflake.yml - dest: ./app/ - """ - ) - ) + with tempfile.TemporaryDirectory() as tmpdir: + assert not Path(tmpdir, "output").exists() + deploy_root = Path(tmpdir, "output", "deploy") + deploy_root.mkdir(parents=True, exist_ok=False) + + override_snowflake_yml_artifacts( + definition_version, + artifacts_section=[ + {"src": "app", "dest": "./"}, + {"src": "snowflake.yml", "dest": "./app/"}, + ], + deploy_root=deploy_root, + ) - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) - assert result.exit_code == 1 - assert_that_result_failed_with_message_containing( - result, "is not a descendent of the project directory!" - ) + assert result.exit_code == 1 + assert_that_result_failed_with_message_containing( + result, "is not a descendent of the project directory!" + ) # Tests restrictions on the src spec that it must be a glob that returns matches @pytest.mark.integration def test_nativeapp_bundle_throws_error_on_incorrect_src_glob(template_setup): - project_root, runner = template_setup - - with pushd(project_root): - # overwrite the snowflake.yml with incorrect glob - with open("snowflake.yml", "w") as f: - f.write( - dedent( - f""" - definition_version: 1 - native_app: - name: myapp - artifacts: - - app/? - """ - ), - ) + project_root, runner, definition_version = template_setup - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) - assert result.exit_code == 1 - assert_that_result_failed_with_message_containing( - result, - "No match was found for the specified source in the project directory", - ) + # incorrect glob + override_snowflake_yml_artifacts(definition_version, artifacts_section=["app/?"]) + + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) + assert result.exit_code == 1 + assert_that_result_failed_with_message_containing( + result, + "No match was found for the specified source in the project directory", + ) # Tests restrictions on the src spec that it must be relative to project root @pytest.mark.integration def test_nativeapp_bundle_throws_error_on_bad_src(template_setup): - project_root, runner = template_setup - - with pushd(project_root): - # overwrite the snowflake.yml with incorrect glob - with open("snowflake.yml", "w") as f: - f.write( - dedent( - f""" - definition_version: 1 - native_app: - name: myapp - artifacts: - - {Path(project_root, "app").absolute()} - """ - ), - ) + project_root, runner, definition_version = template_setup - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) - assert result.exit_code == 1 - assert_that_result_failed_with_message_containing( - result, "Source path must be a relative path" - ) + # absolute path + src_path = Path(project_root, "app").absolute() + override_snowflake_yml_artifacts( + definition_version, artifacts_section=[f"{src_path}"] + ) + + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) + assert result.exit_code == 1 + assert_that_result_failed_with_message_containing( + result, "Source path must be a relative path" + ) # Tests restrictions on the dest spec: It must be within the deploy root, and must be a relative path @pytest.mark.integration def test_nativeapp_bundle_throws_error_on_bad_dest(template_setup): - project_root, runner = template_setup - - with pushd(project_root): - # overwrite the snowflake.yml rules - with open("snowflake.yml", "w") as f: - f.write( - dedent( - f""" - definition_version: 1 - native_app: - name: myapp - artifacts: - - src: app/* - dest: / - """ - ) - ) + project_root, runner, definition_version = template_setup - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) - assert result.exit_code == 1 - assert_that_result_failed_with_message_containing( - result, "The specified destination path is outside of the deploy root" - ) - - with open("snowflake.yml", "w") as f: - f.write( - dedent( - f""" - definition_version: 1 - native_app: - name: myapp - artifacts: - - src: app/* - dest: {Path(project_root, "output", "deploy", "stagepath").absolute()} - """ - ) - ) + override_snowflake_yml_artifacts( + definition_version, artifacts_section=[{"src": "app/*", "dest": "/"}] + ) + + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) + assert result.exit_code == 1 + assert_that_result_failed_with_message_containing( + result, "The specified destination path is outside of the deploy root" + ) - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) - assert result.exit_code == 1 - assert_that_result_failed_with_message_containing( - result, "Destination path must be a relative path" - ) + override_snowflake_yml_artifacts( + definition_version, + artifacts_section=[ + { + "src": "app/*", + "dest": str( + Path(project_root, "output", "deploy", "stagepath").absolute() + ), + } + ], + ) + + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) + assert result.exit_code == 1 + assert_that_result_failed_with_message_containing( + result, "Destination path must be a relative path" + ) # Tests restriction on mapping multiple files to the same destination file @pytest.mark.integration def test_nativeapp_bundle_throws_error_on_too_many_files_to_dest(template_setup): + project_root, runner, definition_version = template_setup + + override_snowflake_yml_artifacts( + definition_version, + artifacts_section=[ + {"src": "app/manifest.yml", "dest": "manifest.yml"}, + {"src": "app/setup_script.sql", "dest": "manifest.yml"}, + ], + ) - project_root, runner = template_setup - with pushd(project_root): - # overwrite the snowflake.yml rules - with open("snowflake.yml", "w") as f: - f.write( - dedent( - f""" - definition_version: 1 - native_app: - name: myapp - artifacts: - - src: app/manifest.yml - dest: manifest.yml - - src: app/setup_script.sql - dest: manifest.yml - """ - ) - ) - - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) - assert result.exit_code == 1 - assert_that_result_failed_with_message_containing( - result, - "Multiple file or directories were mapped to one output destination.", - ) + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) + assert result.exit_code == 1 + assert_that_result_failed_with_message_containing( + result, + "Multiple file or directories were mapped to one output destination.", + ) # Tests that bundle wipes out any existing deploy root to recreate it from scratch on every run @pytest.mark.integration def test_nativeapp_bundle_deletes_existing_deploy_root(template_setup): - project_root, runner = template_setup - - with pushd(project_root) as project_dir: - existing_deploy_root_dest = Path(project_dir, "output", "deploy", "dummy.txt") - existing_deploy_root_dest.mkdir(parents=True, exist_ok=False) - result = runner.invoke_json( - ["app", "bundle"], - env=TEST_ENV, - ) - assert result.exit_code == 0 - assert not existing_deploy_root_dest.exists() + project_root, runner, definition_version = template_setup + + existing_deploy_root_dest = Path(project_root, "output", "deploy", "dummy.txt") + existing_deploy_root_dest.mkdir(parents=True, exist_ok=False) + result = runner.invoke_json( + ["app", "bundle"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + assert not existing_deploy_root_dest.exists() diff --git a/tests_integration/nativeapp/test_deploy.py b/tests_integration/nativeapp/test_deploy.py index b54b4b0fb9..cf4747a96e 100644 --- a/tests_integration/nativeapp/test_deploy.py +++ b/tests_integration/nativeapp/test_deploy.py @@ -28,6 +28,7 @@ not_contains_row_with, pushd, row_from_snowflake_session, + enable_definition_v2_feature_flag, ) from tests_integration.testing_utils import ( assert_that_result_failed_with_message_containing, @@ -46,21 +47,18 @@ def sanitize_deploy_output(output): # Tests a simple flow of executing "snow app deploy", verifying that an application package was created, and an application was not @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy( + definition_version, + project_directory, runner, snowflake_session, - temporary_working_directory, snapshot, print_paths_as_posix, ): project_name = "myapp" - result = runner.invoke_json( - ["app", "init", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_name)): + with project_directory(f"napp_init_{definition_version}"): result = runner.invoke_with_connection( ["app", "deploy"], env=TEST_ENV, @@ -117,6 +115,7 @@ def test_nativeapp_deploy( @pytest.mark.integration +@enable_definition_v2_feature_flag @pytest.mark.parametrize( "command,contains,not_contains", [ @@ -132,24 +131,19 @@ def test_nativeapp_deploy( ["app deploy --no-prune", ["stage/README.md"], []], ], ) +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_prune( command, contains, not_contains, + definition_version, + project_directory, runner, - snowflake_session, - temporary_working_directory, snapshot, print_paths_as_posix, ): project_name = "myapp" - result = runner.invoke_json( - ["app", "init", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_name)): + with project_directory(f"napp_init_{definition_version}"): result = runner.invoke_with_connection_json( ["app", "deploy"], env=TEST_ENV, @@ -198,20 +192,17 @@ def test_nativeapp_deploy_prune( # Tests a simple flow of executing "snow app deploy [files]", verifying that only the specified files are synced to the stage @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_files( + definition_version, + project_directory, runner, - temporary_working_directory, snapshot, print_paths_as_posix, ): project_name = "myapp" - result = runner.invoke_json( - ["app", "init", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_name)): + with project_directory(f"napp_init_{definition_version}"): # sync only two specific files to stage result = runner.invoke_with_connection( [ @@ -258,21 +249,17 @@ def test_nativeapp_deploy_files( # Tests that files inside of a symlinked directory are deployed @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_nested_directories( + definition_version, + project_directory, runner, - temporary_working_directory, snapshot, print_paths_as_posix, ): project_name = "myapp" - project_dir = "app root" - result = runner.invoke_json( - ["app", "init", project_dir, "--name", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_dir)): + with project_directory(f"napp_init_{definition_version}"): # create nested file under app/ touch("app/nested/dir/file.txt") @@ -312,19 +299,15 @@ def test_nativeapp_deploy_nested_directories( # Tests that deploying a directory recursively syncs all of its contents @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_directory( + definition_version, + project_directory, runner, - temporary_working_directory, ): project_name = "myapp" - project_dir = "app root" - result = runner.invoke_json( - ["app", "init", project_dir, "--name", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_dir)): + with project_directory(f"napp_init_{definition_version}"): touch("app/dir/file.txt") result = runner.invoke_with_connection( ["app", "deploy", "app/dir", "--no-recursive", "--no-validate"], @@ -367,19 +350,14 @@ def test_nativeapp_deploy_directory( # Tests that deploying a directory without specifying -r returns an error @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_directory_no_recursive( + definition_version, + project_directory, runner, - temporary_working_directory, ): - project_name = "myapp" - project_dir = "app root" - result = runner.invoke_json( - ["app", "init", project_dir, "--name", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_dir)): + with project_directory(f"napp_init_{definition_version}"): try: touch("app/nested/dir/file.txt") result = runner.invoke_with_connection_json( @@ -399,19 +377,14 @@ def test_nativeapp_deploy_directory_no_recursive( # Tests that specifying an unknown path to deploy results in an error @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_unknown_path( + definition_version, + project_directory, runner, - temporary_working_directory, ): - project_name = "myapp" - project_dir = "app root" - result = runner.invoke_json( - ["app", "init", project_dir, "--name", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_dir)): + with project_directory(f"napp_init_{definition_version}"): try: result = runner.invoke_with_connection_json( ["app", "deploy", "does_not_exist", "--no-validate"], @@ -431,19 +404,14 @@ def test_nativeapp_deploy_unknown_path( # Tests that specifying a path with no deploy artifact results in an error @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_path_with_no_mapping( + definition_version, + project_directory, runner, - temporary_working_directory, ): - project_name = "myapp" - project_dir = "app root" - result = runner.invoke_json( - ["app", "init", project_dir, "--name", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_dir)): + with project_directory(f"napp_init_{definition_version}"): try: result = runner.invoke_with_connection_json( ["app", "deploy", "snowflake.yml", "--no-validate"], @@ -463,19 +431,14 @@ def test_nativeapp_deploy_path_with_no_mapping( # Tests that specifying a path and pruning result in an error @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_rejects_pruning_when_path_is_specified( + definition_version, + project_directory, runner, - temporary_working_directory, ): - project_name = "myapp" - project_dir = "app root" - result = runner.invoke_json( - ["app", "init", project_dir, "--name", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_dir)): + with project_directory(f"napp_init_{definition_version}"): try: os.unlink("app/README.md") result = runner.invoke_with_connection_json( @@ -498,39 +461,19 @@ def test_nativeapp_deploy_rejects_pruning_when_path_is_specified( # Tests that specifying a path with no direct mapping falls back to search for prefix matches @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_looks_for_prefix_matches( + definition_version, + project_directory, runner, - temporary_working_directory, snapshot, print_paths_as_posix, ): project_name = "myapp" - project_dir = "app root" - result = runner.invoke_json( - ["app", "init", project_dir, "--name", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - project_dir = Path(os.getcwd(), project_dir) - with pushd(project_dir): + with project_directory(f"napp_deploy_prefix_matches_{definition_version}"): try: - snowflake_yml = project_dir / "snowflake.yml" - project_definition_file = yaml.load( - snowflake_yml.read_text(), yaml.BaseLoader - ) - project_definition_file["native_app"]["artifacts"].append("src") - project_definition_file["native_app"]["artifacts"].append( - {"src": "lib/parent", "dest": "parent-lib"} - ) - snowflake_yml.write_text(yaml.dump(project_definition_file)) - - touch(str(project_dir / "src/main.py")) - - touch(str(project_dir / "lib/parent/child/a.py")) - touch(str(project_dir / "lib/parent/child/b.py")) - touch(str(project_dir / "lib/parent/child/c/c.py")) - result = runner.invoke_with_connection( ["app", "deploy", "-r", "app"], env=TEST_ENV, @@ -623,21 +566,17 @@ def test_nativeapp_deploy_looks_for_prefix_matches( # Tests that snow app deploy -r . deploys all changes @pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) def test_nativeapp_deploy_dot( + definition_version, + project_directory, runner, - temporary_working_directory, snapshot, print_paths_as_posix, ): project_name = "myapp" - project_dir = "app root" - result = runner.invoke_json( - ["app", "init", project_dir, "--name", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0 - - with pushd(Path(os.getcwd(), project_dir)): + with project_directory(f"napp_init_{definition_version}"): try: result = runner.invoke_with_connection( ["app", "deploy", "-r", "."], diff --git a/tests_integration/nativeapp/test_teardown.py b/tests_integration/nativeapp/test_teardown.py index 9ab2883050..51f9cef64e 100644 --- a/tests_integration/nativeapp/test_teardown.py +++ b/tests_integration/nativeapp/test_teardown.py @@ -12,16 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import uuid -from textwrap import dedent -from unittest import mock from snowflake.cli.api.project.util import generate_user_env from tests.project.fixtures import * from tests_integration.test_utils import ( - pushd, contains_row_with, not_contains_row_with, row_from_snowflake_session, diff --git a/tests_integration/nativeapp/test_validate.py b/tests_integration/nativeapp/test_validate.py index aaabf68dd1..73c69a207b 100644 --- a/tests_integration/nativeapp/test_validate.py +++ b/tests_integration/nativeapp/test_validate.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import uuid from snowflake.cli.api.project.util import generate_user_env from tests.project.fixtures import * from tests_integration.test_utils import ( - pushd, + enable_definition_v2_feature_flag, ) USER_NAME = f"user_{uuid.uuid4().hex}" @@ -26,15 +25,10 @@ @pytest.mark.integration -def test_nativeapp_validate(runner, temporary_working_directory): - project_name = "myapp" - result = runner.invoke_json( - ["app", "init", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0, result.output - - with pushd(Path(os.getcwd(), project_name)): +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) +def test_nativeapp_validate(definition_version, project_directory, runner): + with project_directory(f"napp_init_{definition_version}"): try: # validate the app's setup script result = runner.invoke_with_connection( @@ -52,15 +46,10 @@ def test_nativeapp_validate(runner, temporary_working_directory): @pytest.mark.integration -def test_nativeapp_validate_failing(runner, temporary_working_directory): - project_name = "myapp" - result = runner.invoke_json( - ["app", "init", project_name], - env=TEST_ENV, - ) - assert result.exit_code == 0, result.output - - with pushd(Path(os.getcwd(), project_name)): +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("definition_version", ["v1", "v2"]) +def test_nativeapp_validate_failing(definition_version, project_directory, runner): + with project_directory(f"napp_init_{definition_version}"): # Create invalid SQL file Path("app/setup_script.sql").write_text("Lorem ipsum dolor sit amet") diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/README.md b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/README.md new file mode 100644 index 0000000000..f66bf75c9b --- /dev/null +++ b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains an extremely simple application that is used for +integration testing SnowCLI. diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/manifest.yml b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/manifest.yml new file mode 100644 index 0000000000..5b8ef74e8a --- /dev/null +++ b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/manifest.yml @@ -0,0 +1,9 @@ +# This is a manifest.yml file, a required component of creating a Snowflake Native App. +# This file defines properties required by the application package, including the location of the setup script and version definitions. +# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file. + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/setup_script.sql b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/setup_script.sql new file mode 100644 index 0000000000..7fc3682b6e --- /dev/null +++ b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/app/setup_script.sql @@ -0,0 +1,11 @@ +-- This is the setup script that runs while installing a Snowflake Native App in a consumer account. +-- To write this script, you can familiarize yourself with some of the following concepts: +-- Application Roles +-- Versioned Schemas +-- UDFs/Procs +-- Extension Code +-- Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script for a detailed understanding of this file. + +CREATE OR ALTER VERSIONED SCHEMA core; + +-- The rest of this script is left blank for purposes of your learning and exploration. diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/a.py b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/a.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/b.py b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/b.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/c/c.py b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/lib/parent/child/c/c.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/snowflake.yml b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/snowflake.yml new file mode 100644 index 0000000000..01b20a8f98 --- /dev/null +++ b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/snowflake.yml @@ -0,0 +1,10 @@ +definition_version: 1 +native_app: + name: myapp + source_stage: app_src.stage + artifacts: + - src: app/* + dest: ./ + - src + - src: lib/parent + dest: parent-lib diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/src/main.py b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v1/src/main.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/README.md b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/README.md new file mode 100644 index 0000000000..6a446bcf55 --- /dev/null +++ b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/README.md @@ -0,0 +1,3 @@ +# README + +This is the v2 version of the napp_deploy_prefix_matches_v1 project diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/manifest.yml b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/manifest.yml new file mode 100644 index 0000000000..1b444dab00 --- /dev/null +++ b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/manifest.yml @@ -0,0 +1,7 @@ +# This is the v2 version of the napp_deploy_prefix_matches_v1 project + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/setup_script.sql b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/setup_script.sql new file mode 100644 index 0000000000..352e2b23bf --- /dev/null +++ b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/app/setup_script.sql @@ -0,0 +1,3 @@ +-- This is the v2 version of the napp_deploy_prefix_matches_v1 project + +CREATE OR ALTER VERSIONED SCHEMA core; diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/a.py b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/a.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/b.py b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/b.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/c/c.py b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/lib/parent/child/c/c.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/snowflake.yml b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/snowflake.yml new file mode 100644 index 0000000000..966b0fb35f --- /dev/null +++ b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/snowflake.yml @@ -0,0 +1,19 @@ +# This is the v2 version of the napp_deploy_prefix_matches_v1 project definition + +definition_version: 2 +entities: + pkg: + type: application package + name: myapp_pkg_<% ctx.env.USER %> + artifacts: + - src: app/* + dest: ./ + - src + - src: lib/parent + dest: parent-lib + manifest: app/manifest.yml + app: + type: application + name: myapp_<% ctx.env.USER %> + from: + target: pkg diff --git a/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/src/main.py b/tests_integration/test_data/projects/napp_deploy_prefix_matches_v2/src/main.py new file mode 100644 index 0000000000..e69de29bb2