From 2444bd5e6a4ef83cee54fbcea65d79860c9e3a40 Mon Sep 17 00:00:00 2001 From: Guy Bloom Date: Tue, 25 Jun 2024 15:31:54 -0400 Subject: [PATCH] App post-deploy hook: sql scripts (#1244) * add app post deploy sql script hook * default warehouse and database * clarify release notes * add template support * release notes * add custom schema validation * simplify schema validation --- RELEASE-NOTES.md | 2 + .../project/schemas/native_app/application.py | 14 +- .../cli/plugins/nativeapp/manager.py | 13 ++ .../cli/plugins/nativeapp/run_processor.py | 34 ++++ tests/nativeapp/test_post_deploy.py | 180 ++++++++++++++++++ tests/nativeapp/utils.py | 1 + tests/project/__snapshots__/test_config.ambr | 2 + .../napp_post_deploy/scripts/post_deploy1.sql | 4 + .../napp_post_deploy/scripts/post_deploy2.sql | 1 + .../projects/napp_post_deploy/snowflake.yml | 15 ++ .../snowflake.yml | 11 ++ tests_integration/nativeapp/test_init_run.py | 52 +++++ .../app/README.md | 4 + .../app/manifest.yml | 9 + .../app/setup_script.sql | 11 ++ .../scripts/post_deploy1.sql | 5 + .../scripts/post_deploy2.sql | 3 + .../snowflake.yml | 15 ++ 18 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 tests/nativeapp/test_post_deploy.py create mode 100644 tests/test_data/projects/napp_post_deploy/scripts/post_deploy1.sql create mode 100644 tests/test_data/projects/napp_post_deploy/scripts/post_deploy2.sql create mode 100644 tests/test_data/projects/napp_post_deploy/snowflake.yml create mode 100644 tests/test_data/projects/napp_post_deploy_missing_file/snowflake.yml create mode 100644 tests_integration/test_data/projects/napp_application_post_deploy/app/README.md create mode 100644 tests_integration/test_data/projects/napp_application_post_deploy/app/manifest.yml create mode 100644 tests_integration/test_data/projects/napp_application_post_deploy/app/setup_script.sql create mode 100644 tests_integration/test_data/projects/napp_application_post_deploy/scripts/post_deploy1.sql create mode 100644 tests_integration/test_data/projects/napp_application_post_deploy/scripts/post_deploy2.sql create mode 100644 tests_integration/test_data/projects/napp_application_post_deploy/snowflake.yml diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index a9cc8bfe60..6bd1d632e9 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -22,6 +22,8 @@ ## New additions * Added support for `title` field in Streamlit definition in `snowflake.yml` project file. * Added `--auto-compress` flag to `snow stage copy` command enabling use of gzip to compress files during upload. +* Added new `native_app.application.post_deploy` section to `snowflake.yml` schema to execute actions after the application has been deployed via `snow app run`. + * Added the `sql_script` hook type to run SQL scripts with template support. ## Fixes and improvements * Passing a directory to `snow app deploy` will now deploy any contained file or subfolder specified in the application's artifact rules diff --git a/src/snowflake/cli/api/project/schemas/native_app/application.py b/src/snowflake/cli/api/project/schemas/native_app/application.py index e0fcb0f306..5394a1c5ff 100644 --- a/src/snowflake/cli/api/project/schemas/native_app/application.py +++ b/src/snowflake/cli/api/project/schemas/native_app/application.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Optional +from typing import List, Optional from pydantic import Field from snowflake.cli.api.project.schemas.updatable_model import ( @@ -23,6 +23,14 @@ ) +class SqlScriptHookType(UpdatableModel): + sql_script: str = Field(title="SQL file path relative to the project root") + + +# Currently sql_script is the only supported hook type. Change to a Union once other hook types are added +ApplicationPostDeployHook = SqlScriptHookType + + class Application(UpdatableModel): role: Optional[str] = Field( title="Role to use when creating the application object and consumer-side objects", @@ -40,3 +48,7 @@ class Application(UpdatableModel): title="Whether to enable debug mode when using a named stage to create an application object", default=True, ) + post_deploy: Optional[List[ApplicationPostDeployHook]] = Field( + title="Actions that will be executed after the application object is created/upgraded", + default=None, + ) diff --git a/src/snowflake/cli/plugins/nativeapp/manager.py b/src/snowflake/cli/plugins/nativeapp/manager.py index 51f355fc81..fde4ce8ba0 100644 --- a/src/snowflake/cli/plugins/nativeapp/manager.py +++ b/src/snowflake/cli/plugins/nativeapp/manager.py @@ -31,6 +31,9 @@ default_application, default_role, ) +from snowflake.cli.api.project.schemas.native_app.application import ( + ApplicationPostDeployHook, +) from snowflake.cli.api.project.schemas.native_app.native_app import NativeApp from snowflake.cli.api.project.schemas.native_app.path_mapping import PathMapping from snowflake.cli.api.project.util import ( @@ -265,6 +268,16 @@ def app_role(self) -> str: else: return self._default_role + @cached_property + def app_post_deploy_hooks(self) -> Optional[List[ApplicationPostDeployHook]]: + """ + List of application post deploy hooks. + """ + if self.definition.application and self.definition.application.post_deploy: + return self.definition.application.post_deploy + else: + return None + @cached_property def _default_role(self) -> str: role = default_role() diff --git a/src/snowflake/cli/plugins/nativeapp/run_processor.py b/src/snowflake/cli/plugins/nativeapp/run_processor.py index 782ccf156e..0910dc40c2 100644 --- a/src/snowflake/cli/plugins/nativeapp/run_processor.py +++ b/src/snowflake/cli/plugins/nativeapp/run_processor.py @@ -28,6 +28,7 @@ unquote_identifier, ) from snowflake.cli.api.utils.cursor import find_all_rows +from snowflake.cli.api.utils.rendering import snowflake_sql_jinja_render from snowflake.cli.plugins.nativeapp.artifacts import BundleMap from snowflake.cli.plugins.nativeapp.constants import ( ALLOWED_SPECIAL_COMMENTS, @@ -104,6 +105,7 @@ def _create_dev_app(self, diff: DiffResult) -> None: f"alter application {self.app_name} set debug_mode = {self.debug_mode}" ) + self._execute_post_deploy_hooks() return except ProgrammingError as err: @@ -141,6 +143,38 @@ def _create_dev_app(self, diff: DiffResult) -> None: except ProgrammingError as err: generic_sql_error_handler(err) + self._execute_post_deploy_hooks() + + def _execute_sql_script(self, sql_script_path): + """ + Executing the SQL script in the provided file path after expanding template variables. + "use warehouse" and "use database" will be executed first if they are set in definition file or in the current connection. + """ + with open(sql_script_path) as f: + sql_script = f.read() + try: + if self.application_warehouse: + self._execute_query(f"use warehouse {self.application_warehouse}") + if self._conn.database: + self._execute_query(f"use database {self._conn.database}") + sql_script = snowflake_sql_jinja_render(content=sql_script) + self._execute_queries(sql_script) + except ProgrammingError as err: + generic_sql_error_handler(err) + + def _execute_post_deploy_hooks(self): + post_deploy_script_hooks = self.app_post_deploy_hooks + if post_deploy_script_hooks: + with cc.phase("Executing application post-deploy actions"): + for hook in post_deploy_script_hooks: + if hook.sql_script: + cc.step(f"Executing SQL script: {hook.sql_script}") + self._execute_sql_script(hook.sql_script) + else: + raise ValueError( + f"Unsupported application post-deploy hook type: {hook}" + ) + def get_all_existing_versions(self) -> SnowflakeCursor: """ Get all existing versions, if defined, for an application package. diff --git a/tests/nativeapp/test_post_deploy.py b/tests/nativeapp/test_post_deploy.py new file mode 100644 index 0000000000..3f9368e025 --- /dev/null +++ b/tests/nativeapp/test_post_deploy.py @@ -0,0 +1,180 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from textwrap import dedent +from unittest import mock + +import pytest +from snowflake.cli.api.project.definition_manager import DefinitionManager +from snowflake.cli.api.project.errors import SchemaValidationError +from snowflake.cli.api.project.schemas.native_app.application import ( + ApplicationPostDeployHook, +) +from snowflake.cli.plugins.nativeapp.run_processor import NativeAppRunProcessor + +from tests.nativeapp.patch_utils import mock_connection +from tests.nativeapp.utils import ( + NATIVEAPP_MANAGER_EXECUTE, + NATIVEAPP_MANAGER_EXECUTE_QUERIES, + RUN_PROCESSOR_APP_POST_DEPLOY_HOOKS, +) +from tests.testing_utils.fixtures import MockConnectionCtx + +CLI_GLOBAL_TEMPLATE_CONTEXT = ( + "snowflake.cli.api.cli_global_context._CliGlobalContextAccess.template_context" +) +MOCK_CONNECTION_DB = "tests.testing_utils.fixtures.MockConnectionCtx.database" +MOCK_CONNECTION_WH = "tests.testing_utils.fixtures.MockConnectionCtx.warehouse" + + +def _get_run_processor(working_dir): + dm = DefinitionManager(working_dir) + return NativeAppRunProcessor( + project_definition=dm.project_definition.native_app, + project_root=dm.project_root, + ) + + +@mock.patch(NATIVEAPP_MANAGER_EXECUTE) +@mock.patch(NATIVEAPP_MANAGER_EXECUTE_QUERIES) +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, new_callable=mock.PropertyMock) +@mock_connection() +def test_sql_scripts( + mock_conn, + mock_cli_ctx, + mock_execute_queries, + mock_execute_query, + project_directory, +): + mock_conn.return_value = MockConnectionCtx() + mock_cli_ctx.return_value = { + "ctx": {"native_app": {"name": "myapp"}, "env": {"foo": "bar"}} + } + with project_directory("napp_post_deploy") as project_dir: + processor = _get_run_processor(str(project_dir)) + + processor._execute_post_deploy_hooks() # noqa SLF001 + + assert mock_execute_query.mock_calls == [ + mock.call("use warehouse MockWarehouse"), + mock.call("use database MockDatabase"), + mock.call("use warehouse MockWarehouse"), + mock.call("use database MockDatabase"), + ] + assert mock_execute_queries.mock_calls == [ + # Verify template variables were expanded correctly + mock.call( + dedent( + """\ + -- app post-deploy script (1/2) + + select myapp; + select bar; + """ + ) + ), + mock.call("-- app post-deploy script (2/2)\n"), + ] + + +@mock.patch(NATIVEAPP_MANAGER_EXECUTE) +@mock.patch(NATIVEAPP_MANAGER_EXECUTE_QUERIES) +@mock.patch(CLI_GLOBAL_TEMPLATE_CONTEXT, new_callable=mock.PropertyMock) +@mock_connection() +@mock.patch(MOCK_CONNECTION_DB, new_callable=mock.PropertyMock) +@mock.patch(MOCK_CONNECTION_WH, new_callable=mock.PropertyMock) +def test_sql_scripts_with_no_warehouse_no_database( + mock_conn_wh, + mock_conn_db, + mock_conn, + mock_cli_ctx, + mock_execute_queries, + mock_execute_query, + project_directory, +): + mock_conn_wh.return_value = None + mock_conn_db.return_value = None + mock_conn.return_value = MockConnectionCtx(None) + mock_cli_ctx.return_value = { + "ctx": {"native_app": {"name": "myapp"}, "env": {"foo": "bar"}} + } + with project_directory("napp_post_deploy") as project_dir: + processor = _get_run_processor(str(project_dir)) + + processor._execute_post_deploy_hooks() # noqa SLF001 + + # Verify no "use warehouse" and no "use database" were called + assert mock_execute_query.mock_calls == [] + assert mock_execute_queries.mock_calls == [ + mock.call( + dedent( + """\ + -- app post-deploy script (1/2) + + select myapp; + select bar; + """ + ) + ), + mock.call("-- app post-deploy script (2/2)\n"), + ] + + +@mock_connection() +def test_missing_sql_script( + mock_conn, + project_directory, +): + mock_conn.return_value = MockConnectionCtx() + with project_directory("napp_post_deploy_missing_file") as project_dir: + processor = _get_run_processor(str(project_dir)) + + with pytest.raises(FileNotFoundError) as err: + processor._execute_post_deploy_hooks() # noqa SLF001 + + +@mock.patch(RUN_PROCESSOR_APP_POST_DEPLOY_HOOKS, new_callable=mock.PropertyMock) +@mock_connection() +def test_invalid_hook_type( + mock_conn, + mock_deploy_hooks, + project_directory, +): + mock_hook = mock.Mock() + mock_hook.invalid_type = "invalid_type" + mock_hook.sql_script = None + mock_deploy_hooks.return_value = [mock_hook] + mock_conn.return_value = MockConnectionCtx() + with project_directory("napp_post_deploy") as project_dir: + processor = _get_run_processor(str(project_dir)) + + with pytest.raises(ValueError) as err: + processor._execute_post_deploy_hooks() # noqa SLF001 + assert "Unsupported application post-deploy hook type" in str(err) + + +@pytest.mark.parametrize( + "args,expected_error", + [ + ({"sql_script": "/path"}, None), + ({}, "missing following fields: ('sql_script',)"), + ], +) +def test_post_deploy_hook_schema(args, expected_error): + if expected_error: + with pytest.raises(SchemaValidationError) as err: + ApplicationPostDeployHook(**args) + assert expected_error in str(err) + else: + ApplicationPostDeployHook(**args) diff --git a/tests/nativeapp/utils.py b/tests/nativeapp/utils.py index 6c60ada4b2..e6ef1e87c1 100644 --- a/tests/nativeapp/utils.py +++ b/tests/nativeapp/utils.py @@ -56,6 +56,7 @@ TEARDOWN_PROCESSOR_DROP_GENERIC_OBJECT = f"{TEARDOWN_PROCESSOR}.drop_generic_object" RUN_PROCESSOR_GET_EXISTING_APP_INFO = f"{RUN_PROCESSOR}.get_existing_app_info" +RUN_PROCESSOR_APP_POST_DEPLOY_HOOKS = f"{RUN_PROCESSOR}.app_post_deploy_hooks" FIND_VERSION_FROM_MANIFEST = f"{VERSION_MODULE}.find_version_info_in_manifest_file" diff --git a/tests/project/__snapshots__/test_config.ambr b/tests/project/__snapshots__/test_config.ambr index fe9945707a..b7ecb98f78 100644 --- a/tests/project/__snapshots__/test_config.ambr +++ b/tests/project/__snapshots__/test_config.ambr @@ -106,6 +106,7 @@ 'application': dict({ 'debug': True, 'name': 'myapp_polly', + 'post_deploy': None, 'role': 'myapp_consumer', 'warehouse': None, }), @@ -157,6 +158,7 @@ 'application': dict({ 'debug': True, 'name': 'myapp_polly', + 'post_deploy': None, 'role': 'myapp_consumer', 'warehouse': None, }), diff --git a/tests/test_data/projects/napp_post_deploy/scripts/post_deploy1.sql b/tests/test_data/projects/napp_post_deploy/scripts/post_deploy1.sql new file mode 100644 index 0000000000..c54445c86b --- /dev/null +++ b/tests/test_data/projects/napp_post_deploy/scripts/post_deploy1.sql @@ -0,0 +1,4 @@ +-- app post-deploy script (1/2) + +select &{ ctx.native_app.name }; +select &{ ctx.env.foo }; diff --git a/tests/test_data/projects/napp_post_deploy/scripts/post_deploy2.sql b/tests/test_data/projects/napp_post_deploy/scripts/post_deploy2.sql new file mode 100644 index 0000000000..107b67e216 --- /dev/null +++ b/tests/test_data/projects/napp_post_deploy/scripts/post_deploy2.sql @@ -0,0 +1 @@ +-- app post-deploy script (2/2) diff --git a/tests/test_data/projects/napp_post_deploy/snowflake.yml b/tests/test_data/projects/napp_post_deploy/snowflake.yml new file mode 100644 index 0000000000..fe734b230c --- /dev/null +++ b/tests/test_data/projects/napp_post_deploy/snowflake.yml @@ -0,0 +1,15 @@ +definition_version: 1.1 +native_app: + name: myapp + + artifacts: + - src: app/* + dest: ./ + + application: + post_deploy: + - sql_script: scripts/post_deploy1.sql + - sql_script: scripts/post_deploy2.sql + +env: + foo: bar diff --git a/tests/test_data/projects/napp_post_deploy_missing_file/snowflake.yml b/tests/test_data/projects/napp_post_deploy_missing_file/snowflake.yml new file mode 100644 index 0000000000..4c6c069d4c --- /dev/null +++ b/tests/test_data/projects/napp_post_deploy_missing_file/snowflake.yml @@ -0,0 +1,11 @@ +definition_version: 1 +native_app: + name: myapp + + artifacts: + - src: app/* + dest: ./ + + application: + post_deploy: + - sql_script: scripts/missing.sql diff --git a/tests_integration/nativeapp/test_init_run.py b/tests_integration/nativeapp/test_init_run.py index db2ef2fc37..081ba0e9d7 100644 --- a/tests_integration/nativeapp/test_init_run.py +++ b/tests_integration/nativeapp/test_init_run.py @@ -25,6 +25,7 @@ contains_row_with, not_contains_row_with, row_from_snowflake_session, + rows_from_snowflake_session, ) USER_NAME = f"user_{uuid.uuid4().hex}" @@ -418,3 +419,54 @@ def test_nativeapp_init_from_repo_with_single_template( assert result.exit_code == 0 finally: single_template_repo.close() + + +# Tests that application post-deploy scripts are executed by creating a post_deploy_log table and having each post-deploy script add a record to it +@pytest.mark.integration +def test_nativeapp_app_post_deploy(runner, snowflake_session, project_directory): + project_name = "myapp" + app_name = f"{project_name}_{USER_NAME}" + with project_directory("napp_application_post_deploy") as tmp_dir: + try: + # First run, application is created + result = runner.invoke_with_connection_json( + ["app", "run"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + # Verify both scripts were executed + assert row_from_snowflake_session( + snowflake_session.execute_string( + f"select * from {app_name}.public.post_deploy_log", + ) + ) == [ + {"TEXT": "post-deploy-part-1"}, + {"TEXT": "post-deploy-part-2"}, + ] + + # Second run, application is upgraded + result = runner.invoke_with_connection_json( + ["app", "run"], + env=TEST_ENV, + ) + assert result.exit_code == 0 + + # Verify both scripts were executed + assert row_from_snowflake_session( + snowflake_session.execute_string( + f"select * from {app_name}.public.post_deploy_log", + ) + ) == [ + {"TEXT": "post-deploy-part-1"}, + {"TEXT": "post-deploy-part-2"}, + {"TEXT": "post-deploy-part-1"}, + {"TEXT": "post-deploy-part-2"}, + ] + + finally: + result = runner.invoke_with_connection_json( + ["app", "teardown", "--force"], + env=TEST_ENV, + ) + assert result.exit_code == 0 diff --git a/tests_integration/test_data/projects/napp_application_post_deploy/app/README.md b/tests_integration/test_data/projects/napp_application_post_deploy/app/README.md new file mode 100644 index 0000000000..f66bf75c9b --- /dev/null +++ b/tests_integration/test_data/projects/napp_application_post_deploy/app/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains an extremely simple application that is used for +integration testing SnowCLI. diff --git a/tests_integration/test_data/projects/napp_application_post_deploy/app/manifest.yml b/tests_integration/test_data/projects/napp_application_post_deploy/app/manifest.yml new file mode 100644 index 0000000000..5b8ef74e8a --- /dev/null +++ b/tests_integration/test_data/projects/napp_application_post_deploy/app/manifest.yml @@ -0,0 +1,9 @@ +# This is a manifest.yml file, a required component of creating a Snowflake Native App. +# This file defines properties required by the application package, including the location of the setup script and version definitions. +# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file. + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md diff --git a/tests_integration/test_data/projects/napp_application_post_deploy/app/setup_script.sql b/tests_integration/test_data/projects/napp_application_post_deploy/app/setup_script.sql new file mode 100644 index 0000000000..7fc3682b6e --- /dev/null +++ b/tests_integration/test_data/projects/napp_application_post_deploy/app/setup_script.sql @@ -0,0 +1,11 @@ +-- This is the setup script that runs while installing a Snowflake Native App in a consumer account. +-- To write this script, you can familiarize yourself with some of the following concepts: +-- Application Roles +-- Versioned Schemas +-- UDFs/Procs +-- Extension Code +-- Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script for a detailed understanding of this file. + +CREATE OR ALTER VERSIONED SCHEMA core; + +-- The rest of this script is left blank for purposes of your learning and exploration. diff --git a/tests_integration/test_data/projects/napp_application_post_deploy/scripts/post_deploy1.sql b/tests_integration/test_data/projects/napp_application_post_deploy/scripts/post_deploy1.sql new file mode 100644 index 0000000000..697dbc16e8 --- /dev/null +++ b/tests_integration/test_data/projects/napp_application_post_deploy/scripts/post_deploy1.sql @@ -0,0 +1,5 @@ +-- app post-deploy script (1/2) + +CREATE SCHEMA IF NOT EXISTS &{ ctx.env.schema }; +CREATE TABLE IF NOT EXISTS &{ ctx.env.schema }.post_deploy_log (text VARCHAR); +INSERT INTO &{ ctx.env.schema }.post_deploy_log VALUES('post-deploy-part-1'); diff --git a/tests_integration/test_data/projects/napp_application_post_deploy/scripts/post_deploy2.sql b/tests_integration/test_data/projects/napp_application_post_deploy/scripts/post_deploy2.sql new file mode 100644 index 0000000000..dfccefe9ee --- /dev/null +++ b/tests_integration/test_data/projects/napp_application_post_deploy/scripts/post_deploy2.sql @@ -0,0 +1,3 @@ +-- app post-deploy script (2/2) + +INSERT INTO &{ ctx.env.schema }.post_deploy_log VALUES('post-deploy-part-2'); diff --git a/tests_integration/test_data/projects/napp_application_post_deploy/snowflake.yml b/tests_integration/test_data/projects/napp_application_post_deploy/snowflake.yml new file mode 100644 index 0000000000..18595ca446 --- /dev/null +++ b/tests_integration/test_data/projects/napp_application_post_deploy/snowflake.yml @@ -0,0 +1,15 @@ +definition_version: 1.1 +native_app: + name: myapp + + artifacts: + - src: app/* + dest: ./ + + application: + post_deploy: + - sql_script: scripts/post_deploy1.sql + - sql_script: scripts/post_deploy2.sql + +env: + schema: public