From 3b59d1cb8f6b03ba5358d65ca21e7f2fc8623d44 Mon Sep 17 00:00:00 2001 From: Tomasz Urbaszek Date: Tue, 10 Oct 2023 13:42:36 +0200 Subject: [PATCH] Adjust streamlit to PuPr syntax (#448) Closes: #442 --- RELEASE-NOTES.md | 4 + src/snowcli/cli/common/sql_execution.py | 19 ++ src/snowcli/cli/stage/manager.py | 2 +- src/snowcli/cli/streamlit/commands.py | 152 ++++----- src/snowcli/cli/streamlit/manager.py | 107 +++---- src/snowcli/utils.py | 5 +- .../default_streamlit/environment.yml | 5 + src/templates/default_streamlit/main.py | 3 + .../default_streamlit/pages/my_page.py | 3 + tests/streamlit/test_commands.py | 303 +++++++++--------- .../example_streamlit/environment.yml | 5 + .../projects/example_streamlit/main.py | 3 + .../example_streamlit/pages/my_page.py | 3 + tests/testing_utils/fixtures.py | 14 + tests_integration/test_streamlit.py | 76 ----- 15 files changed, 341 insertions(+), 363 deletions(-) create mode 100644 src/templates/default_streamlit/environment.yml create mode 100644 src/templates/default_streamlit/main.py create mode 100644 src/templates/default_streamlit/pages/my_page.py create mode 100644 tests/test_data/projects/example_streamlit/environment.yml create mode 100644 tests/test_data/projects/example_streamlit/main.py create mode 100644 tests/test_data/projects/example_streamlit/pages/my_page.py diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index d7502b653b..8ad2fd59bb 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -13,11 +13,15 @@ * Update function or procedure will upload function/procedure code to new path on stage. Previous code will remain under old path on stage. * Snowpark command `compute-pool` and its alias `cp` were replaced by `pool` command. * `snow snowpark registry` was replaced with `snow registry` command. +* Removed `snow streamlit create` command. Streamlit can be deployd using `snow streamlit deploy` ## New additions * `--temporary-connection` flag, that allows you to connect, without anything declared in config file +* `snow streamlit init` command that creates a new streamlit project. +* `snow streamlit deploy` support pages and environment.yml files. ## Fixes and improvements +* Adjust streamlit commands to PuPr syntax * Too long texts in table cells are now wrapped instead of cropped * Split global options into separate section in `help` * Avoiding unnecessary replace in function/procedure update diff --git a/src/snowcli/cli/common/sql_execution.py b/src/snowcli/cli/common/sql_execution.py index bf903e67f8..4544ca0644 100644 --- a/src/snowcli/cli/common/sql_execution.py +++ b/src/snowcli/cli/common/sql_execution.py @@ -3,6 +3,7 @@ from textwrap import dedent +from click import ClickException from snowflake.connector.errors import ProgrammingError from snowcli.cli.common.snow_cli_global_context import snow_cli_global_context_manager from snowflake.connector.cursor import DictCursor @@ -89,3 +90,21 @@ def check_schema_exists(self, database: str, schema: str) -> None: Use `snow connection list` to list existing connections """ ) from e + + def to_fully_qualified_name(self, name: str): + current_parts = name.split(".") + if len(current_parts) == 3: + # already fully qualified name + return name.upper() + + if not self._conn.database: + raise ClickException( + "Default database not specified in connection details." + ) + + if len(current_parts) == 2: + # we assume this is schema.object + return f"{self._conn.database}.{name}".upper() + + schema = self._conn.schema or "public" + return f"{self._conn.database}.{schema}.{name}".upper() diff --git a/src/snowcli/cli/stage/manager.py b/src/snowcli/cli/stage/manager.py index 2c2c7ae292..f20409afb1 100644 --- a/src/snowcli/cli/stage/manager.py +++ b/src/snowcli/cli/stage/manager.py @@ -14,7 +14,7 @@ class StageManager(SqlExecutionMixin): @staticmethod def get_standard_stage_name(name: str) -> str: # Handle embedded stages - if name.startswith("snow://"): + if name.startswith("snow://") or name.startswith("@"): return name return f"@{name}" diff --git a/src/snowcli/cli/streamlit/commands.py b/src/snowcli/cli/streamlit/commands.py index d4effa13eb..9c5cea395b 100644 --- a/src/snowcli/cli/streamlit/commands.py +++ b/src/snowcli/cli/streamlit/commands.py @@ -1,25 +1,24 @@ import logging + import typer from pathlib import Path from typing import Optional -from snowcli.cli.common.decorators import global_options_with_connection +from click import ClickException + +from snowcli.cli.common.decorators import global_options_with_connection, global_options from snowcli.cli.common.flags import DEFAULT_CONTEXT_SETTINGS from snowcli.cli.streamlit.manager import StreamlitManager from snowcli.output.decorators import with_output -from snowcli.cli.snowpark_shared import ( - CheckAnacondaForPyPiDependencies, - PackageNativeLibrariesOption, - PyPiDownloadOption, -) + from snowcli.output.types import ( CommandResult, QueryResult, - CollectionResult, SingleQueryResult, MessageResult, MultipleResults, ) +from snowcli.utils import create_project_template app = typer.Typer( context_settings=DEFAULT_CONTEXT_SETTINGS, @@ -29,6 +28,29 @@ log = logging.getLogger(__name__) +StageNameOption: str = typer.Option( + "streamlit", + "--stage", + help="Stage name where Streamlit files will be uploaded.", +) + + +@app.command("init") +@with_output +@global_options +def streamlit_init( + project_name: str = typer.Argument( + "example_streamlit", help="Name of the Streamlit project you want to create." + ), + **options, +) -> CommandResult: + """ + Initializes this directory with a sample set of files for creating a Streamlit dashboard. + """ + create_project_template("default_streamlit", project_directory=project_name) + return MessageResult(f"Initialized the new project in {project_name}/") + + @app.command("list") @with_output @global_options_with_connection @@ -59,40 +81,6 @@ def streamlit_describe( return result -@app.command("create") -@with_output -@global_options_with_connection -def streamlit_create( - name: str = typer.Argument(..., help="Name of streamlit to create."), - file: Path = typer.Option( - "streamlit_app.py", - exists=True, - readable=True, - file_okay=True, - help="Path to the Streamlit Python application (`streamlit_app.py`) file.", - ), - from_stage: Optional[str] = typer.Option( - None, - help="Stage name from which to copy a Streamlit file.", - ), - use_packaging_workaround: bool = typer.Option( - False, - help="Whether to package all code and dependencies into a zip file. Valid values: `true`, `false` (default). You should use this only for a temporary workaround until native support is available.", - ), - **options, -) -> CommandResult: - """ - Creates a new Streamlit application object in Snowflake. The streamlit is created in database and schema configured in the connection. - """ - cursor = StreamlitManager().create( - streamlit_name=name, - file=file, - from_stage=from_stage, - use_packaging_workaround=use_packaging_workaround, - ) - return SingleQueryResult(cursor) - - @app.command("share") @with_output @global_options_with_connection @@ -128,7 +116,7 @@ def streamlit_drop( @with_output @global_options_with_connection def streamlit_deploy( - name: str = typer.Argument(..., help="Name of streamlit to deploy."), + streamlit_name: str = typer.Argument(..., help="Name of Streamlit to deploy."), file: Path = typer.Option( "streamlit_app.py", exists=True, @@ -136,43 +124,63 @@ def streamlit_deploy( file_okay=True, help="Path of the Streamlit app file.", ), - open_: bool = typer.Option( - False, - "--open", - "-o", - help="Whether to open Streamlit in a browser. Valid values: `true`, `false`. Default: `false`.", + stage: Optional[str] = StageNameOption, + environment_file: Path = typer.Option( + None, + "--env-file", + help="Environment file to use.", + file_okay=True, + dir_okay=False, ), - use_packaging_workaround: bool = typer.Option( - False, - help="Whether to package all code and dependencies into a zip file. Valid values: `true`, `false` (default). You should use this only for a temporary workaround until native support is available.", + pages_dir: Path = typer.Option( + None, + "--pages-dir", + help="Directory with Streamlit pages", + file_okay=False, + dir_okay=True, + ), + query_warehouse: Optional[str] = typer.Option( + None, "--query-warehouse", help="Query warehouse for this Streamlit." ), - packaging_workaround_includes_content: bool = typer.Option( + replace: Optional[bool] = typer.Option( False, - help="Whether to package all code and dependencies into a zip file. Valid values: `true`, `false`. Default: `false`.", + "--replace", + help="Replace the Streamlit if it already exists.", + is_flag=True, ), - pypi_download: str = PyPiDownloadOption, - check_anaconda_for_pypi_deps: bool = CheckAnacondaForPyPiDependencies, - package_native_libraries: str = PackageNativeLibrariesOption, - excluded_anaconda_deps: str = typer.Option( - None, - help="List of comma-separated package names from `environment.yml` to exclude in the deployed app, particularly when Streamlit fails to import an Anaconda package at runtime. Be aware that excluding files might the risk of runtime errors).", + open_: bool = typer.Option( + False, "--open", help="Whether to open Streamlit in a browser.", is_flag=True ), **options, ) -> CommandResult: """ - Creates a Streamlit app package for deployment. + Uploads local files to specified stage and creates a Streamlit dashboard using the files. You must specify the + main python file. By default, the command will upload environment.yml and pages/ folder if present. If you + don't provide any stage name then 'streamlit' stage will be used. If provided stage will be created if it does + not exist. + You can modify the behaviour using flags. For details check help information. """ - result = StreamlitManager().deploy( - streamlit_name=name, - file=file, - open_in_browser=open_, - use_packaging_workaround=use_packaging_workaround, - packaging_workaround_includes_content=packaging_workaround_includes_content, - pypi_download=pypi_download, - check_anaconda_for_pypi_deps=check_anaconda_for_pypi_deps, - package_native_libraries=package_native_libraries, - excluded_anaconda_deps=excluded_anaconda_deps, + if environment_file and not environment_file.exists(): + raise ClickException(f"Provided file {environment_file} does not exist") + else: + environment_file = Path("environment.yml") + + if pages_dir and not pages_dir.exists(): + raise ClickException(f"Provided file {pages_dir} does not exist") + else: + pages_dir = Path("pages") + + url = StreamlitManager().deploy( + streamlit_name=streamlit_name, + environment_file=environment_file, + pages_dir=pages_dir, + stage_name=stage, + main_file=file, + replace=replace, + warehouse=query_warehouse, ) - if result is not None: - return MessageResult(result) - return MessageResult("Done") + + if open_: + typer.launch(url) + + return MessageResult(f"Streamlit successfully deployed and available under {url}") diff --git a/src/snowcli/cli/streamlit/manager.py b/src/snowcli/cli/streamlit/manager.py index 9a99dec323..54b9077a99 100644 --- a/src/snowcli/cli/streamlit/manager.py +++ b/src/snowcli/cli/streamlit/manager.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import List, Optional, Tuple +from click import ClickException from snowflake.connector.cursor import SnowflakeCursor from snowcli.cli.common.sql_execution import SqlExecutionMixin @@ -29,34 +30,6 @@ def describe(self, streamlit_name: str) -> Tuple[SnowflakeCursor, SnowflakeCurso ) return description, url - def create( - self, - streamlit_name: str, - file: Path, - from_stage: str, - use_packaging_workaround: bool, - ) -> SnowflakeCursor: - connection = self._conn - if from_stage: - standard_page_name = StageManager.get_standard_stage_name(from_stage) - from_stage_command = f"FROM {standard_page_name}" - else: - from_stage_command = "" - main_file = ( - "streamlit_app_launcher.py" if use_packaging_workaround else file.name - ) - - return self._execute_query( - f""" - create streamlit {streamlit_name} - {from_stage_command} - MAIN_FILE = '{main_file}' - QUERY_WAREHOUSE = {connection.warehouse}; - - alter streamlit {streamlit_name} checkout; - """ - ) - def share(self, streamlit_name: str, to_role: str) -> SnowflakeCursor: return self._execute_query( f"grant usage on streamlit {streamlit_name} to role {to_role}" @@ -65,50 +38,57 @@ def share(self, streamlit_name: str, to_role: str) -> SnowflakeCursor: def drop(self, streamlit_name: str) -> SnowflakeCursor: return self._execute_query(f"drop streamlit {streamlit_name}") + def get_url_from_name(self, streamlit_name: str): + return self._execute_query( + f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{streamlit_name}')" + ).fetchone()[0] + def deploy( self, streamlit_name: str, - file: Path, - open_in_browser: bool, - use_packaging_workaround: bool, - packaging_workaround_includes_content: bool, - pypi_download: str, - check_anaconda_for_pypi_deps: bool, - package_native_libraries: str, - excluded_anaconda_deps: str, + main_file: Path, + environment_file: Optional[Path] = None, + pages_dir: Optional[Path] = None, + stage_name: Optional[str] = None, + warehouse: Optional[str] = None, + replace: Optional[bool] = False, ): stage_manager = StageManager() - # THIS WORKAROUND HAS NOT BEEN TESTED WITH THE NEW STREAMLIT SYNTAX - if use_packaging_workaround: - self._packaging_workaround( - streamlit_name, - file, - packaging_workaround_includes_content, - pypi_download, - check_anaconda_for_pypi_deps, - package_native_libraries, - excluded_anaconda_deps, - stage_manager, - ) + stage_name = stage_name or "streamlit" + stage_name = stage_manager.to_fully_qualified_name(stage_name) - qualified_name = self.qualified_name(streamlit_name) - streamlit_stage_name = f"snow://streamlit/{qualified_name}/default_checkout" - stage_manager.put(str(file), streamlit_stage_name, 4, True) - query_result = self._execute_query( - f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{streamlit_name}')" + stage_manager.create(stage_name=stage_name) + + root_location = stage_manager.get_standard_stage_name( + f"{stage_name}/{streamlit_name}" ) - base_url = query_result.fetchone()[0] - url = self._get_url(base_url, qualified_name) - if open_in_browser: - typer.launch(url) - else: - return url + stage_manager.put(main_file, root_location, 4, True) + + if environment_file and environment_file.exists(): + stage_manager.put(environment_file, root_location, 4, True) + + if pages_dir and pages_dir.exists(): + stage_manager.put(pages_dir / "*", f"{root_location}/pages", 4, True) + + replace_stmt = "OR REPLACE" if replace else "" + use_warehouse_stmt = f"QUERY_WAREHOUSE = {warehouse}" if warehouse else "" + self._execute_query( + f""" + CREATE {replace_stmt} STREAMLIT {streamlit_name} + ROOT_LOCATION = '{root_location}' + MAIN_FILE = '{main_file.name}' + {use_warehouse_stmt} + """ + ) + + return self.get_url(streamlit_name) def _packaging_workaround( self, streamlit_name: str, + stage_name: str, file: Path, packaging_workaround_includes_content: bool, pypi_download: str, @@ -125,11 +105,11 @@ def _packaging_workaround( ) # upload the resulting app.zip file - stage_name = f"{streamlit_name}_stage" + stage_name = stage_name or f"{streamlit_name}_stage" stage_manager.put("app.zip", stage_name, 4, True) main_module = str(file).replace(".py", "") file = generate_streamlit_package_wrapper( - stage_name=f"{streamlit_name}_stage", + stage_name=stage_name, main_module=main_module, extract_zip=packaging_workaround_includes_content, ) @@ -146,7 +126,10 @@ def _packaging_workaround( if env_file: stage_manager.put(str(env_file), stage_name, 4, True) - def _get_url(self, base_url: str, qualified_name: str) -> str: + def get_url(self, streamlit_name: str) -> str: + qualified_name = self.qualified_name(streamlit_name) + base_url = self.get_url_from_name(streamlit_name) + connection = self._conn if not connection.host: diff --git a/src/snowcli/utils.py b/src/snowcli/utils.py index 1e506f75ce..1b85481816 100644 --- a/src/snowcli/utils.py +++ b/src/snowcli/utils.py @@ -596,10 +596,11 @@ class File: relpath: Optional[str] = None -def create_project_template(template_name: str): +def create_project_template(template_name: str, project_directory: str | None = None): + target = project_directory or os.getcwd() shutil.copytree( Path(importlib.util.find_spec("templates").origin).parent / template_name, # type: ignore - f"{os.getcwd()}", + target, dirs_exist_ok=True, ) diff --git a/src/templates/default_streamlit/environment.yml b/src/templates/default_streamlit/environment.yml new file mode 100644 index 0000000000..ac8feac3e8 --- /dev/null +++ b/src/templates/default_streamlit/environment.yml @@ -0,0 +1,5 @@ +name: sf_env +channels: + - snowflake +dependencies: + - pandas diff --git a/src/templates/default_streamlit/main.py b/src/templates/default_streamlit/main.py new file mode 100644 index 0000000000..b699538d7e --- /dev/null +++ b/src/templates/default_streamlit/main.py @@ -0,0 +1,3 @@ +import streamlit as st + +st.title("Example streamlit app") diff --git a/src/templates/default_streamlit/pages/my_page.py b/src/templates/default_streamlit/pages/my_page.py new file mode 100644 index 0000000000..bc3ecbccba --- /dev/null +++ b/src/templates/default_streamlit/pages/my_page.py @@ -0,0 +1,3 @@ +import streamlit as st + +st.title("Example page") diff --git a/tests/streamlit/test_commands.py b/tests/streamlit/test_commands.py index 621176467d..a1e88d45da 100644 --- a/tests/streamlit/test_commands.py +++ b/tests/streamlit/test_commands.py @@ -5,225 +5,228 @@ from unittest import mock from unittest.mock import call +from click import ClickException + from tests.testing_utils.fixtures import * STREAMLIT_NAME = "test_streamlit" @mock.patch("snowflake.connector.connect") -def test_create_streamlit(mock_connector, runner, mock_ctx): +def test_list_streamlit(mock_connector, runner, mock_ctx): + ctx = mock_ctx() + mock_connector.return_value = ctx + + result = runner.invoke(["streamlit", "list"]) + + assert result.exit_code == 0, result.output + assert ctx.get_query() == "show streamlits" + + +@mock.patch("snowflake.connector.connect") +def test_describe_streamlit(mock_connector, runner, mock_ctx): + ctx = mock_ctx() + mock_connector.return_value = ctx + + result = runner.invoke(["streamlit", "describe", STREAMLIT_NAME]) + + assert result.exit_code == 0, result.output + assert ctx.get_queries() == [ + f"describe streamlit {STREAMLIT_NAME}", + f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{STREAMLIT_NAME}')", + ] + + +def _put_query(source: str, dest: str): + return dedent( + f"put file://{source} {dest} auto_compress=false parallel=4 overwrite=True" + ) + + +@mock.patch("snowcli.cli.streamlit.commands.typer") +@mock.patch("snowflake.connector.connect") +def test_deploy_streamlit_single_file( + mock_connector, mock_typer, mock_cursor, runner, mock_ctx +): ctx = mock_ctx() mock_connector.return_value = ctx with NamedTemporaryFile(suffix=".py") as file: - result = runner.invoke_with_config( - ["streamlit", "create", STREAMLIT_NAME, "--file", file.name] + result = runner.invoke( + ["streamlit", "deploy", STREAMLIT_NAME, "--file", file.name, "--open"] ) - assert result.exit_code == 0, result.output - assert ctx.get_query() == dedent( + assert result.exit_code == 0, result.output + assert ctx.get_queries() == [ + "create stage if not exists MOCKDATABASE.MOCKSCHEMA.STREAMLIT", + _put_query(file.name, "@MOCKDATABASE.MOCKSCHEMA.STREAMLIT/test_streamlit"), + dedent( f""" - create streamlit {STREAMLIT_NAME} - - MAIN_FILE = '{os.path.basename(file.name)}' - QUERY_WAREHOUSE = MockWarehouse; - - alter streamlit {STREAMLIT_NAME} checkout; - """ - ) + CREATE STREAMLIT {STREAMLIT_NAME} + ROOT_LOCATION = '@MOCKDATABASE.MOCKSCHEMA.STREAMLIT/{STREAMLIT_NAME}' + MAIN_FILE = '{Path(file.name).name}' + + """ + ), + f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{STREAMLIT_NAME}')", + ] + + mock_typer.launch.assert_called_once_with( + f"https://account.test.region.aws.snowflakecomputing.com/test.region.aws/account/#/streamlit-apps/MOCKDATABASE.MOCKSCHEMA.{STREAMLIT_NAME.upper()}" + ) @mock.patch("snowflake.connector.connect") -def test_create_streamlit_with_use_packaging_workaround( - mock_connector, runner, mock_ctx +def test_deploy_streamlit_all_files_default_stage( + mock_connector, mock_cursor, runner, mock_ctx, project_file ): ctx = mock_ctx() mock_connector.return_value = ctx - with NamedTemporaryFile(suffix=".py") as file: - result = runner.invoke_with_config( - [ - "streamlit", - "create", - STREAMLIT_NAME, - "--file", - file.name, - "--use-packaging-workaround", - ] + with project_file("example_streamlit") as pdir: + result = runner.invoke( + ["streamlit", "deploy", STREAMLIT_NAME, "--file", "main.py"] ) - assert result.exit_code == 0, result.output - assert ctx.get_query() == dedent( + root_path = f"@MOCKDATABASE.MOCKSCHEMA.STREAMLIT/{STREAMLIT_NAME}" + assert result.exit_code == 0, result.output + assert ctx.get_queries() == [ + "create stage if not exists MOCKDATABASE.MOCKSCHEMA.STREAMLIT", + _put_query("main.py", root_path), + _put_query("environment.yml", root_path), + _put_query("pages/*", f"{root_path}/pages"), + dedent( f""" - create streamlit {STREAMLIT_NAME} - - MAIN_FILE = 'streamlit_app_launcher.py' - QUERY_WAREHOUSE = MockWarehouse; - - alter streamlit {STREAMLIT_NAME} checkout; - """ - ) + CREATE STREAMLIT {STREAMLIT_NAME} + ROOT_LOCATION = '@MOCKDATABASE.MOCKSCHEMA.STREAMLIT/{STREAMLIT_NAME}' + MAIN_FILE = 'main.py' + + """ + ), + f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{STREAMLIT_NAME}')", + ] -@pytest.mark.parametrize( - "stage, expected", - [ - ("stage_name", "@stage_name"), - ("snow://stage_dots", "snow://stage_dots"), - ], -) @mock.patch("snowflake.connector.connect") -def test_create_streamlit_with_from_stage( - mock_connector, runner, mock_ctx, stage, expected +def test_deploy_streamlit_all_files_users_stage( + mock_connector, mock_cursor, runner, mock_ctx, project_file ): ctx = mock_ctx() mock_connector.return_value = ctx - with NamedTemporaryFile(suffix=".py") as file: + with project_file("example_streamlit") as pdir: result = runner.invoke( [ "streamlit", - "create", + "deploy", STREAMLIT_NAME, "--file", - file.name, - "--from-stage", - stage, + "main.py", + "--stage", + "MY_FANCY_STAGE", ] ) - assert result.exit_code == 0, result.output - assert ctx.get_query() == dedent( + root_path = f"@MOCKDATABASE.MOCKSCHEMA.MY_FANCY_STAGE/{STREAMLIT_NAME}" + assert result.exit_code == 0, result.output + assert ctx.get_queries() == [ + "create stage if not exists MOCKDATABASE.MOCKSCHEMA.MY_FANCY_STAGE", + _put_query("main.py", root_path), + _put_query("environment.yml", root_path), + _put_query("pages/*", f"{root_path}/pages"), + dedent( f""" - create streamlit {STREAMLIT_NAME} - FROM {expected} - MAIN_FILE = '{os.path.basename(file.name)}' - QUERY_WAREHOUSE = MockWarehouse; - - alter streamlit {STREAMLIT_NAME} checkout; - """ - ) + CREATE STREAMLIT {STREAMLIT_NAME} + ROOT_LOCATION = '@MOCKDATABASE.MOCKSCHEMA.MY_FANCY_STAGE/{STREAMLIT_NAME}' + MAIN_FILE = 'main.py' + + """ + ), + f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{STREAMLIT_NAME}')", + ] @mock.patch("snowflake.connector.connect") -def test_list_streamlit(mock_connector, runner, mock_ctx): +def test_deploy_streamlit_main_and_environment_files( + mock_connector, mock_cursor, runner, mock_ctx, project_file +): ctx = mock_ctx() mock_connector.return_value = ctx - result = runner.invoke(["streamlit", "list"]) - - assert result.exit_code == 0, result.output - assert ctx.get_query() == "show streamlits" - + with project_file("example_streamlit") as pdir: -@mock.patch("snowflake.connector.connect") -def test_describe_streamlit(mock_connector, runner, mock_ctx): - ctx = mock_ctx() - mock_connector.return_value = ctx + (pdir / "pages" / "my_page.py").unlink() + (pdir / "pages").rmdir() - result = runner.invoke(["streamlit", "describe", STREAMLIT_NAME]) + result = runner.invoke( + ["streamlit", "deploy", STREAMLIT_NAME, "--file", "main.py"] + ) + root_path = f"@MOCKDATABASE.MOCKSCHEMA.STREAMLIT/{STREAMLIT_NAME}" assert result.exit_code == 0, result.output assert ctx.get_queries() == [ - f"describe streamlit {STREAMLIT_NAME}", + "create stage if not exists MOCKDATABASE.MOCKSCHEMA.STREAMLIT", + _put_query("main.py", root_path), + _put_query("environment.yml", root_path), + dedent( + f""" + CREATE STREAMLIT {STREAMLIT_NAME} + ROOT_LOCATION = '@MOCKDATABASE.MOCKSCHEMA.STREAMLIT/{STREAMLIT_NAME}' + MAIN_FILE = 'main.py' + + """ + ), f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{STREAMLIT_NAME}')", ] -@mock.patch("snowcli.cli.streamlit.manager.typer") -@mock.patch("snowcli.cli.streamlit.manager.StageManager") @mock.patch("snowflake.connector.connect") -def test_deploy_streamlit( - mock_connector, mock_stage_manager, mock_typer, mock_cursor, runner, mock_ctx +def test_deploy_streamlit_main_and_pages_files( + mock_connector, mock_cursor, runner, mock_ctx, project_file ): - ctx = mock_ctx( - mock_cursor( - rows=["snowflake.com"], columns=["SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME"] - ) - ) + ctx = mock_ctx() mock_connector.return_value = ctx - with NamedTemporaryFile(suffix=".py") as file: + with project_file("example_streamlit") as pdir: + (pdir / "environment.yml").unlink() result = runner.invoke( - ["streamlit", "deploy", STREAMLIT_NAME, "--file", file.name, "--open"] + ["streamlit", "deploy", STREAMLIT_NAME, "--file", "main.py"] ) - assert result.exit_code == 0, result.output - assert ( - ctx.get_query() - == f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{STREAMLIT_NAME}')" - ) - mock_stage_manager().put.assert_called_once_with( - file.name, - f"snow://streamlit/MockDatabase.MockSchema.{STREAMLIT_NAME}/default_checkout", - 4, - True, - ) - mock_typer.launch.assert_called_once_with( - f"https://account.test.region.aws.snowflakecomputing.com/test.region.aws/account/#/streamlit-apps/MOCKDATABASE.MOCKSCHEMA.{STREAMLIT_NAME.upper()}" - ) + root_path = f"@MOCKDATABASE.MOCKSCHEMA.STREAMLIT/{STREAMLIT_NAME}" + assert result.exit_code == 0, result.output + assert ctx.get_queries() == [ + "create stage if not exists MOCKDATABASE.MOCKSCHEMA.STREAMLIT", + _put_query("main.py", root_path), + _put_query("pages/*", f"{root_path}/pages"), + dedent( + f""" + CREATE STREAMLIT {STREAMLIT_NAME} + ROOT_LOCATION = '@MOCKDATABASE.MOCKSCHEMA.STREAMLIT/{STREAMLIT_NAME}' + MAIN_FILE = 'main.py' + """ + ), + f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{STREAMLIT_NAME}')", + ] -@mock.patch("snowcli.cli.streamlit.manager.snowpark_package") -@mock.patch("snowcli.cli.streamlit.manager.StageManager") + +@pytest.mark.parametrize( + "opts", [("--pages-dir", "foo/bar"), ("--env-file", "foo.yml")] +) @mock.patch("snowflake.connector.connect") -def test_deploy_streamlit_with_packaging_workaround( - mock_connector, - mock_stage_manager, - mock_snowpark_package, - mock_cursor, - runner, - mock_ctx, - temp_dir, +def test_deploy_streamlit_nonexisting_file( + mock_connector, mock_cursor, runner, mock_ctx, project_file, opts ): - ctx = mock_ctx( - mock_cursor( - rows=["snowflake.com"], columns=["SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME"] - ) - ) + ctx = mock_ctx() mock_connector.return_value = ctx - with NamedTemporaryFile(suffix=".py") as file: - result = runner.invoke_with_config( - [ - "streamlit", - "deploy", - STREAMLIT_NAME, - "--file", - file.name, - "--use-packaging-workaround", - ] - ) - - assert result.exit_code == 0, result.output - assert ( - ctx.get_query() - == f"call SYSTEM$GENERATE_STREAMLIT_URL_FROM_NAME('{STREAMLIT_NAME}')" + with project_file("example_streamlit") as pdir: + result = runner.invoke( + ["streamlit", "deploy", STREAMLIT_NAME, "--file", "main.py", *opts] ) - mock_snowpark_package.assert_called_once_with("ask", True, "ask") - mock_stage_manager().put.assert_has_calls( - [ - call( - "app.zip", - f"{STREAMLIT_NAME}_stage", - 4, - True, - ), - call( - "streamlit_app_launcher.py", - f"{STREAMLIT_NAME}_stage", - 4, - True, - ), - call( - file.name, - f"snow://streamlit/MockDatabase.MockSchema.{STREAMLIT_NAME}/default_checkout", - 4, - True, - ), - ] - ) + assert f"Provided file {opts[1]} does not exist" in result.output @mock.patch("snowflake.connector.connect") diff --git a/tests/test_data/projects/example_streamlit/environment.yml b/tests/test_data/projects/example_streamlit/environment.yml new file mode 100644 index 0000000000..ac8feac3e8 --- /dev/null +++ b/tests/test_data/projects/example_streamlit/environment.yml @@ -0,0 +1,5 @@ +name: sf_env +channels: + - snowflake +dependencies: + - pandas diff --git a/tests/test_data/projects/example_streamlit/main.py b/tests/test_data/projects/example_streamlit/main.py new file mode 100644 index 0000000000..b699538d7e --- /dev/null +++ b/tests/test_data/projects/example_streamlit/main.py @@ -0,0 +1,3 @@ +import streamlit as st + +st.title("Example streamlit app") diff --git a/tests/test_data/projects/example_streamlit/pages/my_page.py b/tests/test_data/projects/example_streamlit/pages/my_page.py new file mode 100644 index 0000000000..bc3ecbccba --- /dev/null +++ b/tests/test_data/projects/example_streamlit/pages/my_page.py @@ -0,0 +1,3 @@ +import streamlit as st + +st.title("Example page") diff --git a/tests/testing_utils/fixtures.py b/tests/testing_utils/fixtures.py index d409b853d0..6988da2717 100644 --- a/tests/testing_utils/fixtures.py +++ b/tests/testing_utils/fixtures.py @@ -1,4 +1,7 @@ import os +import shutil +from contextlib import contextmanager + import pytest import tempfile @@ -200,3 +203,14 @@ def test_root_path(): def txt_file_in_a_subdir(temp_dir: str) -> Generator: subdir = tempfile.TemporaryDirectory(dir=temp_dir) yield create_temp_file(".txt", subdir.name, []) + + +@pytest.fixture +def project_file(temp_dir, test_root_path): + @contextmanager + def _temporary_project_file(project_name): + test_data_file = test_root_path / "test_data" / "projects" / project_name + shutil.copytree(test_data_file, temp_dir, dirs_exist_ok=True) + yield Path(temp_dir) + + return _temporary_project_file diff --git a/tests_integration/test_streamlit.py b/tests_integration/test_streamlit.py index 013b13b83f..546c91bca5 100644 --- a/tests_integration/test_streamlit.py +++ b/tests_integration/test_streamlit.py @@ -20,11 +20,6 @@ def test_streamlit_create_and_deploy( streamlit_name = "test_streamlit_create_and_deploy_snowcli" streamlit_app_path = test_root_path / "test_files/streamlit.py" - result = runner.invoke_integration( - ["streamlit", "create", streamlit_name, "--file", streamlit_app_path] - ) - assert result.exit_code == 0 - result = runner.invoke_integration( ["streamlit", "deploy", streamlit_name, "--file", streamlit_app_path] ) @@ -69,77 +64,6 @@ def test_streamlit_create_and_deploy( assert row_from_snowflake_session(expect) == [] -@pytest.mark.integration -def test_streamlit_create_from_stage( - runner, snowflake_session, _new_streamlit_role, test_root_path -): - stage_name = "test_streamlit_create_from_stage" - streamlit_name = "test_streamlit_create_from_stage_snowcli" - streamlit_filename = "streamlit.py" - streamlit_app_path = test_root_path / f"test_files/{streamlit_filename}" - - expect = snowflake_session.execute_string( - f"create stage {stage_name}; put file://{streamlit_app_path} @{stage_name} auto_compress=false overwrite=true;" - ) - assert contains_row_with( - rows_from_snowflake_session(expect)[1], - { - "source": streamlit_filename, - "target": streamlit_filename, - "status": "UPLOADED", - }, - ) - - result = runner.invoke_integration( - [ - "streamlit", - "create", - streamlit_name, - "--file", - streamlit_app_path, - "--from-stage", - stage_name, - ] - ) - assert result.exit_code == 0 - - result = runner.invoke_integration(["streamlit", "list"]) - expect = snowflake_session.execute_string( - f"show streamlits like '{streamlit_name}'" - ) - assert contains_row_with(result.json, row_from_snowflake_session(expect)[0]) - - result = runner.invoke_integration(["streamlit", "describe", streamlit_name]) - expect = snowflake_session.execute_string(f"describe streamlit {streamlit_name}") - assert contains_row_with(result.json[0], row_from_snowflake_session(expect)[0]) - expect = snowflake_session.execute_string( - f"call system$generate_streamlit_url_from_name('{streamlit_name}')" - ) - assert contains_row_with(result.json[1], row_from_snowflake_session(expect)[0]) - - result = runner.invoke_integration(["streamlit", "share", streamlit_name, "public"]) - assert contains_row_with( - result.json, - {"status": "Statement executed successfully."}, - ) - expect = snowflake_session.execute_string( - f"use role {_new_streamlit_role}; show streamlits like '{streamlit_name}'; use role integration_tests;" - ) - assert contains_row_with( - rows_from_snowflake_session(expect)[1], {"name": streamlit_name.upper()} - ) - - result = runner.invoke_integration(["streamlit", "drop", streamlit_name]) - assert contains_row_with( - result.json, - {"status": f"{streamlit_name.upper()} successfully dropped."}, - ) - expect = snowflake_session.execute_string( - f"show streamlits like '%{streamlit_name}%'" - ) - assert row_from_snowflake_session(expect) == [] - - @pytest.fixture def _new_streamlit_role(snowflake_session, test_database): role_name = f"snowcli_streamlit_role_{uuid.uuid4().hex}"