From 5ca1847e44c736d3f5286dd1b4d3ef73b55d8607 Mon Sep 17 00:00:00 2001 From: Tomasz Urbaszek Date: Thu, 14 Mar 2024 13:06:18 +0100 Subject: [PATCH] Refactor package lookup (#901) --- RELEASE-NOTES.md | 5 ++ src/snowflake/cli/api/commands/flags.py | 9 +++ .../cli/plugins/snowpark/package/anaconda.py | 3 + .../cli/plugins/snowpark/package/commands.py | 75 ++++++++++++++----- tests/__snapshots__/test_help_messages.ambr | 19 +---- .../snowpark/__snapshots__/test_package.ambr | 12 ++- tests/snowpark/test_package.py | 42 +++++------ 7 files changed, 104 insertions(+), 61 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 6b4900fda5..bcf4c64a2a 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -2,6 +2,11 @@ ## Backward incompatibility +## Deprecations +* `snow snowpark package lookup` no longer performs check against PyPi. Using `--pypi-download` or `--yes` + has no effect and will cause a warning. In this way the command has single responsibility - check if package is + available in Snowflake Anaconda channel. + ## New additions * Added support for fully qualified name (`database.schema.name`) in `name` parameter in streamlit project definition * Added support for fully qualified image repository names in `spcs image-repository` commands. diff --git a/src/snowflake/cli/api/commands/flags.py b/src/snowflake/cli/api/commands/flags.py index e6aa7994f3..7ca365541d 100644 --- a/src/snowflake/cli/api/commands/flags.py +++ b/src/snowflake/cli/api/commands/flags.py @@ -443,3 +443,12 @@ def _callback(project_path: Optional[str]): callback=_callback, show_default=False, ) + + +def deprecated_flag_callback(msg: str): + def _warning_callback(ctx: click.Context, param: click.Parameter, value: Any): + if ctx.get_parameter_source(param.name) != click.core.ParameterSource.DEFAULT: # type: ignore[attr-defined] + cli_console.warning(message=msg) + return value + + return _warning_callback diff --git a/src/snowflake/cli/plugins/snowpark/package/anaconda.py b/src/snowflake/cli/plugins/snowpark/package/anaconda.py index 72e06cf124..45dd8f1ea3 100644 --- a/src/snowflake/cli/plugins/snowpark/package/anaconda.py +++ b/src/snowflake/cli/plugins/snowpark/package/anaconda.py @@ -27,6 +27,9 @@ def is_package_available(self, package: Requirement): return all([parse(spec[1]) <= latest_ver for spec in package.specs]) return True + def package_version(self, package: Requirement): + return self._packages[package.name.lower()].get("version") + @classmethod def from_snowflake(cls): response = requests.get(AnacondaChannel.snowflake_channel_url) diff --git a/src/snowflake/cli/plugins/snowpark/package/commands.py b/src/snowflake/cli/plugins/snowpark/package/commands.py index e743a4eb82..23b576cd87 100644 --- a/src/snowflake/cli/plugins/snowpark/package/commands.py +++ b/src/snowflake/cli/plugins/snowpark/package/commands.py @@ -5,9 +5,13 @@ from textwrap import dedent import typer +from click import ClickException +from requests import HTTPError +from snowflake.cli.api.commands.flags import deprecated_flag_callback from snowflake.cli.api.commands.snow_typer import SnowTyper from snowflake.cli.api.output.types import CommandResult, MessageResult -from snowflake.cli.plugins.snowpark.models import PypiOption +from snowflake.cli.plugins.snowpark.models import PypiOption, Requirement +from snowflake.cli.plugins.snowpark.package.anaconda import AnacondaChannel from snowflake.cli.plugins.snowpark.package.manager import ( cleanup_after_install, create_packages_zip, @@ -26,44 +30,64 @@ ) log = logging.getLogger(__name__) -install_option = typer.Option( + +lookup_install_option = typer.Option( False, "--pypi-download", + hidden=True, + callback=deprecated_flag_callback( + "Using --pypi-download is deprecated. Lookup command no longer checks for package in PyPi." + ), help="Installs packages that are not available on the Snowflake Anaconda channel.", ) -deprecated_install_option = typer.Option( +lookup_deprecated_install_option = typer.Option( False, "--yes", "-y", hidden=True, + callback=deprecated_flag_callback( + "Using --yes is deprecated. Lookup command no longer checks for package in PyPi." + ), help="Installs packages that are not available on the Snowflake Anaconda channel.", ) @app.command("lookup", requires_connection=True) -@cleanup_after_install def package_lookup( - name: str = typer.Argument(..., help="Name of the package."), - install_packages: bool = install_option, - _deprecated_install_option: bool = deprecated_install_option, - allow_native_libraries: PypiOption = PackageNativeLibrariesOption, + package_name: str = typer.Argument( + ..., help="Name of the package.", show_default=False + ), + # todo: remove with 3.0 + _: bool = lookup_install_option, + __: bool = lookup_deprecated_install_option, **options, ) -> CommandResult: """ Checks if a package is available on the Snowflake Anaconda channel. - If the `--pypi-download` flag is provided, this command checks all dependencies of the packages - outside Snowflake channel. """ - if _deprecated_install_option: - install_packages = _deprecated_install_option - - lookup_result = lookup( - name=name, - install_packages=install_packages, - allow_native_libraries=allow_native_libraries, + try: + anaconda = AnacondaChannel.from_snowflake() + except HTTPError as err: + raise ClickException( + f"Accessing Snowflake Anaconda channel failed. Reason {err}" + ) + + package = Requirement.parse(package_name) + if anaconda.is_package_available(package=package): + msg = f"Package `{package_name}` is available in Anaconda." + if version := anaconda.package_version(package=package): + msg += f" Latest available version: {version}." + return MessageResult(msg) + + return MessageResult( + dedent( + f""" + Package `{package_name}` is not available in Anaconda. To prepare Snowpark compatible package run: + snow snowpark package create {package_name} + """ + ) ) - return MessageResult(lookup_result.message) @app.command("upload", requires_connection=True) @@ -95,6 +119,21 @@ def package_upload( return MessageResult(upload(file=file, stage=stage, overwrite=overwrite)) +install_option = typer.Option( + False, + "--pypi-download", + help="Installs packages that are not available on the Snowflake Anaconda channel.", +) + +deprecated_install_option = typer.Option( + False, + "--yes", + "-y", + hidden=True, + help="Installs packages that are not available on the Snowflake Anaconda channel.", +) + + @app.command("create", requires_connection=True) @cleanup_after_install def package_create( diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 475e0532c4..30d2bd6296 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -1772,24 +1772,15 @@ # name: test_help_messages[snowpark.package.lookup] ''' - Usage: default snowpark package lookup [OPTIONS] NAME + Usage: default snowpark package lookup [OPTIONS] PACKAGE_NAME - Checks if a package is available on the Snowflake Anaconda channel. If the - `--pypi-download` flag is provided, this command checks all dependencies of - the packages outside Snowflake channel. + Checks if a package is available on the Snowflake Anaconda channel. ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ - │ * name TEXT Name of the package. [default: None] [required] │ + │ * package_name TEXT Name of the package. [required] │ ╰──────────────────────────────────────────────────────────────────────────────╯ ╭─ Options ────────────────────────────────────────────────────────────────────╮ - │ --pypi-download Installs packages that are │ - │ not available on the │ - │ Snowflake Anaconda channel. │ - │ --allow-native-libraries [yes|no|ask] Allows native libraries, │ - │ when using packages │ - │ installed through PIP │ - │ [default: no] │ - │ --help -h Show this message and exit. │ + │ --help -h Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────╯ ╭─ Connection configuration ───────────────────────────────────────────────────╮ │ --connection,--environment -c TEXT Name of the connection, as defined │ @@ -1933,8 +1924,6 @@ │ create Creates a Python package as a zip file that can be uploaded to a │ │ stage and imported for a Snowpark Python app. │ │ lookup Checks if a package is available on the Snowflake Anaconda channel. │ - │ If the `--pypi-download` flag is provided, this command checks all │ - │ dependencies of the packages outside Snowflake channel. │ │ upload Uploads a Python package zip file to a Snowflake stage so it can be │ │ referenced in the imports of a procedure or function. │ ╰──────────────────────────────────────────────────────────────────────────────╯ diff --git a/tests/snowpark/__snapshots__/test_package.ambr b/tests/snowpark/__snapshots__/test_package.ambr index 08aefb8f35..88c69d8b9e 100644 --- a/tests/snowpark/__snapshots__/test_package.ambr +++ b/tests/snowpark/__snapshots__/test_package.ambr @@ -1,15 +1,15 @@ # serializer version: 1 # name: TestPackage.test_package_lookup[snowflake-connector-python] ''' - Package snowflake-connector-python is available on the Snowflake Anaconda channel. + Package `snowflake-connector-python` is available in Anaconda. Latest available version: 2.7.12. ''' # --- # name: TestPackage.test_package_lookup[some-weird-package-we-dont-know] ''' - Nothing found for some-weird-package-we-dont-know. Most probably, package is not available on Snowflake Anaconda channel. - Please check the package name or try again with --pypi-download option. + Package `some-weird-package-we-dont-know` is not available in Anaconda. To prepare Snowpark compatible package run: + snow snowpark package create some-weird-package-we-dont-know ''' @@ -17,10 +17,8 @@ # name: TestPackage.test_package_lookup_with_install_packages ''' - The package some-other-package is supported, but does depend on the - following Snowflake supported libraries. You should include the - following dependencies in you function or procedure packages list: - snowflake-snowpark-python + Package `some-other-package` is not available in Anaconda. To prepare Snowpark compatible package run: + snow snowpark package create some-other-package ''' diff --git a/tests/snowpark/test_package.py b/tests/snowpark/test_package.py index d2316b3384..345fc14267 100644 --- a/tests/snowpark/test_package.py +++ b/tests/snowpark/test_package.py @@ -6,11 +6,10 @@ import pytest from snowflake.cli.plugins.snowpark.models import ( - PypiOption, Requirement, SplitRequirements, ) -from snowflake.cli.plugins.snowpark.package.utils import NothingFound, NotInAnaconda +from snowflake.cli.plugins.snowpark.package.utils import NotInAnaconda from tests.test_data import test_data @@ -28,7 +27,7 @@ def test_package_lookup( test_data.anaconda_response ) - result = runner.invoke(["snowpark", "package", "lookup", argument, "--yes"]) + result = runner.invoke(["snowpark", "package", "lookup", argument]) assert result.exit_code == 0 assert result.output == snapshot @@ -52,9 +51,7 @@ def test_package_lookup_with_install_packages( ), ) - result = runner.invoke( - ["snowpark", "package", "lookup", "some-other-package", "--yes"] - ) + result = runner.invoke(["snowpark", "package", "lookup", "some-other-package"]) assert result.exit_code == 0 assert result.output == snapshot @@ -71,7 +68,7 @@ def test_package_create( logging.DEBUG, logger="snowflake.cli.plugins.snowpark.package" ): result = runner.invoke( - ["snowpark", "package", "create", "totally-awesome-package", "--yes"] + ["snowpark", "package", "create", "totally-awesome-package"] ) assert result.exit_code == 0 @@ -139,26 +136,29 @@ def test_package_upload_to_path( assert create.args[0] == "create stage if not exists db.schema.stage" assert "db.schema.stage/path/to/file" in put.args[0] - @pytest.mark.parametrize("command", ["lookup", "create"]) @pytest.mark.parametrize( - "flags,expected_value", + "flags", [ - (["--pypi-download"], True), - (["-y"], True), - (["--yes"], True), - (["--pypi-download", "-y"], True), - ([], False), + ["--pypi-download"], + ["-y"], + ["--yes"], + ["--pypi-download", "-y"], ], ) - @mock.patch("snowflake.cli.plugins.snowpark.package.commands.lookup") - def test_install_flag(self, mock_lookup, command, flags, expected_value, runner): - mock_lookup.return_value = NothingFound + @mock.patch("snowflake.cli.plugins.snowpark.package.commands.AnacondaChannel") + def test_lookup_install_flag_are_deprecated(self, _, flags, runner): result = runner.invoke(["snowpark", "package", "lookup", "foo", *flags]) + assert ( + "is deprecated. Lookup command no longer checks for package in PyPi" + in result.output + ) - mock_lookup.assert_called_with( - name="foo", - install_packages=expected_value, - allow_native_libraries=PypiOption.NO, + @mock.patch("snowflake.cli.plugins.snowpark.package.commands.AnacondaChannel") + def test_lookup_install_with_out_flags_does_not_warn(self, _, runner): + result = runner.invoke(["snowpark", "package", "lookup", "foo"]) + assert ( + "is deprecated. Lookup command no longer checks for package in PyPi" + not in result.output ) @staticmethod