Merge branch 'main' into SNOW-1232599-refactor-package-create

RELEASE-NOTES.md

@@ -27,6 +27,7 @@
 * Project definition no longer accept extra fields. Any extra field will cause an error.
 * Changing imports in function/procedure section in `snowflake.yml` will cause the definition update on replace
 * Adding `--pattern` flag to `stage list` command for filtering out results with regex.
+* Fixed snowpark build paths for builds with --project option (fixed empty zip issue).
 
 # v2.1.1
 

src/snowflake/cli/plugins/snowpark/commands.py

@@ -2,7 +2,6 @@
 
 import logging
 from enum import Enum
-from pathlib import Path
 from typing import Dict, List, Optional, Set
 
 import typer
@@ -32,7 +31,7 @@
     FunctionSchema,
     ProcedureSchema,
 )
-from snowflake.cli.api.project.schemas.snowpark.snowpark import Snowpark
+from snowflake.cli.api.secure_path import SecurePath
 from snowflake.cli.plugins.object.manager import ObjectManager
 from snowflake.cli.plugins.object.stage.manager import StageManager
 from snowflake.cli.plugins.snowpark.common import (
@@ -42,6 +41,7 @@
 from snowflake.cli.plugins.snowpark.manager import FunctionManager, ProcedureManager
 from snowflake.cli.plugins.snowpark.models import PypiOption
 from snowflake.cli.plugins.snowpark.package_utils import get_snowflake_packages
+from snowflake.cli.plugins.snowpark.snowpark_package_paths import SnowparkPackagePaths
 from snowflake.cli.plugins.snowpark.snowpark_shared import (
     CheckAnacondaForPyPiDependencies,
     PackageNativeLibrariesOption,
@@ -79,6 +79,10 @@ def deploy(
     All deployed objects use the same artifact which is deployed only once.
     """
     snowpark = cli_context.project_definition
+    paths = SnowparkPackagePaths.for_snowpark_project(
+        project_root=SecurePath(cli_context.project_root),
+        snowpark_project_definition=snowpark,
+    )
 
     procedures = snowpark.procedures
     functions = snowpark.functions
@@ -88,9 +92,7 @@ def deploy(
             "No procedures or functions were specified in the project definition."
         )
 
-    build_artifact_path = _get_snowpark_artifact_path(snowpark)
-
-    if not build_artifact_path.exists():
+    if not paths.artifact_file.exists():
         raise ClickException(
             "Artifact required for deploying the project does not exist in this directory. "
             "Please use build command to create it."
@@ -125,10 +127,12 @@ def deploy(
     packages = get_snowflake_packages()
 
     artifact_stage_directory = get_app_stage_path(stage_name, snowpark.project_name)
-    artifact_stage_target = f"{artifact_stage_directory}/{build_artifact_path.name}"
+    artifact_stage_target = (
+        f"{artifact_stage_directory}/{paths.artifact_file.path.name}"
+    )
 
     stage_manager.put(
-        local_path=build_artifact_path,
+        local_path=paths.artifact_file.path,
         stage_path=artifact_stage_directory,
         overwrite=True,
     )
@@ -143,7 +147,7 @@ def deploy(
             existing_objects=existing_procedures,
             packages=packages,
             stage_artifact_path=artifact_stage_target,
-            source_name=build_artifact_path.name,
+            source_name=paths.artifact_file.path.name,
         )
         deploy_status.append(operation_result)
 
@@ -156,7 +160,7 @@ def deploy(
             existing_objects=existing_functions,
             packages=packages,
             stage_artifact_path=artifact_stage_target,
-            source_name=build_artifact_path.name,
+            source_name=paths.artifact_file.path.name,
         )
         deploy_status.append(operation_result)
 
@@ -304,12 +308,6 @@ def _deploy_single_object(
     }
 
 
-def _get_snowpark_artifact_path(snowpark_definition: Snowpark):
-    source = Path(snowpark_definition.src)
-    artifact_file = Path.cwd() / (source.name + ".zip")
-    return artifact_file
-
-
 @app.command("build")
 @with_project_definition("snowpark")
 def build(
@@ -322,19 +320,19 @@ def build(
     Builds the Snowpark project as a `.zip` archive that can be used by `deploy` command.
     The archive is built using only the `src` directory specified in the project file.
     """
-    snowpark = cli_context.project_definition
-    source = Path(snowpark.src)
-    artifact_file = _get_snowpark_artifact_path(snowpark)
-    log.info("Building package using sources from: %s", source.resolve())
+    paths = SnowparkPackagePaths.for_snowpark_project(
+        project_root=SecurePath(cli_context.project_root),
+        snowpark_project_definition=cli_context.project_definition,
+    )
+    log.info("Building package using sources from: %s", paths.source.path)
 
     snowpark_package(
-        source=source,
-        artifact_file=artifact_file,
+        paths=paths,
         pypi_download=pypi_download,  # type: ignore[arg-type]
         check_anaconda_for_pypi_deps=check_anaconda_for_pypi_deps,
         package_native_libraries=package_native_libraries,  # type: ignore[arg-type]
     )
-    return MessageResult(f"Build done. Artifact path: {artifact_file}")
+    return MessageResult(f"Build done. Artifact path: {paths.artifact_file.path}")
 
 
 class _SnowparkObject(Enum):

src/snowflake/cli/plugins/snowpark/package/commands.py

@@ -12,6 +12,7 @@
 )
 from snowflake.cli.api.commands.snow_typer import SnowTyper
 from snowflake.cli.api.output.types import CommandResult, MessageResult
+from snowflake.cli.api.secure_path import SecurePath
 from snowflake.cli.plugins.snowpark.models import (
     PypiOption,
     Requirement,
@@ -213,11 +214,13 @@ def package_create(
                 f"Package {name} is already available in Snowflake Anaconda Channel."
             )
 
+    packages_dir = SecurePath(".packages")
     packages_are_downloaded, dependencies = download_packages(
         anaconda=anaconda,
         perform_anaconda_check_for_dependencies=not ignore_anaconda,
         package_name=name,
-        file_name=None,
+        requirements_file=None,
+        packages_dir=packages_dir,
         index_url=index_url,
         allow_shared_libraries=allow_shared_libraries_pypi_option,
         skip_version_check=skip_version_check,

src/snowflake/cli/plugins/snowpark/package_utils.py

@@ -2,7 +2,6 @@
 
 import logging
 import os
-from pathlib import Path
 from typing import List
 
 import click
@@ -26,7 +25,7 @@
 
 
 def parse_requirements(
-    requirements_file: str = "requirements.txt",
+    requirements_file: SecurePath = SecurePath("requirements.txt"),
 ) -> List[Requirement]:
     """Reads and parses a Python requirements.txt file.
 
@@ -38,15 +37,14 @@ def parse_requirements(
         list[str]: A flat list of package names, without versions
     """
     reqs: List[Requirement] = []
-    requirements_file_spath = SecurePath(requirements_file)
-    if requirements_file_spath.exists():
-        with requirements_file_spath.open(
+    if requirements_file.exists():
+        with requirements_file.open(
             "r", read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB, encoding="utf-8"
         ) as f:
             for req in requirements.parse(f):
                 reqs.append(req)
     else:
-        log.info("No %s found", requirements_file)
+        log.info("No %s found", requirements_file.path)
 
     return deduplicate_and_sort_reqs(reqs)
 
@@ -103,7 +101,8 @@ def generate_deploy_stage_name(identifier: str) -> str:
 
 def download_packages(
     anaconda: AnacondaChannel | None,
-    file_name: str | None,
+    requirements_file: SecurePath | None,
+    packages_dir: SecurePath,
     perform_anaconda_check_for_dependencies: bool = True,
     package_name: str | None = None,
     index_url: str | None = None,
@@ -122,12 +121,12 @@ def download_packages(
         which are available on the Snowflake Anaconda channel.
         They will not be downloaded into '.packages' directory.
     """
-    if file_name and package_name:
+    if requirements_file and package_name:
         raise ClickException(
             "Could not use package name and requirements file simultaneously"
         )
-    if file_name and not Path(file_name).exists():
-        raise ClickException(f"File {file_name} does not exists.")
+    if requirements_file and not requirements_file.exists():
+        raise ClickException(f"File {requirements_file.path} does not exists.")
     if perform_anaconda_check_for_dependencies and not anaconda:
         raise ClickException(
             "Cannot perform anaconda checks if anaconda channel is not specified."
@@ -136,19 +135,18 @@ def download_packages(
     with Venv() as v, SecurePath.temporary_directory() as downloads_dir:
         if package_name:
             # This is a Windows workaround where use TemporaryDirectory instead of NamedTemporaryFile
-            tmp_requirements = Path(v.directory.name) / "requirements.txt"
-            tmp_requirements.write_text(str(package_name))
-            file_name = str(tmp_requirements)
+            requirements_file = SecurePath(v.directory.name) / "requirements.txt"
+            requirements_file.write_text(str(package_name))
 
         pip_wheel_result = v.pip_wheel(
-            file_name, download_dir=downloads_dir.path, index_url=index_url
+            requirements_file.path, download_dir=downloads_dir.path, index_url=index_url  # type: ignore
         )
         if pip_wheel_result != 0:
             log.info(pip_failed_log_msg, pip_wheel_result)
             return False, None
 
         dependencies = v.get_package_dependencies(
-            file_name, downloads_dir=downloads_dir.path
+            requirements_file, downloads_dir=downloads_dir.path
         )
         dependency_requirements = [d.requirement for d in dependencies]
 
@@ -177,10 +175,9 @@ def download_packages(
         ) and not _confirm_shared_libraries(allow_shared_libraries):
             return False, split_dependencies
         else:
-            packages_dest = SecurePath(".packages")
-            packages_dest.mkdir(exist_ok=True)
+            packages_dir.mkdir(exist_ok=True)
             for package in dependencies_to_be_packed:
-                package.extract_files(packages_dest.path)
+                package.extract_files(packages_dir.path)
             return True, split_dependencies
 
 

src/snowflake/cli/plugins/snowpark/snowpark_package_paths.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+
+from snowflake.cli.api.project.schemas.snowpark.snowpark import Snowpark
+from snowflake.cli.api.secure_path import SecurePath
+
+_DEFINED_REQUIREMENTS = "requirements.txt"
+_REQUIREMENTS_SNOWFLAKE = "requirements.snowflake.txt"
+_REQUIREMENTS_OTHER = "requirements.other.txt"
+_PACKAGES_DIR = ".packages"
+
+
+@dataclass
+class SnowparkPackagePaths:
+    source: SecurePath
+    artifact_file: SecurePath
+    defined_requirements_file: SecurePath = SecurePath(_DEFINED_REQUIREMENTS)
+    snowflake_requirements_file: SecurePath = SecurePath(_REQUIREMENTS_SNOWFLAKE)
+    other_requirements_file: SecurePath = SecurePath(_REQUIREMENTS_OTHER)
+    downloaded_packages_dir: SecurePath = SecurePath(_PACKAGES_DIR)
+
+    @classmethod
+    def for_snowpark_project(
+        cls, project_root: SecurePath, snowpark_project_definition: Snowpark
+    ) -> "SnowparkPackagePaths":
+        defined_source_path = SecurePath(snowpark_project_definition.src)
+        return cls(
+            source=cls._get_snowpark_project_source_absolute_path(
+                project_root=project_root,
+                defined_source_path=defined_source_path,
+            ),
+            artifact_file=cls._get_snowpark_project_artifact_absolute_path(
+                project_root=project_root,
+                defined_source_path=defined_source_path,
+            ),
+            defined_requirements_file=project_root / _DEFINED_REQUIREMENTS,
+            snowflake_requirements_file=project_root / _REQUIREMENTS_SNOWFLAKE,
+            other_requirements_file=project_root / _REQUIREMENTS_OTHER,
+            downloaded_packages_dir=project_root / _PACKAGES_DIR,
+        )
+
+    @classmethod
+    def _get_snowpark_project_source_absolute_path(
+        cls, project_root: SecurePath, defined_source_path: SecurePath
+    ) -> SecurePath:
+        if defined_source_path.path.is_absolute():
+            return defined_source_path
+        return SecurePath((project_root / defined_source_path.path).path.resolve())
+
+    @classmethod
+    def _get_snowpark_project_artifact_absolute_path(
+        cls, project_root: SecurePath, defined_source_path: SecurePath
+    ) -> SecurePath:
+        source_path = cls._get_snowpark_project_source_absolute_path(
+            project_root=project_root, defined_source_path=defined_source_path
+        )
+        artifact_file = project_root / (source_path.path.name + ".zip")
+        return artifact_file