Add support for "execute immediate" for python files (#1018)

.github/workflows/test_integration.yaml

@@ -35,7 +35,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip hatch

.gitignore

@@ -17,6 +17,7 @@ gen_docs/
 /venv/
 .env
 .vscode
+tmp/
 
 ^app.zip
 ^snowflake.yml

RELEASE-NOTES.md

@@ -22,6 +22,7 @@
 ## New additions
 * Added connection option `--token-file-path` allowing passing OAuth token using a file. The function is also
   supported by setting `token_file_path` in connection definition.
+* Support for Python remote execution via `snow stage execute` and `snow git execute` similar to existing EXECUTE IMMEDIATE support.
 
 ## Fixes and improvements
 * The `snow app run` command now allows upgrading to unversioned mode from a versioned or release mode application installation

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
   "setuptools==70.2.0",
   'snowflake.core==0.8.0; python_version < "3.12"',
   "snowflake-connector-python[secure-local-storage]==3.11.0",
+  'snowflake-snowpark-python>=1.15.0;python_version < "3.12"',
   "tomlkit==0.12.5",
   "typer==0.12.3",
   "urllib3>=1.24.3,<2.3",

snyk/requirements.txt

@@ -8,6 +8,7 @@ requirements-parser==0.9.0
 setuptools==70.2.0
 snowflake.core==0.8.0; python_version < "3.12"
 snowflake-connector-python[secure-local-storage]==3.11.0
+snowflake-snowpark-python>=1.15.0;python_version < "3.12"
 tomlkit==0.12.5
 typer==0.12.3
 urllib3>=1.24.3,<2.3

src/snowflake/cli/api/commands/flags.py

@@ -449,7 +449,10 @@ def _password_callback(value: str):
     None,
     "--variable",
     "-D",
-    help="Variables for the template. For example: `-D \"<key>=<value>\"`, string values must be in `''`.",
+    help='Variables for the execution context. For example: `-D "<key>=<value>"`. '
+    "For SQL files variables are use to expand the template and any unknown variable will cause an error. "
+    "For Python files variables are used to update os.environ dictionary. Provided keys are capitalized to adhere to best practices."
+    "In case of SQL files string values must be quoted in `''` (consider embedding quoting in the file).",
     show_default=False,
 )
 
@@ -612,10 +615,11 @@ def __init__(self, key: str, value: str):
 
 def parse_key_value_variables(variables: Optional[List[str]]) -> List[Variable]:
     """Util for parsing key=value input. Useful for commands accepting multiple input options."""
+    if not variables:
+        return []
     result: List[Variable] = []
     if not variables:
         return result
-
     for p in variables:
         if "=" not in p:
             raise ClickException(f"Invalid variable: '{p}'")

src/snowflake/cli/api/identifiers.py

@@ -53,11 +53,17 @@ def name(self) -> str:
         return self._name
 
     @property
-    def identifier(self) -> str:
+    def prefix(self) -> str:
         if self.database:
-            return f"{self.database}.{self.schema if self.schema else 'PUBLIC'}.{self.name}"
+            return f"{self.database}.{self.schema if self.schema else 'PUBLIC'}"
         if self.schema:
-            return f"{self.schema}.{self.name}"
+            return f"{self.schema}"
+        return ""
+
+    @property
+    def identifier(self) -> str:
+        if self.prefix:
+            return f"{self.prefix}.{self.name}"
         return self.name
 
     @property
@@ -96,6 +102,13 @@ def from_string(cls, identifier: str) -> "FQN":
             unqualified_name = unqualified_name + signature
         return cls(name=unqualified_name, schema=schema, database=database)
 
+    @classmethod
+    def from_stage(cls, stage: str) -> "FQN":
+        name = stage
+        if stage.startswith("@"):
+            name = stage[1:]
+        return cls.from_string(name)
+
     @classmethod
     def from_identifier_model(cls, model: ObjectIdentifierBaseModel) -> "FQN":
         """Create an instance from object model."""

src/snowflake/cli/api/sql_execution.py

@@ -41,12 +41,22 @@
 
 class SqlExecutionMixin:
     def __init__(self):
-        pass
+        self._snowpark_session = None
 
     @property
     def _conn(self):
         return cli_context.connection
 
+    @property
+    def snowpark_session(self):
+        if not self._snowpark_session:
+            from snowflake.snowpark.session import Session
+
+            self._snowpark_session = Session.builder.configs(
+                {"connection": self._conn}
+            ).create()
+        return self._snowpark_session
+
     @cached_property
     def _log(self):
         return logging.getLogger(__name__)

src/snowflake/cli/plugins/cortex/commands.py

@@ -24,6 +24,7 @@
 from snowflake.cli.api.cli_global_context import cli_context
 from snowflake.cli.api.commands.flags import readable_file_option
 from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
+from snowflake.cli.api.constants import PYTHON_3_12
 from snowflake.cli.api.output.types import (
     CollectionResult,
     CommandResult,
@@ -45,7 +46,7 @@
     help="Provides access to Snowflake Cortex.",
 )
 
-SEARCH_COMMAND_ENABLED = sys.version_info < (3, 12)
+SEARCH_COMMAND_ENABLED = sys.version_info < PYTHON_3_12
 
 
 @app.command(

src/snowflake/cli/plugins/snowpark/models.py

@@ -33,6 +33,10 @@ class YesNoAsk(Enum):
 class Requirement(requirement.Requirement):
     extra_pattern = re.compile("'([^']*)'")
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.package_name = None
+
     @classmethod
     def parse_line(cls, line: str) -> Requirement:
         if len(line_elements := line.split(";")) > 1:
@@ -44,6 +48,8 @@ def parse_line(cls, line: str) -> Requirement:
                 if "extra" in element and (extras := cls.extra_pattern.search(element)):
                     result.extras.extend(extras.groups())
 
+        result.package_name = result.name
+
         if result.uri and not result.name:
             result.name = get_package_name(result.uri)
         result.name = cls.standardize_name(result.name)