Skip to content

Commit

Permalink
validate and check table name for nones
Browse files Browse the repository at this point in the history
Signed-off-by: tdhooghe <thomas_dhooghe@mckinsey.com>
  • Loading branch information
tdhooghe committed Oct 21, 2024
1 parent 0a14bb7 commit 6b4e536
Showing 1 changed file with 22 additions and 5 deletions.
27 changes: 22 additions & 5 deletions kedro-datasets/kedro_datasets/snowflake/snowpark_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,9 +229,7 @@ def load(self) -> sp.DataFrame:
Returns:
sp.DataFrame: The loaded data as a Snowpark DataFrame.
"""
table_name = [self._database, self._schema, self._table_name]
sp_df = self._session.table(".".join(table_name))
return sp_df
return self._session.table(self._validate_and_get_table_name())

def save(self, data: pd.DataFrame | sp.DataFrame) -> None:
"""
Expand All @@ -240,11 +238,11 @@ def save(self, data: pd.DataFrame | sp.DataFrame) -> None:
Args:
data (pd.DataFrame | sp.DataFrame): The data to save.
"""
breakpoint()
if isinstance(data, pd.DataFrame):
data = self._session.create_dataframe(data)

table_name = [self._database, self._schema, self._table_name]
data.write.save_as_table(table_name, **self._save_args)
data.write.save_as_table(self._validate_and_get_table_name(), **self._save_args)

def _exists(self) -> bool:
"""
Expand All @@ -261,3 +259,22 @@ def _exists(self) -> bool:
except Exception as e:
logger.debug(f"Table {self._table_name} does not exist: {e}")
return False

def _validate_and_get_table_name(self) -> str:
"""
Validate that all parts of the table name are not None and join them into a string.
Args:
parts (list[str | None]): The list containing database, schema, and table name.
Returns:
str: The joined table name in the format 'database.schema.table'.
Raises:
ValueError: If any part of the table name is None.
"""
parts = [self._database, self._schema, self._table_name]
if any(part is None for part in parts):
raise ValueError(f"Table name parts cannot be None: {parts}")

return ".".join(parts)

0 comments on commit 6b4e536

Please sign in to comment.