Merge pull request #6 from znamlab/dev

Dev

.gitignore

@@ -5,4 +5,4 @@ tests/test_data/tests_outputs
 **/__pycache__/
 *.egg-info/
 *checkpoint.ipynb
-build/
+build/

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+exclude: 'conf.py'
+
+# Configuring https://pre-commit.ci/
+ci:
+    autoupdate_schedule: monthly
+
+repos:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.4.0
+      hooks:
+          - id: check-docstring-first
+          - id: check-executables-have-shebangs
+          - id: check-merge-conflict
+          - id: check-toml
+          - id: end-of-file-fixer
+          - id: mixed-line-ending
+            args: [--fix=lf]
+          - id: requirements-txt-fixer
+          - id: trailing-whitespace
+    - repo: https://github.com/psf/black
+      rev: 23.3.0
+      hooks:
+          - id: black
+    - repo: https://github.com/kynan/nbstripout
+      rev: 0.6.1
+      hooks:
+          - id: nbstripout
+            args: [--extra-keys=metadata.language_info.version metadata.kernelspec.name metadata.kernelspec.display_name]

.vscode/settings.json

@@ -10,4 +10,4 @@
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true
-}
+}

CHANGELOG.md

@@ -1,6 +1,12 @@
 
 ## Changelog
 
+
+### [v0.7] - 2024-05-14
+
+- Feature: `pathlib.Path` are automatically converted to strings in the main slurm 
+    python script. This avoid crashing because `PosixPath` and co are not imported.
+
 ### [v0.6] - 2024-04-04
 
 - Change default cpu partition to `ncpu`

ReadMe.md

@@ -1,12 +1,12 @@
 # ZnamUtils
 
-This package contains common utilities for Znamenskiy's lab projects. 
+This package contains common utilities for Znamenskiy's lab projects.
 
 # SlurmIt
 
-`@slurm_it` is a decorator allowing to run a function on slurm. Once set up, running the 
+`@slurm_it` is a decorator allowing to run a function on slurm. Once set up, running the
 decorated function with `use_slurm=False` will run the function and return its normal output.
-Running the decorated function with `use_slurm=True` will create a slurm script and a python 
+Running the decorated function with `use_slurm=True` will create a slurm script and a python
 script, submit the slurm script and return the job id of the slurm job.
 
 ## Usage
@@ -48,10 +48,10 @@ The decorator has five arguments:
 - slurm_options (dict, optional): options to pass to sbatch. Will be used to
     update the default config (see below) if not None. Defaults to None.
 - imports (list, optional): list of imports to add to the python script. Defaults to None.
-- from_imports (dict, optional): dict of imports to add to the python script as "from 
+- from_imports (dict, optional): dict of imports to add to the python script as "from
     key import value". Defaults to None.
 
-The default parameters of SlurmIt are: 
+The default parameters of SlurmIt are:
 ```
 ntasks=1
 time="12:00:00"
@@ -62,8 +62,8 @@ partition="cpu"
 An example of fully custromised decoration would be:
 
 ```python
-@slurm_it(conda_env='myenv', 
-  module_list=['FFmpeg', 'cuda'], 
+@slurm_it(conda_env='myenv',
+  module_list=['FFmpeg', 'cuda'],
   slurm_options=dict(partition="gpu",
   imports=['numpy', 'matplotlib'],
   from_imports={'sklearn': 'svm'}
@@ -73,9 +73,9 @@ def analysis_step(param1, param2):
   return out
 ```
 
-> Note: 
+> Note:
 > `imports` and `from_imports` are useful only if the decorated function require non
-> built-in datatype arguments or if the module containing the function cannot be 
+> built-in datatype arguments or if the module containing the function cannot be
 > accessed from the python script in the same way as it is in the code calling slurm_it
 > (for instance if you use relative imports). Then explicitely setting `from_imports` to
 > import the decorated function is required.
@@ -93,11 +93,11 @@ dependency_type (str, optional): Type of dependence on previous jobs.
 job_dependency (str): job id to depend on
 slurm_folder (str): where to write the slurm script and logs
 scripts_name (str): name of the slurm script and python file
-slurm_options (dict): options to pass to sbatch, will update the default options 
+slurm_options (dict): options to pass to sbatch, will update the default options
   provided in the decorator.
 ```
 
-When `use_slurm = True`, `slurm_folder` must be provided. 
+When `use_slurm = True`, `slurm_folder` must be provided.
 If `scripts_name` is false, the name of the function is used instead.
 
 Calling:
@@ -126,4 +126,3 @@ A collection of utilities to interact with the Slurm scheduler. Used by `slurmit
 # Tests
 
 To run the test, we need to access camp/nemo and slurm. It also requires a flexiznam installation.
-

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="znamutils",
-    version="v0.6",
+    version="v0.6-dev",
     packages=find_packages(exclude=["tests"]),
     url="https://github.com/znamlab/znamutils",
     license="MIT",

znamutils/decorators.py

@@ -56,7 +56,7 @@ def slurm_it(
             instance {'numpy': 'mean'} results in `from numpy import mean`. If `None`,
             the decorated function will be imported from its parent module. Defaults to
             None.
-        print_job_id (bool, optional): Whether to print the job id of the slurm job in 
+        print_job_id (bool, optional): Whether to print the job id of the slurm job in
             the log file. Defaults to False.
 
     Returns:
@@ -100,10 +100,10 @@ def slurm_it(
         ),
     )
     from_imports = from_imports or {func.__module__: func.__name__}
+
     # create the new function with modified signature
     @wraps(func, new_sig=new_sig)
     def new_func(*args, **kwargs):
-
         # pop the slurm only arguments
         use_slurm = kwargs.pop("use_slurm")
         dependency_type = kwargs.pop("dependency_type")
@@ -153,7 +153,6 @@ def new_func(*args, **kwargs):
             arguments=kwargs,
             imports=imports,
             from_imports=from_imports,
-            print_job_id=print_job_id,
         )
 
         return slurm_helper.run_slurm_batch(

znamutils/slurm_helper.py

@@ -104,7 +104,12 @@ def create_slurm_sbatch(
 
 
 def python_script_single_func(
-    target_file, function_name, arguments=None, imports=None, from_imports=None, print_job_id=False,
+    target_file,
+    function_name,
+    arguments=None,
+    imports=None,
+    from_imports=None,
+    path2string=True,
 ):
     """Create a python script that will call a function
 
@@ -118,7 +123,8 @@ def python_script_single_func(
         from_imports (dict, optional): Dictionary of imports to add to the script. Keys
             are the module names, values are the functions to import. For instance
             {'numpy': 'mean'} results in `from numpy import mean`. Defaults to None.
-        print_job_id (bool, optional): Whether to print the job ID. Defaults to False.
+        path2string (bool, optional): Whether to convert arguments that are paths to
+            strings. Defaults to True.
     """
 
     target_file = Path(target_file)
@@ -139,6 +145,8 @@ def python_script_single_func(
         fhandle.write(f"{function_name}(")
         if arguments is not None:
             for k, v in arguments.items():
+                if path2string and isinstance(v, Path):
+                    v = str(v)
                 fhandle.write(f"{k}={repr(v)}, ")
         fhandle.write(")\n")