From 3335e55a2e78370e6710c6657b76145a711f121b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 30 Sep 2024 10:01:12 -0400 Subject: [PATCH] More ability to set global defaults for file source plugin types. --- lib/galaxy/config/__init__.py | 4 ++++ lib/galaxy/config/schemas/config_schema.yml | 18 ++++++++++++++++++ lib/galaxy/files/plugins.py | 19 +++++++++++++++++++ lib/galaxy/files/sources/s3fs.py | 11 +++++++++-- lib/galaxy/files/sources/webdav.py | 14 ++++++++++++-- 5 files changed, 62 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index ecf8f8aebcfb..06dd8ec0678f 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -1236,6 +1236,9 @@ def _load_theme(path: str, theme_dict: dict): else: _load_theme(self.themes_config_file, self.themes) + if self.file_source_temp_dir: + self.file_source_temp_dir = os.path.abspath(self.file_source_temp_dir) + def _process_celery_config(self): if self.celery_conf and self.celery_conf.get("result_backend") is None: # If the result_backend is not set, use a SQLite database in the data directory @@ -1348,6 +1351,7 @@ def check(self): self.template_cache_path, self.tool_data_path, self.user_library_import_dir, + self.file_source_temp_dir, ] for path in paths_to_check: self._ensure_directory(path) diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 5098d923d8ca..005e6b073481 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -4089,3 +4089,21 @@ mapping: per_host: true desc: | Enable the integration of the Galaxy Help Forum in the tool panel. This requires the help_forum_api_url to be set. + + file_source_temp_dir: + type: str + required: false + desc: | + Directory to store temporary files for file sources. This defaults to new_file_path if not set. + + file_source_webdav_use_temp_files: + type: bool + default: false + desc: | + Default value for use_temp_files for webdav plugins that don't explicitly declare this. + + file_source_webdav_s3fs_listings_expiry_time: + type: int + default: 60 + desc: | + Default value for listings_expiry_time for s3fs file source plugins that don't explicitly declare this. diff --git a/lib/galaxy/files/plugins.py b/lib/galaxy/files/plugins.py index 4f9c182f846a..009d232c808d 100644 --- a/lib/galaxy/files/plugins.py +++ b/lib/galaxy/files/plugins.py @@ -24,6 +24,9 @@ class FileSourcePluginsConfig: user_library_import_dir: Optional[str] ftp_upload_dir: Optional[str] ftp_upload_purge: bool + tmp_dir: Optional[str] + webdav_use_temp_files: bool + s3fs_listings_expiry_time: Optional[int] def __init__( self, @@ -33,6 +36,9 @@ def __init__( user_library_import_dir=None, ftp_upload_dir=None, ftp_upload_purge=True, + tmp_dir=None, + webdav_use_temp_files=False, + s3fs_listings_expiry_time=None, ): symlink_allowlist = symlink_allowlist or [] fetch_url_allowlist = fetch_url_allowlist or [] @@ -42,6 +48,9 @@ def __init__( self.user_library_import_dir = user_library_import_dir self.ftp_upload_dir = ftp_upload_dir self.ftp_upload_purge = ftp_upload_purge + self.tmp_dir = tmp_dir + self.webdav_use_temp_files = webdav_use_temp_files + self.s3fs_listings_expiry_time = s3fs_listings_expiry_time @staticmethod def from_app_config(config): @@ -54,6 +63,10 @@ def from_app_config(config): kwds["user_library_import_dir"] = config.user_library_import_dir kwds["ftp_upload_dir"] = config.ftp_upload_dir kwds["ftp_upload_purge"] = config.ftp_upload_purge + kwds["tmp_dir"] = config.file_source_temp_dir + kwds["webdav_use_temp_files"] = config.file_source_webdav_use_temp_files + kwds["s3fs_listings_expiry_time"] = config.file_source_s3fs_listings_expiry_time + return FileSourcePluginsConfig(**kwds) def to_dict(self): @@ -64,6 +77,9 @@ def to_dict(self): "user_library_import_dir": self.user_library_import_dir, "ftp_upload_dir": self.ftp_upload_dir, "ftp_upload_purge": self.ftp_upload_purge, + "tmp_dir": self.tmp_dir, + "webdav_use_temp_files": self.webdav_use_temp_files, + "s3fs_listings_expiry_time": self.s3fs_listings_expiry_time, } @staticmethod @@ -75,6 +91,9 @@ def from_dict(as_dict): user_library_import_dir=as_dict["user_library_import_dir"], ftp_upload_dir=as_dict["ftp_upload_dir"], ftp_upload_purge=as_dict["ftp_upload_purge"], + tmp_dir=as_dict.get("tmp_dir"), + webdav_use_temp_files=as_dict.get("webdav_use_temp_files"), + s3fs_listings_expiry_time=as_dict.get("s3fs_listings_expiry_time"), ) diff --git a/lib/galaxy/files/sources/s3fs.py b/lib/galaxy/files/sources/s3fs.py index 6967f4a5b6a7..3bd5c93a8e52 100644 --- a/lib/galaxy/files/sources/s3fs.py +++ b/lib/galaxy/files/sources/s3fs.py @@ -8,7 +8,10 @@ Tuple, ) -from typing_extensions import Unpack +from typing_extensions import ( + NotRequired, + Unpack, +) from galaxy.files import OptionalUserContext from . import ( @@ -35,16 +38,20 @@ class S3FsFilesSourceProperties(FilesSourceProperties, total=False): endpoint_url: int user: str passwd: str + listings_expiry_time: NotRequired[Optional[int]] client_kwargs: dict # internally computed. Should not be specified in config file class S3FsFilesSource(BaseFilesSource): plugin_type = "s3fs" - def __init__(self, **kwd: Unpack[S3FsFilesSourceProperties]): + def __init__(self, opts: Optional[FilesSourceOptions] = None, **kwd: Unpack[S3FsFilesSourceProperties]): if s3fs is None: raise Exception("Package s3fs unavailable but required for this file source plugin.") props: S3FsFilesSourceProperties = cast(S3FsFilesSourceProperties, self._parse_common_config_opts(kwd)) + if props.get("listings_expiry_time") is None and opts: + if opts.listings_expiry_time: + props["listings_expiry_time"] = opts.s3fs_listings_expiry_time # There is a possibility that the bucket name could be parameterized: e.g. # bucket: ${user.preferences['generic_s3|bucket']} # that's ok, because we evaluate the bucket name again later. The bucket property here will only diff --git a/lib/galaxy/files/sources/webdav.py b/lib/galaxy/files/sources/webdav.py index 2e11a51d98b2..ef7bc710b350 100644 --- a/lib/galaxy/files/sources/webdav.py +++ b/lib/galaxy/files/sources/webdav.py @@ -24,10 +24,20 @@ class WebDavFilesSource(PyFilesystem2FilesSource): def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None): props = self._serialization_props(user_context) use_temp_files = props.pop("use_temp_files", None) + if use_temp_files is None and opts: + use_temp_files = opts.webdav_use_temp_files if use_temp_files is None: # Default to True to avoid memory issues with large files. - props["use_temp_files"] = True - props["temp_path"] = props.get("temp_path", tempfile.TemporaryDirectory(prefix="webdav_")) + use_temp_files = True + + if use_temp_files: + temp_path = props.get("temp_path") + if temp_path is None and opts: + temp_path = opts.tmp_dir + if temp_path is None: + temp_path = tempfile.TemporaryDirectory(prefix="webdav_") + props["temp_path"] = temp_path + props["use_temp_files"] = use_temp_files extra_props: Union[FilesSourceProperties, dict] = opts.extra_props or {} if opts else {} handle = WebDAVFS(**{**props, **extra_props}) return handle