Merge pull request galaxyproject#18049 from jmchilton/files_typing_3

Refactor galaxy.files plugin loading + config handling.

lib/galaxy/files/__init__.py

@@ -16,9 +16,16 @@
     FilesSourceProperties,
     PluginKind,
 )
-from galaxy.util import plugin_config
-from galaxy.util.config_parsers import parse_allowlist_ips
 from galaxy.util.dictifiable import Dictifiable
+from galaxy.util.plugin_config import (
+    plugin_source_from_dict,
+    plugin_source_from_path,
+    PluginConfigSource,
+)
+from .plugins import (
+    FileSourcePluginLoader,
+    FileSourcePluginsConfig,
+)
 
 log = logging.getLogger(__name__)
 
@@ -44,18 +51,18 @@ class ConfiguredFileSources:
 
     def __init__(
         self,
-        file_sources_config: "ConfiguredFileSourcesConfig",
+        file_sources_config: FileSourcePluginsConfig,
         conf_file=None,
         conf_dict=None,
         load_stock_plugins=False,
     ):
         self._file_sources_config = file_sources_config
-        self._plugin_classes = self._file_source_plugins_dict()
+        self._plugin_loader = FileSourcePluginLoader()
         file_sources: List[BaseFilesSource] = []
         if conf_file is not None:
             file_sources = self._load_plugins_from_file(conf_file)
         elif conf_dict is not None:
-            plugin_source = plugin_config.plugin_source_from_dict(conf_dict)
+            plugin_source = plugin_source_from_dict(conf_dict)
             file_sources = self._parse_plugin_source(plugin_source)
         else:
             file_sources = []
@@ -81,7 +88,7 @@ def _ensure_loaded(plugin_type):
                 _ensure_loaded("gxuserimport")
 
             if stock_file_source_conf_dict:
-                stock_plugin_source = plugin_config.plugin_source_from_dict(stock_file_source_conf_dict)
+                stock_plugin_source = plugin_source_from_dict(stock_file_source_conf_dict)
                 # insert at beginning instead of append so FTP and library import appear
                 # at the top of the list (presumably the most common options). Admins can insert
                 # these explicitly for greater control.
@@ -90,25 +97,12 @@ def _ensure_loaded(plugin_type):
         self._file_sources = file_sources
         self.custom_sources_configured = custom_sources_configured
 
-    def _load_plugins_from_file(self, conf_file):
-        plugin_source = plugin_config.plugin_source_from_path(conf_file)
+    def _load_plugins_from_file(self, conf_file: str):
+        plugin_source = plugin_source_from_path(conf_file)
         return self._parse_plugin_source(plugin_source)
 
-    def _file_source_plugins_dict(self):
-        import galaxy.files.sources
-
-        return plugin_config.plugins_dict(galaxy.files.sources, "plugin_type")
-
-    def _parse_plugin_source(self, plugin_source):
-        extra_kwds = {
-            "file_sources_config": self._file_sources_config,
-        }
-        return plugin_config.load_plugins(
-            self._plugin_classes,
-            plugin_source,
-            extra_kwds,
-            dict_to_list_key="id",
-        )
+    def _parse_plugin_source(self, plugin_source: PluginConfigSource):
+        return self._plugin_loader.load_plugins(plugin_source, self._file_sources_config)
 
     def find_best_match(self, url: str) -> Optional[BaseFilesSource]:
         """Returns the best matching file source for handling a particular url. Each filesource scores its own
@@ -204,7 +198,7 @@ def from_app_config(config):
         if not config_file or not os.path.exists(config_file):
             config_file = None
             config_dict = config.file_sources
-        file_sources_config = ConfiguredFileSourcesConfig.from_app_config(config)
+        file_sources_config = FileSourcePluginsConfig.from_app_config(config)
         return ConfiguredFileSources(
             file_sources_config, conf_file=config_file, conf_dict=config_dict, load_stock_plugins=True
         )
@@ -214,10 +208,10 @@ def from_dict(as_dict, load_stock_plugins=False):
         if as_dict is not None:
             sources_as_dict = as_dict["file_sources"]
             config_as_dict = as_dict["config"]
-            file_sources_config = ConfiguredFileSourcesConfig.from_dict(config_as_dict)
+            file_sources_config = FileSourcePluginsConfig.from_dict(config_as_dict)
         else:
             sources_as_dict = []
-            file_sources_config = ConfiguredFileSourcesConfig()
+            file_sources_config = FileSourcePluginsConfig()
         return ConfiguredFileSources(
             file_sources_config, conf_dict=sources_as_dict, load_stock_plugins=load_stock_plugins
         )
@@ -227,61 +221,7 @@ class NullConfiguredFileSources(ConfiguredFileSources):
     def __init__(
         self,
     ):
-        super().__init__(ConfiguredFileSourcesConfig())
-
-
-class ConfiguredFileSourcesConfig:
-    def __init__(
-        self,
-        symlink_allowlist=None,
-        fetch_url_allowlist=None,
-        library_import_dir=None,
-        user_library_import_dir=None,
-        ftp_upload_dir=None,
-        ftp_upload_purge=True,
-    ):
-        symlink_allowlist = symlink_allowlist or []
-        fetch_url_allowlist = fetch_url_allowlist or []
-        self.symlink_allowlist = symlink_allowlist
-        self.fetch_url_allowlist = fetch_url_allowlist
-        self.library_import_dir = library_import_dir
-        self.user_library_import_dir = user_library_import_dir
-        self.ftp_upload_dir = ftp_upload_dir
-        self.ftp_upload_purge = ftp_upload_purge
-
-    @staticmethod
-    def from_app_config(config):
-        # Formalize what we read in from config to create a more clear interface
-        # for this component.
-        kwds = {}
-        kwds["symlink_allowlist"] = config.user_library_import_symlink_allowlist
-        kwds["fetch_url_allowlist"] = [str(ip) for ip in config.fetch_url_allowlist_ips]
-        kwds["library_import_dir"] = config.library_import_dir
-        kwds["user_library_import_dir"] = config.user_library_import_dir
-        kwds["ftp_upload_dir"] = config.ftp_upload_dir
-        kwds["ftp_upload_purge"] = config.ftp_upload_purge
-        return ConfiguredFileSourcesConfig(**kwds)
-
-    def to_dict(self):
-        return {
-            "symlink_allowlist": self.symlink_allowlist,
-            "fetch_url_allowlist": self.fetch_url_allowlist,
-            "library_import_dir": self.library_import_dir,
-            "user_library_import_dir": self.user_library_import_dir,
-            "ftp_upload_dir": self.ftp_upload_dir,
-            "ftp_upload_purge": self.ftp_upload_purge,
-        }
-
-    @staticmethod
-    def from_dict(as_dict):
-        return ConfiguredFileSourcesConfig(
-            symlink_allowlist=as_dict["symlink_allowlist"],
-            fetch_url_allowlist=parse_allowlist_ips(as_dict["fetch_url_allowlist"]),
-            library_import_dir=as_dict["library_import_dir"],
-            user_library_import_dir=as_dict["user_library_import_dir"],
-            ftp_upload_dir=as_dict["ftp_upload_dir"],
-            ftp_upload_purge=as_dict["ftp_upload_purge"],
-        )
+        super().__init__(FileSourcePluginsConfig())
 
 
 class FileSourceDictifiable(Dictifiable):

lib/galaxy/files/plugins.py

@@ -0,0 +1,106 @@
+from typing import (
+    List,
+    Optional,
+    TYPE_CHECKING,
+)
+
+from galaxy.util.config_parsers import parse_allowlist_ips
+from galaxy.util.plugin_config import (
+    load_plugins,
+    PluginConfigSource,
+    plugins_dict,
+)
+
+if TYPE_CHECKING:
+    from galaxy.files.sources import BaseFilesSource
+
+
+class FileSourcePluginsConfig:
+    symlink_allowlist: List[str]
+    fetch_url_allowlist: List[str]
+    library_import_dir: Optional[str]
+    user_library_import_dir: Optional[str]
+    ftp_upload_dir: Optional[str]
+    ftp_upload_purge: bool
+
+    def __init__(
+        self,
+        symlink_allowlist=None,
+        fetch_url_allowlist=None,
+        library_import_dir=None,
+        user_library_import_dir=None,
+        ftp_upload_dir=None,
+        ftp_upload_purge=True,
+    ):
+        symlink_allowlist = symlink_allowlist or []
+        fetch_url_allowlist = fetch_url_allowlist or []
+        self.symlink_allowlist = symlink_allowlist
+        self.fetch_url_allowlist = fetch_url_allowlist
+        self.library_import_dir = library_import_dir
+        self.user_library_import_dir = user_library_import_dir
+        self.ftp_upload_dir = ftp_upload_dir
+        self.ftp_upload_purge = ftp_upload_purge
+
+    @staticmethod
+    def from_app_config(config):
+        # Formalize what we read in from config to create a more clear interface
+        # for this component.
+        kwds = {}
+        kwds["symlink_allowlist"] = config.user_library_import_symlink_allowlist
+        kwds["fetch_url_allowlist"] = [str(ip) for ip in config.fetch_url_allowlist_ips]
+        kwds["library_import_dir"] = config.library_import_dir
+        kwds["user_library_import_dir"] = config.user_library_import_dir
+        kwds["ftp_upload_dir"] = config.ftp_upload_dir
+        kwds["ftp_upload_purge"] = config.ftp_upload_purge
+        return FileSourcePluginsConfig(**kwds)
+
+    def to_dict(self):
+        return {
+            "symlink_allowlist": self.symlink_allowlist,
+            "fetch_url_allowlist": self.fetch_url_allowlist,
+            "library_import_dir": self.library_import_dir,
+            "user_library_import_dir": self.user_library_import_dir,
+            "ftp_upload_dir": self.ftp_upload_dir,
+            "ftp_upload_purge": self.ftp_upload_purge,
+        }
+
+    @staticmethod
+    def from_dict(as_dict):
+        return FileSourcePluginsConfig(
+            symlink_allowlist=as_dict["symlink_allowlist"],
+            fetch_url_allowlist=parse_allowlist_ips(as_dict["fetch_url_allowlist"]),
+            library_import_dir=as_dict["library_import_dir"],
+            user_library_import_dir=as_dict["user_library_import_dir"],
+            ftp_upload_dir=as_dict["ftp_upload_dir"],
+            ftp_upload_purge=as_dict["ftp_upload_purge"],
+        )
+
+
+class FileSourcePluginLoader:
+
+    def __init__(self):
+        self._plugin_classes = self._file_source_plugins_dict()
+
+    def _file_source_plugins_dict(self):
+        import galaxy.files.sources
+
+        return plugins_dict(galaxy.files.sources, "plugin_type")
+
+    def load_plugins(
+        self, plugin_source: PluginConfigSource, file_source_plugin_config: FileSourcePluginsConfig
+    ) -> List["BaseFilesSource"]:
+        extra_kwds = {
+            "file_sources_config": file_source_plugin_config,
+        }
+        return load_plugins(
+            self._plugin_classes,
+            plugin_source,
+            extra_kwds,
+            dict_to_list_key="id",
+        )
+
+
+__all__ = (
+    "FileSourcePluginLoader",
+    "FileSourcePluginsConfig",
+)

lib/galaxy/files/sources/__init__.py

@@ -11,7 +11,6 @@
     ClassVar,
     Optional,
     Set,
-    TYPE_CHECKING,
 )
 
 from typing_extensions import (
@@ -24,6 +23,7 @@
     ConfigurationError,
     ItemAccessibilityException,
 )
+from galaxy.files.plugins import FileSourcePluginsConfig
 from galaxy.util.bool_expressions import (
     BooleanExpressionEvaluator,
     TokenContainedEvaluator,
@@ -33,9 +33,6 @@
 DEFAULT_SCHEME = "gxfiles"
 DEFAULT_WRITABLE = False
 
-if TYPE_CHECKING:
-    from galaxy.files import ConfiguredFileSourcesConfig
-
 
 class PluginKind(str, Enum):
     """Enum to distinguish between different kinds or categories of plugins."""
@@ -78,7 +75,7 @@ class FilesSourceProperties(TypedDict):
     filesource specific properties.
     """
 
-    file_sources_config: NotRequired["ConfiguredFileSourcesConfig"]
+    file_sources_config: NotRequired[FileSourcePluginsConfig]
     id: NotRequired[str]
     label: NotRequired[str]
     doc: NotRequired[Optional[str]]

lib/galaxy/files/unittest_utils/__init__.py

@@ -2,14 +2,12 @@
 import tempfile
 from typing import Tuple
 
-from galaxy.files import (
-    ConfiguredFileSources,
-    ConfiguredFileSourcesConfig,
-)
+from galaxy.files import ConfiguredFileSources
+from galaxy.files.plugins import FileSourcePluginsConfig
 
 
 class TestConfiguredFileSources(ConfiguredFileSources):
-    def __init__(self, file_sources_config: ConfiguredFileSourcesConfig, conf_dict: dict, test_root: str):
+    def __init__(self, file_sources_config: FileSourcePluginsConfig, conf_dict: dict, test_root: str):
         super().__init__(file_sources_config, conf_dict=conf_dict)
         self.test_root = test_root
 
@@ -22,7 +20,7 @@ def __init__(self, root: str):
             "type": "posix",
             "root": root,
         }
-        file_sources_config = ConfiguredFileSourcesConfig({})
+        file_sources_config = FileSourcePluginsConfig({})
         super().__init__(file_sources_config, {"test1": plugin}, root)
 
 

lib/galaxy_test/api/test_drs.py

@@ -13,9 +13,9 @@
 
 from galaxy.files import (
     ConfiguredFileSources,
-    ConfiguredFileSourcesConfig,
     DictFileSourcesUserContext,
 )
+from galaxy.files.plugins import FileSourcePluginsConfig
 from galaxy.files.sources.util import (
     fetch_drs_to_file,
     RetryOptions,
@@ -33,7 +33,7 @@
 
 
 def user_context_fixture():
-    file_sources_config = ConfiguredFileSourcesConfig(fetch_url_allowlist=parse_allowlist_ips(["127.0.0.0/24"]))
+    file_sources_config = FileSourcePluginsConfig(fetch_url_allowlist=parse_allowlist_ips(["127.0.0.0/24"]))
     file_sources = ConfiguredFileSources(file_sources_config, load_stock_plugins=True)
     user_context = DictFileSourcesUserContext(
         preferences={

test/unit/data/test_sniff_file_sources.py

@@ -1,7 +1,7 @@
 import os
 
 from galaxy.datatypes import sniff
-from galaxy.files import ConfiguredFileSourcesConfig
+from galaxy.files.plugins import FileSourcePluginsConfig
 from galaxy.files.unittest_utils import (
     setup_root,
     TestConfiguredFileSources,
@@ -30,7 +30,7 @@ def _download_and_check_file(file_sources):
 
 def _configured_file_sources() -> TestConfiguredFileSources:
     tmp, root = setup_root()
-    file_sources_config = ConfiguredFileSourcesConfig()
+    file_sources_config = FileSourcePluginsConfig()
     plugin = {
         "type": "posix",
     }

test/unit/files/_util.py

@@ -5,9 +5,9 @@
 
 from galaxy.files import (
     ConfiguredFileSources,
-    ConfiguredFileSourcesConfig,
     DictFileSourcesUserContext,
 )
+from galaxy.files.plugins import FileSourcePluginsConfig
 
 TEST_USERNAME = "alice"
 TEST_EMAIL = "alice@galaxyproject.org"
@@ -126,7 +126,7 @@ def write_from(file_sources, uri, content, user_context=None):
 
 
 def configured_file_sources(conf_file):
-    file_sources_config = ConfiguredFileSourcesConfig()
+    file_sources_config = FileSourcePluginsConfig()
     return ConfiguredFileSources(file_sources_config, conf_file=conf_file)