chore: implement reader for redis

ckanext/files/base.py

@@ -23,14 +23,15 @@
 storages = utils.Registry({})
 
 
-def storage_from_settings(settings):
-    # type: (dict[str, types.Any]) -> Storage
+def storage_from_settings(name, settings):
+    # type: (str, dict[str, types.Any]) -> Storage
 
     adapter_type = settings.pop("type", None)
     adapter = adapters.get(adapter_type)  # type: type[Storage] | None
     if not adapter:
         raise exceptions.UnknownAdapterError(adapter_type)
 
+    settings.setdefault("name", name)
     return adapter(**settings)
 
 
@@ -57,6 +58,7 @@ def __iter__(self):
         while True:
             chunk = self.stream.read(self.chunk_size)
             if not chunk:
+                yield chunk
                 break
             self.position += len(chunk)
             self.hashsum.update(chunk)
@@ -161,6 +163,10 @@ def stream(self, data):
         # type: (dict[str, types.Any]) -> types.IO[str] | types.IO[bytes]
         raise NotImplementedError
 
+    def content(self, data):
+        # type: (dict[str, types.Any]) -> bytes | str
+        return self.stream(data).read()
+
 
 class Storage(OptionChecker):
     __metaclass__ = abc.ABCMeta
@@ -195,6 +201,9 @@ def declare_config_options(cls, declaration, key):
             + "\nSupports size suffixes: 42B, 2M, 24KiB, 1GB."
             + " `0` means no restrictions.",
         )
+        declaration.declare(key.name, key[-1]).set_description(
+            "Descriptive name of the storage used for debugging.",
+        )
 
     def compute_capabilities(self):
         # type: () -> types.CapabilityCluster
@@ -252,3 +261,17 @@ def remove(self, data):
             raise exceptions.UnsupportedOperationError("remove", type(self).__name__)
 
         return self.manager.remove(data)
+
+    def stream(self, data):
+        # type: (dict[str, types.Any]) -> types.IO[bytes] | types.IO[str]
+        if not self.supports(Capability.STREAM):
+            raise exceptions.UnsupportedOperationError("stream", type(self).__name__)
+
+        return self.reader.stream(data)
+
+    def content(self, data):
+        # type: (dict[str, types.Any]) -> bytes | str
+        if not self.supports(Capability.STREAM):
+            raise exceptions.UnsupportedOperationError("content", type(self).__name__)
+
+        return self.reader.content(data)

ckanext/files/exceptions.py

@@ -21,7 +21,7 @@ class StorageError(FilesError):
     pass
 
 
-class UploadError(FilesError):
+class UploadError(StorageError):
     pass
 
 
@@ -122,3 +122,18 @@ def __init__(self, strategy):
 
     def __str__(self):
         return "Unknown name strategy {}".format(self.strategy)
+
+
+class MissingFileError(StorageError):
+    """File does not exist."""
+
+    def __init__(self, storage, filename):
+        # type: (str, str) -> None
+        self.storage = storage
+        self.filename = filename
+
+    def __str__(self):
+        return "File {} does not exist inside {} storage".format(
+            self.filename,
+            self.storage,
+        )

ckanext/files/logic/action.py

@@ -1,4 +1,3 @@
-import six
 from werkzeug.utils import secure_filename
 
 import ckan.plugins.toolkit as tk
@@ -11,8 +10,8 @@
 
 from . import schema
 
-if six.PY3:
-    from typing import Any  # isort: skip # noqa: F401
+from ckanext.files import types  # isort: skip # noqa: F401
+
 
 _actions, action = make_collector()
 
@@ -24,7 +23,7 @@ def get_actions():
 @action
 @validate(schema.file_create)
 def files_file_create(context, data_dict):
-    # type: (Any, dict[str, Any]) -> dict[str, Any]
+    # type: (types.Any, dict[str, types.Any]) -> dict[str, types.Any]
     tk.check_access("files_file_create", context, data_dict)
     _ensure_name(data_dict)
 
@@ -54,7 +53,7 @@ def files_file_create(context, data_dict):
 
 
 def _ensure_name(data_dict, name_field="name", upload_field="upload"):
-    # type: (dict[str, Any], str, str) -> None
+    # type: (dict[str, types.Any], str, str) -> None
     if name_field in data_dict:
         return
     name = data_dict[upload_field].filename
@@ -74,7 +73,7 @@ def _ensure_name(data_dict, name_field="name", upload_field="upload"):
 @action
 @validate(schema.file_delete)
 def files_file_delete(context, data_dict):
-    # type: (Any, dict[str, Any]) -> bool
+    # type: (types.Any, dict[str, types.Any]) -> bool
     tk.check_access("files_file_delete", context, data_dict)
 
     data_dict["id"]
@@ -97,7 +96,7 @@ def files_file_delete(context, data_dict):
 @action
 @validate(schema.file_show)
 def files_file_show(context, data_dict):
-    # type: (Any, dict[str, Any]) -> dict[str, Any]
+    # type: (types.Any, dict[str, types.Any]) -> dict[str, types.Any]
     tk.check_access("files_file_show", context, data_dict)
 
     data_dict["id"]
@@ -116,7 +115,7 @@ def files_file_show(context, data_dict):
 @action
 @validate(schema.upload_initialize)
 def files_upload_initialize(context, data_dict):
-    # type: (Any, dict[str, Any]) -> dict[str, Any]
+    # type: (types.Any, dict[str, types.Any]) -> dict[str, types.Any]
     tk.check_access("files_upload_initialize", context, data_dict)
     _ensure_name(data_dict)
     extras = data_dict.get("__extras", {})
@@ -151,7 +150,7 @@ def files_upload_initialize(context, data_dict):
 @action
 @validate(schema.upload_show)
 def files_upload_show(context, data_dict):
-    # type: (Any, dict[str, Any]) -> dict[str, Any]
+    # type: (types.Any, dict[str, types.Any]) -> dict[str, types.Any]
     tk.check_access("files_upload_show", context, data_dict)
 
     upload = context["session"].get(Upload, data_dict["id"])
@@ -168,7 +167,7 @@ def files_upload_show(context, data_dict):
 @action
 @validate(schema.upload_update)
 def files_upload_update(context, data_dict):
-    # type: (Any, dict[str, Any]) -> dict[str, Any]
+    # type: (types.Any, dict[str, types.Any]) -> dict[str, types.Any]
     tk.check_access("files_upload_update", context, data_dict)
 
     extras = data_dict.get("__extras", {})
@@ -189,7 +188,7 @@ def files_upload_update(context, data_dict):
 @action
 @validate(schema.upload_complete)
 def files_upload_complete(context, data_dict):
-    # type: (Any, dict[str, Any]) -> dict[str, Any]
+    # type: (types.Any, dict[str, types.Any]) -> dict[str, types.Any]
     tk.check_access("files_upload_complete", context, data_dict)
 
     extras = data_dict.get("__extras", {})

ckanext/files/logic/auth.py

@@ -1,11 +1,8 @@
-import six
-
 from ckan import authz
 
 from ckanext.files.utils import make_collector
 
-if six.PY3:
-    from typing import Any  # isort: skip # noqa: F401
+from ckanext.files import types  # isort: skip # noqa: F401
 
 
 _auth_functions, auth = make_collector()
@@ -17,47 +14,47 @@ def get_auth_functions():
 
 @auth
 def files_manage_files(context, data_dict):
-    # type: (Any, dict[str, Any]) -> Any
+    # type: (types.Any, dict[str, types.Any]) -> types.Any
     return {"success": False}
 
 
 @auth
 def files_file_create(context, data_dict):
-    # type: (Any, dict[str, Any]) -> Any
+    # type: (types.Any, dict[str, types.Any]) -> types.Any
     return authz.is_authorized("files_manage_files", context, data_dict)
 
 
 @auth
 def files_file_delete(context, data_dict):
-    # type: (Any, dict[str, Any]) -> Any
+    # type: (types.Any, dict[str, types.Any]) -> types.Any
     return authz.is_authorized("files_manage_files", context, data_dict)
 
 
 @auth
 def files_file_show(context, data_dict):
-    # type: (Any, dict[str, Any]) -> Any
+    # type: (types.Any, dict[str, types.Any]) -> types.Any
     return authz.is_authorized("files_manage_files", context, data_dict)
 
 
 @auth
 def files_upload_show(context, data_dict):
-    # type: (Any, dict[str, Any]) -> Any
+    # type: (types.Any, dict[str, types.Any]) -> types.Any
     return authz.is_authorized("files_manage_files", context, data_dict)
 
 
 @auth
 def files_upload_initialize(context, data_dict):
-    # type: (Any, dict[str, Any]) -> Any
+    # type: (types.Any, dict[str, types.Any]) -> types.Any
     return authz.is_authorized("files_manage_files", context, data_dict)
 
 
 @auth
 def files_upload_update(context, data_dict):
-    # type: (Any, dict[str, Any]) -> Any
+    # type: (types.Any, dict[str, types.Any]) -> types.Any
     return authz.is_authorized("files_manage_files", context, data_dict)
 
 
 @auth
 def files_upload_complete(context, data_dict):
-    # type: (Any, dict[str, Any]) -> Any
+    # type: (types.Any, dict[str, types.Any]) -> types.Any
     return authz.is_authorized("files_manage_files", context, data_dict)

ckanext/files/logic/schema.py

@@ -1,16 +1,13 @@
-import six
-
 from ckan.logic.schema import validator_args
 
 from ckanext.files import config
 
-if six.PY3:
-    from typing import Any  # isort: skip # noqa: F401
+from ckanext.files import types  # isort: skip # noqa: F401
 
 
 @validator_args
 def file_create(ignore_empty, unicode_safe, default, files_into_upload, not_missing):
-    # type: (Any, Any, Any, Any, Any) -> Any
+    # type: (types.Any, types.Any, types.Any, types.Any, types.Any) -> types.Any
 
     # name is checked inside action, using "upload" as source if empty
     return {
@@ -22,23 +19,23 @@ def file_create(ignore_empty, unicode_safe, default, files_into_upload, not_miss
 
 @validator_args
 def file_delete(not_empty, unicode_safe):
-    # type: (Any, Any) -> Any
+    # type: (types.Any, types.Any) -> types.Any
     return {
         "id": [not_empty, unicode_safe],
     }
 
 
 @validator_args
 def file_show(not_empty, unicode_safe):
-    # type: (Any, Any) -> Any
+    # type: (types.Any, types.Any) -> types.Any
     return {
         "id": [not_empty, unicode_safe],
     }
 
 
 @validator_args
 def upload_initialize(ignore_empty, unicode_safe, default, int_validator, not_missing):
-    # type: (Any, Any, Any, Any, Any) -> Any
+    # type: (types.Any, types.Any, types.Any, types.Any, types.Any) -> types.Any
 
     # name is checked inside action, using "upload" as source if empty
     return {
@@ -49,23 +46,23 @@ def upload_initialize(ignore_empty, unicode_safe, default, int_validator, not_mi
 
 @validator_args
 def upload_show(not_empty, unicode_safe):
-    # type: (Any, Any) -> Any
+    # type: (types.Any, types.Any) -> types.Any
     return {
         "id": [not_empty, unicode_safe],
     }
 
 
 @validator_args
 def upload_update(not_empty, unicode_safe):
-    # type: (Any, Any) -> Any
+    # type: (types.Any, types.Any) -> types.Any
     return {
         "id": [not_empty, unicode_safe],
     }
 
 
 @validator_args
 def upload_complete(not_empty, unicode_safe):
-    # type: (Any, Any) -> Any
+    # type: (types.Any, types.Any) -> types.Any
     return {
         "id": [not_empty, unicode_safe],
     }

ckanext/files/logic/validators.py

@@ -25,7 +25,7 @@ def get_validators():
 @validator
 def files_into_upload(value):
     # type: (Any) -> types.Upload
-    """Try converting value into werkzeug.FileStorage object"""
+    """Convert value into werkzeug.FileStorage object"""
     if isinstance(value, FileStorage):
         if not value.content_length:
             value.headers["content-length"] = str(value.stream.seek(0, 2))
@@ -68,7 +68,12 @@ def files_into_upload(value):
 @validator
 def files_parse_filesize(value):
     # type: (Any) -> int
+    """Convert human-readable filesize into an integer."""
+
     if isinstance(value, int):
         return value
 
-    return utils.parse_filesize(value)
+    try:
+        return utils.parse_filesize(value)
+    except ValueError:
+        raise tk.Invalid("Wrong filesize string: {}".format(value))  # noqa: B904

ckanext/files/plugin.py

@@ -1,3 +1,4 @@
+import json
 import os
 
 import ckan.plugins as p
@@ -44,13 +45,25 @@ def declare_config_options(self, declaration, key):
 
             _register_adapters()
             for name, settings in config.storages().items():
+                storage_key = key.from_string(config.STORAGE_PREFIX + name)
+
+                if tk.check_ckan_version("2.10.3"):
+                    available_adapters = json.dumps(
+                        list(base.adapters),
+                        separators=(",", ":"),
+                    )
+
+                    declaration.declare(storage_key.type).append_validators(
+                        "one_of({})".format(available_adapters),
+                    )
+
                 adapter = base.adapters.get(settings.get("type"))
                 if not adapter:
                     continue
 
                 adapter.declare_config_options(
                     declaration,
-                    key.from_string(config.STORAGE_PREFIX + name),
+                    storage_key,
                 )
 
     # IFiles
@@ -123,7 +136,7 @@ def _initialize_storages():
     base.storages.reset()
     for name, settings in config.storages().items():
         try:
-            storage = base.storage_from_settings(settings)
+            storage = base.storage_from_settings(name, settings)
         except exceptions.UnknownAdapterError as err:
             raise CkanConfigurationException(str(err))  # noqa: B904