Add HttpRequest storage class

mandible/metadata_mapper/storage/__init__.py

@@ -8,9 +8,16 @@
     StorageError,
 )
 
+try:
+    from .http_request import HttpRequest
+except ImportError:
+    from .storage import HttpRequest
+
+
 __all__ = (
     "Dummy",
     "FilteredStorage",
+    "HttpRequest",
     "LocalFile",
     "S3File",
     "STORAGE_REGISTRY",

mandible/metadata_mapper/storage/http_request.py

@@ -0,0 +1,48 @@
+import io
+from dataclasses import dataclass
+from typing import IO, Any, Optional, Union
+
+import requests
+
+from mandible.metadata_mapper.context import Context
+
+from .storage import Storage
+
+# TODO(reweeden): drop python 3.8 support
+
+
+@dataclass
+class HttpRequest(Storage):
+    """A storage which returns the body of an HTTP response"""
+
+    url: str
+    method: str = "GET"
+    params: Optional[dict] = None
+    data: Optional[Union[dict, bytes]] = None
+    json: Optional[Any] = None
+    headers: Optional[dict] = None
+    cookies: Optional[dict] = None
+    timeout: Optional[Union[float, tuple[float, float]]] = None
+    allow_redirects: bool = True
+
+    def open_file(self, context: Context) -> IO[bytes]:
+        response = requests.request(
+            self.method,
+            self.url,
+            params=self.params,
+            data=self.data,
+            json=self.json,
+            headers=self.headers,
+            cookies=self.cookies,
+            timeout=self.timeout,
+            allow_redirects=self.allow_redirects,
+            stream=True,
+        )
+
+        # TODO(reweeden): Using response.content causes the entire response
+        # payload to be loaded into memory immediately. Ideally, we would
+        # optimize here by returning a file-like object that could load the
+        # response lazily. Requests does provide a response.raw file-like
+        # object, however, this doesn't preform the content decoding that you
+        # get when using response.content.
+        return io.BytesIO(response.content)

mandible/metadata_mapper/storage/storage.py

@@ -31,6 +31,33 @@ def open_file(self, context: Context) -> IO[bytes]:
         pass
 
 
+# Define placeholders for when extras are not installed
+
+
+@dataclass
+class _PlaceholderBase(Storage, register=False):
+    """
+    Base class for defining placeholder implementations for classes that
+    require extra dependencies to be installed
+    """
+    def __init__(self, dep: str):
+        raise Exception(
+            f"{dep} must be installed to use the {self.__class__.__name__} "
+            "format class"
+        )
+
+    def open_file(self, context: Context) -> IO[bytes]:
+        pass
+
+
+@dataclass
+class HttpRequest(_PlaceholderBase):
+    def __init__(self):
+        super().__init__("requests")
+
+
+# Define storages that don't require extra dependencies
+
 @dataclass
 class Dummy(Storage):
     """A dummy storage that returns a hardcoded byte stream.

pyproject.toml

@@ -24,10 +24,12 @@ lxml = { version = "^4.9.2", optional = true }
 # Numpy is pinned to a minimum version by h5py. Unpinning here means our
 # requirements will always match those of h5py.
 numpy = { version = "*", optional = true }
+requests = { version = "^2.32.3", optional = true }
 
 [tool.poetry.extras]
-all = ["h5py", "numpy", "jsonpath-ng", "lxml"]
+all = ["h5py", "numpy", "requests", "jsonpath-ng", "lxml"]
 h5 = ["h5py", "numpy"]
+http = ["requests"]
 jsonpath = ["jsonpath-ng"]
 xml = ["lxml"]
 
@@ -45,6 +47,7 @@ markers = [
     "h5: requires the 'h5' extra to be installed",
     "jsonpath: requires the 'jsonpath' extra to be installed",
     "xml: requires the 'xml' extra to be installed",
+    "http: requires the 'http' extra to be installed",
 ]
 
 [tool.isort]

tests/test_storage.py

@@ -7,6 +7,7 @@
 from mandible.metadata_mapper.storage import (
     STORAGE_REGISTRY,
     Dummy,
+    HttpRequest,
     LocalFile,
     S3File,
     Storage,
@@ -17,6 +18,7 @@
 def test_registry():
     assert STORAGE_REGISTRY == {
         "Dummy": Dummy,
+        "HttpRequest": HttpRequest,
         "LocalFile": LocalFile,
         "S3File": S3File,
     }

tox.ini

@@ -29,5 +29,5 @@ extras =
     Xnone:
     Xall: all
 commands =
-    Xnone: pytest tests/ -m "not h5 and not xml and not jsonpath" {posargs}
+    Xnone: pytest tests/ -m "not h5 and not xml and not jsonpath and not http" {posargs}
     Xall: pytest tests/ {posargs}