Merge pull request #1264 from girder/max-annotation-input-file-length

Added a max_annotation_input_file_length setting

CHANGELOG.md

@@ -11,6 +11,7 @@
 - Presets from Large Image Configuration file ([#1248](../../pull/1248), [#1256](../../pull/1256))
 - Reduce memory allocation during some region tiling operations ([#1261](../../pull/1261))
 - Reduce memory allocation during some styling operations ([#1262](../../pull/1262), [#1263](../../pull/1263))
+- Added a max_annotation_input_file_length setting ([#1264](../../pull/1264))
 
 ### Changes
 - Minor code changes based on suggestions from ruff linting ([#1257](../../pull/1257))

docs/config_options.rst

@@ -29,6 +29,8 @@ Configuration parameters:
 
 - ``icc_correction``: If this is True or undefined, ICC color correction will be applied for tile sources that have ICC profile information.  If False, correction will not be applied.  If the style used to open a tilesource specifies ICC correction explicitly (on or off), then this setting is not used.  This may also be a string with one of the intents defined by the PIL.ImageCms.Intents enum.  ``True`` is the same as ``perceptual``.
 
+- ``max_annotation_input_file_length``: When an annotation file is uploaded through Girder, it is loaded into memory, validated, and then added to the database.  This is the maximum number of bytes that will be read directly.  Files larger than this are ignored.  If unspecified, this defaults to the larger of 1 GByte and 1/16th of the system virtual memory.
+
 
 Configuration from Python
 -------------------------
@@ -63,6 +65,9 @@ For the Girder plugin, these can also be set in the ``girder.cfg`` file in a ``l
   # The bioformats tilesource won't read files that end in a comma-separated
   # list of extensions
   source_bioformats_ignored_names = r'(^[!.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi))$'
+  # The maximum size of an annotation file that will be ingested into girder
+  # via direct load
+  max_annotation_input_file_length = 1 * 1024 ** 3
 
 Logging from Python
 -------------------

girder_annotation/girder_large_image_annotation/handlers.py

@@ -5,6 +5,7 @@
 import cachetools
 import orjson
 
+import large_image.config
 from girder import logger
 from girder.constants import AccessType
 from girder.models.file import File
@@ -121,8 +122,11 @@ def process_annotations(event):  # noqa: C901
         logger.error('Could not load models from the database')
         return
     try:
-        if file['size'] > 1 * 1024 ** 3:
-            msg = 'File is larger than will be read into memory.'
+        if file['size'] > int(large_image.config.getConfig(
+                'max_annotation_input_file_length', 1024 ** 3)):
+            msg = ('File is larger than will be read into memory.  If your '
+                   'server will permit it, increase the '
+                   'max_annotation_input_file_length setting.')
             raise Exception(msg)
         data = []
         with File().open(file) as fptr:

large_image/config.py

@@ -1,5 +1,10 @@
 import logging
 
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
 # Default logger
 fallbackLogger = logging.getLogger('large_image')
 fallbackLogger.setLevel(logging.INFO)
@@ -33,6 +38,11 @@
 
     # Should ICC color correction be applied by default
     'icc_correction': True,
+
+    # The maximum size of an annotation file that will be ingested into girder
+    # via direct load
+    'max_annotation_input_file_length': 1 * 1024 ** 3 if not psutil else max(
+        1 * 1024 ** 3, psutil.virtual_memory().total // 16),
 }
 
 

test/test_files/sample.girder.cfg

@@ -17,3 +17,6 @@ max_small_image_size = 4096
 # The bioformats tilesource won't read files that end in a comma-separated
 # list of extensions
 source_bioformats_ignored_names = r'(^[!.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi))$'
+# The maximum size of an annotation file that will be ingested into girder
+# via direct load
+max_annotation_input_file_length = 1 * 1024 ** 3