Harmonize the girder code paths for image conversion

CHANGELOG.md

@@ -4,7 +4,8 @@
 
 ### Improvements
 - Allow editing metadata in item lists ([#1235](../../pull/1235))
-- Frame selector hotkeys for channel or bands  ([#1233](../../pull/1233))
+- Frame selector hotkeys for channel or bands ([#1233](../../pull/1233))
+- More consistency in how local and remote image conversions are performed in Girder ([#1238](../../pull/1238))
 
 ## 1.23.1
 

girder/girder_large_image/__init__.py

@@ -95,6 +95,11 @@ def _postUpload(event):
         if fileObj['name'].endswith('.geo.tiff'):
             item['largeImage']['sourceName'] = 'gdal'
         Item().save(item)
+        # If the job looks finished, update it once more to force notifications
+        if 'jobId' in item['largeImage'] and item['largeImage'].get('notify'):
+            job = Job().load(item['largeImage']['jobId'], force=True)
+            if job and job['status'] == JobStatus.SUCCESS:
+                Job().save(job)
 
 
 def _updateJob(event):

girder/girder_large_image/models/image_item.py

@@ -58,7 +58,7 @@ def initialize(self):
         ])
 
     def createImageItem(self, item, fileObj, user=None, token=None,
-                        createJob=True, notify=False, **kwargs):
+                        createJob=True, notify=False, localJob=None, **kwargs):
         logger.info('createImageItem called on item %s (%s)', item['_id'], item['name'])
         # Using setdefault ensures that 'largeImage' is in the item
         if 'fileId' in item.setdefault('largeImage', {}):
@@ -97,7 +97,10 @@ def createImageItem(self, item, fileObj, user=None, token=None,
                 item['_id'], item['name'])
             # No source was successful
             del item['largeImage']['fileId']
-            job = self._createLargeImageJob(item, fileObj, user, token, **kwargs)
+            if not localJob:
+                job = self._createLargeImageJob(item, fileObj, user, token, **kwargs)
+            else:
+                job = self._createLargeImageLocalJob(item, fileObj, user, **kwargs)
             item['largeImage']['expected'] = True
             item['largeImage']['notify'] = notify
             item['largeImage']['originalId'] = fileObj['_id']
@@ -108,18 +111,20 @@ def createImageItem(self, item, fileObj, user=None, token=None,
         self.save(item)
         return job
 
-    def _createLargeImageJob(self, item, fileObj, user, token, **kwargs):
+    def _createLargeImageJob(
+            self, item, fileObj, user, token, toFolder=False, folderId=None, name=None, **kwargs):
         import large_image_tasks.tasks
         from girder_worker_utils.transforms.common import TemporaryDirectory
         from girder_worker_utils.transforms.contrib.girder_io import GirderFileIdAllowDirect
-        from girder_worker_utils.transforms.girder_io import GirderUploadToItem
+        from girder_worker_utils.transforms.girder_io import (GirderUploadToFolder,
+                                                              GirderUploadToItem)
 
         try:
             localPath = File().getLocalFilePath(fileObj)
         except (FilePathException, AttributeError):
             localPath = None
         job = large_image_tasks.tasks.create_tiff.apply_async(kwargs=dict(
-            girder_job_title='TIFF Conversion: %s' % fileObj['name'],
+            girder_job_title='Large Image Conversion: %s' % fileObj['name'],
             girder_job_other_fields={'meta': {
                 'creator': 'large_image',
                 'itemId': str(item['_id']),
@@ -129,68 +134,52 @@ def _createLargeImageJob(self, item, fileObj, user, token, **kwargs):
             inputName=fileObj['name'],
             outputDir=TemporaryDirectory(),
             girder_result_hooks=[
-                GirderUploadToItem(str(item['_id']), False),
+                GirderUploadToItem(str(item['_id']), False)
+                if not toFolder else
+                GirderUploadToFolder(
+                    str(folderId or item['folderId']),
+                    upload_kwargs=dict(filename=name),
+                ),
             ],
             **kwargs,
         ), countdown=int(kwargs['countdown']) if kwargs.get('countdown') else None)
         return job.job
 
-    def convertImage(self, item, fileObj, user=None, token=None, localJob=True, **kwargs):
-        if fileObj['itemId'] != item['_id']:
-            raise TileGeneralError(
-                'The provided file must be in the provided item.')
-        if not localJob:
-            return self._convertImageViaWorker(item, fileObj, user, token, **kwargs)
-        # local job
+    def _createLargeImageLocalJob(
+            self, item, fileObj, user=None, toFolder=False, folderId=None, name=None, **kwargs):
         job = Job().createLocalJob(
             module='large_image_tasks.tasks',
             function='convert_image_job',
             kwargs={
                 'itemId': str(item['_id']),
                 'fileId': str(fileObj['_id']),
                 'userId': str(user['_id']) if user else None,
+                'toFolder': toFolder,
                 **kwargs,
             },
-            title='Convert a file to a large image file.',
-            type='large_image_convert_image',
+            title='Large Image Conversion: %s' % fileObj['name'],
+            type='large_image_tiff',
             user=user,
             public=True,
             asynchronous=True,
         )
+        # For consistency with the non-local job
+        job['meta'] = {
+            'creator': 'large_image',
+            'itemId': str(item['_id']),
+            'task': 'createImageItem',
+        }
+        job = Job().save(job)
         Job().scheduleJob(job)
         return job
 
-    def _convertImageViaWorker(
-            self, item, fileObj, user=None, token=None, folderId=None,
-            name=None, **kwargs):
-        import large_image_tasks.tasks
-        from girder_worker_utils.transforms.common import TemporaryDirectory
-        from girder_worker_utils.transforms.contrib.girder_io import GirderFileIdAllowDirect
-        from girder_worker_utils.transforms.girder_io import GirderUploadToFolder
-
-        try:
-            localPath = File().getLocalFilePath(fileObj)
-        except (FilePathException, AttributeError):
-            localPath = None
-        job = large_image_tasks.tasks.create_tiff.apply_async(kwargs=dict(
-            girder_job_title='TIFF Conversion: %s' % fileObj['name'],
-            girder_job_other_fields={'meta': {
-                'creator': 'large_image',
-                'itemId': str(item['_id']),
-                'task': 'convertImage',
-            }},
-            inputFile=GirderFileIdAllowDirect(str(fileObj['_id']), fileObj['name'], localPath),
-            inputName=fileObj['name'],
-            outputDir=TemporaryDirectory(),
-            girder_result_hooks=[
-                GirderUploadToFolder(
-                    str(folderId or item['folderId']),
-                    upload_kwargs=dict(filename=name),
-                ),
-            ],
-            **kwargs,
-        ), countdown=int(kwargs['countdown']) if kwargs.get('countdown') else None)
-        return job.job
+    def convertImage(self, item, fileObj, user=None, token=None, localJob=True, **kwargs):
+        if fileObj['itemId'] != item['_id']:
+            raise TileGeneralError(
+                'The provided file must be in the provided item.')
+        if not localJob:
+            return self._createLargeImageJob(item, fileObj, user, token, toFolder=True, **kwargs)
+        return self._createLargeImageLocalJob(item, fileObj, user, toFolder=True, **kwargs)
 
     @classmethod
     def _tileFromHash(cls, item, x, y, z, mayRedirect=False, **kwargs):

girder/girder_large_image/rest/tiles.py

@@ -198,6 +198,8 @@ def __init__(self, apiRoot):
         .param('notify', 'If a job is required to create the large image, '
                'a nofication can be sent when it is complete.',
                dataType='boolean', default=True, required=False)
+        .param('localJob', 'If true, run as a local job; if false, run via '
+               'the remote worker', dataType='boolean', required=False)
         .param('tileSize', 'Tile size', dataType='int', default=256,
                required=False)
         .param('compression', 'Internal compression format', required=False,
@@ -215,9 +217,9 @@ def __init__(self, apiRoot):
         .param('cr', 'JP2K target compression ratio where 1 is lossless',
                dataType='int', required=False)
         .param('concurrent', 'Suggested number of maximum concurrent '
-               'processes to use during conversion.  Values <= 0 use the '
-               'number of logical cpus less that value.  Default is -2.',
-               dataType='int', required=False)
+               'processes to use during conversion.  Values less than or '
+               'equal to 0 use the number of logical cpus less that value.  '
+               'Default is -2.', dataType='int', required=False)
     )
     @access.user(scope=TokenScope.DATA_WRITE)
     @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)
@@ -237,11 +239,14 @@ def createTiles(self, item, params):
         token = self.getCurrentToken()
         notify = self.boolParam('notify', params, default=True)
         params.pop('notify', None)
+        localJob = self.boolParam('localJob', params, default=None)
+        params.pop('localJob', None)
         try:
             return self.imageItemModel.createImageItem(
                 item, largeImageFile, user, token,
                 createJob='always' if self.boolParam('force', params, default=False) else True,
                 notify=notify,
+                localJob=localJob,
                 **params)
         except TileGeneralError as e:
             raise RestException(e.args[0])
@@ -256,8 +261,7 @@ def createTiles(self, item, params):
         .param('folderId', 'The destination folder.', required=False)
         .param('name', 'A new name for the output item.', required=False)
         .param('localJob', 'If true, run as a local job; if false, run via '
-               'the remote worker', dataType='boolean', default=True,
-               required=False)
+               'the remote worker', dataType='boolean', required=False)
         .param('tileSize', 'Tile size', dataType='int', default=256,
                required=False)
         .param('onlyFrame', 'Only convert a specific 0-based frame of a '
@@ -280,9 +284,9 @@ def createTiles(self, item, params):
         .param('cr', 'JP2K target compression ratio where 1 is lossless',
                dataType='int', required=False)
         .param('concurrent', 'Suggested number of maximum concurrent '
-               'processes to use during conversion.  Values <= 0 use the '
-               'number of logical cpus less that value.  Default is -2.',
-               dataType='int', required=False)
+               'processes to use during conversion.  Values less than or '
+               'equal to 0 use the number of logical cpus less that value.  '
+               'Default is -2.', dataType='int', required=False)
     )
     @access.user(scope=TokenScope.DATA_WRITE)
     @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)

girder/girder_large_image/web_client/views/fileList.js

@@ -53,7 +53,7 @@ wrap(FileListWidget, 'render', function (render) {
         restRequest({
             type: 'POST',
             url: 'item/' + this.parentItem.id + '/tiles',
-            data: {fileId: fileId, notify: true, force: !!(e.originalEvent || {}).ctrlKey},
+            data: {fileId: fileId, notify: true, force: !!(e.originalEvent || {}).ctrlKey, localJob: true},
             error: function (error) {
                 if (error.status !== 0) {
                     events.trigger('g:alert', {

girder_annotation/girder_large_image_annotation/web_client/views/annotationListWidget.js

@@ -59,7 +59,7 @@ const AnnotationListWidget = View.extend({
         }).done((createResp) => {
             this.createResp = createResp;
             largeImageConfig.getConfigFile(this.model.get('folderId')).done((val) => {
-                this._liconfig = val;
+                this._liconfig = val || {};
                 this._confList = this._liconfig.annotationList || {
                     columns: [{
                         type: 'record',

utilities/tasks/large_image_tasks/tasks.py

@@ -101,12 +101,18 @@ def convert_image_job(job):
     from girder.models.user import User
 
     kwargs = job['kwargs']
+    toFolder = kwargs.pop('toFolder', True)
     item = Item().load(kwargs.pop('itemId'), force=True)
     fileObj = File().load(kwargs.pop('fileId'), force=True)
     userId = kwargs.pop('userId', None)
     user = User().load(userId, force=True) if userId else None
-    folder = Folder().load(kwargs.pop('folderId', item['folderId']),
-                           user=user, level=AccessType.WRITE)
+    if toFolder:
+        parentType = 'folder'
+        parent = Folder().load(kwargs.pop('folderId', item['folderId']),
+                               user=user, level=AccessType.WRITE)
+    else:
+        parentType = 'item'
+        parent = item
     name = kwargs.pop('name', None)
 
     job = Job().updateJob(
@@ -138,8 +144,8 @@ def convert_image_job(job):
                     fobj,
                     size=os.path.getsize(dest),
                     name=name or os.path.basename(dest),
-                    parentType='folder',
-                    parent=folder,
+                    parentType=parentType,
+                    parent=parent,
                     user=user,
                 )
                 job = Job().load(job['_id'], force=True)