diff --git a/CHANGELOG.md b/CHANGELOG.md index ee9a4f528..132fad808 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Reduce updates when showing item lists; add a waiting spinner ([#1653](../../pull/1653)) - Update item lists check for large images when toggling recurse ([#1654](../../pull/1654)) - Support named item lists ([#1665](../../pull/1665)) +- Add options to group within item lists ([#1666](../../pull/1666)) ### Changes diff --git a/docs/conf.py b/docs/conf.py index dfe19fc15..6e592799c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,8 +13,6 @@ import os import sys -import sphinx_rtd_theme - docs_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.abspath(os.path.join(docs_dir, '..', '..'))) @@ -62,7 +60,6 @@ # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] pygments_style = 'sphinx' diff --git a/docs/girder_config_options.rst b/docs/girder_config_options.rst index 720aa56a0..5c113a227 100644 --- a/docs/girder_config_options.rst +++ b/docs/girder_config_options.rst @@ -61,11 +61,35 @@ This is used to specify how items appear in item lists. There are two settings, itemList: # layout does not need to be specified. layout: + # The default list (with flatten: false) shows only the items in the + # current folder; flattening the list shows items in the current folder + # and all subfolders. This can also be "only", in which case the + # flatten option will start enabled and, when flattened, the folder + # list will be hidden. + flatten: true # The default layout is a list. This can optionally be "grid" mode: grid # max-width is only used in grid mode. It is the maximum width in # pixels for grid entries. It defaults to 250. max-width: 250 + # group does not need to be specified. Instead of listing items + # directly, multiple items can be grouped together. + group: + # keys is a single metadata value reference (see the column metadata + # records), or a list of such records. + keys: dicom.PatientID + # counts is optional. If specified, the left side is either a metadata + # value references or "_id" to just count total items. The right side + # is where, conceptually, the count is stored in the item.meta record. + # to show a column of the counts, add a metadata column with a value + # equal to this. That is, in this example, all items with the same + # meta.dicom.PatientID are grouped as a single row, and two count + # columns are generated. The unique values for each group row of + # meta.dicom.StudyInstanceUID and counted and that count is added to + # meta._count.studiescount. + counts: + dicom.StudyInstanceUID: _count.studiescount + dicom.SeriesInstanceUID: _count.seriescount # show these columns in order from left to right. Each column has a # "type" and "value". It optionally has a "title" used for the column # header, and a "format" used for searching and filtering. The "label", diff --git a/girder/girder_large_image/rest/__init__.py b/girder/girder_large_image/rest/__init__.py index f06174673..05ebfd072 100644 --- a/girder/girder_large_image/rest/__init__.py +++ b/girder/girder_large_image/rest/__init__.py @@ -1,3 +1,4 @@ +import collections import json from girder import logger @@ -9,7 +10,7 @@ from girder.models.item import Item -def addSystemEndpoints(apiRoot): +def addSystemEndpoints(apiRoot): # noqa """ This adds endpoints to routes that already exist in Girder. @@ -29,6 +30,9 @@ def altItemFind(self, folderId, text, name, limit, offset, sort, filters=None): if text and text.startswith('_recurse_:'): recurse = True text = text.split('_recurse_:', 1)[1] + group = None + if text and text.startswith('_group_:') and len(text.split(':', 2)) >= 3: + _, group, text = text.split(':', 2) if filters is None and text and text.startswith('_filter_:'): try: filters = json.loads(text.split('_filter_:', 1)[1].strip()) @@ -40,9 +44,10 @@ def altItemFind(self, folderId, text, name, limit, offset, sort, filters=None): logger.debug('Item find filters: %s', json.dumps(filters)) except Exception: pass - if recurse: + if recurse or group: return _itemFindRecursive( - self, origItemFind, folderId, text, name, limit, offset, sort, filters) + self, origItemFind, folderId, text, name, limit, offset, sort, + filters, recurse, group) return origItemFind(folderId, text, name, limit, offset, sort, filters) @boundHandler(apiRoot.item) @@ -58,7 +63,55 @@ def altFolderFind(self, parentType, parentId, text, name, limit, offset, sort, f altFolderFind._origFunc = origFolderFind -def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, sort, filters): +def _groupingPipeline(initialPipeline, cbase, grouping, sort=None): + """ + Modify the recursive pipeline to add grouping and counts. + + :param initialPipeline: a pipeline to extend. + :param cbase: a unique value for each grouping set. + :param grouping: a dictionary where 'keys' is a list of data to group by + and, optionally, 'counts' is a dictionary of data to count as keys and + names where to add the results. For instance, this could be + {'keys': ['meta.dicom.PatientID'], 'counts': { + 'meta.dicom.StudyInstanceUID': 'meta._count.studycount', + 'meta.dicom.SeriesInstanceUID': 'meta._count.seriescount'}} + :param sort: an optional lost of (key, direction) tuples + """ + for gidx, gr in enumerate(grouping['keys']): + grsort = [(gr, 1)] + (sort or []) + initialPipeline.extend([{ + '$match': {gr: {'$exists': True}}, + }, { + '$sort': collections.OrderedDict(grsort), + }, { + '$group': { + '_id': f'${gr}', + 'firstOrder': {'$first': '$$ROOT'}, + }, + }]) + groupStep = initialPipeline[-1]['$group'] + if not gidx and grouping['counts']: + for cidx, (ckey, cval) in enumerate(grouping['counts'].items()): + groupStep[f'count_{cbase}_{cidx}'] = {'$addToSet': f'${ckey}'} + cparts = cval.split('.') + centry = {cparts[-1]: {'$size': f'$count_{cbase}_{cidx}'}} + for cidx in range(len(cparts) - 2, -1, -1): + centry = { + cparts[cidx]: { + '$mergeObjects': [ + '$firstOrder.' + '.'.join(cparts[:cidx + 1]), + centry, + ], + }, + } + initialPipeline.append({'$set': {'firstOrder': { + '$mergeObjects': ['$firstOrder', centry]}}}) + initialPipeline.append({'$replaceRoot': {'newRoot': '$firstOrder'}}) + + +def _itemFindRecursive( # noqa + self, origItemFind, folderId, text, name, limit, offset, sort, filters, + recurse=True, group=None): """ If a recursive search within a folderId is specified, use an aggregation to find all folders that are descendants of the specified folder. If there @@ -73,20 +126,23 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, from bson.objectid import ObjectId if folderId: - pipeline = [ - {'$match': {'_id': ObjectId(folderId)}}, - {'$graphLookup': { - 'from': 'folder', - 'connectFromField': '_id', - 'connectToField': 'parentId', - 'depthField': '_depth', - 'as': '_folder', - 'startWith': '$_id', - }}, - {'$group': {'_id': '$_folder._id'}}, - ] - children = [ObjectId(folderId)] + next(Folder().collection.aggregate(pipeline))['_id'] - if len(children) > 1: + if recurse: + pipeline = [ + {'$match': {'_id': ObjectId(folderId)}}, + {'$graphLookup': { + 'from': 'folder', + 'connectFromField': '_id', + 'connectToField': 'parentId', + 'depthField': '_depth', + 'as': '_folder', + 'startWith': '$_id', + }}, + {'$group': {'_id': '$_folder._id'}}, + ] + children = [ObjectId(folderId)] + next(Folder().collection.aggregate(pipeline))['_id'] + else: + children = [ObjectId(folderId)] + if len(children) > 1 or group: filters = (filters.copy() if filters else {}) if text: filters['$text'] = { @@ -98,6 +154,69 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, user = self.getCurrentUser() if isinstance(sort, list): sort.append(('parentId', 1)) + + # This is taken from girder.utility.acl_mixin.findWithPermissions, + # except it adds a grouping stage + initialPipeline = [ + {'$match': filters}, + {'$lookup': { + 'from': 'folder', + 'localField': Item().resourceParent, + 'foreignField': '_id', + 'as': '__parent', + }}, + {'$match': Item().permissionClauses(user, AccessType.READ, '__parent.')}, + {'$project': {'__parent': False}}, + ] + if group is not None: + if not isinstance(group, list): + group = [gr for gr in group.split(',') if gr] + groups = [] + idx = 0 + while idx < len(group): + if group[idx] != '_count_': + if not len(groups) or groups[-1]['counts']: + groups.append({'keys': [], 'counts': {}}) + groups[-1]['keys'].append(group[idx]) + idx += 1 + else: + if idx + 3 <= len(group): + groups[-1]['counts'][group[idx + 1]] = group[idx + 2] + idx += 3 + for gidx, grouping in enumerate(groups): + _groupingPipeline(initialPipeline, gidx, grouping, sort) + fullPipeline = initialPipeline + countPipeline = initialPipeline + [ + {'$count': 'count'}, + ] + if sort is not None: + fullPipeline.append({'$sort': collections.OrderedDict(sort)}) + if limit: + fullPipeline.append({'$limit': limit + (offset or 0)}) + if offset: + fullPipeline.append({'$skip': offset}) + + logger.debug('Find item pipeline %r', fullPipeline) + + options = { + 'allowDiskUse': True, + 'cursor': {'batchSize': 0}, + } + result = Item().collection.aggregate(fullPipeline, **options) + + def count(): + try: + return next(iter( + Item().collection.aggregate(countPipeline, **options)))['count'] + except StopIteration: + # If there are no values, this won't return the count, in + # which case it is zero. + return 0 + + result.count = count + result.fromAggregate = True + return result + return Item().findWithPermissions(filters, offset, limit, sort=sort, user=user) return origItemFind(folderId, text, name, limit, offset, sort, filters) diff --git a/girder/girder_large_image/web_client/templates/itemList.pug b/girder/girder_large_image/web_client/templates/itemList.pug index 66f4b38b1..6545904fe 100644 --- a/girder/girder_large_image/web_client/templates/itemList.pug +++ b/girder/girder_large_image/web_client/templates/itemList.pug @@ -92,7 +92,7 @@ ul.g-item-list.li-item-list(layout_mode=(itemList.layout || {}).mode || '') != String(value).replace(/&/g, '&').replace(//, '>').replace(/"/, '"').replace(/'/, ''').replace(/\./g, '.­').replace(/_/g, '_­') else = value - if value + if value && column.format !== 'count' span.li-item-list-cell-filter(title="Only show items that match this metadata value exactly", filter-value=value, column-value=column.value) i.icon-filter if (hasMore && !paginated) diff --git a/girder/girder_large_image/web_client/views/itemList.js b/girder/girder_large_image/web_client/views/itemList.js index 86d55ffd5..4a8bdbf4f 100644 --- a/girder/girder_large_image/web_client/views/itemList.js +++ b/girder/girder_large_image/web_client/views/itemList.js @@ -30,12 +30,16 @@ wrap(HierarchyWidget, 'initialize', function (initialize, settings) { wrap(HierarchyWidget, 'render', function (render) { render.call(this); + if (this.parentModel.resourceName !== 'folder') { + this.$('.g-folder-list-container').toggleClass('hidden', false); + } if (!this.$('#flattenitemlist').length && this.$('.g-item-list-container').length && this.itemListView && this.itemListView.setFlatten) { $('button.g-checked-actions-button').parent().after( '
' ); - if ((this.itemListView || {})._recurse) { + if ((this.itemListView || {})._recurse && this.parentModel.resourceName === 'folder') { this.$('#flattenitemlist').prop('checked', true); + this.$('.g-folder-list-container').toggleClass('hidden', this.itemListView._hideFoldersOnFlatten); } this.events['click #flattenitemlist'] = (evt) => { this.itemListView.setFlatten(this.$('#flattenitemlist').is(':checked')); @@ -96,6 +100,16 @@ wrap(ItemListWidget, 'initialize', function (initialize, settings) { this.render(); return; } + if (!_.isEqual(val, this._liconfig) && !this.$el.closest('.modal-dialog').length && val) { + this._liconfig = val; + const list = this._confList(); + if (list.layout && list.layout.flatten !== undefined) { + this._recurse = !!list.layout.flatten; + this.parentView.$('#flattenitemlist').prop('checked', this._recurse); + } + this._hideFoldersOnFlatten = !!(list.layout && list.layout.flatten === 'only'); + this.parentView.$('.g-folder-list-container').toggleClass('hidden', this._hideFoldersOnFlatten); + } delete this._lastSort; this._liconfig = val; const curRoute = Backbone.history.fragment; @@ -138,6 +152,7 @@ wrap(ItemListWidget, 'initialize', function (initialize, settings) { this.setFlatten = (flatten) => { if (!!flatten !== !!this._recurse) { this._recurse = !!flatten; + this.parentView.$('.g-folder-list-container').toggleClass('hidden', this._hideFoldersOnFlatten && this._recurse); this._setFilter(); this.render(); } @@ -335,6 +350,23 @@ wrap(ItemListWidget, 'render', function (render) { filter = '_filter_:' + JSON.stringify(filter); } } + const group = (this._confList() || {}).group || undefined; + if (group) { + if (group.keys.length) { + let grouping = '_group_:meta.' + group.keys.join(',meta.'); + if (group.counts) { + for (let [gkey, gval] of Object.entries(group.counts)) { + if (!gkey.includes(',') && !gkey.includes(':') && !gval.includes(',') && !gval.includes(':')) { + if (gkey !== '_id') { + gkey = `meta.${gkey}`; + } + grouping += `,_count_,${gkey},meta.${gval}`; + } + } + } + filter = grouping + ':' + (filter || ''); + } + } if (this._recurse) { filter = '_recurse_:' + (filter || ''); } @@ -485,9 +517,7 @@ function sortColumn(evt) { } } -function itemListCellFilter(evt) { - evt.preventDefault(); - const cell = $(evt.target).closest('.li-item-list-cell-filter'); +function addCellToFilter(cell, update) { let filter = this._generalFilter || ''; let val = cell.attr('filter-value'); let col = cell.attr('column-value'); @@ -499,7 +529,15 @@ function itemListCellFilter(evt) { filter = filter.trim(); this.$el.closest('.g-hierarchy-widget').find('.li-item-list-filter-input').val(filter); this._generalFilter = filter; - this._setFilter(); + if (update !== false) { + this._setFilter(); + } +} + +function itemListCellFilter(evt) { + evt.preventDefault(); + const cell = $(evt.target).closest('.li-item-list-cell-filter'); + addCellToFilter.call(this, cell); addToRoute({filter: this._generalFilter}); this._setSort(); return false;