diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e322b0eae..79264e2e1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,11 @@ Changes ------- Unreleased ========== +2019-12-05: v0.7.1 +^^^^^^^^^^^^^^^^^^ +* :racehorse: Resolve some query efficiency problems +* :fire: :books: Remove some deprecated text and illustrations from README + 2019-11-25: v0.6.6 ^^^^^^^^^^^^^^^^^^ * :ambulance: Fix `POST response/applet/activity` route diff --git a/README.rst b/README.rst index ec24d8112..8feb7cd63 100644 --- a/README.rst +++ b/README.rst @@ -87,83 +87,6 @@ Data Structure **Girder for MindLogger** uses `reprolib `_ and has the following underlying data structure: *Note: This project is still in version 0 and these descriptions and diagrams may sometimes diverge from the actual data structure as we develop.* -Diagram -####### -|ERD| - The above `entity-relationship diagram `_ was created with `dia 0.97+git `_. - -Glossary -######## - -Activity -^^^^^^^^ -An "individual assessment", as defined in `reprolib `_:`Activity `_. - -Activity Set -^^^^^^^^^^^^ -A "collection[…] of `activities <#activity>`_ as defined in `reprolib `_:`ActivitySet `_. - -Applet -^^^^^^ -A document assigning one or more `activity sets <#activity-set>`_ to one or more `users <#user>`_ with or without scheduling and other constraints. - -Applet-specific User ID -^^^^^^^^^^^^^^^^^^^^^^^ -An identifier for a given `user <#user>`_ (or `reviewer <#reviewer>`_ or `subject <#subject>`_) for an `applet <#applet>`_ that does not expose that user's other data to anyone authorized to view information related to that applet. - -Context -^^^^^^^ - A set of rules for interpreting a JSON-LD document [from this database] as specified in The Context of the JSON-LD Syntax specification." - -This definition comes from `JSON-LD 1.1 `_ `context `_. - -Coordinator -^^^^^^^^^^^ - -Icon -^^^^ - -Illustration -^^^^^^^^^^^^ - -Manager -^^^^^^^ -An individual responsible for setting schedules, `subjects <#subject>`_ and other constraints as well as inviting other managers, `users <#user>`_ and `reviewers <#reviewer>`_ to an `applet <#applet>`_. - -Protected health information -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Any information about health status, provision of health care, or payment for health care that […] can be linked to a specific `individual <#user>`_. - -This definition comes from the Wikipedia article `Protected health information `_. - -Response -^^^^^^^^ -Data collected when a `user <#user>`_ responds to an `activity <#activity>`_. - -Reviewer -^^^^^^^^ -An individual authorized to review `user <#user>`_ `responses <#response>`_ to `activities <#activity>`_ in an `applet <#applet>`_. - -Screen -^^^^^^ -One or more "elements of individual assessments" displayed in a single screen or page view, as defined in `reprolib `_:`Item `_ and `Issue #85 `_. - -Skin -^^^^ -Color scheme and other branding and appearance-related metadata. - -Subject -^^^^^^^ -The person being informed about by the `user <#user>`_ `responding <#response>`_ to an `activity <#activity>`_. For self-report, the same user as the informant. - -Text -^^^^ -Copy included in the mobile and web app, including "About MindLogger" and helper text. - -User -^^^^ -An individual using a MindLogger mobile application or MindLogger web application to `respond <#response>`_ to `activities <#activity>`_. - Links ----- - `reprolib specification `_ @@ -187,10 +110,6 @@ For questions, comments, or to get in touch with the maintainers, head to their We'd love for you to `contribute to Girder `_. -.. |ERD| image:: ./docs/images/Mindlogger-DB-ER.png - :alt: MindLogger database entity-relationship diagram - :target: ./docs/images/Mindlogger-DB-ER.dia - .. |logo| image:: ./girderformindlogger/web_client/src/assets/ML-logo_25px.png :alt: Girder for MindLogger :target: https://api.mindlogger.org diff --git a/girderformindlogger/api/v1/applet.py b/girderformindlogger/api/v1/applet.py index e4d52917c..71d0cb73b 100644 --- a/girderformindlogger/api/v1/applet.py +++ b/girderformindlogger/api/v1/applet.py @@ -19,6 +19,7 @@ import itertools import re +import threading import uuid import requests from ..describe import Description, autoDescribeRoute @@ -162,36 +163,24 @@ def assignGroup(self, folder, group, role, subject): ) def createApplet(self, protocolUrl=None, name=None, informant=None): thisUser = self.getCurrentUser() - # get an activity set from a URL - protocol = ProtocolModel().getFromUrl( - protocolUrl, - 'protocol', - thisUser, - refreshCache=False - )[0] - protocol = protocol.get('protocol', protocol) - # create an applet for it - applet=AppletModel().createApplet( - name=name if name is not None and len(name) else ProtocolModel( - ).preferredName( - protocol - ), - protocol={ - '_id': 'protocol/{}'.format(protocol.get('_id')), - 'url': protocol.get( - 'meta', - {} - ).get( - 'protocol', - {} - ).get('url', protocolUrl) - }, - user=thisUser, - constraints={ - 'informantRelationship': informant - } if informant is not None else None + thread = threading.Thread( + target=AppletModel().createAppletFromUrl, + kwargs={ + 'name': name, + 'protocolUrl': protocolUrl, + 'user': thisUser, + 'constraints': { + 'informantRelationship': informant + } if informant is not None else None + } ) - return(applet) + thread.start() + return({ + "message": "The applet is being created. Please check back in " + "several mintutes to see it. If you have an email " + "address associated with your account, you will receive " + "an email when your applet is ready." + }) @access.user(scope=TokenScope.DATA_WRITE) @autoDescribeRoute( @@ -239,8 +228,6 @@ def updateInformant(self, applet, informant): .errorResponse('Write access was denied for this applet.', 403) ) def deactivateApplet(self, folder): - import threading - applet = folder user = Applet().getCurrentUser() applet['meta']['applet']['deleted'] = True @@ -251,8 +238,10 @@ def deactivateApplet(self, folder): applet.get('_id') ) thread = threading.Thread( - target=AppletModel().updateAllUserCaches(applet, user) + target=AppletModel().updateUserCacheAllUsersAllRoles, + args=(applet, user) ) + thread.start() else: message = 'Could not deactivate applet {} ({}).'.format( AppletModel().preferredName(applet), @@ -264,7 +253,12 @@ def deactivateApplet(self, folder): @access.user(scope=TokenScope.DATA_READ) @autoDescribeRoute( Description('Get an applet by ID.') - .modelParam('id', model=AppletModel, level=AccessType.READ) + .modelParam( + 'id', + model=AppletModel, + level=AccessType.READ, + destName='applet' + ) .param( 'refreshCache', 'Reparse JSON-LD', @@ -274,8 +268,7 @@ def deactivateApplet(self, folder): .errorResponse('Invalid applet ID.') .errorResponse('Read access was denied for this applet.', 403) ) - def getApplet(self, folder, refreshCache=False): - applet = folder + def getApplet(self, applet, refreshCache=False): user = self.getCurrentUser() return( jsonld_expander.formatLdObject( @@ -414,8 +407,7 @@ def invite(self, applet, role="user", idCode=None, profile=None): .errorResponse('Read access was denied for this applet.', 403) ) def setConstraints(self, folder, activity, schedule, **kwargs): - import threading - thisUser = Applet().getCurrentUser() + thisUser = self.getCurrentUser() applet = jsonld_expander.formatLdObject( _setConstraints(folder, activity, schedule, thisUser), 'applet', diff --git a/girderformindlogger/api/v1/protocol.py b/girderformindlogger/api/v1/protocol.py index a927176b9..3cac2be5e 100644 --- a/girderformindlogger/api/v1/protocol.py +++ b/girderformindlogger/api/v1/protocol.py @@ -57,7 +57,7 @@ def __init__(self): def getProtocol(self, folder): try: protocol = folder - user = Protocol().getCurrentUser() + user = self.getCurrentUser() return( jsonld_expander.formatLdObject( protocol, diff --git a/girderformindlogger/api/v1/screen.py b/girderformindlogger/api/v1/screen.py index 02e2b3b3c..bd2f0d9de 100644 --- a/girderformindlogger/api/v1/screen.py +++ b/girderformindlogger/api/v1/screen.py @@ -59,7 +59,7 @@ def __init__(self): .errorResponse('Read access was denied for the screen.', 403) ) def createScreen(self, activity, screenName=None): - thisUser = Screen().getCurrentUser() + thisUser = self.getCurrentUser() activity = ActivityModel().load( activity, level=AccessType.WRITE, diff --git a/girderformindlogger/api/v1/user.py b/girderformindlogger/api/v1/user.py index c8b33c5ca..a95d6111f 100644 --- a/girderformindlogger/api/v1/user.py +++ b/girderformindlogger/api/v1/user.py @@ -40,7 +40,6 @@ def __init__(self): self.route('GET', (':id',), self.getUserByID) self.route('GET', (':id', 'access'), self.getUserAccess) self.route('PUT', (':id', 'access'), self.updateUserAccess) - self.route('GET', (':id', 'applets'), self.getUserApplets) self.route('PUT', (':id', 'code'), self.updateIDCode) self.route('DELETE', (':id', 'code'), self.removeIDCode) self.route('GET', ('applets',), self.getOwnApplets) @@ -114,26 +113,29 @@ def getGroupInvites(self): fields=userfields )) ] - output.append({ - '_id': groupId, - 'applets': [{ - 'name': applet.get('cached', {}).get('applet', {}).get( - 'skos:prefLabel', - '' - ), - 'image': applet.get('cached', {}).get('applet', {}).get( - 'schema:image', - '' - ), - 'description': applet.get('cached', {}).get('applet', { - }).get( - 'schema:description', - '' - ), - 'managers': applet.get('managers'), - 'reviewers': applet.get('reviewers') - } for applet in applets] - }) + appletC = jsonld_expander.loadCache( + applet.get('cached') + ) if 'cached' in applet else applet + output.append({ + '_id': groupId, + 'applets': [{ + 'name': appletC.get('applet', {}).get( + 'skos:prefLabel', + '' + ), + 'image': appletC.get('applet', {}).get( + 'schema:image', + '' + ), + 'description': appletC.get('applet', { + }).get( + 'schema:description', + '' + ), + 'managers': applet.get('managers'), + 'reviewers': applet.get('reviewers') + } for applet in applets] + }) return(output) @access.user @@ -332,102 +334,6 @@ def updateUserAccess(self, user, access): save=True ) - @access.public(scope=TokenScope.DATA_READ) - @autoDescribeRoute( - Description('Get all applets for a user by that user\'s ID and role.') - .modelParam('id', model=UserModel, level=AccessType.READ) - .param( - 'role', - 'One of ' + str(USER_ROLES.keys()), - required=False, - default='user' - ) - .param( - 'ids_only', - 'If true, only returns an Array of the IDs of assigned applets. ' - 'Otherwise, returns an Array of Objects keyed with "applet" ' - '"protocol", "activities" and "items" with expanded JSON-LD as ' - 'values.', - required=False, - default=False, - dataType='boolean' - ) - .errorResponse('ID was invalid.') - .errorResponse( - 'You do not have permission to see any of this user\'s applets.', - 403 - ) - .deprecated() - ) - def getUserApplets(self, user, role, ids_only): - from bson.objectid import ObjectId - reviewer = self.getCurrentUser() - if reviewer is None: - raise AccessException("You must be logged in to get user applets.") - if user.get('_id') != reviewer.get('_id') and user.get( - '_id' - ) is not None: - raise AccessException("You can only get your own applets.") - role = role.lower() - if role not in USER_ROLES.keys(): - raise RestException( - 'Invalid user role.', - 'role' - ) - try: - applets = AppletModel().getAppletsForUser(role, user, active=True) - if len(applets)==0: - return([]) - if ids_only==True: - return([applet.get('_id') for applet in applets]) - return( - [ - { - **jsonld_expander.formatLdObject( - applet, - 'applet', - reviewer, - refreshCache=False - ), - "users": AppletModel().getAppletUsers(applet, user), - "groups": AppletModel().getAppletGroups( - applet, - arrayOfObjects=True - ) - } if role=="manager" else { - **jsonld_expander.formatLdObject( - applet, - 'applet', - reviewer, - dropErrors=True - ), - "groups": [ - group for group in AppletModel( - ).getAppletGroups(applet).get(role) if ObjectId( - group - ) in [ - *user.get('groups', []), - *user.get('formerGroups', []), - *[invite['groupId'] for invite in [ - *user.get('groupInvites', []), - *user.get('declinedInvites', []) - ]] - ] - ] - } for applet in applets if ( - applet is not None and not applet.get( - 'meta', - {} - ).get( - 'applet', - {} - ).get('deleted') - ) - ] - ) - except Exception as e: - return(e) - @access.public(scope=TokenScope.DATA_READ) @autoDescribeRoute( Description('Get all your applets by role.') @@ -510,7 +416,11 @@ def getOwnApplets( ) ) try: - if 'cached' in reviewer and 'applets' in reviewer[ + if 'cached' in reviewer: + reviewer['cached'] = jsonld_expander.loadCache( + reviewer['cached'] + ) + if 'applets' in reviewer[ 'cached' ] and role in reviewer['cached']['applets'] and isinstance( reviewer['cached']['applets'][role], diff --git a/girderformindlogger/constants.py b/girderformindlogger/constants.py index cb175f739..ecfedfec6 100644 --- a/girderformindlogger/constants.py +++ b/girderformindlogger/constants.py @@ -159,7 +159,7 @@ def MODELS(): 'user': UserModel }) -NONES = {"None", None, "none"} +NONES = {"None", None, "none", ""} REPROLIB_CANONICAL = "://".join([ "https", @@ -186,6 +186,11 @@ def MODELS(): 'screen': 'reprolib:schemas/Field' } +REPROLIB_TYPES_REVERSED = { + v.split('/')[1]: k for k, v in REPROLIB_TYPES.items() +} + + SPECIAL_SUBJECTS = {"ALL", "NONE"} USER_ROLES = { diff --git a/girderformindlogger/models/__init__.py b/girderformindlogger/models/__init__.py index f17d31029..8147372b4 100644 --- a/girderformindlogger/models/__init__.py +++ b/girderformindlogger/models/__init__.py @@ -122,76 +122,118 @@ def pluralize(modelType): def cycleModels(IRIset, modelType=None): - from girderformindlogger.constants import HIERARCHY + from girderformindlogger.constants import HIERARCHY, REPROLIB_TYPES + from girderformindlogger.models.folder import Folder as FolderModel + from girderformindlogger.models.item import Item as ItemModel + from girderformindlogger.utility.jsonld_expander import reprolibCanonize cachedDoc = None primary = [modelType] if isinstance(modelType, str) else [ ] if modelType is None else modelType secondary = [m for m in HIERARCHY if m not in primary] - for i in primary: - cachedDoc = lookForCached(i, IRIset) - if cachedDoc is not None: - modelType = i - break + + del modelType + + if len(primary): + query = { + '$and': [ + { + '$or': [{ + 'meta.{}.@type'.format(modelType): { + "$in": [ + t for t in [ + reprolibCanonize( + REPROLIB_TYPES[modelType] + ), + 'reproschema:{}'.format(suffix), + 'reprolib:{}'.format(suffix), + suffix + ] if t is not None + ] for suffix in [ + REPROLIB_TYPES[modelType].split('/')[-1] + ] + } + } for modelType in primary if modelType in REPROLIB_TYPES] + }, + { + '$or': [{ + 'meta.{}.url'.format(modelType): { + '$in': list(IRIset) + } + } for modelType in primary if modelType in REPROLIB_TYPES] + } + ] + } + cachedDoc = (FolderModel() if not any([ + 'screen' in primary, + 'item' in primary + ]) else ItemModel()).findOne(query) if cachedDoc is None: - for i in secondary: - cachedDoc = lookForCached(i, IRIset) - if cachedDoc is not None: - modelType = i - break + query = { + '$and': [ + { + '$or': [{ + 'meta.{}.@type'.format(modelType): { + "$in": [ + t for t in [ + reprolibCanonize( + REPROLIB_TYPES[modelType] + ), + 'reproschema:{}'.format(suffix), + 'reprolib:{}'.format(suffix), + suffix + ] if t is not None + ] for suffix in [ + REPROLIB_TYPES[modelType].split('/')[-1] + ] + } + } for modelType in secondary if modelType in REPROLIB_TYPES] + }, + { + '$or': [{ + 'meta.{}.url'.format(modelType): { + '$in': list(IRIset) + } + } for modelType in secondary if modelType in REPROLIB_TYPES] + } + ] + } + cachedDoc = FolderModel().findOne(query) + if cachedDoc is None: + cachedDoc = ItemModel().findOne(query) + if cachedDoc is None: return(None, None) - print("Found {}/{}".format(modelType, str(cachedDoc['_id']))) - return(modelType, cachedDoc) + modelType = [ + rt for rt in list( + REPROLIB_TYPES.keys() + ) if '@type' in cachedDoc.get('meta', {}).get(rt, {}) + ] + modelType = modelType[0] if len(modelType) else None -def lookForCached(modelType, IRIset): - from girderformindlogger.constants import MODELS, REPROLIB_TYPES - from girderformindlogger.utility.jsonld_expander import reprolibCanonize - - MODELS = MODELS() - query = { - 'meta.{}.url'.format(modelType): { - '$in': list(IRIset) - } - } - if modelType in REPROLIB_TYPES.keys(): - suffix = REPROLIB_TYPES[modelType].split('/')[-1] - query['meta.{}.@type'.format(modelType)] = { - "$in": [t for t in [ - reprolibCanonize( - REPROLIB_TYPES[modelType] - ), - 'reproschema:{}'.format(suffix), - 'reprolib:{}'.format(suffix), - suffix - ] if t is not None] - } - print("Looking for cached {} {}".format(modelType, str(IRIset))) - cachedDoc = MODELS[modelType]().findOne(query) - return(cachedDoc) + print("Found {}/{}".format(modelType, str(cachedDoc['_id']))) + return(modelType, cachedDoc) def smartImport(IRI, user=None, refreshCache=False, modelType=None): from girderformindlogger.constants import MODELS - from girderformindlogger.utility.jsonld_expander import reprolibCanonize + from girderformindlogger.utility.jsonld_expander import loadCache, \ + reprolibCanonize MODELS = MODELS() - mt1 = "screen" if modelType is None else modelType + mt1 = "screen" if modelType in [ + None, + "external JSON-LD document" + ] else modelType model, modelType = MODELS[mt1]().getFromUrl( IRI, user=user, - refreshCache=refreshCache + refreshCache=refreshCache, + thread=False ) - if mt1!=modelType: - MODELS[mt1]().remove(model) - model, modelType = MODELS[modelType]().getFromUrl( - IRI, - user=user, - refreshCache=refreshCache - ) return(( modelType, - model, + loadCache(model.get('cached', model)), reprolibCanonize(IRI) )) diff --git a/girderformindlogger/models/activity.py b/girderformindlogger/models/activity.py index f98dc3d25..b63b0a78f 100644 --- a/girderformindlogger/models/activity.py +++ b/girderformindlogger/models/activity.py @@ -160,7 +160,7 @@ def load(self, id, level=AccessType.ADMIN, user=None, objectId=True, url = doc.get('meta', {}).get('url') if url: return( - ActivityModel.getFromUrl( + self.getFromUrl( url, 'activity', user, diff --git a/girderformindlogger/models/applet.py b/girderformindlogger/models/applet.py index a50aa4e09..d8e9cc929 100644 --- a/girderformindlogger/models/applet.py +++ b/girderformindlogger/models/applet.py @@ -37,7 +37,8 @@ from girderformindlogger.models.group import Group as GroupModel from girderformindlogger.models.protoUser import ProtoUser as ProtoUserModel from girderformindlogger.models.user import User as UserModel -from girderformindlogger.utility.progress import noProgress, setResponseTimeLimit +from girderformindlogger.utility.progress import noProgress, \ + setResponseTimeLimit class Applet(Folder): @@ -134,26 +135,66 @@ def createApplet( currentUser=user, force=False ) - return(jsonld_expander.formatLdObject( applet, 'applet', user )) - return(self.formatThenUpdate( - applet, - user - )) - return({ - "_id": applet.get("_id"), - "applet": { - **self.unexpanded(applet), - "name": self.preferredName(applet), - "note - loading": "Your applet is being expanded on the " - "server. Check back in a few minutes to see the full content." - }, - "protocol": protocol - }) + + def createAppletFromUrl( + self, + name, + protocolUrl, + user=None, + roles=None, + constraints=None, + sendEmail=True + ): + from girderformindlogger.models.protocol import Protocol + # get a protocol from a URL + protocol = Protocol().getFromUrl( + protocolUrl, + 'protocol', + user, + thread=False + ) + protocol = protocol[0].get('protocol', protocol[0]) + name = name if name is not None and len(name) else Protocol( + ).preferredName( + protocol + ) + applet = self.createApplet( + name=name, + protocol={ + '_id': 'protocol/{}'.format( + str(protocol.get('_id')).split('/')[-1] + ), + 'url': protocol.get( + 'meta', + {} + ).get( + 'protocol', + {} + ).get('url', protocolUrl) + }, + user=user, + roles=roles, + constraints=constraints + ) + emailMessage = "Your applet, {}, has been successfully created. The " \ + "applet's ID is {}".format( + name, + str(applet.get('applet', applet).get('_id') + ) + ) + if sendEmail and 'email' in user: + from girderformindlogger.utility.mail_utils import sendMail + sendMail( + subject=name, + text=emailMessage, + to=[user['email']] + ) + print(emailMessage) def formatThenUpdate(self, applet, user): from girderformindlogger.utility import jsonld_expander @@ -186,6 +227,9 @@ def updateRelationship(self, applet, relationship): :type relationship: str :returns: updated Applet """ + from bson.json_util import dumps + from girderformindlogger.utility.jsonld_expander import loadCache + if not isinstance(relationship, str): raise TypeError("Applet relationship must be defined as a string.") if 'meta' not in applet: @@ -193,17 +237,21 @@ def updateRelationship(self, applet, relationship): if 'applet' not in applet['meta']: applet['meta']['applet'] = {} applet['meta']['applet']['informantRelationship'] = relationship - if 'cached' in applet and 'applet' in applet['cached']: + if 'cached' in applet: + applet['cached'] = loadCache(applet['cached']) + if 'applet' in applet['cached']: applet['cached']['applet']['informantRelationship'] = relationship + applet['cached'] = dumps(applet['cached']) return(self.save(applet, validate=False)) def unexpanded(self, applet): + from girderformindlogger.utility.jsonld_expander import loadCache return({ **( - applet.get( + loadCache(applet.get( 'cached', {} - ).get('applet') if isinstance( + )).get('applet') if isinstance( applet, dict ) and 'cached' in applet else { @@ -304,7 +352,7 @@ def updateUserCache(self, role, user, active=True, refreshCache=False): from girderformindlogger.utility import jsonld_expander applets=self.getAppletsForUser(role, user, active) - user['cached'] = user.get('cached', {}) + user['cached'] = jsonld_expander.loadCache(user.get('cached', {})) user['cached']['applets'] = user['cached'].get('applets', {}) user['cached']['applets'][role] = user['cached']['applets'].get( role, @@ -329,9 +377,8 @@ def updateUserCache(self, role, user, active=True, refreshCache=False): applet, 'applet', user, - dropErrors=True, - responseDates=True if role=="user" else False, - refreshCache=refreshCache + refreshCache=refreshCache, + responseDates=(role=="user") ), "groups": [ group for group in self.getAppletGroups(applet).get( @@ -377,8 +424,13 @@ def getAppletsForUser(self, role, user, active=True): :type active: bool :returns: list of dicts """ - if "userId" in user: - user = UserModel().load(id=ObjectId(user["userId"]), force=True) + user = UserModel().load( + id=ObjectId(user["userId"]), + force=True + ) if "userId" in user else UserModel().load( + id=ObjectId(user["_id"]), + force=True + ) if "_id" in user else user applets = [ *list(self.find( { @@ -508,16 +560,6 @@ def getAppletUsers(self, applet, user=None): print(sys.exc_info()) return({traceback.print_tb(sys.exc_info()[2])}) - - def importUrl(self, url, user=None, refreshCache=False): - """ - Gets an applet from a given URL, checks against the database, stores - and returns that applet. - - Deprecated. - """ - return(self.getFromUrl(url, 'applet', user, refreshCache)[0]) - def load(self, id, level=AccessType.ADMIN, user=None, objectId=True, force=False, fields=None, exc=False): """ diff --git a/girderformindlogger/models/folder.py b/girderformindlogger/models/folder.py index c1e317d53..a0c93a32f 100644 --- a/girderformindlogger/models/folder.py +++ b/girderformindlogger/models/folder.py @@ -25,8 +25,25 @@ class Folder(AccessControlledModel): def initialize(self): self.name = 'folder' - self.ensureIndices(('parentId', 'name', 'lowerName', - ([('parentId', 1), ('name', 1)], {}))) + self.ensureIndices( + ( + 'parentId', + 'name', + 'lowerName', + 'meta.activity.@type', + 'meta.protocol.@type', + 'meta.protocol.url', + 'meta.activity.url', + ([ + ('parentId', 1), + ('name', 1), + ('meta.protocol.@type', 1), + ('meta.activity.@type', 1), + ('meta.protocol.url', 1), + ('meta.activity.url', 1) + ], {}) + ) + ) self.ensureTextIndex({ 'name': 10, 'description': 1 diff --git a/girderformindlogger/models/item.py b/girderformindlogger/models/item.py index 00c5f9085..31b84628a 100644 --- a/girderformindlogger/models/item.py +++ b/girderformindlogger/models/item.py @@ -23,8 +23,21 @@ class Item(acl_mixin.AccessControlMixin, Model): def initialize(self): self.name = 'item' - self.ensureIndices(('folderId', 'name', 'lowerName', - ([('folderId', 1), ('name', 1)], {}))) + self.ensureIndices( + ( + 'folderId', + 'name', + 'lowerName', + 'meta.screen.@type', + 'meta.screen.url', + ([ + ('folderId', 1), + ('name', 1), + ('meta.screen.@type', 1), + ('meta.screen.url', 1) + ], {}) + ) + ) self.ensureTextIndex({ 'name': 10, 'description': 1 diff --git a/girderformindlogger/models/model_base.py b/girderformindlogger/models/model_base.py index 4355ace62..6e9552d3d 100644 --- a/girderformindlogger/models/model_base.py +++ b/girderformindlogger/models/model_base.py @@ -11,8 +11,9 @@ from pymongo.errors import WriteError from dictdiffer import diff from girderformindlogger import events, logprint, logger, auditLogger -from girderformindlogger.constants import AccessType, CoreEventHandler, \ - SortDir, ACCESS_FLAGS, PREFERRED_NAMES, TEXT_SCORE_SORT_MAX, USER_ROLES +from girderformindlogger.constants import ACCESS_FLAGS, AccessType, \ + CoreEventHandler, MODELS, PREFERRED_NAMES, REPROLIB_TYPES_REVERSED, \ + SortDir, TEXT_SCORE_SORT_MAX, USER_ROLES from girderformindlogger.external.mongodb_proxy import MongoProxy from girderformindlogger.models import getDbConnection from girderformindlogger.exceptions import AccessException, \ @@ -198,7 +199,34 @@ def filter(self, doc, user=None, additionalKeys=None): return self.filterDocument(doc, allow=keys) - def getFromUrl(self, url, modelType=None, user=None, refreshCache=False): + def getModelType(self, model): + if '@type' in model: + mt = model['@type'] if isinstance( + model['@type'], + str + ) else model['@type'][0] if isinstance( + model['@type'], + list + ) and len(model['@type']) else None + return(REPROLIB_TYPES_REVERSED[ + mt.split(':')[-1].split('/')[-1] + ]) + if 'meta' in model: + for k in MODELS(): + if k in model['meta']: + mt = self.getModelType(model['meta'][k]) + if mt is not None: + return(mt) + return(None) + + def getFromUrl( + self, + url, + modelType=None, + user=None, + refreshCache=False, + thread=False + ): """ Loads from a URL and saves to the DB, returning the loaded model. @@ -210,11 +238,12 @@ def getFromUrl(self, url, modelType=None, user=None, refreshCache=False): :type refreshCache: bool :returns: (dict, str) or (None, None) """ + import threading from . import cycleModels - from girderformindlogger.utility import firstLower, loadJSON + from girderformindlogger.utility import loadJSON from girderformindlogger.utility.jsonld_expander import camelCase, \ - contextualize, formatLdObject, reprolibCanonize, snake_case, \ - _fixUpFormat + contextualize, formatLdObject, importAndCompareModelType, \ + loadCache, reprolibCanonize, snake_case primary = [modelType] if isinstance(modelType, str) else [ ] if modelType is None else modelType @@ -242,143 +271,37 @@ def getFromUrl(self, url, modelType=None, user=None, refreshCache=False): ] else " {}".format(modelType) ) ) - model = contextualize(loadJSON( - url, - modelType - )) - if model is None: - return(None, None) - atType = model.get('@type', '').split('/')[-1].split(':')[-1] - else: - model = cachedDoc - model = _fixUpFormat(model) - modelType = firstLower(atType) if len(atType) else modelType - modelType = 'screen' if modelType.lower( - )=='field' else 'protocol' if modelType.lower( - )=='activityset' else modelType - changedModel = (atType != modelType and len(atType)) - prefName = self.preferredName(model) - if cachedDoc and not changedModel: - if not refreshCache: - return(cachedDoc, modelType) - provenenceProps = [ - 'schema:isBasedOn', - 'prov:wasRevisionOf' - ] - cachedId = str(cachedDoc.get('_id')) - cachedDocObj = cachedDoc.get('meta', {}).get( - snake_case(modelType), - cachedDoc.get('meta', {}).get( - camelCase(modelType), - cachedDoc.get('meta', {}).get( - modelType, - {} - ) + compact = loadJSON(url, modelType) + if thread: + thread = threading.Thread( + target=importAndCompareModelType, + args=(contextualize(compact),), + kwargs={'url': url, 'user': user, 'modelType': modelType} ) - ) - for prop in ['url', *provenenceProps]: - cachedDocObj.pop(prop, None) - else: - cachedId = None - cachedDocObj = {} - if not cachedDocObj or len(list(diff(cachedDocObj, model))): - if cachedId is not None: - for prop in provenenceProps: - model[prop] = { - '@id': '/'.join([ - snake_case(modelType), - cachedId - ]) - } - docCollection=self.getModelCollection(modelType) - if self.name in ['folder', 'item']: - if self.name=='item': - from girderformindlogger.models.folder import Folder as \ - FolderModel - docFolder = ( - FolderModel() if self.name=='item' else self - ).createFolder( - name=prefName, - parent=docCollection, - parentType='collection', - public=True, - creator=user, - allowRename=True, - reuseExisting=False + thread.start() + return( + { + "message": "This JSON LD document is not cached and must " + "be loaded. Please check back in several " + "minutes." + }, + self.getModelType(compact) ) - if self.name=='folder': - return( - formatLdObject( - self.setMetadata( - docFolder, - { - modelType: { - **model, - 'schema:url': url, - 'url': url - } - } - ), - modelType, - user - ), - modelType - ) - elif self.name=='item': - return( - formatLdObject( - self.setMetadata( - self.createItem( - name=prefName if prefName else str(len(list( - FolderModel().childItems( - FolderModel().load( - docFolder, - level=None, - user=user, - force=True - ) - ) - )) + 1), - creator=user, - folder=docFolder, - reuseExisting=False - ), - { - modelType: { - **model, - 'url': url - } - } - ), - modelType, - user - ), - modelType - ) - return(cachedDoc, modelType) - - def getModelCollection(self, modelType): - """ - Returns the Collection named for the given modelType, creating if not - already extant. - - :param modelType: 'activity', 'screen', etc. - :type modelType: str - :returns: dict - """ - from girderformindlogger.models import pluralize - from girderformindlogger.models.collection import Collection - name = pluralize(modelType).title() - collection = Collection().findOne( - {'name': name} - ) - if not collection: - collection = Collection().createCollection( - name=name, - public=True, - reuseExisting=True + model, modelType = importAndCompareModelType( + contextualize(compact), + url=url, + user=user, + modelType=modelType ) - return(collection) + else: + model = cachedDoc + modelType = self.getModelType(model) + if "cached" in model: + r = loadCache(model["cached"]) + return(r, self.getModelType(r)) + else: + return(model, modelType) + return(model, modelType) def _createIndex(self, index): if isinstance(index, (list, tuple)): diff --git a/girderformindlogger/models/protocol.py b/girderformindlogger/models/protocol.py index b15cc8c8c..719e8b862 100644 --- a/girderformindlogger/models/protocol.py +++ b/girderformindlogger/models/protocol.py @@ -41,7 +41,11 @@ def importUrl(self, url, user=None, refreshCache=False): Gets an activity set from a given URL, checks against the database, stores and returns that activity set. """ - return(self.getFromUrl(url, 'protocol', user, refreshCache)[0]) + return( + self.getFromUrl(url, 'protocol', user, refreshCache, thread=False)[ + 0 + ] + ) def load(self, id, level=AccessType.ADMIN, user=None, objectId=True, force=False, fields=None, exc=False): diff --git a/girderformindlogger/models/screen.py b/girderformindlogger/models/screen.py index 320a7582f..a8a0b2c1a 100644 --- a/girderformindlogger/models/screen.py +++ b/girderformindlogger/models/screen.py @@ -117,13 +117,28 @@ def importUrl(self, url, user=None, refreshCache=False): return(self.getFromUrl(url, 'screen', user, refreshCache)[0]) - def load(self, id, level=AccessType.ADMIN, user=None, refreshCache=False): - doc = super(Item, self).load(id=id, level=level, user=user) + def load( + self, + id, + level=AccessType.ADMIN, + user=None, + refreshCache=False, + **kwargs + ): + doc = super(Item, self).load(id=id, level=level, user=user, **kwargs) try: url = doc.get('meta', {}).get('url') except AttributeError: url = None if url: - return(self.getFromUrl(url, 'screen', user, refreshCache)[0]) + return( + self.getFromUrl( + url, + 'screen', + user, + refreshCache, + thread=False + )[0] + ) else: return(doc) diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index c3cccc25e..8b7d3d513 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -3,7 +3,7 @@ from datetime import datetime from girderformindlogger.constants import AccessType, DEFINED_RELATIONS, \ HIERARCHY, KEYS_TO_DELANGUAGETAG, KEYS_TO_DEREFERENCE, KEYS_TO_EXPAND, \ - MODELS, REPROLIB_CANONICAL, REPROLIB_PREFIXES + MODELS, NONES, REPROLIB_CANONICAL, REPROLIB_PREFIXES from girderformindlogger.exceptions import AccessException, \ ResourcePathNotFound, ValidationException from girderformindlogger.models.activity import Activity as ActivityModel @@ -18,6 +18,106 @@ from pyld import jsonld +def getModelCollection(modelType): + """ + Returns the Collection named for the given modelType, creating if not + already extant. + + :param modelType: 'activity', 'screen', etc. + :type modelType: str + :returns: dict + """ + from girderformindlogger.models import pluralize + name = pluralize(modelType).title() + collection = CollectionModel().findOne( + {'name': name} + ) + if not collection: + collection = CollectionModel().createCollection( + name=name, + public=True, + reuseExisting=True + ) + return(collection) + + +def importAndCompareModelType(model, url, user, modelType): + import threading + from girderformindlogger.utility import firstLower + + if model is None: + return(None, None) + atType = model.get('@type', '').split('/')[-1].split(':')[-1] + modelType = firstLower(atType) if len(atType) else modelType + modelType = 'screen' if modelType.lower( + )=='field' else 'protocol' if modelType.lower( + )=='activityset' else modelType + changedModel = ( + (atType != modelType and len(atType)) or (" " in modelType) + ) + modelType = firstLower(atType) if changedModel else modelType + modelType = 'screen' if modelType.lower( + )=='field' else 'protocol' if modelType.lower( + )=='activityset' else modelType + modelClass = MODELS()[modelType]() + prefName = modelClass.preferredName(model) + cachedDocObj = {} + print("Loaded {}".format(": ".join([modelType, prefName]))) + docCollection=getModelCollection(modelType) + if modelClass.name in ['folder', 'item']: + docFolder = FolderModel().createFolder( + name=prefName, + parent=docCollection, + parentType='collection', + public=True, + creator=user, + allowRename=True, + reuseExisting=(modelType!='applet') + ) + if modelClass.name=='folder': + newModel = modelClass.setMetadata( + docFolder, + { + modelType: { + **model, + 'schema:url': url, + 'url': url + } + } + ) + elif modelClass.name=='item': + newModel = modelClass.setMetadata( + modelClass.createItem( + name=prefName if prefName else str(len(list( + FolderModel().childItems( + FolderModel().load( + docFolder, + level=None, + user=user, + force=True + ) + ) + )) + 1), + creator=user, + folder=docFolder, + reuseExisting=True + ), + { + modelType: { + **model, + 'url': url + } + } + ) + formatted = _fixUpFormat(formatLdObject( + newModel, + mesoPrefix=modelType, + user=user, + )) + createCache(newModel, formatted, modelType, user) + return(formatted, modelType) + + def _createContextForStr(s): sp = s.split('/') k = '_'.join( @@ -86,21 +186,6 @@ def inferRelationships(person): for related in person['schema:knows'][rel]: if related not in person['schema:knows'][ep]: person['schema:knows'][ep].append(related) - # if "owl:inverseOf" in DEFINED_RELATIONS[rel]: - # for related in [ - # Profile().load( - # p, - # force=True - # ) for p in person['schema:knows'][rel] - # ]: - # if 'schema:knows' not in related: - # related['schema:knows'] = {} - # for io in DEFINED_RELATIONS[rel]["owl:inverseOf"]: - # if io not in related['schema:knows']: - # related['schema:knows'][io] = [] - # if person['_id'] not in related['schema:knows'][io]: - # related['schema:knows'][io].append(person['_id']) - # inferRelationships(related) if any([ bool( rp not in start.get('schema:knows', {}).get(rel, []) @@ -377,6 +462,80 @@ def checkURL(s): return(False) +def compactKeys(obj): + context = obj.get('@context', []) + if not isinstance(context, list): + context = [context] + newObj = {} + for k in list(obj.keys()): + if "." in k: + c, nk = _createContext(k) + if c not in context: + context.append(c) + nc, newObj[nk] = _deepCompactKeys(obj[k]) + for c in nc: + if c not in context: + context.append(c) + else: + nc, newObj[k] = _deepCompactKeys(obj[k]) + for c in nc: + if c not in context: + context.append(c) + newObj['@context'] = context + return(newObj) + + +def _deepCompactKeys(obj): + context = [] + if not isinstance(obj, dict): + return(context, obj) + newObj = {} + for k in list(obj.keys()): + if "." in k: + c, nk = _createContext(k) + if c not in context: + context.append(c) + nc, newObj[nk] = _deepCompactKeys(obj[k]) + for c in nc: + if c not in context: + context.append(c) + else: + nc, newObj[k] = _deepCompactKeys(obj[k]) + for c in nc: + if c not in context: + context.append(c) + return(context, newObj) + + +def _createContext(key): + s = key.split('/') + k = s[-1] + key = '{}/'.format('/'.join(s[0:-1])) + return({key.split('://')[-1].replace('.', '_dot_'): key}, k) + + +def createCache(obj, formatted, modelType, user): + obj = MODELS()[modelType]().load(obj['_id'], force=True) + if "cached" in obj: + oc = obj.get("oldCache", []) + obj["oldCache"] = (oc if oc is not None else []).append(obj["cached"]) + if modelType in NONES: + print("No modelType!") + print(obj) + obj["cached"] = json_util.dumps({ + **formatted, + "prov:generatedAtTime": xsdNow() + }) + return(MODELS()[modelType]().save(obj, validate=False)) + + +def loadCache(obj): + if isinstance(obj, dict): + return(obj) + else: + return(json_util.loads(obj)) + + def _fixUpFormat(obj): if isinstance(obj, dict): newObj = {} @@ -417,7 +576,8 @@ def formatLdObject( keepUndefined=False, dropErrors=False, refreshCache=False, - responseDates=False): + responseDates=False +): """ Function to take a compacted JSON-LD Object within a Girder for Mindlogger database and return an exapanded JSON-LD Object including an _id. @@ -442,186 +602,177 @@ def formatLdObject( """ from girderformindlogger.models import pluralize + refreshCache = refreshCache if refreshCache is not None else False + try: if obj is None: return(None) - elif isinstance(obj, dict) and 'meta' not in obj.keys(): - return(_fixUpFormat(obj)) - elif isinstance(obj, dict) and "cached" in obj and not refreshCache: - returnObj = obj["cached"] - else: - mesoPrefix = camelCase(mesoPrefix) - if type(obj)==list: - return(_fixUpFormat([ - formatLdObject( - o, - mesoPrefix, - user=user - ) for o in obj if o is not None - ])) - if not type(obj)==dict and not dropErrors: - raise TypeError("JSON-LD must be an Object or Array.") - newObj = obj.get('meta', obj) - newObj = newObj.get(mesoPrefix, newObj) - newObj = expand(newObj, keepUndefined=keepUndefined) - if type(newObj)==list and len(newObj)==1: - try: - newObj = newObj[0] - except: - raise ValidationException(str(newObj)) - if type(newObj)!=dict: - newObj = {} - objID = str(obj.get('_id', 'undefined')) - if objID=='undefined': - raise ResourcePathNotFound() - newObj['_id'] = "/".join([snake_case(mesoPrefix), objID]) - if mesoPrefix=='applet': - protocolUrl = obj.get('meta', {}).get('protocol', obj).get( - 'http://schema.org/url', - obj.get('meta', {}).get('protocol', obj).get('url') - ) - protocol = _fixUpFormat(formatLdObject( - ProtocolModel().getFromUrl( - protocolUrl, - 'protocol', - user - )[0], + if isinstance(obj, dict): + oc = obj.get("cached") + if all([ + not refreshCache, + oc is not None + ]): + return(loadCache(oc)) + if 'meta' not in obj.keys(): + return(_fixUpFormat(obj)) + mesoPrefix = camelCase(mesoPrefix) + if type(obj)==list: + return(_fixUpFormat([ + formatLdObject( + o, + mesoPrefix, + user=user + ) for o in obj if o is not None + ])) + if not type(obj)==dict and not dropErrors: + raise TypeError("JSON-LD must be an Object or Array.") + newObj = obj.get('meta', obj) + newObj = newObj.get(mesoPrefix, newObj) + newObj = expand(newObj, keepUndefined=keepUndefined) + if type(newObj)==list and len(newObj)==1: + try: + newObj = newObj[0] + except: + raise ValidationException(str(newObj)) + if type(newObj)!=dict: + newObj = {} + objID = str(obj.get('_id', 'undefined')) + if objID=='undefined': + raise ResourcePathNotFound() + newObj['_id'] = "/".join([snake_case(mesoPrefix), objID]) + if mesoPrefix=='applet': + protocolUrl = obj.get('meta', {}).get('protocol', obj).get( + 'http://schema.org/url', + obj.get('meta', {}).get('protocol', obj).get('url') + ) + protocol = ProtocolModel().getFromUrl( + protocolUrl, + 'protocol', + user, + thread=False + )[0] if protocolUrl is not None else {} + protocol = formatLdObject(protocol, 'protocol', user) + applet = {} + applet['activities'] = protocol.pop('activities', {}) + applet['items'] = protocol.pop('items', {}) + applet['protocol'] = { + key: protocol.get( 'protocol', - user - )) if protocolUrl is not None else {} - applet = {} - applet['activities'] = protocol.pop('activities', {}) - applet['items'] = protocol.pop('items', {}) - applet['protocol'] = { - key: protocol.get( - 'protocol', - protocol.get( - 'activitySet', - {} - ) - ).pop( - key - ) for key in [ - '@type', - '_id', - 'http://schema.org/url', - 'url' - ] if key in list(protocol.get('protocol', {}).keys()) - } - - applet['applet'] = { - **protocol.pop('protocol', {}), - **obj.get('meta', {}).get(mesoPrefix, {}), - '_id': "/".join([snake_case(mesoPrefix), objID]), - 'url': "#".join([ - obj.get('meta', {}).get('protocol', {}).get("url", "") - ]) - } - applet = _fixUpFormat(applet) - obj["cached"] = { - **applet, - "prov:generatedAtTime": xsdNow() - } - AppletModel().save(obj, validate=False) - returnObj = applet - elif mesoPrefix=='protocol': - protocol = { - 'protocol': newObj, - 'activities': {}, - "items": {} - } - activitiesNow = set() - itemsNow = set() - try: - protocol = componentImport( - newObj, - deepcopy(protocol), - user, - refreshCache=refreshCache - ) - except: - protocol = componentImport( - newObj, - deepcopy(protocol), - user, - refreshCache=True + protocol.get( + 'activitySet', + {} ) - newActivities = [ - a for a in protocol.get('activities', {}).keys( - ) if a not in activitiesNow - ] - newItems = [ - i for i in protocol.get('items', {}).keys( - ) if i not in itemsNow - ] - while(any([len(newActivities), len(newItems)])): - activitiesNow = set( - protocol.get('activities', {}).keys() - ) - itemsNow = set(protocol.get('items', {}).keys()) - for activityURL in newActivities: - activity = protocol['activities'][activityURL] - activity = activity.get( - 'meta', - {} - ).get('activity', activity) - try: - protocol = componentImport( - deepcopy(activity), - deepcopy(protocol), - user, - refreshCache=refreshCache - ) - except: - protocol = componentImport( - deepcopy(activity), - deepcopy(protocol), - user, - refreshCache=True - ) - for itemURL in newItems: - activity = protocol['items'][itemURL] - activity = activity.get( - 'meta', - {} - ).get('screen', activity) - try: - protocol = componentImport( - deepcopy(activity), - deepcopy(protocol), - user, - refreshCache=refreshCache - ) - except: - protocol = componentImport( - deepcopy(activity), - deepcopy(protocol), - user, - refreshCache=True - ) - newActivities = list( - set( - protocol.get('activities', {}).keys() - ) - activitiesNow - ) - newItems = list( - set( - protocol.get('items', {}).keys() - ) - itemsNow - ) - return(_fixUpFormat(protocol)) - else: - return(_fixUpFormat(newObj)) - if responseDates and mesoPrefix=="applet": + ).pop( + key + ) for key in [ + '@type', + '_id', + 'http://schema.org/url', + 'url' + ] if key in list(protocol.get('protocol', {}).keys()) + } + applet['applet'] = { + **protocol.pop('protocol', {}), + **obj.get('meta', {}).get(mesoPrefix, {}), + '_id': "/".join([snake_case(mesoPrefix), objID]), + 'url': "#".join([ + obj.get('meta', {}).get('protocol', {}).get("url", "") + ]) + } + createCache(obj, applet, 'applet', user) + return(applet) + elif mesoPrefix=='protocol': + protocol = { + 'protocol': newObj, + 'activities': {}, + "items": {} + } + activitiesNow = set() + itemsNow = set() try: - returnObj["applet"]["responseDates"] = responseDateList( - obj.get('_id'), - user.get('_id'), - user + protocol = componentImport( + newObj, + deepcopy(protocol), + user, + refreshCache=refreshCache ) except: - returnObj["applet"]["responseDates"] = [] - return(_fixUpFormat(returnObj)) + print("636") + protocol = componentImport( + newObj, + deepcopy(protocol), + user, + refreshCache=True + ) + newActivities = [ + a for a in protocol.get('activities', {}).keys( + ) if a not in activitiesNow + ] + newItems = [ + i for i in protocol.get('items', {}).keys( + ) if i not in itemsNow + ] + while(any([len(newActivities), len(newItems)])): + activitiesNow = set( + protocol.get('activities', {}).keys() + ) + itemsNow = set(protocol.get('items', {}).keys()) + for activityURL in newActivities: + activity = protocol['activities'][activityURL] + activity = activity.get( + 'meta', + {} + ).get('activity', activity) + try: + protocol = componentImport( + deepcopy(activity), + deepcopy(protocol), + user, + refreshCache=refreshCache + ) + except: + print("670") + protocol = componentImport( + deepcopy(activity), + deepcopy(protocol), + user, + refreshCache=True + ) + for itemURL in newItems: + activity = protocol['items'][itemURL] + activity = activity.get( + 'meta', + {} + ).get('screen', activity) + try: + protocol = componentImport( + deepcopy(activity), + deepcopy(protocol), + user, + refreshCache=refreshCache + ) + except: + print("691") + protocol = componentImport( + deepcopy(activity), + deepcopy(protocol), + user, + refreshCache=True + ) + newActivities = list( + set( + protocol.get('activities', {}).keys() + ) - activitiesNow + ) + newItems = list( + set( + protocol.get('items', {}).keys() + ) - itemsNow + ) + return(_fixUpFormat(protocol)) + else: + return(_fixUpFormat(newObj)) except: if refreshCache==False: return(_fixUpFormat(formatLdObject( @@ -655,7 +806,7 @@ def componentImport( from girderformindlogger.utility import firstLower updatedProtocol = deepcopy(protocol) - obj2 = expand(obj.copy()) + obj2 = {k: v for k, v in expand(obj.copy()).items() if v is not None} try: for order in obj2.get( "reprolib:terms/order", diff --git a/girderformindlogger/utility/response.py b/girderformindlogger/utility/response.py index f1b2c5980..f0709cc8f 100644 --- a/girderformindlogger/utility/response.py +++ b/girderformindlogger/utility/response.py @@ -399,7 +399,7 @@ def last7Days( referenceDate=None ): from bson import json_util - from .jsonld_expander import reprolibCanonize, reprolibPrefix + from .jsonld_expander import loadCache, reprolibCanonize, reprolibPrefix referenceDate = delocalize( datetime.now( tzlocal.get_localzone() @@ -407,10 +407,7 @@ def last7Days( ) # we need to get the activities - cachedApplet = appletInfo['cached'] if isinstance( - appletInfo['cached'], - dict - ) else json_util.loads(appletInfo['cached']) + cachedApplet = loadCache(appletInfo['cached']) listOfActivities = [ reprolibPrefix(activity) for activity in list( cachedApplet['activities'].keys() diff --git a/test/testLib.py b/test/testLib.py index 2c33580ca..6f05e3ce2 100644 --- a/test/testLib.py +++ b/test/testLib.py @@ -149,31 +149,39 @@ def addApplet(new_user, protocolUrl): # TODO: create an activity-set that JUST for testing. # make sure it has all the weird qualities that can break + userAppletsToStart = AppletModel().getAppletsForUser( + 'manager', + currentUser, + active=True + ) + + userApplets = userAppletsToStart.copy() # for now, lets do the mindlogger demo - protocol = {} - protocol.update(ProtocolModel().getFromUrl( + protocol = ProtocolModel().getFromUrl( protocolUrl, 'protocol', currentUser - )[0]) + )[0] randomAS = np.random.randint(1000000) - ar = AppletModel().createApplet( + ar = AppletModel().createAppletFromUrl( name="testProtocol{}".format(randomAS), - protocol={ - '_id': 'protocol/{}'.format(protocol.get('_id')), - 'url': protocol.get( - 'meta', - {} - ).get( - 'protocol', - {} - ).get('url', protocolUrl) - }, - user=currentUser + protocolUrl=protocolUrl, + user=currentUser, + sendEmail=False ) - assert ar['applet']['_id'], 'there is no ID!' + while len(userApplets)==len(userAppletsToStart): + nightyNight(sleepInterval) + userApplets = AppletModel().getAppletsForUser( + 'manager', + currentUser, + active=True + ) + + print(len(userApplets)) + ar = jsonld_expander.loadCache(userApplets[-1]['cached']) + assert jsonld_expander.reprolibCanonize( ar['protocol']['url'] ) == jsonld_expander.reprolibCanonize(protocolUrl), \ @@ -182,16 +190,7 @@ def addApplet(new_user, protocolUrl): protocolUrl ) - userApplets = AppletModel().getAppletsForUser( - 'user', - currentUser, - active=True - ) - timer = sleepInterval - while(timer > 0): - print("😴 {}".format(str(timer))) - time.sleep(1) - timer = timer - 1 + assert ar['applet']['_id'], 'there is no ID!' assert getAppletById( new_user, @@ -200,6 +199,13 @@ def addApplet(new_user, protocolUrl): return ar +def nightyNight(timer): + while(timer > 0): + print("😴 {}".format(str(timer))) + time.sleep(1) + timer = timer - 1 + + def getAppletsUser(user, n=1): """ count applets for the user, and assert the length is a given amount. @@ -496,26 +502,6 @@ def checkForInvite(user, appletObject): fields=userfields )) ] - pendingInvitesForUser.append({ - '_id': groupId, - 'applets': [{ - 'name': applet.get('cached', {}).get('applet', {}).get( - 'skos:prefLabel', - '' - ), - 'image': applet.get('cached', {}).get('applet', {}).get( - 'schema:image', - '' - ), - 'description': applet.get('cached', {}).get('applet', { - }).get( - 'schema:description', - '' - ), - 'managers': applet.get('managers'), - 'reviewers': applet.get('reviewers') - } for applet in applets] - }) groupId = appletObject['roles']['user']['groups'][0]['id'] assert len(pendingInvitesForUser), "this user has no invites" assert pendingInvitesForUser[0]['_id'] == groupId, "this user doesn't have the invite you expect" @@ -986,10 +972,10 @@ def step06(user, appletObject): # accept the applet invite # print('accept the applet invite') - def step08(userB, appletObject): - acceptAppletInvite(userB, appletObject, 'user') - - tryExceptTester(step08, [userB, appletObject], 'accept the applet invite') + # def step08(userB, appletObject): + # acceptAppletInvite(userB, appletObject, 'user') + # + # tryExceptTester(step08, [userB, appletObject], 'accept the applet invite') # invite someone that doesn't have an account yet # print('\033[0;37;40m invite someone that doesn\'t have an account yet')