From c8bd3751b10bb3d6080df39f0906c03ef0f68357 Mon Sep 17 00:00:00 2001 From: Jarvanerp Date: Mon, 7 Nov 2016 17:44:32 +0100 Subject: [PATCH 1/5] Add implementation of serializer for datatype retrieval call. --- .../db/RestExportService.groovy | 95 ++++++++----------- .../export/DataTypeRetrieved.groovy | 13 +++ .../ontology/AbstractI2b2MetadataTests.groovy | 1 - 3 files changed, 51 insertions(+), 58 deletions(-) create mode 100644 src/groovy/org/transmartproject/export/DataTypeRetrieved.groovy diff --git a/grails-app/services/org/transmartproject/db/RestExportService.groovy b/grails-app/services/org/transmartproject/db/RestExportService.groovy index f7bcaa1b..570cc0b5 100644 --- a/grails-app/services/org/transmartproject/db/RestExportService.groovy +++ b/grails-app/services/org/transmartproject/db/RestExportService.groovy @@ -8,6 +8,7 @@ import org.transmartproject.export.Tasks.DataExportFetchTask import org.transmartproject.export.Tasks.DataExportFetchTaskFactory import org.transmartproject.core.dataquery.highdim.HighDimensionResource import org.transmartproject.core.ontology.ConceptsResource +import org.transmartproject.export.DataTypeRetrieved import static org.transmartproject.core.ontology.OntologyTerm.VisualAttributes.HIGH_DIMENSIONAL @@ -22,52 +23,23 @@ class RestExportService { ConceptsResource conceptsResourceService + int cohortNumberID + List export(arguments) { DataExportFetchTask task = dataExportFetchTaskFactory.createTask(arguments) task.getTsv() } - def formatDataTypes(List datatypes){ - //Maybe make this a marshaller. - def returnDataTypeList = [] - def dataTypesList = [] - datatypes.each { dataTypeMap -> - dataTypeMap.each {key, value-> - if (key in dataTypesList){ - returnDataTypeList.each { map -> - if (key in map.values()){ - value.each{it.remove('datatypeCode')} - map.get('cohorts').add([concepts:value]) - } - } - } else{ - dataTypesList.add(key) - def datatypeCode = value.collect({it -> - it.get('datatypeCode')}) - value.each{it.remove('datatypeCode')} - def datatypeMap = [dataType:key, - dataTypeCode: datatypeCode[0], - cohorts:[[concepts:value]]] - returnDataTypeList.add(datatypeMap) - } - } - } - returnDataTypeList - } - - - def getDataTypes(List conceptKeysList){ - List cohortDataTypes = [] - Map datatypesMap = [:] + public def getDataTypes(List conceptKeysList, List dataTypes, Integer cohortNumber){ + cohortNumberID = cohortNumber conceptKeysList.each { conceptKey -> OntologyTerm concept = conceptsResourceService.getByKey(conceptKey) - datatypesMap = getHighDimDataType(concept, datatypesMap) + dataTypes = getDataType(concept, dataTypes) } - cohortDataTypes += datatypesMap - cohortDataTypes + dataTypes } - def getHighDimDataType(OntologyTerm term, Map datatypesMap) { + private def getDataType(OntologyTerm term, List dataTypes) { // Retrieve all descendant terms that have the HIGH_DIMENSIONAL attribute def terms = term.getAllDescendants() + term def highDimTerms = terms.findAll { it.visualAttributes.contains(HIGH_DIMENSIONAL) } @@ -83,34 +55,43 @@ class RestExportService { }) ] ) - //datatypes contains the number of patients for each datatype. + //TODO: List with datatype objects. This part gets moved to the serializationhelper def datatypes = highDimensionResourceService.getSubResourcesAssayMultiMap([constraint]) datatypes.collect({ key, value -> - String datatype = key.dataTypeDescription - String datatypeCode = key.dataTypeName - if (datatype in datatypesMap.keySet()){ - // term.getPatientCount() can also be value.size(). They give different numbers but I'm not sure - // what's the difference - datatypesMap[datatype].add([numOfPatients: term.getPatientCount(), conceptPath: term.fullName , datatypeCode: datatypeCode]) - } else{ - datatypesMap[datatype] = [[numOfPatients: term.getPatientCount(), conceptPath: term.fullName, datatypeCode: datatypeCode]] - } + dataTypes = addDataType(term, dataTypes, key) }) - datatypesMap + dataTypes } else { // No high dimensional data found for this term, this means it is clinical data - String datatype = "Clinical data" - String datatypeCode = "clinical" - if (datatype in datatypesMap.keySet()){ - // term.getPatientCount() can also be value.size(). They give different numbers but I'm not sure - // what's the difference - datatypesMap[datatype].add([numOfPatients: term.patientCount, conceptPath: term.fullName , datatypeCode: datatypeCode]) - } else{ - datatypesMap[datatype] = [[numOfPatients: term.patientCount, conceptPath: term.fullName, datatypeCode: datatypeCode]] - } - datatypesMap + dataTypes = addDataType(term, dataTypes) + dataTypes + } + } + + private List addDataType(OntologyTerm term, List dataTypes, datatype = null) { + String dataTypeString = datatype ? datatype.dataTypeDescription :"Clinical data" + String dataTypeCode = datatype ? datatype.dataTypeName : "clinical" + List tempDataTypes = dataTypes.collect {it.dataType} + if (dataTypeString in tempDataTypes){ + int index = tempDataTypes.indexOf(dataTypeString) + DataTypeRetrieved dataType = dataTypes[index] + addOntologyTerm(term, dataType) + } else{ + DataTypeRetrieved dataType = new DataTypeRetrieved(dataType: dataTypeString, dataTypeCode: dataTypeCode) + addOntologyTerm(term, dataType) + dataTypes.add(dataType) + } + dataTypes + } + + private void addOntologyTerm(OntologyTerm term, DataTypeRetrieved dataType) { + if(cohortNumberID in dataType.OntologyTermsMap.keySet()) { + dataType.OntologyTermsMap[cohortNumberID].add(term) + } else { + dataType.OntologyTermsMap[cohortNumberID] = [term] } + } } diff --git a/src/groovy/org/transmartproject/export/DataTypeRetrieved.groovy b/src/groovy/org/transmartproject/export/DataTypeRetrieved.groovy new file mode 100644 index 00000000..6d199a07 --- /dev/null +++ b/src/groovy/org/transmartproject/export/DataTypeRetrieved.groovy @@ -0,0 +1,13 @@ +package org.transmartproject.export + +import org.transmartproject.core.ontology.OntologyTerm + +class DataTypeRetrieved { + + Map> OntologyTermsMap = [:] + + String dataType + + String dataTypeCode + +} diff --git a/transmart-core-db-tests/test/unit/org/transmartproject/db/ontology/AbstractI2b2MetadataTests.groovy b/transmart-core-db-tests/test/unit/org/transmartproject/db/ontology/AbstractI2b2MetadataTests.groovy index d7ca58dd..6db3a72a 100644 --- a/transmart-core-db-tests/test/unit/org/transmartproject/db/ontology/AbstractI2b2MetadataTests.groovy +++ b/transmart-core-db-tests/test/unit/org/transmartproject/db/ontology/AbstractI2b2MetadataTests.groovy @@ -62,7 +62,6 @@ class AbstractI2b2MetadataTests { 04/15/2007 01:23:14 UA-BLD Occult Blood - Enum GRP 5794-3 HL From f024b233aa6685fa2e4f282ecf0011533730fb2a Mon Sep 17 00:00:00 2001 From: Jarvanerp Date: Wed, 9 Nov 2016 14:12:11 +0100 Subject: [PATCH 2/5] Changes to RestExportService based on feedback --- .../db/RestExportService.groovy | 72 ++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/grails-app/services/org/transmartproject/db/RestExportService.groovy b/grails-app/services/org/transmartproject/db/RestExportService.groovy index 570cc0b5..2bccff63 100644 --- a/grails-app/services/org/transmartproject/db/RestExportService.groovy +++ b/grails-app/services/org/transmartproject/db/RestExportService.groovy @@ -1,14 +1,18 @@ package org.transmartproject.db import grails.transaction.Transactional +import groovy.json.JsonException +import groovy.json.JsonSlurper import org.springframework.beans.factory.annotation.Autowired +import org.transmartproject.core.dataquery.highdim.HighDimensionResource import org.transmartproject.core.dataquery.highdim.assayconstraints.AssayConstraint +import org.transmartproject.core.exceptions.InvalidArgumentsException +import org.transmartproject.core.exceptions.NoSuchResourceException +import org.transmartproject.core.ontology.ConceptsResource import org.transmartproject.core.ontology.OntologyTerm +import org.transmartproject.export.DataTypeRetrieved import org.transmartproject.export.Tasks.DataExportFetchTask import org.transmartproject.export.Tasks.DataExportFetchTaskFactory -import org.transmartproject.core.dataquery.highdim.HighDimensionResource -import org.transmartproject.core.ontology.ConceptsResource -import org.transmartproject.export.DataTypeRetrieved import static org.transmartproject.core.ontology.OntologyTerm.VisualAttributes.HIGH_DIMENSIONAL @@ -25,21 +29,44 @@ class RestExportService { int cohortNumberID + List dataTypes + List export(arguments) { DataExportFetchTask task = dataExportFetchTaskFactory.createTask(arguments) task.getTsv() } - public def getDataTypes(List conceptKeysList, List dataTypes, Integer cohortNumber){ - cohortNumberID = cohortNumber - conceptKeysList.each { conceptKey -> - OntologyTerm concept = conceptsResourceService.getByKey(conceptKey) - dataTypes = getDataType(concept, dataTypes) + public def retrieveDataTypes(params) { + if (!(params.containsKey('concepts'))) { + throw new NoSuchResourceException("No parameter named concepts was given.") + } + + if (params.get('concepts') == "") { + throw new InvalidArgumentsException("Parameter concepts has no value.") + } + + def jsonSlurper = new JsonSlurper() + def conceptParameters = params.get('concepts').decodeURL() + dataTypes = [] + try { + def conceptArguments = jsonSlurper.parseText(conceptParameters) + int cohortNumber = 1 + conceptArguments.each { it -> + List conceptKeysList = it.conceptKeys + cohortNumberID = cohortNumber + conceptKeysList.collect { conceptKey -> + getDataType(conceptKey) + } + cohortNumber += 1 + } + dataTypes + } catch (JsonException e) { + throw new InvalidArgumentsException("Given parameter was non valid JSON.") } - dataTypes } - private def getDataType(OntologyTerm term, List dataTypes) { + private void getDataType(String conceptKey) { + OntologyTerm term = conceptsResourceService.getByKey(conceptKey) // Retrieve all descendant terms that have the HIGH_DIMENSIONAL attribute def terms = term.getAllDescendants() + term def highDimTerms = terms.findAll { it.visualAttributes.contains(HIGH_DIMENSIONAL) } @@ -55,38 +82,33 @@ class RestExportService { }) ] ) - //TODO: List with datatype objects. This part gets moved to the serializationhelper def datatypes = highDimensionResourceService.getSubResourcesAssayMultiMap([constraint]) datatypes.collect({ key, value -> - dataTypes = addDataType(term, dataTypes, key) + addDataType(term, key) }) - dataTypes - } - else { + } else { // No high dimensional data found for this term, this means it is clinical data - dataTypes = addDataType(term, dataTypes) - dataTypes + addDataType(term) } } - private List addDataType(OntologyTerm term, List dataTypes, datatype = null) { - String dataTypeString = datatype ? datatype.dataTypeDescription :"Clinical data" - String dataTypeCode = datatype ? datatype.dataTypeName : "clinical" - List tempDataTypes = dataTypes.collect {it.dataType} - if (dataTypeString in tempDataTypes){ + private void addDataType(OntologyTerm term, datatype = null) { + String dataTypeString = datatype ? datatype.dataTypeDescription : "Clinical data" + String dataTypeCode = datatype ? datatype.dataTypeName : "clinical" + List tempDataTypes = dataTypes.collect { it.dataType } + if (dataTypeString in tempDataTypes) { int index = tempDataTypes.indexOf(dataTypeString) DataTypeRetrieved dataType = dataTypes[index] addOntologyTerm(term, dataType) - } else{ + } else { DataTypeRetrieved dataType = new DataTypeRetrieved(dataType: dataTypeString, dataTypeCode: dataTypeCode) addOntologyTerm(term, dataType) dataTypes.add(dataType) } - dataTypes } private void addOntologyTerm(OntologyTerm term, DataTypeRetrieved dataType) { - if(cohortNumberID in dataType.OntologyTermsMap.keySet()) { + if (cohortNumberID in dataType.OntologyTermsMap.keySet()) { dataType.OntologyTermsMap[cohortNumberID].add(term) } else { dataType.OntologyTermsMap[cohortNumberID] = [term] From bc5ca6f36216cf94bfab9eebdbe0437ed555557d Mon Sep 17 00:00:00 2001 From: Jarvanerp Date: Wed, 9 Nov 2016 15:18:18 +0100 Subject: [PATCH 3/5] Change input from JSONArray to JSONObject --- .../services/org/transmartproject/db/RestExportService.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grails-app/services/org/transmartproject/db/RestExportService.groovy b/grails-app/services/org/transmartproject/db/RestExportService.groovy index 2bccff63..4f5e3f00 100644 --- a/grails-app/services/org/transmartproject/db/RestExportService.groovy +++ b/grails-app/services/org/transmartproject/db/RestExportService.groovy @@ -36,7 +36,7 @@ class RestExportService { task.getTsv() } - public def retrieveDataTypes(params) { + public List retrieveDataTypes(params) { if (!(params.containsKey('concepts'))) { throw new NoSuchResourceException("No parameter named concepts was given.") } @@ -51,7 +51,7 @@ class RestExportService { try { def conceptArguments = jsonSlurper.parseText(conceptParameters) int cohortNumber = 1 - conceptArguments.each { it -> + conceptArguments.cohorts.each { it -> List conceptKeysList = it.conceptKeys cohortNumberID = cohortNumber conceptKeysList.collect { conceptKey -> From fb81fbc565c30f6843e6d4fecdcd687e73c6d922 Mon Sep 17 00:00:00 2001 From: Jarvanerp Date: Thu, 10 Nov 2016 16:21:42 +0100 Subject: [PATCH 4/5] Make changes based on feedback --- .../db/RestExportService.groovy | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/grails-app/services/org/transmartproject/db/RestExportService.groovy b/grails-app/services/org/transmartproject/db/RestExportService.groovy index 4f5e3f00..dcb4baa9 100644 --- a/grails-app/services/org/transmartproject/db/RestExportService.groovy +++ b/grails-app/services/org/transmartproject/db/RestExportService.groovy @@ -27,10 +27,6 @@ class RestExportService { ConceptsResource conceptsResourceService - int cohortNumberID - - List dataTypes - List export(arguments) { DataExportFetchTask task = dataExportFetchTaskFactory.createTask(arguments) task.getTsv() @@ -47,17 +43,13 @@ class RestExportService { def jsonSlurper = new JsonSlurper() def conceptParameters = params.get('concepts').decodeURL() - dataTypes = [] + List dataTypes = [] try { def conceptArguments = jsonSlurper.parseText(conceptParameters) - int cohortNumber = 1 - conceptArguments.cohorts.each { it -> - List conceptKeysList = it.conceptKeys - cohortNumberID = cohortNumber - conceptKeysList.collect { conceptKey -> - getDataType(conceptKey) - } + int cohortNumber = 0 + conceptArguments.cohorts.collect { it -> cohortNumber += 1 + getDataTypes(it, dataTypes, cohortNumber) } dataTypes } catch (JsonException e) { @@ -65,7 +57,13 @@ class RestExportService { } } - private void getDataType(String conceptKey) { + private List getDataTypes(Map conceptKeysList, List dataTypes, int cohortNumber) { + conceptKeysList.conceptKeys.collect { conceptKey -> + getDataType(conceptKey, dataTypes, cohortNumber) + } + } + + private void getDataType(String conceptKey, List dataTypes, int cohortNumber) { OntologyTerm term = conceptsResourceService.getByKey(conceptKey) // Retrieve all descendant terms that have the HIGH_DIMENSIONAL attribute def terms = term.getAllDescendants() + term @@ -84,30 +82,30 @@ class RestExportService { ) def datatypes = highDimensionResourceService.getSubResourcesAssayMultiMap([constraint]) datatypes.collect({ key, value -> - addDataType(term, key) + addDataType(term, dataTypes, cohortNumber, key) }) } else { // No high dimensional data found for this term, this means it is clinical data - addDataType(term) + addDataType(term, dataTypes, cohortNumber) } } - private void addDataType(OntologyTerm term, datatype = null) { + private void addDataType(OntologyTerm term, List dataTypes, int cohortNumber, datatype = null) { String dataTypeString = datatype ? datatype.dataTypeDescription : "Clinical data" String dataTypeCode = datatype ? datatype.dataTypeName : "clinical" List tempDataTypes = dataTypes.collect { it.dataType } if (dataTypeString in tempDataTypes) { int index = tempDataTypes.indexOf(dataTypeString) DataTypeRetrieved dataType = dataTypes[index] - addOntologyTerm(term, dataType) + addOntologyTerm(term, dataType, cohortNumber) } else { DataTypeRetrieved dataType = new DataTypeRetrieved(dataType: dataTypeString, dataTypeCode: dataTypeCode) - addOntologyTerm(term, dataType) + addOntologyTerm(term, dataType, cohortNumber) dataTypes.add(dataType) } } - private void addOntologyTerm(OntologyTerm term, DataTypeRetrieved dataType) { + private void addOntologyTerm(OntologyTerm term, DataTypeRetrieved dataType, int cohortNumberID) { if (cohortNumberID in dataType.OntologyTermsMap.keySet()) { dataType.OntologyTermsMap[cohortNumberID].add(term) } else { From 2da53ac2762a9bfe338b97491b3e6938eb071160 Mon Sep 17 00:00:00 2001 From: Jarvanerp Date: Fri, 11 Nov 2016 11:21:53 +0100 Subject: [PATCH 5/5] Change name of DataTypeRetrieved to Datatypes for frontend consistency --- .../transmartproject/db/RestExportService.groovy | 14 +++++++------- .../{DataTypeRetrieved.groovy => Datatypes.groovy} | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) rename src/groovy/org/transmartproject/export/{DataTypeRetrieved.groovy => Datatypes.groovy} (88%) diff --git a/grails-app/services/org/transmartproject/db/RestExportService.groovy b/grails-app/services/org/transmartproject/db/RestExportService.groovy index dcb4baa9..4196ad69 100644 --- a/grails-app/services/org/transmartproject/db/RestExportService.groovy +++ b/grails-app/services/org/transmartproject/db/RestExportService.groovy @@ -10,7 +10,7 @@ import org.transmartproject.core.exceptions.InvalidArgumentsException import org.transmartproject.core.exceptions.NoSuchResourceException import org.transmartproject.core.ontology.ConceptsResource import org.transmartproject.core.ontology.OntologyTerm -import org.transmartproject.export.DataTypeRetrieved +import org.transmartproject.export.Datatypes import org.transmartproject.export.Tasks.DataExportFetchTask import org.transmartproject.export.Tasks.DataExportFetchTaskFactory @@ -32,7 +32,7 @@ class RestExportService { task.getTsv() } - public List retrieveDataTypes(params) { + public List retrieveDataTypes(params) { if (!(params.containsKey('concepts'))) { throw new NoSuchResourceException("No parameter named concepts was given.") } @@ -43,7 +43,7 @@ class RestExportService { def jsonSlurper = new JsonSlurper() def conceptParameters = params.get('concepts').decodeURL() - List dataTypes = [] + List dataTypes = [] try { def conceptArguments = jsonSlurper.parseText(conceptParameters) int cohortNumber = 0 @@ -57,7 +57,7 @@ class RestExportService { } } - private List getDataTypes(Map conceptKeysList, List dataTypes, int cohortNumber) { + private List getDataTypes(Map conceptKeysList, List dataTypes, int cohortNumber) { conceptKeysList.conceptKeys.collect { conceptKey -> getDataType(conceptKey, dataTypes, cohortNumber) } @@ -96,16 +96,16 @@ class RestExportService { List tempDataTypes = dataTypes.collect { it.dataType } if (dataTypeString in tempDataTypes) { int index = tempDataTypes.indexOf(dataTypeString) - DataTypeRetrieved dataType = dataTypes[index] + Datatypes dataType = dataTypes[index] addOntologyTerm(term, dataType, cohortNumber) } else { - DataTypeRetrieved dataType = new DataTypeRetrieved(dataType: dataTypeString, dataTypeCode: dataTypeCode) + Datatypes dataType = new Datatypes(dataType: dataTypeString, dataTypeCode: dataTypeCode) addOntologyTerm(term, dataType, cohortNumber) dataTypes.add(dataType) } } - private void addOntologyTerm(OntologyTerm term, DataTypeRetrieved dataType, int cohortNumberID) { + private void addOntologyTerm(OntologyTerm term, Datatypes dataType, int cohortNumberID) { if (cohortNumberID in dataType.OntologyTermsMap.keySet()) { dataType.OntologyTermsMap[cohortNumberID].add(term) } else { diff --git a/src/groovy/org/transmartproject/export/DataTypeRetrieved.groovy b/src/groovy/org/transmartproject/export/Datatypes.groovy similarity index 88% rename from src/groovy/org/transmartproject/export/DataTypeRetrieved.groovy rename to src/groovy/org/transmartproject/export/Datatypes.groovy index 6d199a07..3220fd5e 100644 --- a/src/groovy/org/transmartproject/export/DataTypeRetrieved.groovy +++ b/src/groovy/org/transmartproject/export/Datatypes.groovy @@ -2,7 +2,7 @@ package org.transmartproject.export import org.transmartproject.core.ontology.OntologyTerm -class DataTypeRetrieved { +class Datatypes { Map> OntologyTermsMap = [:]