From cdf5a77eaf67a46a7aeea9c091d51af50e43054c Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Thu, 25 Feb 2016 19:08:22 -0500 Subject: [PATCH 01/27] feat(getData): add detection if JSON is flat or nested #SOCRFW-225 --- app/partials/analysis/getData/main.jade | 2 +- app/scripts/analysis/getData/getData.coffee | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/partials/analysis/getData/main.jade b/app/partials/analysis/getData/main.jade index 0606413b..e561c614 100644 --- a/app/partials/analysis/getData/main.jade +++ b/app/partials/analysis/getData/main.jade @@ -44,4 +44,4 @@ div(ng-controller='getDataMainCtrl') button.btn.btn-primary(ng-click="getJsonByUrl()") Parse br - handsontable(purpose="json") \ No newline at end of file + handsontable(purpose="json") diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index 5de01be0..0bc2663a 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -312,6 +312,9 @@ getData = angular.module('app_analysis_getData', [ (getDataEventMngr, $scope, showState, jsonParser, state) -> console.log 'getDataMainCtrl executed' + # https://coffeescript-cookbook.github.io/chapters/arrays/check-type-is-array + typeIsArray = Array.isArray || ( value ) -> return {}.toString.call( value ) is '[object Array]' + # available SOCR Datasets $scope.socrDatasets = [ id: 'IRIS' @@ -383,16 +386,22 @@ getData = angular.module('app_analysis_getData', [ else console.log 'rejected:' + msg - $scope.getJsonByUrl = -> + $scope.getJsonByUrl = (type) -> d3.json $scope.jsonUrl, (dataResults) -> if dataResults?.length > 0 + # check if JSON contains "flat data" - 2d array + if typeIsArray dataResults and typeIsArray dataResults[0] and not typeIsArray dataResults[0][0] + type = 'flat' + else + type = 'nested' _data = columnHeader: dataResults.shift() data: [null, dataResults] # purpose is helps in pin pointing which # handsontable directive to update. purpose: 'json' + type: type console.log 'resolved' # pass a message to update the handsontable div # data is the formatted data which plugs into the @@ -527,6 +536,7 @@ getData = angular.module('app_analysis_getData', [ #check if data is in the right format # if arg? and typeof arg.data is 'object' and typeof arg.columns is 'object' if arg? and typeof arg.data is 'object' + # TODO: not to pass nested data to ht, but save in db obj = data: arg.data[1] # startRows: Object.keys(arg.data[1]).length From 56555cbc53fc8cf9b93cd357f46a4de5ecb8f994 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 7 Mar 2016 16:08:11 -0500 Subject: [PATCH 02/27] feat(getData): detect if arbitary JSON is mappable to 2d data table !WIP #SOCRFW-225 --- app/scripts/analysis/getData/getData.coffee | 71 ++++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index 0bc2663a..a4828aef 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -312,9 +312,6 @@ getData = angular.module('app_analysis_getData', [ (getDataEventMngr, $scope, showState, jsonParser, state) -> console.log 'getDataMainCtrl executed' - # https://coffeescript-cookbook.github.io/chapters/arrays/check-type-is-array - typeIsArray = Array.isArray || ( value ) -> return {}.toString.call( value ) is '[object Array]' - # available SOCR Datasets $scope.socrDatasets = [ id: 'IRIS' @@ -456,6 +453,16 @@ getData = angular.module('app_analysis_getData', [ .factory('app_analysis_getData_dataAdaptor', [ () -> + # https://coffeescript-cookbook.github.io/chapters/arrays/check-type-is-array + typeIsArray = Array.isArray || ( value ) -> return {}.toString.call(value) is '[object Array]' + + haveSameKeys = (obj1, obj2) -> + if Object.keys(obj1).length is Object.keys(obj2).length + res = (k of obj2 for k of obj1) + res.every (e) -> e is true + else + false + # accepts handsontable table as input and returns dataFrame _toDataFrame = (tableData, nSpareCols, nSpareRows) -> @@ -480,6 +487,64 @@ getData = angular.module('app_analysis_getData', [ _toHandsontable = () -> # TODO: implement for poping up data when coming back from analysis tabs + # tries to convert JSON to 2d flat data table, + # returns coverted data or false if not possible + _jsonToFlatTable = (data) -> + # check if JSON contains "flat data" - 2d array + if data? and typeof data is 'object' + if typeIsArray data + if data[0]? + # non-empty array + if (data.every (el) -> typeof el is 'object') + # array or object + if (data.every (el) -> typeIsArray el) + # array of arrays + if (data[0][0]? and data.every (col) -> col.every (el) -> typeof el in ['number', 'string']) + # array of arrays of (numbers or strings) + data + else + # empty 2d array or non-string values + false + else + # array of arbitrary objects + # http://stackoverflow.com/a/21266395/1237809 + if (not not data.reduce((prev, next) -> if haveSameKeys prev, next then next else {})) and +# (not not data.reduce((prev, next) -> if haveSameKeys prev, next then next else {})) + # array of objects with the same keys - make them columns + cols = Object.keys data[0] + # reorder values according to keys order + data = (cols.map((col) -> row[col]) for row in data) + # insert keys as a header + data.splice 0, 0, cols + else + # 1d array of strings or numbers + if (data.every (el) -> typeof el in ['number', 'string']) + data + else + # empty array + false + else + # arbitrary object + if Object.keys(data)? and Object.keys(data).length > 0 + ks = Object.keys(data) + vals = ks.map (k) -> data[k] + if vals.every (el) -> typeof el in ['number', 'string'] + # 1d object + data = [ks, vals] + else if values.every (el) -> typeof el is 'object' + # object of arrays or objects + if (vals.every (el) -> typeIsArray el) and + (not not vals.reduce((prev, next) -> if prev.length is next.length then next else [])) + # object of arrays of the same length + vals = (t[i] for t in vals for i of vals) # transpose + vals.splice 0, 0, ks # add header + else + # object of arbitrary objects + if not not vals.reduce((prev, next) -> if haveSameKeys prev, next then next else {}) + # object of objects with the same keys - make them columns + else + false + toDataFrame: _toDataFrame toHandsontable: _toHandsontable ]) From fd2bfec848257517049da03ae0cd9a472a1d798d Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 7 Mar 2016 21:33:52 -0500 Subject: [PATCH 03/27] feat(getData): WIP parse arbitrary JSON to 2d flat data table #SOCRFW-225 --- app/scripts/analysis/getData/getData.coffee | 79 +++++++++++++++++---- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index a4828aef..1a6ebdb8 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -308,8 +308,9 @@ getData = angular.module('app_analysis_getData', [ '$scope' 'showState' 'app_analysis_getData_jsonParser' + 'app_analysis_getData_dataAdaptor' '$state' - (getDataEventMngr, $scope, showState, jsonParser, state) -> + (getDataEventMngr, $scope, showState, jsonParser, dataAdaptor, state) -> console.log 'getDataMainCtrl executed' # available SOCR Datasets @@ -386,15 +387,16 @@ getData = angular.module('app_analysis_getData', [ $scope.getJsonByUrl = (type) -> d3.json $scope.jsonUrl, (dataResults) -> - if dataResults?.length > 0 + if dataResults? + res = dataAdaptor.jsonToFlatTable dataResults # check if JSON contains "flat data" - 2d array - if typeIsArray dataResults and typeIsArray dataResults[0] and not typeIsArray dataResults[0][0] + if res type = 'flat' else type = 'nested' _data = - columnHeader: dataResults.shift() - data: [null, dataResults] + columnHeader: if res.length > 1 then res.shift() else [] + data: [null, res] # purpose is helps in pin pointing which # handsontable directive to update. purpose: 'json' @@ -463,6 +465,10 @@ getData = angular.module('app_analysis_getData', [ else false + isNumStringArray = (arr) -> + console.log arr + arr.every (el) -> typeof el in ['number', 'string'] + # accepts handsontable table as input and returns dataFrame _toDataFrame = (tableData, nSpareCols, nSpareRows) -> @@ -488,7 +494,7 @@ getData = angular.module('app_analysis_getData', [ # TODO: implement for poping up data when coming back from analysis tabs # tries to convert JSON to 2d flat data table, - # returns coverted data or false if not possible + # returns coverted data or false if not possible / object is empty _jsonToFlatTable = (data) -> # check if JSON contains "flat data" - 2d array if data? and typeof data is 'object' @@ -508,14 +514,38 @@ getData = angular.module('app_analysis_getData', [ else # array of arbitrary objects # http://stackoverflow.com/a/21266395/1237809 - if (not not data.reduce((prev, next) -> if haveSameKeys prev, next then next else {})) and -# (not not data.reduce((prev, next) -> if haveSameKeys prev, next then next else {})) + if (not not data.reduce((prev, next) -> + # check if objects have same keys + if haveSameKeys prev, next + prevValues = Object.keys(prev).map (k) -> prev[k] + console.log prevValues + nextValues = Object.keys(prev).map (k) -> next[k] + console.log nextValues + # check that values are + if ((prevValues.length is nextValues.length) and + (isNumStringArray prevValues) and + (isNumStringArray nextValues) + ) +# if ((prevValues.length is nextValues.length) and +# (((isNumStringArray prevValues) and +# (isNumStringArray nextValues)) or +# ((prevValues.map(isNumStringArray).every (e) -> e is true) and +# (nextValues.map(isNumStringArray).every (e) -> e is true)) +# ) +# ) + next + else NaN + else NaN + )) # array of objects with the same keys - make them columns cols = Object.keys data[0] # reorder values according to keys order data = (cols.map((col) -> row[col]) for row in data) # insert keys as a header data.splice 0, 0, cols + data + else + false else # 1d array of strings or numbers if (data.every (el) -> typeof el in ['number', 'string']) @@ -528,25 +558,44 @@ getData = angular.module('app_analysis_getData', [ if Object.keys(data)? and Object.keys(data).length > 0 ks = Object.keys(data) vals = ks.map (k) -> data[k] - if vals.every (el) -> typeof el in ['number', 'string'] + if (vals.every (el) -> typeof el in ['number', 'string']) # 1d object data = [ks, vals] - else if values.every (el) -> typeof el is 'object' + else if (vals.every (el) -> typeof el is 'object') # object of arrays or objects - if (vals.every (el) -> typeIsArray el) and - (not not vals.reduce((prev, next) -> if prev.length is next.length then next else [])) + if ((vals.every (el) -> typeIsArray el) and + (not not vals.reduce((prev, next) -> + if (prev.length is next.length) then next else []))) # object of arrays of the same length vals = (t[i] for t in vals for i of vals) # transpose vals.splice 0, 0, ks # add header else # object of arbitrary objects - if not not vals.reduce((prev, next) -> if haveSameKeys prev, next then next else {}) - # object of objects with the same keys - make them columns + if (not not vals.reduce((prev, next) -> + # check if objects have same keys + if haveSameKeys prev, next + prevValues = Object.keys(prev).map (k) -> prev[k] + nextValues = Object.keys(prev).map (k) -> next[k] + # check that values are + if ((prevValues.length is nextValues.length) and + (isNumStringArray prevValues) and + (isNumStringArray nextValues) + ) + next + else NaN + else NaN + )) + subKs = Object.keys vals[0] + data = ([sk].concat(vals.map((val)-> val[sk])) for sk in subKs) + # insert keys as a header + data.splice 0, 0, [""].concat ks + data else false toDataFrame: _toDataFrame toHandsontable: _toHandsontable + jsonToFlatTable: _jsonToFlatTable ]) @@ -600,7 +649,7 @@ getData = angular.module('app_analysis_getData', [ #check if data is in the right format # if arg? and typeof arg.data is 'object' and typeof arg.columns is 'object' - if arg? and typeof arg.data is 'object' + if arg? and typeof arg.data is 'object' and arg.type is 'flat' # TODO: not to pass nested data to ht, but save in db obj = data: arg.data[1] From c0e72475459755079045d08f2229bb4407ba5ec3 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 8 Mar 2016 16:58:07 -0500 Subject: [PATCH 04/27] feat(getData): parse 2D-mappable JSON files #SOCRFW-225 --- app/scripts/analysis/getData/getData.coffee | 170 +++++++++----------- 1 file changed, 78 insertions(+), 92 deletions(-) diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index 1a6ebdb8..0ba47392 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -387,7 +387,8 @@ getData = angular.module('app_analysis_getData', [ $scope.getJsonByUrl = (type) -> d3.json $scope.jsonUrl, (dataResults) -> - if dataResults? + # check that data object is not empty + if dataResults? and Object.keys(dataResults)?.length > 0 res = dataAdaptor.jsonToFlatTable dataResults # check if JSON contains "flat data" - 2d array if res @@ -401,7 +402,6 @@ getData = angular.module('app_analysis_getData', [ # handsontable directive to update. purpose: 'json' type: type - console.log 'resolved' # pass a message to update the handsontable div # data is the formatted data which plugs into the # handontable. @@ -409,7 +409,7 @@ getData = angular.module('app_analysis_getData', [ # switch the accordion from getJson to grid # $scope.$emit("change in showStates","grid") else - console.log 'rejected:' + msg + console.log 'GETDATA: request failed' try _showState = new showState(['grid', 'socrData', 'worldBank', 'generate', 'jsonParse'], $scope) @@ -494,104 +494,90 @@ getData = angular.module('app_analysis_getData', [ # TODO: implement for poping up data when coming back from analysis tabs # tries to convert JSON to 2d flat data table, - # returns coverted data or false if not possible / object is empty + # assumes JSON object is not empty - has values, + # returns coverted data or false if not possible _jsonToFlatTable = (data) -> # check if JSON contains "flat data" - 2d array if data? and typeof data is 'object' if typeIsArray data - if data[0]? - # non-empty array - if (data.every (el) -> typeof el is 'object') - # array or object - if (data.every (el) -> typeIsArray el) - # array of arrays - if (data[0][0]? and data.every (col) -> col.every (el) -> typeof el in ['number', 'string']) - # array of arrays of (numbers or strings) - data - else - # empty 2d array or non-string values - false + # non-empty array + if not (data.every (el) -> typeof el is 'object') + # 1d array of strings or numbers + if (data.every (el) -> typeof el in ['number', 'string']) + data + else + # array of arrays or objects + if (data.every (el) -> typeIsArray el) + # array of arrays + if (data.every (col) -> col.every (el) -> typeof el in ['number', 'string']) + # array of arrays of (numbers or strings) + data else - # array of arbitrary objects - # http://stackoverflow.com/a/21266395/1237809 - if (not not data.reduce((prev, next) -> - # check if objects have same keys - if haveSameKeys prev, next - prevValues = Object.keys(prev).map (k) -> prev[k] - console.log prevValues - nextValues = Object.keys(prev).map (k) -> next[k] - console.log nextValues - # check that values are - if ((prevValues.length is nextValues.length) and - (isNumStringArray prevValues) and - (isNumStringArray nextValues) - ) -# if ((prevValues.length is nextValues.length) and -# (((isNumStringArray prevValues) and -# (isNumStringArray nextValues)) or -# ((prevValues.map(isNumStringArray).every (e) -> e is true) and -# (nextValues.map(isNumStringArray).every (e) -> e is true)) -# ) -# ) - next - else NaN - else NaN - )) - # array of objects with the same keys - make them columns - cols = Object.keys data[0] - # reorder values according to keys order - data = (cols.map((col) -> row[col]) for row in data) - # insert keys as a header - data.splice 0, 0, cols - data - else - false + # non-string values + false else - # 1d array of strings or numbers - if (data.every (el) -> typeof el in ['number', 'string']) + # array of arbitrary objects + # http://stackoverflow.com/a/21266395/1237809 + if (not not data.reduce((prev, next) -> + # check if objects have same keys + if haveSameKeys prev, next + prevValues = Object.keys(prev).map (k) -> prev[k] + nextValues = Object.keys(prev).map (k) -> next[k] + # check that values are numeric/string + if ((prevValues.length is nextValues.length) and + (isNumStringArray prevValues) and + (isNumStringArray nextValues) + ) + next + else NaN + else NaN + )) + # array of objects with the same keys - make them columns + cols = Object.keys data[0] + # reorder values according to keys order + data = (cols.map((col) -> row[col]) for row in data) + # insert keys as a header + data.splice 0, 0, cols data - else - # empty array - false + else + false else # arbitrary object - if Object.keys(data)? and Object.keys(data).length > 0 - ks = Object.keys(data) - vals = ks.map (k) -> data[k] - if (vals.every (el) -> typeof el in ['number', 'string']) - # 1d object - data = [ks, vals] - else if (vals.every (el) -> typeof el is 'object') - # object of arrays or objects - if ((vals.every (el) -> typeIsArray el) and - (not not vals.reduce((prev, next) -> - if (prev.length is next.length) then next else []))) - # object of arrays of the same length - vals = (t[i] for t in vals for i of vals) # transpose - vals.splice 0, 0, ks # add header - else - # object of arbitrary objects - if (not not vals.reduce((prev, next) -> - # check if objects have same keys - if haveSameKeys prev, next - prevValues = Object.keys(prev).map (k) -> prev[k] - nextValues = Object.keys(prev).map (k) -> next[k] - # check that values are - if ((prevValues.length is nextValues.length) and - (isNumStringArray prevValues) and - (isNumStringArray nextValues) - ) - next - else NaN - else NaN - )) - subKs = Object.keys vals[0] - data = ([sk].concat(vals.map((val)-> val[sk])) for sk in subKs) - # insert keys as a header - data.splice 0, 0, [""].concat ks - data - else - false + ks = Object.keys(data) + vals = ks.map (k) -> data[k] + if (vals.every (el) -> typeof el in ['number', 'string']) + # 1d object + data = [ks, vals] + else if (vals.every (el) -> typeof el is 'object') + # object of arrays or objects + if (vals.every (row) -> typeIsArray row) and + (vals.every (row) -> row.every (el) -> typeof el in ['number', 'string']) + # object of arrays + vals = (t[i] for t in vals for i of vals) # transpose + vals.splice 0, 0, ks # add header + vals + else + # object of arbitrary objects + if (not not vals.reduce((prev, next) -> + # check if objects have same keys + if haveSameKeys prev, next + prevValues = Object.keys(prev).map (k) -> prev[k] + nextValues = Object.keys(prev).map (k) -> next[k] + # check that values are + if ((prevValues.length is nextValues.length) and + (isNumStringArray prevValues) and + (isNumStringArray nextValues) + ) + next + else NaN + else NaN + )) + subKs = Object.keys vals[0] + data = ([sk].concat(vals.map((val)-> val[sk])) for sk in subKs) + # insert keys as a header + data.splice 0, 0, [""].concat ks + data + else false toDataFrame: _toDataFrame toHandsontable: _toHandsontable From 10417ceafb6fe57d8abd070a7833d22488a1fbaa Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Thu, 10 Mar 2016 18:50:39 -0500 Subject: [PATCH 05/27] feat(getData, db): allow saving arbitrary object to db #SOCRFW-227 --- app/scripts/analysis/getData/getData.coffee | 82 +++++++++++--------- app/scripts/db/db.coffee | 86 +++++++++++++++------ 2 files changed, 107 insertions(+), 61 deletions(-) diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index 0ba47392..df135ef8 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -97,10 +97,11 @@ getData = angular.module('app_analysis_getData', [ sb.publish msg: 'save data' - data: + dat6 dataFrame: data tableName: $stateParams.projectId + ':' + $stateParams.forkId promise: deferred + type: if data.type is 'nested' then data.type else 'flat' msgScope: ['getData'] callback: -> console.log 'handsontable data updated to db' @@ -237,13 +238,22 @@ getData = angular.module('app_analysis_getData', [ '$stateParams' 'app_analysis_getData_inputCache' ($q, $scope, getDataEventMngr, jsonParser, $stateParams, inputCache) -> - # get the sandbox made for this module - # sb = getDataSb.getSb() - # console.log 'sandbox created' $scope.jsonUrl = '' flag = true $scope.selected = null + passReceivedData = (data) -> + if data.type is 'nested' + inputCache.set data + else + # default data type is 2d 'flat' table + data.type = 'flat' + # pass a message to update the handsontable div + # data is the formatted data which plugs into the + # handontable. + # TODO: getData module shouldn't know about controllers listening for handsontable update + $scope.$emit 'update handsontable', data + # showGrid $scope.show = (val) -> switch val @@ -255,7 +265,7 @@ getData = angular.module('app_analysis_getData', [ data = default: true purpose: 'json' - $scope.$emit 'update handsontable', data + passReceivedData data $scope.$emit 'change in showStates', 'grid' when 'socrData' @@ -276,7 +286,6 @@ getData = angular.module('app_analysis_getData', [ # getJson $scope.getJson = -> - console.log 123 console.log $scope.jsonUrl if $scope.jsonUrl is '' @@ -290,7 +299,7 @@ getData = angular.module('app_analysis_getData', [ # Pass a message to update the handsontable div. # data is the formatted data which plugs into the # handontable. - $scope.$emit 'update handsontable', data + passReceivedData data $scope.$emit 'get Data from handsontable', inputCache , (msg) -> @@ -309,10 +318,24 @@ getData = angular.module('app_analysis_getData', [ 'showState' 'app_analysis_getData_jsonParser' 'app_analysis_getData_dataAdaptor' + 'app_analysis_getData_inputCache' '$state' - (getDataEventMngr, $scope, showState, jsonParser, dataAdaptor, state) -> + (getDataEventMngr, $scope, showState, jsonParser, dataAdaptor, inputCache, state) -> console.log 'getDataMainCtrl executed' + passReceivedData = (data) -> + if data.type is 'nested' + inputCache.set data + else + # default data type is 2d 'flat' table + data.type = 'flat' + # pass a message to update the handsontable div + # data is the formatted data which plugs into the + # handontable. + # TODO: getData module shouldn't know about controllers listening for handsontable update + $scope.$emit 'update handsontable', data + + # available SOCR Datasets $scope.socrDatasets = [ id: 'IRIS' @@ -342,14 +365,7 @@ getData = angular.module('app_analysis_getData', [ .then( (data) -> console.log 'resolved' - # Pass a message to update the handsontable div. - # data is the formatted data which plugs into the - # handontable. - - # TODO: getData module shouldn't know about controllers listening for handsontable update - $scope.$emit 'update handsontable', data - # Switch the accordion from getJson to grid. - #$scope.$emit("change in showStates","grid") + passReceivedData data , (msg) -> console.log 'rejected:' + msg @@ -375,12 +391,7 @@ getData = angular.module('app_analysis_getData', [ # handsontable directive to update. purpose: 'json' console.log 'resolved' - # pass a message to update the handsontable div - # data is the formatted data which plugs into the - # handontable. - $scope.$emit 'update handsontable', _data - # switch the accordion from getJson to grid - # $scope.$emit("change in showStates","grid") + passReceivedData _data else console.log 'rejected:' + msg @@ -392,22 +403,18 @@ getData = angular.module('app_analysis_getData', [ res = dataAdaptor.jsonToFlatTable dataResults # check if JSON contains "flat data" - 2d array if res - type = 'flat' + _data = + columnHeader: if res.length > 1 then res.shift() else [] + data: [null, res] + # purpose is helps in pin pointing which + # handsontable directive to update. + purpose: 'json' + type: 'flat' else - type = 'nested' - _data = - columnHeader: if res.length > 1 then res.shift() else [] - data: [null, res] - # purpose is helps in pin pointing which - # handsontable directive to update. - purpose: 'json' - type: type - # pass a message to update the handsontable div - # data is the formatted data which plugs into the - # handontable. - $scope.$emit 'update handsontable', _data - # switch the accordion from getJson to grid - # $scope.$emit("change in showStates","grid") + _data = + data: dataResults + type: 'nested' + passReceivedData _data else console.log 'GETDATA: request failed' @@ -616,7 +623,6 @@ getData = angular.module('app_analysis_getData', [ # retrieves data from handsontable object _format = (obj) -> - data = obj.getData() header = obj.getColHeader() nCols = obj.countCols() diff --git a/app/scripts/db/db.coffee b/app/scripts/db/db.coffee index 963004d2..32572b16 100644 --- a/app/scripts/db/db.coffee +++ b/app/scripts/db/db.coffee @@ -106,6 +106,20 @@ db.factory 'app_database_dataAdaptor', [ toDataFrame: _toDataFrame ] +db.factory 'app_database_nested', [ + () -> + _nestedObj = null + + _save = (obj) -> + _nestedObj = obj + + _get = () -> + _nestedObj + + save: _save + get: _get +] + db.service 'app_database_dv', -> # contains references to all the tables created. @@ -262,20 +276,60 @@ db.service 'app_database_dv', -> db.factory 'app_database_handler', [ '$q' 'app_database_dv' + 'app_database_nested' 'app_database_dataAdaptor' - ($q, _db, dataAdaptor) -> + ($q, _db, nestedDb, dataAdaptor) -> - # set all the callbacks here. + # set all the callbacks here _setSb = ((_db) -> window.db = _db - (sb) -> - #registering database callbacks for all possible incoming messages. + _lastDataType = '' + + _saveData = (data) -> + # convert from the universal dataFrame object to datavore table or keep as is + if data.type? + _lastDataType = data.type + switch data.type + when 'flat' + dvData = dataAdaptor.toDvTable data.dataFrame + res = _db.create dvData, data.tableName + res + when 'nested' + nestedDb.save data.data + true + else console.log '%cDATABASE: data type is unknown' , 'color:green' + else console.log '%cDATABASE: data type is unknown' , 'color:green' + + _getData = (data) -> +# if data.type + switch _lastDataType + when 'flat' + _data = _db.get data.tableName + # convert data to DataFrame if returning it + _data = dataAdaptor.toDataFrame _data + _data + when 'nested' + _data = nestedDb.get() + _data + else console.log '%cDATABASE: data type is unknown' , 'color:green' +# else console.log '%cDATABASE: data type is unknown' , 'color:green' + + (sb) -> + # registering database callbacks for all possible incoming messages # TODO: add wrapper layer on top of _db methods? _methods = [ - {incoming: 'save table', outgoing: 'table saved', event: _db.create} - {incoming: 'get table', outgoing: 'take table', event: _db.get} - {incoming: 'add listener', outgoing: 'listener added', event: _db.addListener} + incoming: 'save table' + outgoing: 'table saved' + event: _saveData + , + incoming: 'get table' + outgoing: 'take table' + event: _getData + , + incoming: 'add listener' + outgoing: 'listener added' + event: _db.addListener ] _status = _methods.map (method) -> @@ -283,23 +337,9 @@ db.factory 'app_database_handler', [ msg: method['incoming'] msgScope: ['database'] listener: (msg, data) -> - console.log "%cDATABASE: listener called for"+msg , "color:green" - - # convert from the universal dataFrame object to datavore table - dvTableData = if msg is 'save table' then dataAdaptor.toDvTable data.dataFrame else data - - # arrange arguments for a callback - # @todo need to find a better way for this. - _data = switch - when msg is 'save table' then [ dvTableData, data.tableName ] - when msg is 'get table' then [ data.tableName ] - else data - + console.log "%cDATABASE: listener called for" + msg , "color:green" # invoke callback - _data = method.event.apply null, _data - - # convert data to DataFrame if returning it - _data = dataAdaptor.toDataFrame _data if msg is 'get table' + _data = method.event.apply null, [data] # all publish calls should pass a promise in the data object # if promise is not defined, create one and pass it along From 4744caa28fb03f1fea51a57ded090b341e37a408 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Fri, 11 Mar 2016 12:38:08 -0500 Subject: [PATCH 06/27] fix(getData): fix typo in inputCache message --- app/scripts/analysis/getData/getData.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index df135ef8..eb71c196 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -97,7 +97,7 @@ getData = angular.module('app_analysis_getData', [ sb.publish msg: 'save data' - dat6 + data: dataFrame: data tableName: $stateParams.projectId + ':' + $stateParams.forkId promise: deferred From 1ea492d6c3df182498d39300136c3955c5b013b8 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 14 Mar 2016 18:00:26 -0400 Subject: [PATCH 07/27] fix(db): add data type when return latest dataset #SOCRFW-229 --- app/scripts/db/db.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/scripts/db/db.coffee b/app/scripts/db/db.coffee index 32572b16..4c06fca4 100644 --- a/app/scripts/db/db.coffee +++ b/app/scripts/db/db.coffee @@ -308,10 +308,13 @@ db.factory 'app_database_handler', [ _data = _db.get data.tableName # convert data to DataFrame if returning it _data = dataAdaptor.toDataFrame _data + _data.dataType = 'flat' _data when 'nested' _data = nestedDb.get() - _data + _data = + data: _data + dataType: 'nested' else console.log '%cDATABASE: data type is unknown' , 'color:green' # else console.log '%cDATABASE: data type is unknown' , 'color:green' From 3274a2066c5470b3cf25a3ec0a12eccf080fbac5 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 14 Mar 2016 19:16:20 -0400 Subject: [PATCH 08/27] feat(wrangler): check if data is flat table before wrangling, hide UI otherwise #SOCRFW-230 --- .../analysis/wrangleData/wrangler.jade | 24 +++-- .../analysis/wrangleData/wrangleData.coffee | 101 ++++++++++-------- 2 files changed, 70 insertions(+), 55 deletions(-) diff --git a/app/partials/analysis/wrangleData/wrangler.jade b/app/partials/analysis/wrangleData/wrangler.jade index ef7ba308..3205cf72 100644 --- a/app/partials/analysis/wrangleData/wrangler.jade +++ b/app/partials/analysis/wrangleData/wrangler.jade @@ -1,10 +1,14 @@ -#dt_example(style='height: 400px') - .ui-layout-north#wranglerNorthPanel - #wranglerDashboard - .ui-layout-west#profilerWestPanel - #transformEditor.transformEditor - .ui-layout-center#profilerCenterPanel - #table - #preview - .spacer - .ui-layout-south \ No newline at end of file +div + #dt_example(style='height: 400px', ng-hide="dataType != 'flat'") + .ui-layout-north#wranglerNorthPanel + #wranglerDashboard + .ui-layout-west#profilerWestPanel + #transformEditor.transformEditor + .ui-layout-center#profilerCenterPanel + #table + #preview + .spacer + .ui-layout-south + div.lead.bg-danger(ng-hide="dataType == 'flat'") + | Data Wrangler doesn't support current dataset. + | Only "flat" data tables are supported. diff --git a/app/scripts/analysis/wrangleData/wrangleData.coffee b/app/scripts/analysis/wrangleData/wrangleData.coffee index 885bfabc..f658e6db 100644 --- a/app/scripts/analysis/wrangleData/wrangleData.coffee +++ b/app/scripts/analysis/wrangleData/wrangleData.coffee @@ -151,17 +151,21 @@ wrangleData = angular.module('app_analysis_wrangleData', []) _initial_transforms = [] _table = [] + _csvData = [] - _start = (viewContainers) -> + _init = () -> data = dataRetriever.getData() - csvData = dataAdaptor.toCsvString data - _table = _wrangle csvData, viewContainers + if data.dataType is 'flat' + _csvData = dataAdaptor.toCsvString data + true + else + false - _wrangle = (csvData, viewContainers) -> + _wrangle = (viewContainers) -> # TODO: abstract from using dv directly #SOCRFW-143 - table = dv.table csvData + table = dv.table _csvData - _initial_transforms = dw.raw_inference(csvData).transforms + _initial_transforms = dw.raw_inference(_csvData).transforms dw.wrangler table: table @@ -208,7 +212,8 @@ wrangleData = angular.module('app_analysis_wrangleData', []) ), 1000 true - start: _start + init: _init + start: _wrangle saveData: _saveDataToDb ]) @@ -232,8 +237,14 @@ wrangleData = angular.module('app_analysis_wrangleData', []) 'app_analysis_wrangleData_manager' ($scope, $rootScope, wrangler, msgManager) -> - # TODO: isolate dw from global scope - w = dw.wrangle() + $scope.dataType = '' + + data = wrangler.init() + if data + $scope.dataType = 'flat' + + # TODO: isolate dw from global scope + w = dw.wrangle() # listen to state change and save data when exiting Wrangle Data stateListener = $rootScope.$on '$stateChangeStart', (event, toState, toParams, fromState, fromParams) -> @@ -244,7 +255,8 @@ wrangleData = angular.module('app_analysis_wrangleData', []) msgManager.broadcast 'wrangler:done' # unsubscribe stateListener() - console.log 'wrangleDataMainCtrl executed' + + console.log 'wrangleDataMainCtrl executed' ]) .directive 'datawrangler', [ @@ -255,48 +267,47 @@ wrangleData = angular.module('app_analysis_wrangleData', []) restrict: 'E' transclude: true templateUrl: '../partials/analysis/wrangleData/wrangler.html' + replace: true # replace the directive element with the output of the template # the controller for the directive controller: ($scope) -> - myLayout = $('#dt_example').layout - north: - spacing_open: 0 - resizable: false - slidable: false - fxName: 'none' - south: - spacing_open: 0 - resizable: false - slidable: false - fxName: 'none' - west: - minSize: 310 - - container = $('#table') - previewContainer = $('#preview') - transformContainer = $('#transformEditor') - dashboardContainer = $("#wranglerDashboard") - - wrangler.start - tableContainer: container - transformContainer: transformContainer - previewContainer: previewContainer - dashboardContainer: dashboardContainer - - # TODO: find correct programmatic way to invoke header propagation - # assuming there always is a header in data, propagate it in Wrangler - $('#table .odd .rowHeader').first().mouseup().mousedown() - d3.select('div.menu_option.Promote')[0][0].__onmousedown() - $('div.suggestion.selected').click() - - replace: true # replace the directive element with the output of the template - # The link method does the work of setting the directive # up, things like bindings, jquery calls, etc are done in here - # It is run before the controller link: (scope, elem, attr) -> - # useful to identify which handsontable instance to update scope.purpose = attr.purpose + + # check if received dataset is flat + if scope.dataType? and scope.dataType is 'flat' + myLayout = $('#dt_example').layout + north: + spacing_open: 0 + resizable: false + slidable: false + fxName: 'none' + south: + spacing_open: 0 + resizable: false + slidable: false + fxName: 'none' + west: + minSize: 310 + + container = $('#table') + previewContainer = $('#preview') + transformContainer = $('#transformEditor') + dashboardContainer = $("#wranglerDashboard") + + wrangler.start + tableContainer: container + transformContainer: transformContainer + previewContainer: previewContainer + dashboardContainer: dashboardContainer + + # TODO: find correct programmatic way to invoke header propagation + # assuming there always is a header in data, propagate it in Wrangler + $('#table .odd .rowHeader').first().mouseup().mousedown() + d3.select('div.menu_option.Promote')[0][0].__onmousedown() + $('div.suggestion.selected').click() ] From b417948fe70bdd131cd497e0de0c0caeff89fecf Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 15 Mar 2016 14:58:44 -0400 Subject: [PATCH 09/27] feat(kmeans): check if data is flat 2d table, otherwise hide main area #SOCRFW-231 --- .../tools/machineLearning/kMeans/main.jade | 32 +++++++++++-------- .../machineLearning/kMeans/kmeans.coffee | 26 ++++++++++----- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/app/partials/analysis/tools/machineLearning/kMeans/main.jade b/app/partials/analysis/tools/machineLearning/kMeans/main.jade index 6b27e7d6..65bd0728 100644 --- a/app/partials/analysis/tools/machineLearning/kMeans/main.jade +++ b/app/partials/analysis/tools/machineLearning/kMeans/main.jade @@ -1,16 +1,20 @@ div(ng-controller="kMeansMainCtrl") h2 2D k-means Clustering - p. - k-means clustering aims to partition n observations into k clusters in which each observation belongs - to the cluster with the nearest mean, serving as a prototype of the cluster. - div(ng-show="showresults").kmeans-results - p.lead Average accuracy: {{avgAccuracy}} - div.table-responsive - table.table.table-bordered.table-condensed - tr - th Label - th Accuracy - tr(ng-repeat="(label, acc) in accs") - td {{label}} - td {{acc.toFixed(2)}} - app-kmeans + div(ng-hide="dataType != 'flat'") + p. + k-means clustering aims to partition n observations into k clusters in which each observation belongs + to the cluster with the nearest mean, serving as a prototype of the cluster. + div(ng-show="showresults").kmeans-results + p.lead Average accuracy: {{avgAccuracy}} + div.table-responsive + table.table.table-bordered.table-condensed + tr + th Label + th Accuracy + tr(ng-repeat="(label, acc) in accs") + td {{label}} + td {{acc.toFixed(2)}} + app-kmeans + div.lead.bg-danger(ng-hide="dataType == 'flat'") + | 2D k-means Clustering doesn't support current dataset. + | Only "flat" data tables are supported. diff --git a/app/scripts/analysis/tools/machineLearning/kMeans/kmeans.coffee b/app/scripts/analysis/tools/machineLearning/kMeans/kmeans.coffee index c840ab5b..f1e4adf4 100644 --- a/app/scripts/analysis/tools/machineLearning/kMeans/kmeans.coffee +++ b/app/scripts/analysis/tools/machineLearning/kMeans/kmeans.coffee @@ -61,6 +61,7 @@ kMeans = angular.module('app_analysis_kMeans', []) $scope.showresults = off $scope.avgAccuracy = '' $scope.accs = {} + $scope.dataType = '' prettifyArrayOutput = (arr) -> if arr? @@ -90,6 +91,9 @@ kMeans = angular.module('app_analysis_kMeans', []) $scope.$on 'kmeans:updateDataPoints', (event, dataPoints) -> _update dataPoints + $scope.$on 'kmeans:updateDataType', (event, dataType) -> + $scope.dataType = dataType + _finish = (results=null) -> msgManager.broadcast 'kmeans:done', results showResults results @@ -224,20 +228,26 @@ kMeans = angular.module('app_analysis_kMeans', []) $scope.running = 'hidden' kmeans.run data, $scope.k, $scope.dist, $scope.initMethod + parseData = (data) -> + updateSidebarControls(data) + updateDataPoints(data) + $scope.detectKValue = -> + detectedK = detectKValue data + setDetectedKValue detectedK + $scope.run = -> + _data = parseDataForKMeans data + callKMeans _data + # subscribe for incoming message with data subscribeForData = -> token = sb.subscribe msg: 'take data' msgScope: ['kMeans'] listener: (msg, data) -> - updateSidebarControls(data) - updateDataPoints(data) - $scope.detectKValue = -> - detectedK = detectKValue data - setDetectedKValue detectedK - $scope.run = -> - _data = parseDataForKMeans data - callKMeans _data + if data.dataType? and data.dataType is 'flat' + $timeout -> + msgManager.broadcast 'kmeans:updateDataType', data.dataType + parseData data # ask core for data sendDataRequest = (deferred, token) -> From c55b5bcfd525baf74efb43338dfe8caf8aa37d5d Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 15 Mar 2016 18:54:04 -0400 Subject: [PATCH 10/27] fix(db, getData): use dataType flag as a filed of dataFrame for dataset type #SOCRFW-229 --- app/scripts/analysis/getData/getData.coffee | 16 ++++----- app/scripts/db/db.coffee | 39 +++++++++++---------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index eb71c196..d33b860e 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -101,7 +101,6 @@ getData = angular.module('app_analysis_getData', [ dataFrame: data tableName: $stateParams.projectId + ':' + $stateParams.forkId promise: deferred - type: if data.type is 'nested' then data.type else 'flat' msgScope: ['getData'] callback: -> console.log 'handsontable data updated to db' @@ -243,11 +242,11 @@ getData = angular.module('app_analysis_getData', [ $scope.selected = null passReceivedData = (data) -> - if data.type is 'nested' + if data.dataType is 'nested' inputCache.set data else # default data type is 2d 'flat' table - data.type = 'flat' + data.dataType = 'flat' # pass a message to update the handsontable div # data is the formatted data which plugs into the # handontable. @@ -324,11 +323,11 @@ getData = angular.module('app_analysis_getData', [ console.log 'getDataMainCtrl executed' passReceivedData = (data) -> - if data.type is 'nested' + if data.dataType is 'nested' inputCache.set data else # default data type is 2d 'flat' table - data.type = 'flat' + data.dataType = 'flat' # pass a message to update the handsontable div # data is the formatted data which plugs into the # handontable. @@ -409,11 +408,11 @@ getData = angular.module('app_analysis_getData', [ # purpose is helps in pin pointing which # handsontable directive to update. purpose: 'json' - type: 'flat' + dataType: 'flat' else _data = data: dataResults - type: 'nested' + dataType: 'nested' passReceivedData _data else console.log 'GETDATA: request failed' @@ -496,6 +495,7 @@ getData = angular.module('app_analysis_getData', [ header: tableData.header nRows: tableData.nRows - nSpareRows nCols: tableData.nCols - nSpareCols + dataType: 'flat' _toHandsontable = () -> # TODO: implement for poping up data when coming back from analysis tabs @@ -641,7 +641,7 @@ getData = angular.module('app_analysis_getData', [ #check if data is in the right format # if arg? and typeof arg.data is 'object' and typeof arg.columns is 'object' - if arg? and typeof arg.data is 'object' and arg.type is 'flat' + if arg? and typeof arg.data is 'object' and arg.dataType is 'flat' # TODO: not to pass nested data to ht, but save in db obj = data: arg.data[1] diff --git a/app/scripts/db/db.coffee b/app/scripts/db/db.coffee index 4c06fca4..83296ff9 100644 --- a/app/scripts/db/db.coffee +++ b/app/scripts/db/db.coffee @@ -101,6 +101,7 @@ db.factory 'app_database_dataAdaptor', [ types: _types nRows: _nRows nCols: _nCols + dataType: 'flat' toDvTable: _toDvTable toDataFrame: _toDataFrame @@ -286,23 +287,25 @@ db.factory 'app_database_handler', [ _lastDataType = '' - _saveData = (data) -> - # convert from the universal dataFrame object to datavore table or keep as is - if data.type? - _lastDataType = data.type - switch data.type - when 'flat' - dvData = dataAdaptor.toDvTable data.dataFrame - res = _db.create dvData, data.tableName - res - when 'nested' - nestedDb.save data.data - true - else console.log '%cDATABASE: data type is unknown' , 'color:green' - else console.log '%cDATABASE: data type is unknown' , 'color:green' + _saveData = (obj) -> + if obj.dataFrame? + dataFrame = obj.dataFrame + # convert from the universal dataFrame object to datavore table or keep as is + if dataFrame.dataType? + _lastDataType = dataFrame.dataType + switch dataFrame.dataType + when 'flat' + dvData = dataAdaptor.toDvTable dataFrame + res = _db.create dvData, obj.tableName + res + when 'nested' + nestedDb.save obj.data + true + else console.log '%cDATABASE: data type is unknown' , 'color:green' + else console.log '%cDATABASE: data type is unknown' , 'color:green' + else console.log '%cDATABASE: nothing to save' , 'color:green' _getData = (data) -> -# if data.type switch _lastDataType when 'flat' _data = _db.get data.tableName @@ -339,14 +342,14 @@ db.factory 'app_database_handler', [ sb.subscribe msg: method['incoming'] msgScope: ['database'] - listener: (msg, data) -> + listener: (msg, obj) -> console.log "%cDATABASE: listener called for" + msg , "color:green" # invoke callback - _data = method.event.apply null, [data] + _data = method.event.apply null, [obj] # all publish calls should pass a promise in the data object # if promise is not defined, create one and pass it along - deferred = data.promise + deferred = obj.promise if typeof deferred isnt 'undefined' if _data isnt false then deferred.resolve() else deferred.reject() else From 47209af99ad45f0f71900ea96c78b8c62bc412c9 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 15 Mar 2016 19:00:48 -0400 Subject: [PATCH 11/27] fix(wrangler): save data on exit if its flat, otherwise dont #SOCRFW-230 --- app/scripts/analysis/wrangleData/wrangleData.coffee | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/scripts/analysis/wrangleData/wrangleData.coffee b/app/scripts/analysis/wrangleData/wrangleData.coffee index f658e6db..ff63a849 100644 --- a/app/scripts/analysis/wrangleData/wrangleData.coffee +++ b/app/scripts/analysis/wrangleData/wrangleData.coffee @@ -133,6 +133,7 @@ wrangleData = angular.module('app_analysis_wrangleData', []) types: _types nRows: _nRows nCols: _nCols + dataType: 'flat' toDvTable: _toDvTable toDataFrame: _toDataFrame @@ -161,6 +162,9 @@ wrangleData = angular.module('app_analysis_wrangleData', []) else false + _start = (viewContainers) -> + _table = _wrangle(viewContainers) + _wrangle = (viewContainers) -> # TODO: abstract from using dv directly #SOCRFW-143 table = dv.table _csvData @@ -213,7 +217,7 @@ wrangleData = angular.module('app_analysis_wrangleData', []) true init: _init - start: _wrangle + start: _start saveData: _saveDataToDb ]) @@ -249,8 +253,9 @@ wrangleData = angular.module('app_analysis_wrangleData', []) # listen to state change and save data when exiting Wrangle Data stateListener = $rootScope.$on '$stateChangeStart', (event, toState, toParams, fromState, fromParams) -> if fromState.name? and fromState.name is 'wrangleData' - # save data to db on exit from wrangler - wrangler.saveData() + if $scope.dataType is 'flat' + # save data to db on exit from wrangler + wrangler.saveData() # signal to show sidebar msgManager.broadcast 'wrangler:done' # unsubscribe From 4cdab98331c41eb94ce0e82387c4d81f2794adc7 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 15 Mar 2016 19:01:59 -0400 Subject: [PATCH 12/27] feat(instrPerfEval): check if data table is flat, otherwise disable controls #SOCRFW-232 --- .../psychometrics/instrPerfEval/main.jade | 148 ++++++++-------- .../psychometrics/instrPerfEval/sidebar.jade | 2 +- .../instrPerfEval/instrPerfEval.coffee | 161 ++++++++++-------- 3 files changed, 169 insertions(+), 142 deletions(-) diff --git a/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade b/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade index d2812819..3a1f1a28 100644 --- a/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade +++ b/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade @@ -1,73 +1,79 @@ div(ng-controller="instrPerfEvalMainCtrl") h2 Instrument Performance Evaluation - p. - Cronbach’s Alpha (α) is a measure of internal consistency or reliability of a psychometric instrument and measures - how well a set of items measure a single, one-dimensional latent aspect of individuals. - p.lead Cronbach's α: {{cronAlpha}} - div.table-responsive - table.table.table-bordered.table-striped - thead - tr - th Cronbach's alpha - th Internal consistency - tbody - tr.success - td α ≥ 0.9 - td Excellent (High-Stakes testing) - tr.success - td 0.7 ≤ α < 0.9 - td Good (Low-Stakes testing) - tr.info - td 0.6 ≤ α < 0.7 - td Acceptable - tr.warning - td 0.5 ≤ α < 0.6 - td Poor - tr.danger - td α < 0.5 - td Unacceptable - p. - Cronbach's α coefficient is a point estimate of the reliability. - Its standard error is important to construct an interval estimation of its true value - and to obtain statistical inference about its significance. - There are parametric and non-parametric methods to estimate the variance of Cronbach's α, - and compute confidence intervals. - p Cronbach’s Alpha confidence intervals - p.bg-info ID confidence interval: {{cronAlphaIdInterval}} - p.bg-info Koning and Franses confidence interval: {{cronAlphaKfInterval}} - p.bg-info Bootstrap confidence interval: {{cronAlphaBootstrapInterval}} - p.bg-info Logit confidence interval: {{cronAlphaLogitInterval}} - p.bg-info Asymptotically distribution-free (ADF) interval: {{cronAlphaAdfInterval}} - br - p Other metrics of reliability: - p. - The Intra-class correlation coefficient (ICC) assesses the consistency, or reproducibility, - of quantitative measurements made by different observers measuring the same quantity. - Broadly speaking, the ICC is defined as the ratio of between-cluster variance to total variance. - p.bg-info Intraclass correlation coefficient (ICC): {{icc}} - p. - In Split-Half Reliability assessment, the test is split in half (e.g., odd / even) creating "equivalent forms". - The two "forms" are correlated with each other and the correlation coefficient is adjusted - to reflect the entire test length, using the Spearman-Brown Prophecy formula. - p.bg-info Split-Half Reliability coefficient: {{splitHalfCoef}} - p. - The Kuder–Richardson Formula 20 (KR-20) is a very reliable internal reliability estimate which simulates - calculating split-half reliability for every possible combination of items. - The Cronbach's α and KR-20 are similar ― KR-20 is a derivative of the Cronbach's α with the advantage that - it can handle both dichotomous and continuous variables, however, KR-20 can't be used - when multiple-choice questions involve partial credit and require systematic item-based analysis. - p.bg-info Kuder–Richardson Formula 20 (KR-20): {{kr20}} - br - p - small - p References: - p 1.  - a(href='http://wiki.socr.umich.edu/index.php/SMHS_Cronbachs'). - Scientific Methods for Health Sciences - Instrument Performance Evaluation: Cronbach's α - p. - 2. TSAGRIS, MICHAIL, CONSTANTINOS C. FRANGOS, and CHRISTOS C. FRANGOS. - "Confidence intervals for Cronbach’s reliability coefficient." - p. - 3. Maydeu-Olivares, Alberto, Donna L. Coffman, and Wolfgang M. Hartmann. - "Asymptotically distribution-free (ADF) interval estimation of coefficient alpha." - Psychological methods 12.2 (2007): 157. + div(ng-hide="dataType != 'flat'") + p. + Cronbach’s Alpha (α) is a measure of internal consistency or reliability of a psychometric instrument and measures + how well a set of items measure a single, one-dimensional latent aspect of individuals. + p.lead Cronbach's α: {{cronAlpha}} + div.table-responsive + table.table.table-bordered.table-striped + thead + tr + th Cronbach's alpha + th Internal consistency + tbody + tr.success + td α ≥ 0.9 + td Excellent (High-Stakes testing) + tr.success + td 0.7 ≤ α < 0.9 + td Good (Low-Stakes testing) + tr.info + td 0.6 ≤ α < 0.7 + td Acceptable + tr.warning + td 0.5 ≤ α < 0.6 + td Poor + tr.danger + td α < 0.5 + td Unacceptable + p. + Cronbach's α coefficient is a point estimate of the reliability. + Its standard error is important to construct an interval estimation of its true value + and to obtain statistical inference about its significance. + There are parametric and non-parametric methods to estimate the variance of Cronbach's α, + and compute confidence intervals. + p Cronbach’s Alpha confidence intervals + p.bg-info ID confidence interval: {{cronAlphaIdInterval}} + p.bg-info Koning and Franses confidence interval: {{cronAlphaKfInterval}} + p.bg-info Bootstrap confidence interval: {{cronAlphaBootstrapInterval}} + p.bg-info Logit confidence interval: {{cronAlphaLogitInterval}} + p.bg-info Asymptotically distribution-free (ADF) interval: {{cronAlphaAdfInterval}} + br + p Other metrics of reliability: + p. + The Intra-class correlation coefficient (ICC) assesses the consistency, or reproducibility, + of quantitative measurements made by different observers measuring the same quantity. + Broadly speaking, the ICC is defined as the ratio of between-cluster variance to total variance. + p.bg-info Intraclass correlation coefficient (ICC): {{icc}} + p. + In Split-Half Reliability assessment, the test is split in half (e.g., odd / even) creating "equivalent forms". + The two "forms" are correlated with each other and the correlation coefficient is adjusted + to reflect the entire test length, using the Spearman-Brown Prophecy formula. + p.bg-info Split-Half Reliability coefficient: {{splitHalfCoef}} + p. + The Kuder–Richardson Formula 20 (KR-20) is a very reliable internal reliability estimate which simulates + calculating split-half reliability for every possible combination of items. + The Cronbach's α and KR-20 are similar ― KR-20 is a derivative of the Cronbach's α with the advantage that + it can handle both dichotomous and continuous variables, however, KR-20 can't be used + when multiple-choice questions involve partial credit and require systematic item-based analysis. + p.bg-info Kuder–Richardson Formula 20 (KR-20): {{kr20}} + br + p + small + p References: + p 1.  + a(href='http://wiki.socr.umich.edu/index.php/SMHS_Cronbachs'). + Scientific Methods for Health Sciences - Instrument Performance Evaluation: Cronbach's α + p. + 2. TSAGRIS, MICHAIL, CONSTANTINOS C. FRANGOS, and CHRISTOS C. FRANGOS. + "Confidence intervals for Cronbach’s reliability coefficient." + p. + 3. Maydeu-Olivares, Alberto, Donna L. Coffman, and Wolfgang M. Hartmann. + "Asymptotically distribution-free (ADF) interval estimation of coefficient alpha." + Psychological methods 12.2 (2007): 157. + + + div.lead.bg-danger(ng-hide="dataType == 'flat'") + | Instrument Performance Evaluation doesn't support current dataset. + | Only "flat" data tables are supported. diff --git a/app/partials/analysis/tools/psychometrics/instrPerfEval/sidebar.jade b/app/partials/analysis/tools/psychometrics/instrPerfEval/sidebar.jade index a2fe336b..6cff0775 100644 --- a/app/partials/analysis/tools/psychometrics/instrPerfEval/sidebar.jade +++ b/app/partials/analysis/tools/psychometrics/instrPerfEval/sidebar.jade @@ -1,6 +1,6 @@ div(ng-controller="instrPerfEvalSidebarCtrl") form - fieldset + fieldset(ng-disabled="!perfeval") legend Data parameters label Number of columns  input(ng-model="nCols").input-small diff --git a/app/scripts/analysis/tools/psychometrics/instrPerfEval/instrPerfEval.coffee b/app/scripts/analysis/tools/psychometrics/instrPerfEval/instrPerfEval.coffee index 94c75e28..28f03cf3 100644 --- a/app/scripts/analysis/tools/psychometrics/instrPerfEval/instrPerfEval.coffee +++ b/app/scripts/analysis/tools/psychometrics/instrPerfEval/instrPerfEval.coffee @@ -3,59 +3,69 @@ instrPerfEval = angular.module('app_analysis_instrPerfEval', []) .factory('app_analysis_instrPerfEval_constructor', [ - 'app_analysis_instrPerfEval_manager' - (manager) -> - (sb) -> + 'app_analysis_instrPerfEval_manager' + (manager) -> + (sb) -> - manager.setSb sb unless !sb? - _msgList = manager.getMsgList() + manager.setSb sb unless !sb? + _msgList = manager.getMsgList() - init: (opt) -> - console.log 'instrPerfEval init invoked' + init: (opt) -> + console.log 'instrPerfEval init invoked' - destroy: () -> + destroy: () -> - msgList: _msgList + msgList: _msgList ]) .factory('app_analysis_instrPerfEval_manager', [ - () -> - _sb = null + '$rootScope' + ($rootScope) -> + _sb = null - _msgList = - outgoing: ['get table'] - incoming: ['take table'] - scope: ['instrPerfEval'] + _msgList = + outgoing: ['get table'] + incoming: ['take table'] + scope: ['instrPerfEval'] - _setSb = (sb) -> - _sb = sb + _setSb = (sb) -> + _sb = sb - _getSb = () -> - _sb + _getSb = () -> + _sb - _getMsgList = () -> - _msgList + _getMsgList = () -> + _msgList - getSb: _getSb - setSb: _setSb - getMsgList: _getMsgList + # wrapper function for controller communications + _broadcast = (msg, data) -> + $rootScope.$broadcast msg, data + + getSb: _getSb + setSb: _setSb + getMsgList: _getMsgList + broadcast: _broadcast ]) .controller('instrPerfEvalMainCtrl', [ - 'app_analysis_instrPerfEval_manager' - 'app_analysis_instrPerfEval_alphaCalculator' - '$scope' - (ctrlMngr, alphaCalculator, $scope) -> - console.log 'instrPerfEvalViewMainCtrl executed' + 'app_analysis_instrPerfEval_manager' + 'app_analysis_instrPerfEval_alphaCalculator' + '$scope' + (ctrlMngr, alphaCalculator, $scope) -> + console.log 'instrPerfEvalViewMainCtrl executed' - prettifyArrayOutput = (arr) -> - if arr? - arr = arr.map (x) -> x.toFixed 3 - '[' + arr.toString().split(',').join('; ') + ']' + $scope.dataType = '' - data = alphaCalculator.getAlpha() + prettifyArrayOutput = (arr) -> + if arr? + arr = arr.map (x) -> x.toFixed 3 + '[' + arr.toString().split(',').join('; ') + ']' + calculateMetrics = () -> + + data = alphaCalculator.getAlpha() cAlpha = Number data.cronAlpha + if not isNaN(cAlpha) $scope.cronAlpha = cAlpha.toFixed(3) $scope.cronAlphaIdInterval = prettifyArrayOutput(data.idInterval) @@ -68,44 +78,55 @@ instrPerfEval = angular.module('app_analysis_instrPerfEval', []) $scope.kr20 = if data.kr20 is 'Not a binary data' then data.kr20 else Number(data.kr20).toFixed(3) $scope.splitHalfCoef = Number(data.adjRCorrCoef).toFixed(3) + + $scope.$on 'instrPerfEval:updateDataType', (event, dataType) -> + $scope.dataType = dataType +# if $scope.dataType is 'flat' + + calculateMetrics() ]) .controller('instrPerfEvalSidebarCtrl', [ - 'app_analysis_instrPerfEval_manager' - 'app_analysis_instrPerfEval_alphaCalculator' - '$scope' - '$stateParams' - '$q' - (ctrlMngr, alphaCalculator, $scope, $stateParams, $q) -> - console.log 'instrPerfEvalViewSidebarCtrl executed' - - sb = ctrlMngr.getSb() - - $scope.nCols = '5' - $scope.nRows = '5' - - deferred = $q.defer() - - $scope.confLevel = 0.95 - - # subscribe for incoming message with data - token = sb.subscribe - msg: 'take table' - msgScope: ['instrPerfEval'] - listener: (msg, data) -> - _data = data - $scope.nRows = _data.data?.length - $scope.nCols = _data.data[0]?.length - console.log data - alphaCalculator.calculate data, $scope.confLevel - - sb.publish - msg: 'get table' - msgScope: ['instrPerfEval'] - callback: -> sb.unsubscribe token - data: - tableName: $stateParams.projectId + ':' + $stateParams.forkId - promise: deferred + 'app_analysis_instrPerfEval_manager' + 'app_analysis_instrPerfEval_alphaCalculator' + '$scope' + '$stateParams' + '$q' + '$timeout' + (msgMngr, alphaCalculator, $scope, $stateParams, $q, $timeout) -> + console.log 'instrPerfEvalViewSidebarCtrl executed' + + sb = msgMngr.getSb() + deferred = $q.defer() + + $scope.nCols = '5' + $scope.nRows = '5' + $scope.confLevel = 0.95 + $scope.perfeval = off + + parseData = (obj) -> + $scope.nRows = obj.data?.length + $scope.nCols = obj.data[0]?.length + $scope.perfeval = on + alphaCalculator.calculate obj, $scope.confLevel + + # subscribe for incoming message with data + token = sb.subscribe + msg: 'take table' + msgScope: ['instrPerfEval'] + listener: (msg, data) -> + if data.dataType? and data.dataType is 'flat' + $timeout -> + msgMngr.broadcast 'instrPerfEval:updateDataType', data.dataType + parseData data + + sb.publish + msg: 'get table' + msgScope: ['instrPerfEval'] + callback: -> sb.unsubscribe token + data: + tableName: $stateParams.projectId + ':' + $stateParams.forkId + promise: deferred ]) .factory('app_analysis_instrPerfEval_alphaCalculator', [ @@ -326,4 +347,4 @@ instrPerfEval = angular.module('app_analysis_instrPerfEval', []) calculate: _calculate getAlpha: _getAlpha -]) \ No newline at end of file +]) From e12ecbdfd0893b1c1bd1e8359860f45723c24e60 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 21 Mar 2016 20:25:27 -0400 Subject: [PATCH 13/27] feat(eventMngr): abstract supported data types as parameter of event manager #SOCRFW-233 --- app/scripts/core/eventMngr.coffee | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/app/scripts/core/eventMngr.coffee b/app/scripts/core/eventMngr.coffee index 38684dcf..0bc40dda 100644 --- a/app/scripts/core/eventMngr.coffee +++ b/app/scripts/core/eventMngr.coffee @@ -8,18 +8,13 @@ eventMngr = angular.module('app_eventMngr', ['app_mediator', 'app_utils']) (pubSub, utils) -> incomeCallbacks = {} -# _defaultEventManager = (msg, data) -> -# try -# #_data = msgList.income[msg].method.apply null,data -# _data = incomeCallbacks[msg] data -# #last item in data is a promise. -# data[data.length - 1].resolve _data if _data isnt false -# catch e -# console.log e.message -# -# pubSub.publish -# msg: msg -# data: _data + # supported data types + DATA_TYPES = + 'FLAT': 'FLAT' + 'NESTED': 'NESTED' + + _getSupportedDataTypes = () -> + DATA_TYPES # Serialized subscription for a list of events _subscribeForEvents = (events, listener) -> @@ -33,7 +28,8 @@ eventMngr = angular.module('app_eventMngr', ['app_mediator', 'app_utils']) context: events.context subscribeForEvents: _subscribeForEvents + getSupportedDataTypes: _getSupportedDataTypes publish: pubSub.publish subscribe: pubSub.subscribe unsubscribe: pubSub.unsubscribe -]) \ No newline at end of file +]) From 22be1cf995b9851227316ccdac2fb2ccafce0e21 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Thu, 24 Mar 2016 17:48:47 -0400 Subject: [PATCH 14/27] refactor(db): WIP remove dependency of db_manager from db_handler #SOCRFW-238 --- app/scripts/analysis/getData/getData.coffee | 41 +++-- app/scripts/db/db.coffee | 188 +++++++++++--------- 2 files changed, 132 insertions(+), 97 deletions(-) diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index d33b860e..3d359c7c 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -54,9 +54,16 @@ getData = angular.module('app_analysis_getData', [ _getMsgList = () -> _msgList + _getDataTypes = () -> + if _sb + _sb.getSupportedDataTypes() + else + false + getSb: _getSb setSb: _setSb getMsgList: _getMsgList + getDataTypes: _getDataTypes ]) # ### @@ -236,17 +243,19 @@ getData = angular.module('app_analysis_getData', [ 'app_analysis_getData_jsonParser' '$stateParams' 'app_analysis_getData_inputCache' - ($q, $scope, getDataEventMngr, jsonParser, $stateParams, inputCache) -> + ($q, $scope, eventManager, jsonParser, $stateParams, inputCache) -> $scope.jsonUrl = '' flag = true $scope.selected = null + DATA_TYPES = eventManager.getDataTypes() + passReceivedData = (data) -> - if data.dataType is 'nested' + if data.dataType is DATA_TYPES.NESTED inputCache.set data else # default data type is 2d 'flat' table - data.dataType = 'flat' + data.dataType = DATA_TYPES.FLAT # pass a message to update the handsontable div # data is the formatted data which plugs into the # handontable. @@ -319,15 +328,17 @@ getData = angular.module('app_analysis_getData', [ 'app_analysis_getData_dataAdaptor' 'app_analysis_getData_inputCache' '$state' - (getDataEventMngr, $scope, showState, jsonParser, dataAdaptor, inputCache, state) -> + (eventManager, $scope, showState, jsonParser, dataAdaptor, inputCache, state) -> console.log 'getDataMainCtrl executed' + DATA_TYPES = eventManager.getDataTypes() + passReceivedData = (data) -> - if data.dataType is 'nested' + if data.dataType is DATA_TYPES.NESTED inputCache.set data else # default data type is 2d 'flat' table - data.dataType = 'flat' + data.dataType = DATA_TYPES.FLAT # pass a message to update the handsontable div # data is the formatted data which plugs into the # handontable. @@ -408,11 +419,11 @@ getData = angular.module('app_analysis_getData', [ # purpose is helps in pin pointing which # handsontable directive to update. purpose: 'json' - dataType: 'flat' + dataType: DATA_TYPES.FLAT else _data = data: dataResults - dataType: 'nested' + dataType: DATA_TYPES.NESTED passReceivedData _data else console.log 'GETDATA: request failed' @@ -459,7 +470,10 @@ getData = angular.module('app_analysis_getData', [ # @description: Reformats data from input table format to the universal dataFrame object. # ### .factory('app_analysis_getData_dataAdaptor', [ - () -> + 'app_analysis_getData_manager' + (eventManager) -> + + DATA_TYPES = eventManager.getDataTypes() # https://coffeescript-cookbook.github.io/chapters/arrays/check-type-is-array typeIsArray = Array.isArray || ( value ) -> return {}.toString.call(value) is '[object Array]' @@ -495,7 +509,7 @@ getData = angular.module('app_analysis_getData', [ header: tableData.header nRows: tableData.nRows - nSpareRows nCols: tableData.nCols - nSpareCols - dataType: 'flat' + dataType: DATA_TYPES.FLAT _toHandsontable = () -> # TODO: implement for poping up data when coming back from analysis tabs @@ -593,10 +607,11 @@ getData = angular.module('app_analysis_getData', [ .directive 'handsontable', [ + 'app_analysis_getData_manager' 'app_analysis_getData_inputCache' 'app_analysis_getData_dataAdaptor' '$exceptionHandler' - (inputCache, dataAdaptor, $exceptionHandler) -> + (eventManager, inputCache, dataAdaptor, $exceptionHandler) -> restrict: 'E' transclude: true @@ -637,11 +652,13 @@ getData = angular.module('app_analysis_getData', [ scope.update = (evt, arg) -> console.log 'handsontable: update called' + DATA_TYPES = eventManager.getDataTypes() + currHeight = elem.height() #check if data is in the right format # if arg? and typeof arg.data is 'object' and typeof arg.columns is 'object' - if arg? and typeof arg.data is 'object' and arg.dataType is 'flat' + if arg? and typeof arg.data is 'object' and arg.dataType is DATA_TYPES.FLAT # TODO: not to pass nested data to ht, but save in db obj = data: arg.data[1] diff --git a/app/scripts/db/db.coffee b/app/scripts/db/db.coffee index 83296ff9..ab587035 100644 --- a/app/scripts/db/db.coffee +++ b/app/scripts/db/db.coffee @@ -33,8 +33,7 @@ db.factory 'app_database_constructor', [ ] db.factory 'app_database_manager', [ - 'app_database_handler' - (database) -> + () -> _sb = null #_msgList = # incoming:['create table','get table','delete table'], @@ -48,7 +47,7 @@ db.factory 'app_database_manager', [ _setSb = (sb) -> _sb = sb - database.setSb sb +# database.setSb sb _getSb = -> _sb @@ -56,9 +55,16 @@ db.factory 'app_database_manager', [ _getMsgList = -> _msgList + _getDataTypes = () -> + if _sb + _sb.getSupportedDataTypes() + else + false + getSb: _getSb setSb: _setSb getMsgList: _getMsgList + getDataTypes: _getDataTypes ] # ### @@ -67,7 +73,10 @@ db.factory 'app_database_manager', [ # @description: Reformats data from the universal dataFrame object to datavore format # ### db.factory 'app_database_dataAdaptor', [ - () -> + 'app_database_manager' + (eventManager) -> + + DATA_TYPES = eventManager.getDataTypes() _toDvTable = (dataFrame) -> @@ -101,7 +110,7 @@ db.factory 'app_database_dataAdaptor', [ types: _types nRows: _nRows nCols: _nCols - dataType: 'flat' + dataType: DATA_TYPES.FLAT toDvTable: _toDvTable toDataFrame: _toDataFrame @@ -279,89 +288,98 @@ db.factory 'app_database_handler', [ 'app_database_dv' 'app_database_nested' 'app_database_dataAdaptor' - ($q, _db, nestedDb, dataAdaptor) -> + 'app_database_manager' + ($q, _db, nestedDb, dataAdaptor, eventManager) -> + + sb = null + _lastDataType = '' + sb = eventManager.getSb() # set all the callbacks here - _setSb = ((_db) -> - window.db = _db - - _lastDataType = '' - - _saveData = (obj) -> - if obj.dataFrame? - dataFrame = obj.dataFrame - # convert from the universal dataFrame object to datavore table or keep as is - if dataFrame.dataType? - _lastDataType = dataFrame.dataType - switch dataFrame.dataType - when 'flat' - dvData = dataAdaptor.toDvTable dataFrame - res = _db.create dvData, obj.tableName - res - when 'nested' - nestedDb.save obj.data - true - else console.log '%cDATABASE: data type is unknown' , 'color:green' - else console.log '%cDATABASE: data type is unknown' , 'color:green' - else console.log '%cDATABASE: nothing to save' , 'color:green' - - _getData = (data) -> - switch _lastDataType - when 'flat' - _data = _db.get data.tableName - # convert data to DataFrame if returning it - _data = dataAdaptor.toDataFrame _data - _data.dataType = 'flat' - _data - when 'nested' - _data = nestedDb.get() - _data = - data: _data - dataType: 'nested' - else console.log '%cDATABASE: data type is unknown' , 'color:green' +# _setSb = ((_db) -> + window.db = _db + + DATA_TYPES = sb.getDataTypes() + + _getLastDataType = () -> + _lastDataType + + _saveData = (obj) -> + if obj.dataFrame? + dataFrame = obj.dataFrame + # convert from the universal dataFrame object to datavore table or keep as is + if dataFrame.dataType? + _lastDataType = dataFrame.dataType + switch dataFrame.dataType + when DATA_TYPES.FLAT + dvData = dataAdaptor.toDvTable dataFrame + res = _db.create dvData, obj.tableName + res + when DATA_TYPES.NESTED + nestedDb.save obj.data + true + else console.log '%cDATABASE: data type is unknown' , 'color:green' + else console.log '%cDATABASE: data type is unknown' , 'color:green' + else console.log '%cDATABASE: nothing to save' , 'color:green' + + _getData = (data) -> + switch _lastDataType + when DATA_TYPES.FLAT + _data = _db.get data.tableName + # convert data to DataFrame if returning it + _data = dataAdaptor.toDataFrame _data + _data.dataType = DATA_TYPES.FLAT + _data + when DATA_TYPES.NESTED + _data = nestedDb.get() + _data = + data: _data + dataType: DATA_TYPES.NESTED + else console.log '%cDATABASE: data type is unknown' , 'color:green' # else console.log '%cDATABASE: data type is unknown' , 'color:green' - (sb) -> - # registering database callbacks for all possible incoming messages - # TODO: add wrapper layer on top of _db methods? - _methods = [ - incoming: 'save table' - outgoing: 'table saved' - event: _saveData - , - incoming: 'get table' - outgoing: 'take table' - event: _getData - , - incoming: 'add listener' - outgoing: 'listener added' - event: _db.addListener - ] - - _status = _methods.map (method) -> - sb.subscribe - msg: method['incoming'] - msgScope: ['database'] - listener: (msg, obj) -> - console.log "%cDATABASE: listener called for" + msg , "color:green" - # invoke callback - _data = method.event.apply null, [obj] - - # all publish calls should pass a promise in the data object - # if promise is not defined, create one and pass it along - deferred = obj.promise - if typeof deferred isnt 'undefined' - if _data isnt false then deferred.resolve() else deferred.reject() - else - _data.promise = $q.defer() - - console.log '%cDATABASE: listener response: ' + _data, 'color:green' - - sb.publish - msg: method.outgoing - data: _data - msgScope: ['database'] - )(_db) + (sb) -> + # registering database callbacks for all possible incoming messages + # TODO: add wrapper layer on top of _db methods? + _methods = [ + incoming: 'save table' + outgoing: 'table saved' + event: _saveData + , + incoming: 'get table' + outgoing: 'take table' + event: _getData + , + incoming: 'add listener' + outgoing: 'listener added' + event: _db.addListener + ] + + _status = _methods.map (method) -> + sb.subscribe + msg: method['incoming'] + msgScope: ['database'] + listener: (msg, obj) -> + console.log "%cDATABASE: listener called for" + msg , "color:green" + # invoke callback + _data = method.event.apply null, [obj] + + # all publish calls should pass a promise in the data object + # if promise is not defined, create one and pass it along + deferred = obj.promise + if typeof deferred isnt 'undefined' + if _data isnt false then deferred.resolve() else deferred.reject() + else + _data.promise = $q.defer() + + console.log '%cDATABASE: listener response: ' + _data, 'color:green' + + sb.publish + msg: method.outgoing + data: _data + msgScope: ['database'] +# )(_db) - setSb: _setSb +# setSb: _setSb + getLastDataType: _getLastDataType ] From df7874aea833fd637258861b83b3ab470f735208 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 28 Mar 2016 20:01:41 -0400 Subject: [PATCH 15/27] fix(db): WIP initialize db in run block to avoid circular dependency #SOCRFW-238' --- app/scripts/db/db.coffee | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/app/scripts/db/db.coffee b/app/scripts/db/db.coffee index ab587035..69a98165 100644 --- a/app/scripts/db/db.coffee +++ b/app/scripts/db/db.coffee @@ -285,22 +285,17 @@ db.service 'app_database_dv', -> db.factory 'app_database_handler', [ '$q' + '$timeout' 'app_database_dv' 'app_database_nested' 'app_database_dataAdaptor' 'app_database_manager' - ($q, _db, nestedDb, dataAdaptor, eventManager) -> + ($q, $timeout, _db, nestedDb, dataAdaptor, eventManager) -> sb = null + DATA_TYPES = null _lastDataType = '' - sb = eventManager.getSb() - # set all the callbacks here -# _setSb = ((_db) -> - window.db = _db - - DATA_TYPES = sb.getDataTypes() - _getLastDataType = () -> _lastDataType @@ -336,9 +331,8 @@ db.factory 'app_database_handler', [ data: _data dataType: DATA_TYPES.NESTED else console.log '%cDATABASE: data type is unknown' , 'color:green' -# else console.log '%cDATABASE: data type is unknown' , 'color:green' - (sb) -> + setDbListeners = () -> # registering database callbacks for all possible incoming messages # TODO: add wrapper layer on top of _db methods? _methods = [ @@ -375,11 +369,23 @@ db.factory 'app_database_handler', [ console.log '%cDATABASE: listener response: ' + _data, 'color:green' sb.publish - msg: method.outgoing + msg: method['outgoing'] data: _data msgScope: ['database'] -# )(_db) -# setSb: _setSb - getLastDataType: _getLastDataType - ] + _initDb = () -> + $timeout -> + window.db = _db + sb = eventManager.getSb() + DATA_TYPES = sb.getDataTypes() + setDbListeners() + + initDb: _initDb +] + +db.run [ + 'app_database_handler' + (handler) -> + console.log 'DB HANDLEEEEEEEEEEEEEER' + handler.initDb() +] From ca9a6d1884e337b547d7a0b99ce7bda550dfa58c Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 28 Mar 2016 22:43:40 -0400 Subject: [PATCH 16/27] fix(db): get supported data types via event manager #SOCRFW-226 --- app/scripts/db/db.coffee | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/scripts/db/db.coffee b/app/scripts/db/db.coffee index 69a98165..b4c3a39f 100644 --- a/app/scripts/db/db.coffee +++ b/app/scripts/db/db.coffee @@ -332,7 +332,7 @@ db.factory 'app_database_handler', [ dataType: DATA_TYPES.NESTED else console.log '%cDATABASE: data type is unknown' , 'color:green' - setDbListeners = () -> + _setDbListeners = () -> # registering database callbacks for all possible incoming messages # TODO: add wrapper layer on top of _db methods? _methods = [ @@ -377,8 +377,8 @@ db.factory 'app_database_handler', [ $timeout -> window.db = _db sb = eventManager.getSb() - DATA_TYPES = sb.getDataTypes() - setDbListeners() + DATA_TYPES = eventManager.getDataTypes() + _setDbListeners() initDb: _initDb ] @@ -386,6 +386,5 @@ db.factory 'app_database_handler', [ db.run [ 'app_database_handler' (handler) -> - console.log 'DB HANDLEEEEEEEEEEEEEER' handler.initDb() ] From b488c3662872bdc21580d3427a4cbe49f69cd298 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 28 Mar 2016 23:12:08 -0400 Subject: [PATCH 17/27] fix(wrangler): use data types from sandbox #SOCRFW-226 --- .../analysis/wrangleData/wrangler.jade | 4 +-- .../analysis/wrangleData/wrangleData.coffee | 31 ++++++++++++++----- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/app/partials/analysis/wrangleData/wrangler.jade b/app/partials/analysis/wrangleData/wrangler.jade index 3205cf72..b54c31fd 100644 --- a/app/partials/analysis/wrangleData/wrangler.jade +++ b/app/partials/analysis/wrangleData/wrangler.jade @@ -1,5 +1,5 @@ div - #dt_example(style='height: 400px', ng-hide="dataType != 'flat'") + #dt_example(style='height: 400px', ng-hide="dataType != DATA_TYPES.FLAT") .ui-layout-north#wranglerNorthPanel #wranglerDashboard .ui-layout-west#profilerWestPanel @@ -9,6 +9,6 @@ div #preview .spacer .ui-layout-south - div.lead.bg-danger(ng-hide="dataType == 'flat'") + div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") | Data Wrangler doesn't support current dataset. | Only "flat" data tables are supported. diff --git a/app/scripts/analysis/wrangleData/wrangleData.coffee b/app/scripts/analysis/wrangleData/wrangleData.coffee index ff63a849..7addafea 100644 --- a/app/scripts/analysis/wrangleData/wrangleData.coffee +++ b/app/scripts/analysis/wrangleData/wrangleData.coffee @@ -42,6 +42,12 @@ wrangleData = angular.module('app_analysis_wrangleData', []) _getMsgList = () -> _msgList + _getSupportedDataTypes = () -> + if _sb + _sb.getSupportedDataTypes() + else + false + # wrapper function for controller communications _broadcast = (msg, data) -> $rootScope.$broadcast msg, data @@ -50,6 +56,7 @@ wrangleData = angular.module('app_analysis_wrangleData', []) setSb: _setSb getMsgList: _getMsgList broadcast: _broadcast + getSupportedDataTypes: _getSupportedDataTypes ]) .factory('app_analysis_wrangleData_dataRetriever', [ @@ -87,7 +94,10 @@ wrangleData = angular.module('app_analysis_wrangleData', []) ]) .factory('app_analysis_wrangleData_dataAdaptor', [ - () -> + 'app_analysis_wrangleData_manager' + (eventManager) -> + + DATA_TYPES = eventManager.getSupportedDataTypes() _toCsvString = (dataFrame) -> @@ -133,7 +143,7 @@ wrangleData = angular.module('app_analysis_wrangleData', []) types: _types nRows: _nRows nCols: _nCols - dataType: 'flat' + dataType: DATA_TYPES.FLAT toDvTable: _toDvTable toDataFrame: _toDataFrame @@ -154,9 +164,11 @@ wrangleData = angular.module('app_analysis_wrangleData', []) _table = [] _csvData = [] + DATA_TYPES = manager.getSupportedDataTypes() + _init = () -> data = dataRetriever.getData() - if data.dataType is 'flat' + if data.dataType is DATA_TYPES.FLAT _csvData = dataAdaptor.toCsvString data true else @@ -241,11 +253,13 @@ wrangleData = angular.module('app_analysis_wrangleData', []) 'app_analysis_wrangleData_manager' ($scope, $rootScope, wrangler, msgManager) -> + DATA_TYPES = msgManager.getSupportedDataTypes() + $scope.DATA_TYPES = DATA_TYPES $scope.dataType = '' data = wrangler.init() if data - $scope.dataType = 'flat' + $scope.dataType = DATA_TYPES.FLAT # TODO: isolate dw from global scope w = dw.wrangle() @@ -253,7 +267,7 @@ wrangleData = angular.module('app_analysis_wrangleData', []) # listen to state change and save data when exiting Wrangle Data stateListener = $rootScope.$on '$stateChangeStart', (event, toState, toParams, fromState, fromParams) -> if fromState.name? and fromState.name is 'wrangleData' - if $scope.dataType is 'flat' + if $scope.dataType is DATA_TYPES.FLAT # save data to db on exit from wrangler wrangler.saveData() # signal to show sidebar @@ -267,7 +281,8 @@ wrangleData = angular.module('app_analysis_wrangleData', []) .directive 'datawrangler', [ '$exceptionHandler' 'app_analysis_wrangleData_wrangler' - ($exceptionHandler, wrangler) -> + 'app_analysis_wrangleData_manager' + ($exceptionHandler, wrangler, msgManager) -> restrict: 'E' transclude: true @@ -283,8 +298,10 @@ wrangleData = angular.module('app_analysis_wrangleData', []) # useful to identify which handsontable instance to update scope.purpose = attr.purpose + DATA_TYPES = msgManager.getSupportedDataTypes() + # check if received dataset is flat - if scope.dataType? and scope.dataType is 'flat' + if scope.dataType? and scope.dataType is DATA_TYPES.FLAT myLayout = $('#dt_example').layout north: spacing_open: 0 From ba06ef9c3faa4c3d743997273fadfaf1b8d51e91 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 28 Mar 2016 23:22:25 -0400 Subject: [PATCH 18/27] fix(instrPerfEval): use data types from sandbox #SOCRFW-226 --- .../tools/psychometrics/instrPerfEval/main.jade | 4 ++-- .../instrPerfEval/instrPerfEval.coffee | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade b/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade index 3a1f1a28..377dbf94 100644 --- a/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade +++ b/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade @@ -1,6 +1,6 @@ div(ng-controller="instrPerfEvalMainCtrl") h2 Instrument Performance Evaluation - div(ng-hide="dataType != 'flat'") + div(ng-hide="dataType != DATA_TYPES.FLAT") p. Cronbach’s Alpha (α) is a measure of internal consistency or reliability of a psychometric instrument and measures how well a set of items measure a single, one-dimensional latent aspect of individuals. @@ -74,6 +74,6 @@ div(ng-controller="instrPerfEvalMainCtrl") Psychological methods 12.2 (2007): 157. - div.lead.bg-danger(ng-hide="dataType == 'flat'") + div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") | Instrument Performance Evaluation doesn't support current dataset. | Only "flat" data tables are supported. diff --git a/app/scripts/analysis/tools/psychometrics/instrPerfEval/instrPerfEval.coffee b/app/scripts/analysis/tools/psychometrics/instrPerfEval/instrPerfEval.coffee index 28f03cf3..06eb6e98 100644 --- a/app/scripts/analysis/tools/psychometrics/instrPerfEval/instrPerfEval.coffee +++ b/app/scripts/analysis/tools/psychometrics/instrPerfEval/instrPerfEval.coffee @@ -37,6 +37,12 @@ instrPerfEval = angular.module('app_analysis_instrPerfEval', []) _getMsgList = () -> _msgList + _getSupportedDataTypes = () -> + if _sb + _sb.getSupportedDataTypes() + else + false + # wrapper function for controller communications _broadcast = (msg, data) -> $rootScope.$broadcast msg, data @@ -45,7 +51,8 @@ instrPerfEval = angular.module('app_analysis_instrPerfEval', []) setSb: _setSb getMsgList: _getMsgList broadcast: _broadcast - ]) + getSupportedDataTypes: _getSupportedDataTypes +]) .controller('instrPerfEvalMainCtrl', [ 'app_analysis_instrPerfEval_manager' @@ -54,6 +61,7 @@ instrPerfEval = angular.module('app_analysis_instrPerfEval', []) (ctrlMngr, alphaCalculator, $scope) -> console.log 'instrPerfEvalViewMainCtrl executed' + $scope.DATA_TYPES = ctrlMngr.getSupportedDataTypes() $scope.dataType = '' prettifyArrayOutput = (arr) -> @@ -96,6 +104,7 @@ instrPerfEval = angular.module('app_analysis_instrPerfEval', []) (msgMngr, alphaCalculator, $scope, $stateParams, $q, $timeout) -> console.log 'instrPerfEvalViewSidebarCtrl executed' + DATA_TYPES = msgMngr.getSupportedDataTypes() sb = msgMngr.getSb() deferred = $q.defer() @@ -115,7 +124,7 @@ instrPerfEval = angular.module('app_analysis_instrPerfEval', []) msg: 'take table' msgScope: ['instrPerfEval'] listener: (msg, data) -> - if data.dataType? and data.dataType is 'flat' + if data.dataType? and data.dataType is DATA_TYPES.FLAT $timeout -> msgMngr.broadcast 'instrPerfEval:updateDataType', data.dataType parseData data From ee814bcd63762381892cfab54bdb9d410ab1725c Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Mon, 28 Mar 2016 23:29:00 -0400 Subject: [PATCH 19/27] fix(kmeans): use data types from sandbox # #SOCRFW-226 --- .../analysis/tools/machineLearning/kMeans/main.jade | 4 ++-- .../tools/machineLearning/kMeans/kmeans.coffee | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/partials/analysis/tools/machineLearning/kMeans/main.jade b/app/partials/analysis/tools/machineLearning/kMeans/main.jade index 65bd0728..c2661cbf 100644 --- a/app/partials/analysis/tools/machineLearning/kMeans/main.jade +++ b/app/partials/analysis/tools/machineLearning/kMeans/main.jade @@ -1,6 +1,6 @@ div(ng-controller="kMeansMainCtrl") h2 2D k-means Clustering - div(ng-hide="dataType != 'flat'") + div(ng-hide="dataType != DATA_TYPES.FLAT") p. k-means clustering aims to partition n observations into k clusters in which each observation belongs to the cluster with the nearest mean, serving as a prototype of the cluster. @@ -15,6 +15,6 @@ div(ng-controller="kMeansMainCtrl") td {{label}} td {{acc.toFixed(2)}} app-kmeans - div.lead.bg-danger(ng-hide="dataType == 'flat'") + div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") | 2D k-means Clustering doesn't support current dataset. | Only "flat" data tables are supported. diff --git a/app/scripts/analysis/tools/machineLearning/kMeans/kmeans.coffee b/app/scripts/analysis/tools/machineLearning/kMeans/kmeans.coffee index f1e4adf4..ad8f5b05 100644 --- a/app/scripts/analysis/tools/machineLearning/kMeans/kmeans.coffee +++ b/app/scripts/analysis/tools/machineLearning/kMeans/kmeans.coffee @@ -37,6 +37,12 @@ kMeans = angular.module('app_analysis_kMeans', []) _getMsgList = () -> _msgList + _getSupportedDataTypes = () -> + if _sb + _sb.getSupportedDataTypes() + else + false + # wrapper function for controller communications _broadcast = (msg, data) -> $rootScope.$broadcast msg, data @@ -45,6 +51,7 @@ kMeans = angular.module('app_analysis_kMeans', []) setSb: _setSb getMsgList: _getMsgList broadcast: _broadcast + getSupportedDataTypes: _getSupportedDataTypes ]) .controller('kMeansMainCtrl', [ @@ -62,6 +69,7 @@ kMeans = angular.module('app_analysis_kMeans', []) $scope.avgAccuracy = '' $scope.accs = {} $scope.dataType = '' + $scope.DATA_TYPES = msgManager.getSupportedDataTypes() prettifyArrayOutput = (arr) -> if arr? @@ -116,6 +124,8 @@ kMeans = angular.module('app_analysis_kMeans', []) (msgManager, kmeans, $scope, $stateParams, $q, $timeout) -> console.log 'kMeansSidebarCtrl executed' + DATA_TYPES = msgManager.getSupportedDataTypes() + DEFAULT_CONTROL_VALUES = k: 2 distance: 'Euclidean' @@ -244,7 +254,7 @@ kMeans = angular.module('app_analysis_kMeans', []) msg: 'take data' msgScope: ['kMeans'] listener: (msg, data) -> - if data.dataType? and data.dataType is 'flat' + if data.dataType? and data.dataType is DATA_TYPES.FLAT $timeout -> msgManager.broadcast 'kmeans:updateDataType', data.dataType parseData data From caea4c4692a5ab9496cff3684200de0a5e87114f Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 29 Mar 2016 19:05:58 -0400 Subject: [PATCH 20/27] feat(getData): hide handsontable when data is nested; update popup messages #SOCRFW-228 --- app/partials/analysis/getData/main.jade | 5 +- app/scripts/analysis/getData/getData.coffee | 195 +++++++++++--------- 2 files changed, 110 insertions(+), 90 deletions(-) diff --git a/app/partials/analysis/getData/main.jade b/app/partials/analysis/getData/main.jade index e561c614..a08ff693 100644 --- a/app/partials/analysis/getData/main.jade +++ b/app/partials/analysis/getData/main.jade @@ -44,4 +44,7 @@ div(ng-controller='getDataMainCtrl') button.btn.btn-primary(ng-click="getJsonByUrl()") Parse br - handsontable(purpose="json") + div.lead.bg-danger(ng-hide="dataType != DATA_TYPES.NESTED") + | Visual representation of hierarchical data currently is not available. + div(ng-class="{'vis-hidden': dataType != DATA_TYPES.FLAT}") + handsontable(purpose="json") diff --git a/app/scripts/analysis/getData/getData.coffee b/app/scripts/analysis/getData/getData.coffee index 3d359c7c..fd71ede9 100644 --- a/app/scripts/analysis/getData/getData.coffee +++ b/app/scripts/analysis/getData/getData.coffee @@ -54,7 +54,7 @@ getData = angular.module('app_analysis_getData', [ _getMsgList = () -> _msgList - _getDataTypes = () -> + _getSupportedDataTypes = () -> if _sb _sb.getSupportedDataTypes() else @@ -63,7 +63,7 @@ getData = angular.module('app_analysis_getData', [ getSb: _getSb setSb: _setSb getMsgList: _getMsgList - getDataTypes: _getDataTypes + getSupportedDataTypes: _getSupportedDataTypes ]) # ### @@ -81,6 +81,7 @@ getData = angular.module('app_analysis_getData', [ '$timeout' (manager, $q, $stateParams, $rootScope, $timeout) -> + DATA_TYPES = manager.getSupportedDataTypes() sb = manager.getSb() _data = {} _timer = null @@ -90,12 +91,15 @@ getData = angular.module('app_analysis_getData', [ _data _saveDataToDb = (data, deferred) -> + + msgEnding = if data.dataType is DATA_TYPES.FLAT then ' as 2D data table' else ' as hierarchical object' + $rootScope.$broadcast 'app:push notification', initial: msg: 'Data is being saved in the database...' type: 'alert-info' success: - msg: 'Successfully loaded data into database' + msg: 'Successfully loaded data into database' + msgEnding type: 'alert-success' failure: msg: 'Error in Database' @@ -248,7 +252,7 @@ getData = angular.module('app_analysis_getData', [ flag = true $scope.selected = null - DATA_TYPES = eventManager.getDataTypes() + DATA_TYPES = eventManager.getSupportedDataTypes() passReceivedData = (data) -> if data.dataType is DATA_TYPES.NESTED @@ -331,14 +335,18 @@ getData = angular.module('app_analysis_getData', [ (eventManager, $scope, showState, jsonParser, dataAdaptor, inputCache, state) -> console.log 'getDataMainCtrl executed' - DATA_TYPES = eventManager.getDataTypes() + DATA_TYPES = eventManager.getSupportedDataTypes() + $scope.DATA_TYPES = DATA_TYPES + $scope.dataType = '' passReceivedData = (data) -> if data.dataType is DATA_TYPES.NESTED + $scope.dataType = DATA_TYPES.NESTED inputCache.set data else # default data type is 2d 'flat' table data.dataType = DATA_TYPES.FLAT + $scope.dataType = DATA_TYPES.FLAT # pass a message to update the handsontable div # data is the formatted data which plugs into the # handontable. @@ -473,7 +481,7 @@ getData = angular.module('app_analysis_getData', [ 'app_analysis_getData_manager' (eventManager) -> - DATA_TYPES = eventManager.getDataTypes() + DATA_TYPES = eventManager.getSupportedDataTypes() # https://coffeescript-cookbook.github.io/chapters/arrays/check-type-is-array typeIsArray = Array.isArray || ( value ) -> return {}.toString.call(value) is '[object Array]' @@ -611,13 +619,15 @@ getData = angular.module('app_analysis_getData', [ 'app_analysis_getData_inputCache' 'app_analysis_getData_dataAdaptor' '$exceptionHandler' - (eventManager, inputCache, dataAdaptor, $exceptionHandler) -> + '$timeout' + (eventManager, inputCache, dataAdaptor, $exceptionHandler, $timeout) -> + restrict: 'E' transclude: true # to the name attribute on the directive element. # the template for the directive. - template: "
" + template: "
" #the controller for the directive controller: ($scope) -> @@ -629,86 +639,93 @@ getData = angular.module('app_analysis_getData', [ # It is run before the controller link: (scope, elem, attr) -> - N_SPARE_COLS = 1 - N_SPARE_ROWS = 1 - DEFAULT_ROW_HEIGHT = 24 - - # useful to identify which handsontable instance to update - scope.purpose = attr.purpose - - # retrieves data from handsontable object - _format = (obj) -> - data = obj.getData() - header = obj.getColHeader() - nCols = obj.countCols() - nRows = obj.countRows() - - table = - data: data - header: header - nCols: nCols - nRows: nRows - - scope.update = (evt, arg) -> - console.log 'handsontable: update called' - - DATA_TYPES = eventManager.getDataTypes() - - currHeight = elem.height() - - #check if data is in the right format -# if arg? and typeof arg.data is 'object' and typeof arg.columns is 'object' - if arg? and typeof arg.data is 'object' and arg.dataType is DATA_TYPES.FLAT - # TODO: not to pass nested data to ht, but save in db - obj = - data: arg.data[1] -# startRows: Object.keys(arg.data[1]).length -# startCols: arg.columns.length - colHeaders: arg.columnHeader -# columns: arg.columns - minSpareRows: N_SPARE_ROWS - minSpareCols: N_SPARE_COLS - allowInsertRow: true - allowInsertColumn: true - else if arg.default is true - obj = - data: [ - ['Copy', 'paste', 'your', 'data', 'here'] - ] - colHeaders: true - minSpareRows: N_SPARE_ROWS - minSpareCols: N_SPARE_COLS - allowInsertRow: true - allowInsertColumn: true - rowHeaders: false - else - $exceptionHandler - message: 'handsontable configuration is missing' - - obj['change'] = true - obj['afterChange'] = (change, source) -> - # saving data to be globally accessible. - # only place from where data is saved before DB: inputCache. - # onSave, data is picked up from inputCache. - if source is 'loadData' or 'paste' - ht = $(this)[0] - tableData = _format ht - dataFrame = dataAdaptor.toDataFrame tableData, N_SPARE_COLS, N_SPARE_ROWS - inputCache.set dataFrame - ht.updateSettings - height: Math.max currHeight, ht.countRows() * DEFAULT_ROW_HEIGHT + $timeout -> + N_SPARE_COLS = 1 + N_SPARE_ROWS = 1 + # from handsontable defaults + # https://docs.handsontable.com/0.24.1/demo-stretching.html + DEFAULT_ROW_HEIGHT = 23 + DEFAULT_COL_WIDTH = 47 + + # useful to identify which handsontable instance to update + scope.purpose = attr.purpose + + # retrieves data from handsontable object + _format = (obj) -> + data = obj.getData() + header = obj.getColHeader() + nCols = obj.countCols() + nRows = obj.countRows() + + table = + data: data + header: header + nCols: nCols + nRows: nRows + + scope.update = (evt, arg) -> + console.log 'handsontable: update called' + + DATA_TYPES = eventManager.getSupportedDataTypes() + + currHeight = elem.height() + currWidth = elem.width() + + #check if data is in the right format + # if arg? and typeof arg.data is 'object' and typeof arg.columns is 'object' + if arg? and typeof arg.data is 'object' and arg.dataType is DATA_TYPES.FLAT + # TODO: not to pass nested data to ht, but save in db + obj = + data: arg.data[1] + # startRows: Object.keys(arg.data[1]).length + # startCols: arg.columns.length + colHeaders: arg.columnHeader + # columns: arg.columns + minSpareRows: N_SPARE_ROWS + minSpareCols: N_SPARE_COLS + allowInsertRow: true + allowInsertColumn: true + stretchH: "all" + else if arg.default is true + obj = + data: [ + ['Copy', 'paste', 'your', 'data', 'here'] + ] + colHeaders: true + minSpareRows: N_SPARE_ROWS + minSpareCols: N_SPARE_COLS + allowInsertRow: true + allowInsertColumn: true + rowHeaders: false else - inputCache.set source - - try - # hook for pushing data changes to handsontable - # TODO: get rid of tight coupling :-/ - ht = elem.handsontable obj - window['inputCache'] = inputCache.ht = $(ht[0]).data('handsontable') - catch e - $exceptionHandler e - - # subscribing to handsontable update. - scope.$on attr.purpose + ':load data to handsontable', scope.update - console.log 'handsontable directive linked' + $exceptionHandler + message: 'handsontable configuration is missing' + + obj['change'] = true + obj['afterChange'] = (change, source) -> + # saving data to be globally accessible. + # only place from where data is saved before DB: inputCache. + # onSave, data is picked up from inputCache. + if source is 'loadData' or 'paste' + ht = $(this)[0] + tableData = _format ht + dataFrame = dataAdaptor.toDataFrame tableData, N_SPARE_COLS, N_SPARE_ROWS + inputCache.set dataFrame + ht.updateSettings + height: Math.max currHeight, ht.countRows() * DEFAULT_ROW_HEIGHT + width: Math.max currWidth, ht.countCols() * DEFAULT_COL_WIDTH + else + inputCache.set source + + try + # hook for pushing data changes to handsontable + # TODO: get rid of tight coupling :-/ + ht = elem.handsontable obj + window['inputCache'] = inputCache.ht = $(ht[0]).data('handsontable') + catch e + $exceptionHandler e + + # subscribing to handsontable update + scope.$on attr.purpose + ':load data to handsontable', scope.update + console.log 'handsontable directive linked' ] From 1fad181fe81b65202de2c468bb7e3527d8b17734 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 29 Mar 2016 19:07:24 -0400 Subject: [PATCH 21/27] feat(wrangler): update popup messages #SOCRFW-228 --- app/partials/analysis/wrangleData/wrangler.jade | 8 ++++---- app/scripts/analysis/wrangleData/wrangleData.coffee | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/partials/analysis/wrangleData/wrangler.jade b/app/partials/analysis/wrangleData/wrangler.jade index b54c31fd..6b726e80 100644 --- a/app/partials/analysis/wrangleData/wrangler.jade +++ b/app/partials/analysis/wrangleData/wrangler.jade @@ -1,5 +1,8 @@ div - #dt_example(style='height: 400px', ng-hide="dataType != DATA_TYPES.FLAT") + div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") + | Data Wrangler doesn't support current dataset. + | Only "flat" data tables are supported. + #dt_example(style='height: 400px', ng-hide="dataType != DATA_TYPES.FLAT", ng-cloak) .ui-layout-north#wranglerNorthPanel #wranglerDashboard .ui-layout-west#profilerWestPanel @@ -9,6 +12,3 @@ div #preview .spacer .ui-layout-south - div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") - | Data Wrangler doesn't support current dataset. - | Only "flat" data tables are supported. diff --git a/app/scripts/analysis/wrangleData/wrangleData.coffee b/app/scripts/analysis/wrangleData/wrangleData.coffee index 7addafea..f6b26b3c 100644 --- a/app/scripts/analysis/wrangleData/wrangleData.coffee +++ b/app/scripts/analysis/wrangleData/wrangleData.coffee @@ -213,12 +213,14 @@ wrangleData = angular.module('app_analysis_wrangleData', []) _timer = $timeout ( -> + msgEnding = if dataFrame.dataType is DATA_TYPES.FLAT then ' as 2D data table' else ' as hierarchical object' + $rootScope.$broadcast 'app:push notification', initial: msg: 'Data is being saved in the database...' type: 'alert-info' success: - msg: 'Successfully loaded data into database' + msg: 'Successfully loaded data into database' + msgEnding type: 'alert-success' failure: msg: 'Error in Database' From e0368cd17639ec3326cf49ae154bb4b6c60405bc Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 29 Mar 2016 19:08:52 -0400 Subject: [PATCH 22/27] refactor(db): rename getDataTypes to getSupportedDataTypes for consistency #SOCRFW-228 --- app/scripts/db/db.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/scripts/db/db.coffee b/app/scripts/db/db.coffee index b4c3a39f..7b05ef73 100644 --- a/app/scripts/db/db.coffee +++ b/app/scripts/db/db.coffee @@ -64,7 +64,7 @@ db.factory 'app_database_manager', [ getSb: _getSb setSb: _setSb getMsgList: _getMsgList - getDataTypes: _getDataTypes + getSupportedDataTypes: _getDataTypes ] # ### @@ -76,7 +76,7 @@ db.factory 'app_database_dataAdaptor', [ 'app_database_manager' (eventManager) -> - DATA_TYPES = eventManager.getDataTypes() + DATA_TYPES = eventManager.getSupportedDataTypes() _toDvTable = (dataFrame) -> @@ -377,7 +377,7 @@ db.factory 'app_database_handler', [ $timeout -> window.db = _db sb = eventManager.getSb() - DATA_TYPES = eventManager.getDataTypes() + DATA_TYPES = eventManager.getSupportedDataTypes() _setDbListeners() initDb: _initDb From e72b1025e37a1f7b252012ef016c2a1075dbdb86 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 29 Mar 2016 19:09:57 -0400 Subject: [PATCH 23/27] refactor(kmeans): move nested data notification up #SOCRFW-228 --- .../analysis/tools/machineLearning/kMeans/main.jade | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/partials/analysis/tools/machineLearning/kMeans/main.jade b/app/partials/analysis/tools/machineLearning/kMeans/main.jade index c2661cbf..d8a592cc 100644 --- a/app/partials/analysis/tools/machineLearning/kMeans/main.jade +++ b/app/partials/analysis/tools/machineLearning/kMeans/main.jade @@ -1,5 +1,8 @@ div(ng-controller="kMeansMainCtrl") h2 2D k-means Clustering + div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") + | 2D k-means Clustering doesn't support current dataset. + | Only "flat" data tables are supported. div(ng-hide="dataType != DATA_TYPES.FLAT") p. k-means clustering aims to partition n observations into k clusters in which each observation belongs @@ -15,6 +18,3 @@ div(ng-controller="kMeansMainCtrl") td {{label}} td {{acc.toFixed(2)}} app-kmeans - div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") - | 2D k-means Clustering doesn't support current dataset. - | Only "flat" data tables are supported. From a381f5b3ea8a292d3b3bbbea48c75faf9771cfe6 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 29 Mar 2016 19:10:31 -0400 Subject: [PATCH 24/27] refactor(instrPerfEval): move nested data notification up #SOCRFW-228 --- .../analysis/tools/psychometrics/instrPerfEval/main.jade | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade b/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade index 377dbf94..065059c0 100644 --- a/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade +++ b/app/partials/analysis/tools/psychometrics/instrPerfEval/main.jade @@ -1,5 +1,8 @@ div(ng-controller="instrPerfEvalMainCtrl") h2 Instrument Performance Evaluation + div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") + | Instrument Performance Evaluation doesn't support current dataset. + | Only "flat" data tables are supported. div(ng-hide="dataType != DATA_TYPES.FLAT") p. Cronbach’s Alpha (α) is a measure of internal consistency or reliability of a psychometric instrument and measures @@ -72,8 +75,3 @@ div(ng-controller="instrPerfEvalMainCtrl") 3. Maydeu-Olivares, Alberto, Donna L. Coffman, and Wolfgang M. Hartmann. "Asymptotically distribution-free (ADF) interval estimation of coefficient alpha." Psychological methods 12.2 (2007): 157. - - - div.lead.bg-danger(ng-hide="dataType == DATA_TYPES.FLAT") - | Instrument Performance Evaluation doesn't support current dataset. - | Only "flat" data tables are supported. From a461f5ace9a360023a4bf51060b19b91f72e923e Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 29 Mar 2016 19:11:05 -0400 Subject: [PATCH 25/27] feat(styles): add style for visibility #SOCRFW-228 --- app/styles/custom.less | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/styles/custom.less b/app/styles/custom.less index ad9284b4..024f3706 100644 --- a/app/styles/custom.less +++ b/app/styles/custom.less @@ -21,6 +21,10 @@ filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=3); } +.vis-hidden{ + visibility: hidden; +} + #welcome-screen { height: 100%; width: 100%; From cccab0f7a5b95cd2a57bb6958ec426d98226c3c0 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 29 Mar 2016 19:55:33 -0400 Subject: [PATCH 26/27] chore: update dependencies #SOCRFW-240 --- Gruntfile.js | 18 ++++++++++-------- bower.json | 6 +++--- package.json | 45 +++++++++++++++++++++++---------------------- 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index f0b29624..534cafc0 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -12,6 +12,8 @@ module.exports = function (grunt) { // Time how long tasks take. Can help when optimizing build times require('time-grunt')(grunt); + var serveStatic = require('serve-static'); + // Automatically load required Grunt tasks require('jit-grunt')(grunt, { useminPrepare: 'grunt-usemin', @@ -82,16 +84,16 @@ module.exports = function (grunt) { open: true, middleware: function (connect) { return [ - connect.static('.tmp'), + serveStatic('.tmp'), connect().use( '/bower_components', - connect.static('./bower_components') + serveStatic('./bower_components') ), connect().use( '/app/styles', - connect.static('./app/styles') + serveStatic('./app/styles') ), - connect.static(appConfig.app) + serveStatic(appConfig.app) ]; } } @@ -101,13 +103,13 @@ module.exports = function (grunt) { port: 9001, middleware: function (connect) { return [ - connect.static('.tmp'), - connect.static('test'), + serveStatic('.tmp'), + serveStatic('test'), connect().use( '/bower_components', - connect.static('./bower_components') + serveStatic('./bower_components') ), - connect.static(appConfig.app) + serveStatic(appConfig.app) ]; } } diff --git a/bower.json b/bower.json index 73270917..8b6220e9 100644 --- a/bower.json +++ b/bower.json @@ -19,11 +19,11 @@ "angular-sanitize": "^1.5.0", "angular-touch": "^1.5.0", "angular-ui": "^0.4.0", - "angular-ui-router": "~0.2.15", - "ngHandsontable": "angular-ui-handsontable#~0.8.0", + "angular-ui-router": "~0.2.18", + "ngHandsontable": "angular-ui-handsontable#~0.9.0", "wrangler": "https://github.com/alxndrkalinin/wrangler.git", "bootstrap": "^3.3.6", - "angular-bootstrap": "~1.1.2", + "angular-bootstrap": "^1.1.0", "jstat": "~1.5.2", "jsfeat": "~0.0.8" }, diff --git a/package.json b/package.json index a4f4e60f..b34b88e9 100644 --- a/package.json +++ b/package.json @@ -11,41 +11,42 @@ "devDependencies": { "coffee-script": "*", "grunt": "^0.4.5", - "grunt-angular-templates": "^0.5.7", - "grunt-autoprefixer": "^2.0.0", - "grunt-concurrent": "^1.0.0", - "grunt-contrib-clean": "^0.6.0", - "grunt-contrib-coffee": "^0.12.0", - "grunt-contrib-concat": "^0.5.0", - "grunt-contrib-connect": "^0.9.0", - "grunt-contrib-copy": "^0.7.0", - "grunt-contrib-cssmin": "^0.12.0", - "grunt-contrib-htmlmin": "^0.4.0", - "grunt-contrib-imagemin": "^0.9.2", - "grunt-contrib-jade": "^0.15.0", - "grunt-contrib-jshint": "^0.11.0", - "grunt-contrib-less": "^1.0.1", - "grunt-contrib-uglify": "^0.7.0", - "grunt-contrib-watch": "^0.6.1", + "grunt-angular-templates": "^1.0.3", + "grunt-autoprefixer": "^3.0.4", + "grunt-concurrent": "^2.2.1", + "grunt-contrib-clean": "^1.0.0", + "grunt-contrib-coffee": "^1.0.0", + "grunt-contrib-concat": "^1.0.0", + "grunt-contrib-connect": "^1.0.1", + "grunt-contrib-copy": "^1.0.0", + "grunt-contrib-cssmin": "^1.0.1", + "grunt-contrib-htmlmin": "^1.1.0", + "grunt-contrib-imagemin": "^1.0.0", + "grunt-contrib-jade": "^1.0.0", + "grunt-contrib-jshint": "^1.0.0", + "grunt-contrib-less": "^1.2.0", + "grunt-contrib-uglify": "^1.0.1", + "grunt-contrib-watch": "^1.0.0", "grunt-filerev": "^2.1.2", "grunt-google-cdn": "^0.4.3", "grunt-include-source": "^0.7.0", "grunt-karma": "*", "grunt-newer": "^1.1.0", - "grunt-ng-annotate": "^0.9.2", - "grunt-svgmin": "^2.0.0", + "grunt-ng-annotate": "^2.0.1", + "grunt-svgmin": "^3.0.0", "grunt-usemin": "^3.0.0", "grunt-wiredep": "^2.0.0", + "serve-static": "^1.10.2", "jade": "^1.11.0", "jasmine-core": "^2.4.1", - "jit-grunt": "^0.9.1", - "jshint-stylish": "^1.0.0", + "jit-grunt": "^0.10.0", + "jshint-stylish": "^2.1.0", "karma": "^0.13.0", "karma-coffee-preprocessor": "*", "karma-jasmine": "*", - "karma-phantomjs2-launcher": "^0.4.0", + "karma-phantomjs2-launcher": "~0.4.0", "less": "^2.5.3", - "phantomjs2": "^2.0.2", + "phantomjs2": "~2.0.2", "time-grunt": "^1.0.0" }, "engines": { From 82a95097d03162bc2915e6018be42ca83b254e01 Mon Sep 17 00:00:00 2001 From: alxndrkalinin Date: Tue, 29 Mar 2016 20:01:01 -0400 Subject: [PATCH 27/27] chore: update links to gemnasium and travis; version bump to 0.1.5-alpha.6 #SOCRFW-239 --- README.md | 4 ++-- bower.json | 2 +- package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 69d54c9d..a9d3a9a4 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ A scalable and highly flexible HTML5/JS platform to build and run in-browser app * Web site: http://socr.umich.edu * Issue-tracking and project management: https://socredu.atlassian.net/browse/SOCRFW -[![Build Status](https://travis-ci.org/SOCRedu/SOCR-framework.svg?branch=master)](https://travis-ci.org/SOCRedu/SOCR-framework) -[![Dependency Status](https://gemnasium.com/SOCRedu/SOCR-framework.png?branch=master)](https://gemnasium.com/SOCRedu/SOCR-framework) +[![Build Status](https://travis-ci.org/SOCR/SOCRAT.svg?branch=master)](https://travis-ci.org/SOCR/SOCRAT) +[![Dependency Status](https://gemnasium.com/SOCR/SOCRAT.png?branch=master)](https://gemnasium.com/SOCR/SOCRAT) Installation ------------ diff --git a/bower.json b/bower.json index 8b6220e9..17257b9b 100644 --- a/bower.json +++ b/bower.json @@ -1,7 +1,7 @@ { "author": "Alexandr Kalinin, Selvam Palanimalai", "name": "SOCRAT", - "version": "v0.1.5-alpha.5", + "version": "v0.1.5-alpha.6", "description": "Flexible HTML5/JS toolkit for interactive data analysis and visuzalization", "homepage": "https://github.com/SOCR/SOCRAT", "dependencies": { diff --git a/package.json b/package.json index b34b88e9..e5c594ee 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "author": "Alexandr Kalinin, Selvam Palanimalai", "name": "SOCRAT", - "version": "v0.1.5-alpha.5", + "version": "v0.1.5-alpha.6", "description": "Flexible HTML5/JS toolkit for interactive data analysis and visuzalization", "homepage": "https://github.com/SOCR/SOCRAT", "repository": {