From c720651b3066e19a72a61db8210996c71a5c6ce4 Mon Sep 17 00:00:00 2001 From: Nick Nicholas Date: Tue, 26 Sep 2017 16:26:35 +1000 Subject: [PATCH 1/5] manifest of records, napval --- app/napval/public/index.html | 6 + app/napval/public/js/nias2.js | 622 +++++++++++++++++----------------- napval/validation_server.go | 99 +++++- 3 files changed, 423 insertions(+), 304 deletions(-) diff --git a/app/napval/public/index.html b/app/napval/public/index.html index 07aeec6..9ced879 100644 --- a/app/napval/public/index.html +++ b/app/napval/public/index.html @@ -143,6 +143,12 @@

Conditions of Data Use

+

diff --git a/app/napval/public/js/nias2.js b/app/napval/public/js/nias2.js index 7dacd4e..7b0496c 100644 --- a/app/napval/public/js/nias2.js +++ b/app/napval/public/js/nias2.js @@ -7,63 +7,63 @@ var display_results_once = false; // instantiate interaction listeners $(document).ready(function() -{ + { - hideResults(); - hideProgress(); - $('.modal').modal(); - // $('.modal').modal(); - $('.modal').modal('open'); + hideResults(); + hideProgress(); + $('.modal').modal(); + // $('.modal').modal(); + $('.modal').modal('open'); - // handler for downloading error report - $(function() - { - $("#csv-download,#csv-download2").click(function(e) - { - CsvDownload(); - }); - }); + // handler for downloading error report + $(function() + { + $("#csv-download,#csv-download2").click(function(e) + { + CsvDownload(); + }); + }); - // handler for validation '>' button - $("#validate").click(function() - { - Validate(); - }); + // handler for validation '>' button + $("#validate").click(function() + { + Validate(); + }); - // handler for csv - sifxml conversion - $("#xml-convert,#xml-convert2").click(function() - { - XmlConvert(); - }); + // handler for csv - sifxml conversion + $("#xml-convert,#xml-convert2").click(function() + { + XmlConvert(); + }); -}); + }); // get the validation results in csv format function CsvDownload() { - var filesList = $('#upload').prop('files'); - if (filesList.length < 1) - { - var toastContent = $('
No file selected
'); - Materialize.toast(toastContent, 3000, 'rounded green'); - return - } - - if (!txID) - { - var toastContent = $('
No results to download
'); - Materialize.toast(toastContent, 3000, 'rounded green'); - return - - } - - var fname = filesList[0].name + var filesList = $('#upload').prop('files'); + if (filesList.length < 1) + { + var toastContent = $('
No file selected
'); + Materialize.toast(toastContent, 3000, 'rounded green'); + return + } + + if (!txID) + { + var toastContent = $('
No results to download
'); + Materialize.toast(toastContent, 3000, 'rounded green'); + return + + } + + var fname = filesList[0].name var url = "/naplan/reg/report/" + txID + "/" + fname; - window.location.href = url; + window.location.href = url; } @@ -72,45 +72,45 @@ function CsvDownload() function Validate() { - var filesList = $('#upload').prop('files'); - if (filesList.length < 1) - { - var toastContent = $('
No file selected
'); - Materialize.toast(toastContent, 4000, 'rounded green'); - return - } + var filesList = $('#upload').prop('files'); + if (filesList.length < 1) + { + var toastContent = $('
No file selected
'); + Materialize.toast(toastContent, 4000, 'rounded green'); + return + } - // reset ui elements for new validation run - hideResults(); - hideProgress(); //clear any previous data - showProgress(filesList[0].size); - display_results_once = false; + // reset ui elements for new validation run + hideResults(); + hideProgress(); //clear any previous data + showProgress(filesList[0].size); + display_results_once = false; - Materialize.toast('Sending file to server...', 4000, 'rounded') + Materialize.toast('Sending file to server...', 4000, 'rounded') $('#validateform').prop('action' , '/naplan/reg/validate'); - $('#upload').fileupload( - { + $('#upload').fileupload( + { autoUpload: false - }); + }); - $('#upload').bind('fileuploaddone', function(e, data) - { + $('#upload').bind('fileuploaddone', function(e, data) + { txTotal = data.result.Records - txID = data.result.TxID - console.log(txID + ": " + txTotal) - $('#upload').fileupload('destroy'); + txID = data.result.TxID + console.log(txID + ": " + txTotal) + $('#upload').fileupload('destroy'); startStreamSocket(); Materialize.toast('Analysis results generating...', 4000, 'rounded') - }); - $('#upload').fileupload('send', - { + }); + $('#upload').fileupload('send', + { sequentialUploads: true, files: filesList[0] - }); + }); } @@ -118,42 +118,42 @@ function Validate() // convert the input file from csv to xml function XmlConvert() { - var filesList = $('#upload').prop('files'); - if (filesList.length < 1) - { - var toastContent = $('
No file selected
'); - Materialize.toast(toastContent, 3000, 'rounded green'); - return - } - - $('#validateform').prop('action' , '/naplan/reg/convert'); - /* - $('#upload').fileupload( - { - autoUpload: false - }); - $('#upload').fileupload('send', - { - sequentialUploads: true, - files: filesList[0] - }); - */ - var x = $('#validateform').submit(); - - - /* - // console.log(filesList); - if(navigator.userAgent.toLowerCase().indexOf('firefox') > -1){ - // Firefox treats files as read only - var fileArray = [filesList[0].name]; - var fc = multiFileInput = document.getElementById('fileConvert'); - fc.mozSetFileNameArray(fileArray, fileArray.length); - } else { - $('#fileConvert').prop('files', filesList); - } - var x = $('#convForm').submit(); - */ - // console.log(x); + var filesList = $('#upload').prop('files'); + if (filesList.length < 1) + { + var toastContent = $('
No file selected
'); + Materialize.toast(toastContent, 3000, 'rounded green'); + return + } + + $('#validateform').prop('action' , '/naplan/reg/convert'); + /* + $('#upload').fileupload( + { + autoUpload: false + }); + $('#upload').fileupload('send', + { + sequentialUploads: true, + files: filesList[0] + }); + */ + var x = $('#validateform').submit(); + + + /* + // console.log(filesList); + if(navigator.userAgent.toLowerCase().indexOf('firefox') > -1){ + // Firefox treats files as read only + var fileArray = [filesList[0].name]; + var fc = multiFileInput = document.getElementById('fileConvert'); + fc.mozSetFileNameArray(fileArray, fileArray.length); + } else { + $('#fileConvert').prop('files', filesList); + } + var x = $('#convForm').submit(); + */ + // console.log(x); } @@ -165,51 +165,51 @@ function XmlConvert() // function startStreamSocket() { - var loc = window.location; - var uri = 'ws:'; + var loc = window.location; + var uri = 'ws:'; - if (loc.protocol === 'https:') - { - uri = 'wss:'; - } - uri += '//' + loc.host; - uri += '/naplan/reg/stream/' + txID; + if (loc.protocol === 'https:') + { + uri = 'wss:'; + } + uri += '//' + loc.host; + uri += '/naplan/reg/stream/' + txID; - console.log(uri) + console.log(uri) websocket = new WebSocket(uri); - msgs_rcvd = 0; - report_data = []; - - websocket.onopen = function(evt) - { - onOpen(evt) - }; - websocket.onclose = function(evt) - { - onClose(evt) - }; - websocket.onmessage = function(evt) - { - onMessage(evt) - }; - websocket.onerror = function(evt) - { - onError(evt) - }; + msgs_rcvd = 0; + report_data = []; + + websocket.onopen = function(evt) + { + onOpen(evt) + }; + websocket.onclose = function(evt) + { + onClose(evt) + }; + websocket.onmessage = function(evt) + { + onMessage(evt) + }; + websocket.onerror = function(evt) + { + onError(evt) + }; } function onOpen(evt) { - console.log('Stream ' + txID + ' Connected'); + console.log('Stream ' + txID + ' Connected'); } function onClose(evt) { - console.log('Stream ' + txID + ' Disconnected'); - console.log('Validation messages received: ' + msgs_rcvd); - console.log('Mesages in report data:' + report_data.length); + console.log('Stream ' + txID + ' Disconnected'); + console.log('Validation messages received: ' + msgs_rcvd); + console.log('Mesages in report data:' + report_data.length); } // @@ -219,67 +219,81 @@ function onClose(evt) function onMessage(evt) { - var msg = JSON.parse(evt.data); + var msg = JSON.parse(evt.data); + + var results = document.getElementById('results-message'); + var progress = document.getElementById('progress-message'); + + if (msg.Type == "result") + { + msgs_rcvd++; + // console.log("results message received") + report_data.push(msg.Payload); - var results = document.getElementById('results-message'); - var progress = document.getElementById('progress-message'); + } - if (msg.Type == "result") + if (msg.Type == "progress") + { + progress.innerHTML = '

' + "Records validated: " + msg.Payload.Progress + '

'; + // console.log(msg) + // console.log("messages received: " + msgs_rcvd) + if (msg.Payload.TxComplete) { - msgs_rcvd++; - // console.log("results message received") - report_data.push(msg.Payload); + console.log("transaction " + txID + " complete"); + websocket.close(); + // console.log("socket closed"); + Materialize.toast('All records validated', 4000, 'rounded'); } - if (msg.Type == "progress") + if (msg.Payload.TxComplete) { - progress.innerHTML = '

' + "Records validated: " + msg.Payload.Progress + '

'; - // console.log(msg) - // console.log("messages received: " + msgs_rcvd) - if (msg.Payload.TxComplete) - { - console.log("transaction " + txID + " complete"); - websocket.close(); - // console.log("socket closed"); - - Materialize.toast('All records validated', 4000, 'rounded'); - } + progress.innerHTML = '

' + "Records validated: " + msg.Payload.Progress + " of " + msg.Payload.Size + '

'; + var manifestref = "/naplan/reg/manifest/" + txID; + var xhttp = new XMLHttpRequest(); + xhttp.open("GET", manifestref, false); + xhttp.setRequestHeader("Content-type", "application/json"); + xhttp.send(); + var prgdata = JSON.parse(xhttp.responseText); + var manifest_container = document.getElementById('manifest'); + var manifest = "
Breakdown of schools
"; + manifest_container.innerHTML = manifest; - if (msg.Payload.TxComplete) + results.innerHTML = '

' + "Results are ready for review" + '

' + if (msgs_rcvd < 1) { - progress.innerHTML = '

' + "Records validated: " + msg.Payload.Progress + " of " + msg.Payload.Size + '

'; - results.innerHTML = '

' + "Results are ready for review" + '

' - if (msgs_rcvd < 1) - { - results.innerHTML = '

' + "No validation errors found." + '

' - } - else - { - renderResultsOnce(report_data); - } - - completeProgress(); + results.innerHTML = '

' + "No validation errors found." + '

' } - else if (msg.Payload.UIComplete) + else { - results.innerHTML = '

' + "Results are ready for review" + '

' - renderResultsOnce(report_data); + renderResultsOnce(report_data); } + completeProgress(); + } + else if (msg.Payload.UIComplete) + { + results.innerHTML = '

' + "Results are ready for review" + '

' + renderResultsOnce(report_data); } + } + } function onError(evt) { - console.log('ERROR: ' + evt.data); + console.log('ERROR: ' + evt.data); } function doSend(message) { - console.log("SENT: " + message); - websocket.send(message); + console.log("SENT: " + message); + websocket.send(message); } // // end of results stream websocket handlers @@ -289,12 +303,12 @@ function doSend(message) function renderResultsOnce(data) { - if (display_results_once == true) - { - return - } - renderAnalysis(data); - display_results_once = true; + if (display_results_once == true) + { + return + } + renderAnalysis(data); + display_results_once = true; } @@ -308,146 +322,146 @@ function renderResultsOnce(data) function renderAnalysis(data) { - // $("#results_container").toggleClass("hide"); + // $("#results_container").toggleClass("hide"); - var errorsBarChart = dc.barChart("#errors-chart"); - var errorsByTypeChart = dc.rowChart('#errors-by-type-chart'); - var dataTable = dc.dataTable('.dc-data-table'); - var verrorsCount = dc.dataCount('.dc-data-count'); - var errorData = data; + var errorsBarChart = dc.barChart("#errors-chart"); + var errorsByTypeChart = dc.rowChart('#errors-by-type-chart'); + var dataTable = dc.dataTable('.dc-data-table'); + var verrorsCount = dc.dataCount('.dc-data-count'); + var errorData = data; - // normalize/parse data so dc can correctly sort & bin them - errorData.forEach(function(d) - { + // normalize/parse data so dc can correctly sort & bin them + errorData.forEach(function(d) + { d.originalLine = +d.originalLine; - }); - // console.log(errorData); + }); + // console.log(errorData); - var ndx = crossfilter(errorData); - var all = ndx.groupAll(); + var ndx = crossfilter(errorData); + var all = ndx.groupAll(); - var lineDim = ndx.dimension(function(d) - { + var lineDim = ndx.dimension(function(d) + { return d.originalLine; - }); + }); - var typeDim = ndx.dimension(function(d) - { + var typeDim = ndx.dimension(function(d) + { return d.validationType; - }); - var validationTypesGroup = typeDim.group(); + }); + var validationTypesGroup = typeDim.group(); - var allDim = ndx.dimension(function(d) - { + var allDim = ndx.dimension(function(d) + { return d; - }); + }); - // var countPerLine = lineDim.group().reduceCount(function(d) { - // return d.originalLine; - // }); + // var countPerLine = lineDim.group().reduceCount(function(d) { + // return d.originalLine; + // }); - var lineGroup = lineDim.group(); + var lineGroup = lineDim.group(); - // var countPerLine = lineDim.group().reduceSum(function(d) {return d.OriginalLine;}); + // var countPerLine = lineDim.group().reduceSum(function(d) {return d.OriginalLine;}); - errorsBarChart - .width(350) - .height(180) - .margins( + errorsBarChart + .width(350) + .height(180) + .margins( { - top: 20, - right: 0, - bottom: 0, - left: 0 + top: 20, + right: 0, + bottom: 0, + left: 0 }) - .gap(1) - .x(d3.scale.linear().domain([0, 200000])) - .elasticX(true) - .elasticY(true) - .dimension(lineDim) - .colors(["#4caf50"]) - .group(lineGroup); - - errorsBarChart.dimension(lineDim); - - errorsByTypeChart - .width(350) - .height(180) - .margins( + .gap(1) + .x(d3.scale.linear().domain([0, 200000])) + .elasticX(true) + .elasticY(true) + .dimension(lineDim) + .colors(["#4caf50"]) + .group(lineGroup); + + errorsBarChart.dimension(lineDim); + + errorsByTypeChart + .width(350) + .height(180) + .margins( { - top: 20, - left: 10, - right: 10, - bottom: 20 + top: 20, + left: 10, + right: 10, + bottom: 20 }) - .group(validationTypesGroup) - .dimension(typeDim) - .title(function(d) + .group(validationTypesGroup) + .dimension(typeDim) + .title(function(d) { - return d.value; + return d.value; }) - .colors(["#4caf50"]) - .elasticX(true) - .xAxis().ticks(4); + .colors(["#4caf50"]) + .elasticX(true) + .xAxis().ticks(4); - verrorsCount - .dimension(ndx) - .group(all); + verrorsCount + .dimension(ndx) + .group(all); - dataTable + dataTable // .width(900) // .height(800) - .dimension(allDim) - .group(function(d) + .dimension(allDim) + .group(function(d) { - // return 'dc.js insists on putting a row here so I remove it using JS'; - // return d.originalLine; - // return 'Errors ordered by original file line number (table shows first 100 errors)' - return '' + // return 'dc.js insists on putting a row here so I remove it using JS'; + // return d.originalLine; + // return 'Errors ordered by original file line number (table shows first 100 errors)' + return '' }) - .size(100) - .columns([ - // function (d) { return d.txID; }, - function(d) - { - return d.originalLine; - }, - function(d) - { - return d.validationType; - }, - function(d) - { - return d.errField; - }, - function(d) - { - return d.description; - }, - function(d) - { - return d.severity.substring(0,1).toUpperCase() - } - ]) - .sortBy(function(d) + .size(100) + .columns([ + // function (d) { return d.txID; }, + function(d) { - return d.originalLine; + return d.originalLine; + }, + function(d) + { + return d.validationType; + }, + function(d) + { + return d.errField; + }, + function(d) + { + return d.description; + }, + function(d) + { + return d.severity.substring(0,1).toUpperCase() + } + ]) + .sortBy(function(d) + { + return d.originalLine; }) - .order(d3.ascending) - .on('renderlet', function(table) + .order(d3.ascending) + .on('renderlet', function(table) { - // each time table is rendered remove nasty extra row dc.js insists on adding - // table.select('tr.dc-table-group').remove(); - table.selectAll('.dc-table-group').classed('info', true); + // each time table is rendered remove nasty extra row dc.js insists on adding + // table.select('tr.dc-table-group').remove(); + table.selectAll('.dc-table-group').classed('info', true); }); - dc.renderAll(); + dc.renderAll(); - // dc.redrawAll(); + // dc.redrawAll(); - showResults(); + showResults(); } @@ -458,44 +472,46 @@ function renderAnalysis(data) // function hideResults() { - $("#results_container").addClass("hide"); + $("#results_container").addClass("hide"); } function showResults() { - $("#results_container").removeClass("hide"); + $("#results_container").removeClass("hide"); } function hideProgress() { - $("#progress").addClass("hide"); - $("#upload-message").empty(); - $("#results-message").empty(); - $("#progress-message").empty(); + $("#progress").addClass("hide"); + $("#upload-message").empty(); + $("#results-message").empty(); + $("#progress-message").empty(); + $("#manifest_accordion").addClass("hide"); } function completeProgress() { - $("#progress").addClass("hide"); - $("#upload-message").empty(); + $("#progress").addClass("hide"); + $("#upload-message").empty(); + $("#manifest_accordion").removeClass("hide"); } function showProgress(fileSizeBytes) { - var prgETA = calculateETA(fileSizeBytes); - $("#progress").removeClass("hide"); - $("#upload-message").text("Validating input file..." + prgETA); + var prgETA = calculateETA(fileSizeBytes); + $("#progress").removeClass("hide"); + $("#upload-message").text("Validating input file..." + prgETA); } function calculateETA(fileSizeBytes) { - var ttc_seconds = ((fileSizeBytes / 1024000) * 4); - if (ttc_seconds <= 60) - { - return "estimated analysis time: " + ttc_seconds.toFixed(0) + " seconds." - } - var ttc_minutes = (ttc_seconds / 60); - return "estimated analysis time: " + ttc_minutes.toFixed(0) + " minutes." + var ttc_seconds = ((fileSizeBytes / 1024000) * 4); + if (ttc_seconds <= 60) + { + return "estimated analysis time: " + ttc_seconds.toFixed(0) + " seconds." + } + var ttc_minutes = (ttc_seconds / 60); + return "estimated analysis time: " + ttc_minutes.toFixed(0) + " minutes." } diff --git a/napval/validation_server.go b/napval/validation_server.go index 9d78af9..2a23456 100644 --- a/napval/validation_server.go +++ b/napval/validation_server.go @@ -29,6 +29,7 @@ import ( "github.com/wildducktheories/go-csv" "golang.org/x/net/websocket" //"time" + "encoding/gob" ) var naplanconfig = LoadNAPLANConfig() @@ -54,6 +55,11 @@ func publish(msg *lib.NiasMessage) { } +// message type for reporting student/school tally +type StudentSchoolTally struct { + Tally map[string]int +} + // // read csv file as stream and post records onto processing queue // @@ -66,6 +72,7 @@ func enqueueCSVforNAPLANValidation(file multipart.File) (lib.IngestResponse, err i := 0 txid := nuid.Next() + studentcount := make(map[string]int, 0) for record := range reader.C() { i = i + 1 @@ -77,6 +84,7 @@ func enqueueCSVforNAPLANValidation(file multipart.File) (lib.IngestResponse, err if decode_err != nil { return ir, decode_err } + studentcount[regr.ASLSchoolId]++ msg := &lib.NiasMessage{} msg.Body = *regr @@ -97,7 +105,31 @@ func enqueueCSVforNAPLANValidation(file multipart.File) (lib.IngestResponse, err // update the tx tracker tt.SetTxSize(txid, i) + enqueueManifest(studentcount, txid) return ir, nil +} + +func enqueueManifest(studentcount map[string]int, txid string) { + tt.SetTxSize("manifest:"+txid, 1) + msg := &lib.NiasMessage{} + msg.Body = StudentSchoolTally{Tally: studentcount} + msg.SeqNo = strconv.Itoa(1) + msg.TxID = "manifest:" + txid + msg.MsgID = nuid.Next() + err := stan_conn.Publish("manifest:"+txid, lib.EncodeNiasMessage(msg)) + if err != nil { + log.Println("publish to store error: ", err) + } + tt.IncrementTracker(msg.TxID) + // get status of transaction and add message to stream + // if a notable status change has occurred + sigChange, msg1 := tt.GetStatusReport(msg.TxID) + if sigChange { + err := stan_conn.Publish("manifest:"+txid, lib.EncodeNiasMessage(msg1)) + if err != nil { + log.Println("publish to store error: ", err) + } + } } @@ -108,6 +140,7 @@ func enqueueXMLforNAPLANValidation(file multipart.File) (lib.IngestResponse, err ir := lib.IngestResponse{} + studentcount := make(map[string]int, 0) decoder := xml.NewDecoder(file) total := 0 txid := nuid.Next() @@ -129,6 +162,7 @@ func enqueueXMLforNAPLANValidation(file multipart.File) (lib.IngestResponse, err if decode_err != nil { return ir, decode_err } + studentcount[rr.ASLSchoolId]++ msg := &lib.NiasMessage{} msg.Body = rr @@ -153,6 +187,7 @@ func enqueueXMLforNAPLANValidation(file multipart.File) (lib.IngestResponse, err // update the tx tracker tt.SetTxSize(txid, total) + enqueueManifest(studentcount, txid) return ir, nil } @@ -161,7 +196,7 @@ func enqueueXMLforNAPLANValidation(file multipart.File) (lib.IngestResponse, err // start the server // func (vws *ValidationWebServer) Run(nats_cfg lib.NATSConfig) { - + gob.Register(StudentSchoolTally{}) log.Println("NAPLAN: Connecting to message bus") req_ec = lib.CreateNATSConnection(nats_cfg) @@ -401,6 +436,68 @@ func (vws *ValidationWebServer) Run(nats_cfg lib.NATSConfig) { }) + // get manifest of schools and student counts for a transaction + e.GET("/naplan/reg/manifest/:txid", func(c echo.Context) error { + //e.GET("/naplan/reg/manifest", func(c echo.Context) error { + + txID := c.Param("txid") + log.Println(txID) + + c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + c.Response().WriteHeader(http.StatusOK) + + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + mcb := func(m *stan.Msg) { + + msg := lib.DecodeNiasMessage(m.Data) + + // convenience type to send results/progress data only + // to web clients + type VMessage struct { + Type string + Payload interface{} + } + + switch t := msg.Body.(type) { + case StudentSchoolTally: + ve := msg.Body.(StudentSchoolTally) + if err := json.NewEncoder(c.Response()).Encode(ve); err != nil { + //if err := enc.Encode(ve); err != nil { + log.Println("error encoding json niasmessage: ", err) + } + c.Response().Flush() + case lib.TxStatusUpdate: + txsu := msg.Body.(lib.TxStatusUpdate) + if txsu.TxComplete { + log.Println("Finished...") + txComplete <- true + } + default: + _ = t + vmsg := VMessage{Type: "unknown", Payload: ""} + log.Printf("unknown message type in handler: %v ", vmsg) + log.Printf("message decoded from stan is (type %v):\n\n %+v\n\n", t, msg) + } + + //log.Printf("message decoded from stan is:\n\n %+v\n\n", msg) + + } + + sub, err := stan_conn.Subscribe("manifest:"+txID, mcb, stan.DeliverAllAvailable()) + defer sub.Unsubscribe() + if err != nil { + log.Println("stan subsciption error results-download: ", err) + return err + } + + <-txComplete + + return nil + }) + // // get validation analysis results - non-websocket, just json stream // From 96b2083174914c5dbe5102e7e422584692f8b59d Mon Sep 17 00:00:00 2001 From: Nick Nicholas Date: Wed, 27 Sep 2017 11:17:11 +1000 Subject: [PATCH 2/5] check identity data matching across schools --- app/napcomp/in/registration/students.csv | 2 +- app/napval/napval.toml | 3 + lib/config.go | 1 + napval/idservice3.go | 47 +++++++-- xml/registrationrecord.go | 126 +++++++++++++++++++++++ 5 files changed, 168 insertions(+), 11 deletions(-) diff --git a/app/napcomp/in/registration/students.csv b/app/napcomp/in/registration/students.csv index e169bd1..6501694 100644 --- a/app/napcomp/in/registration/students.csv +++ b/app/napcomp/in/registration/students.csv @@ -599,4 +599,4 @@ ywxno726,61757,83962,7149,39746,91421,35978,R100000042R,62662,18043,89978,18963, mhnmb320,7547,67896,93569,90337,33538,10613,R100000123K,47680,33991,7839,11099,17493,6275,84961,R100000123K,Adams,Mildred,Mildred,,2001-12-11,2,1101,Y,1,101,4,Y,1201,9,9,0.53,9F,49453,6053,1,1,qnkmm683,49453,Y,N,Y,1,5,3,9601,4,7,3,1201,1624 S. Taylorville Rd.,,BLACKFORD,5275,SA bbcev920,32583,17070,89576,84657,44158,69881,R100000130P,13902,79392,68324,79756,93375,48991,14445,R100000130P,Abe,Claude,Claude,R,2001-09-12,1,1101,X,1,101,3,N,1201,9,9,0.18,9B,49360,22782,1,1,geuuu654,49360,Y,Y,Y,2,7,4,1201,4,5,4,1201,1210 Bethlehem Pike,,BOOKABIE,5690,SA dhevd393,94764,55144,38637,90446,14514,75693,R100000011H,9769,80987,91020,71475,48478,24612,88724,R100000011H,Abbott,Walter,Walter,E,2007-04-26,1,1101,N,1,101,4,N,1201,3,3,0.17,3E,49559,16481,1,2,armnc164,49559,Y,N,Y,1,6,2,1201,3,8,1,5203,3443 N. Southport Ave.,,FRANKLYN,5421,SA -dhevd393,94764,55144,38637,90446,14514,75693,R100000011H,9769,80987,91020,71475,48478,24612,88724,R6555003423,Tong,Bing,Bang,E,2007-04-26,1,1101,N,1,101,4,N,1201,3,3,0.17,3E,49559,16481,1,2,armnc164,49559,Y,N,Y,1,6,2,1201,3,8,1,5203,3443 N. Southport Ave.,,FRANKLYN,5421,SA \ No newline at end of file +dhevd493,94764,55144,38637,90446,14514,75693,R100100011H,9769,80987,91020,71475,48478,24612,88724,R6555003423,Youngman,Genaro,Genaro,L,2003-07-19,1,1101,N,1,101,4,N,1201,3,3,0.17,3E,49559,16481,1,2,armnc164,49559,Y,N,Y,1,6,2,1201,3,8,1,5203,3443 N. Southport Ave.,,FRANKLYN,5421,SA \ No newline at end of file diff --git a/app/napval/napval.toml b/app/napval/napval.toml index a14656d..9fc0175 100644 --- a/app/napval/napval.toml +++ b/app/napval/napval.toml @@ -9,6 +9,9 @@ TestYear = "2017" # ValidationRoute = ["schema", "local","schema2", "dob", "id","asl", "psi", "numericvalid"] ValidationRoute = ["schema", "schema2", "dob", "id","asl", "psi", "numericvalid"] +# Data match fields for matching across schools +StudentMatch = ["FamilyName", "GivenName", "BirthDate"] + # Webserver port WebServerPort = "1325" diff --git a/lib/config.go b/lib/config.go index b8d2430..466640a 100644 --- a/lib/config.go +++ b/lib/config.go @@ -13,6 +13,7 @@ type NIASConfig struct { WebServerPort string NATSPort string ValidationRoute []string + StudentMatch []string SSFRoute []string SMSRoute []string PoolSize int // number of service processors diff --git a/napval/idservice3.go b/napval/idservice3.go index a87b1b1..13483cb 100644 --- a/napval/idservice3.go +++ b/napval/idservice3.go @@ -50,10 +50,11 @@ type IDService3 struct { // transaction id lookups type TransactionIDs struct { - Locations cmap.ConcurrentMap - SimpleKeysLocalId *set.Set - SimpleKeysPSI *set.Set - ExtendedKeys *set.Set + Locations cmap.ConcurrentMap + SimpleKeysLocalId *set.Set + SimpleKeysPSI *set.Set + ExtendedKeys *set.Set + CrossSchoolMatches *set.Set } // create a new id service instance @@ -91,9 +92,10 @@ func (ids *IDService3) HandleMessage(req *lib.NiasMessage) ([]lib.NiasMessage, e // see if dataset exists for this transaction, create if not ids.Transactions.SetIfAbsent(req.TxID, TransactionIDs{Locations: cmap.New(), - SimpleKeysLocalId: set.New(), - SimpleKeysPSI: set.New(), - ExtendedKeys: set.New()}) + SimpleKeysLocalId: set.New(), + SimpleKeysPSI: set.New(), + ExtendedKeys: set.New(), + CrossSchoolMatches: set.New()}) // retrieve the transaction dataset from the store tdata, ok := ids.Transactions.Get(req.TxID) @@ -129,8 +131,10 @@ func (ids *IDService3) HandleMessage(req *lib.NiasMessage) ([]lib.NiasMessage, e simpleKey1 := fmt.Sprintf("%v", k11) simpleKey2 := fmt.Sprintf("%v", k12) complexKey := fmt.Sprintf("%v", k2) + crossSchoolKey := rr.FieldsKey(config.StudentMatch) + //log.Printf("simplekey: %s\nsimplekey2: %s\ncompllexkey: %s", simpleKey1, simpleKey2, complexKey) - var simpleRecordExists1, simpleRecordExists2, complexRecordExists bool + var simpleRecordExists1, simpleRecordExists2, complexRecordExists, crossSchoolRecordExists bool if simpleRecordExists1 = (tids.SimpleKeysLocalId.Has(simpleKey1) && len(rr.LocalId) > 0); !simpleRecordExists1 { tids.SimpleKeysLocalId.Add(simpleKey1) @@ -142,17 +146,40 @@ func (ids *IDService3) HandleMessage(req *lib.NiasMessage) ([]lib.NiasMessage, e if complexRecordExists = (tids.ExtendedKeys.Has(complexKey) && len(rr.LocalId) > 0); !complexRecordExists { tids.ExtendedKeys.Add(complexKey) } + if crossSchoolRecordExists = (tids.CrossSchoolMatches.Has(crossSchoolKey) && len(crossSchoolKey) > 0); !crossSchoolRecordExists { + tids.CrossSchoolMatches.Add(crossSchoolKey) + } tids.Locations.SetIfAbsent(simpleKey1, req.SeqNo) tids.Locations.SetIfAbsent(simpleKey2, req.SeqNo) tids.Locations.SetIfAbsent(complexKey, req.SeqNo) + tids.Locations.SetIfAbsent(crossSchoolKey, req.SeqNo) // if record is new then just return - if !complexRecordExists && !simpleRecordExists1 && !simpleRecordExists2 { + if !complexRecordExists && !simpleRecordExists1 && !simpleRecordExists2 && !crossSchoolRecordExists { return responses, nil } // if we have seen it before then construct validation error - if complexRecordExists { + if crossSchoolRecordExists { + loc, _ := tids.Locations.Get(crossSchoolKey) + ol, _ := loc.(string) + desc := "Potential duplicate of record: " + ol + "\n" + + fmt.Sprintf("based on matching: %v", config.StudentMatch) + ve := ValidationError{ + Description: desc, + Field: "Multiple (see description)", + OriginalLine: req.SeqNo, + Vtype: "identity", + Severity: "warning", + } + r := lib.NiasMessage{} + r.TxID = req.TxID + r.SeqNo = req.SeqNo + // r.Target = VALIDATION_PREFIX + r.Body = ve + responses = append(responses, r) + log.Printf("%v\n", responses) + } else if complexRecordExists { loc, _ := tids.Locations.Get(complexKey) ol, _ := loc.(string) desc := "Potential duplicate of record: " + ol + "\n" + diff --git a/xml/registrationrecord.go b/xml/registrationrecord.go index b8c279c..847e96f 100644 --- a/xml/registrationrecord.go +++ b/xml/registrationrecord.go @@ -189,6 +189,132 @@ func (r *RegistrationRecord) Unflatten() RegistrationRecord { return *r } +// return key based on concatenation of named fields +// avoiding reflection for performance reasons +func (r RegistrationRecord) FieldsKey(keys []string) string { + ret := "" + for _, k := range keys { + switch k { + case "RefId": + ret += r.RefId + case "LocalId": + ret += r.LocalId + case "StateProvinceId": + ret += r.StateProvinceId + case "FamilyName": + ret += r.FamilyName + case "GivenName": + ret += r.GivenName + case "MiddleName": + ret += r.MiddleName + case "PreferredName": + ret += r.PreferredName + case "IndigenousStatus": + ret += r.IndigenousStatus + case "Sex": + ret += r.Sex + case "BirthDate": + ret += r.BirthDate + case "CountryOfBirth": + ret += r.CountryOfBirth + case "StudentLOTE": + ret += r.StudentLOTE + case "VisaCode": + ret += r.VisaCode + case "LBOTE": + ret += r.LBOTE + case "AddressLine1": + ret += r.AddressLine1 + case "AddressLine2": + ret += r.AddressLine2 + case "Locality": + ret += r.Locality + case "StateTerritory": + ret += r.StateTerritory + case "Postcode": + ret += r.Postcode + case "SchoolLocalId": + ret += r.SchoolLocalId + case "YearLevel": + ret += r.YearLevel + case "FTE": + ret += r.FTE + case "Parent1LOTE": + ret += r.Parent1LOTE + case "Parent2LOTE": + ret += r.Parent2LOTE + case "Parent1Occupation": + ret += r.Parent1Occupation + case "Parent2Occupation": + ret += r.Parent2Occupation + case "Parent1SchoolEducation": + ret += r.Parent1SchoolEducation + case "Parent2SchoolEducation": + ret += r.Parent2SchoolEducation + case "Parent1NonSchoolEducation": + ret += r.Parent1NonSchoolEducation + case "Parent2NonSchoolEducation": + ret += r.Parent2NonSchoolEducation + case "LocalCampusId": + ret += r.LocalCampusId + case "ASLSchoolId": + ret += r.ASLSchoolId + case "TestLevel": + ret += r.TestLevel + case "Homegroup": + ret += r.Homegroup + case "ClassGroup": + ret += r.ClassGroup + case "MainSchoolFlag": + ret += r.MainSchoolFlag + case "FFPOS": + ret += r.FFPOS + case "ReportingSchoolId": + ret += r.ReportingSchoolId + case "OtherSchoolId": + ret += r.OtherSchoolId + case "EducationSupport": + ret += r.EducationSupport + case "HomeSchooledStudent": + ret += r.HomeSchooledStudent + case "Sensitive": + ret += r.Sensitive + case "OfflineDelivery": + ret += r.OfflineDelivery + case "DiocesanId": + ret += r.DiocesanId + case "JurisdictionId": + ret += r.JurisdictionId + case "NationalId": + ret += r.NationalId + case "OtherId": + ret += r.OtherId + case "PlatformId": + ret += r.PlatformId + case "PreviousDiocesanId": + ret += r.PreviousDiocesanId + case "PreviousNationalId": + ret += r.PreviousNationalId + case "PreviousOtherId": + ret += r.PreviousOtherId + case "PreviousPlatformId": + ret += r.PreviousPlatformId + case "PreviousSectorId": + ret += r.PreviousSectorId + case "PreviousLocalId": + ret += r.PreviousLocalId + case "PreviousStateProvinceId": + ret += r.PreviousStateProvinceId + case "SectorId": + ret += r.SectorId + case "TAAId": + ret += r.TAAId + } + ret += ":::" + } + return ret +} + // convenience method to return otherid by type func (r RegistrationRecord) GetOtherId(idtype string) string { From 7ba50d4701a436bb46560515c40f9ebd5145dba0 Mon Sep 17 00:00:00 2001 From: Nick Nicholas Date: Wed, 27 Sep 2017 12:08:18 +1000 Subject: [PATCH 3/5] check that database is empty before running naprrql --- app/naprrql/naprrql.go | 3 ++- napcomp/napcomp.go | 8 ++++---- naprrql/datastore.go | 21 ++++++++++++++++----- naprrql/ingest.go | 2 +- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/app/naprrql/naprrql.go b/app/naprrql/naprrql.go index dd9571d..865268b 100644 --- a/app/naprrql/naprrql.go +++ b/app/naprrql/naprrql.go @@ -92,6 +92,7 @@ func ingestData() { // launch the webserver // func startWebServer(silent bool) { + naprrql.GetDB(false) // Verify that the database is populated; if not will abort go naprrql.RunQLServer() if !silent { fmt.Printf("\n\nBrowse to follwing locations:\n") @@ -136,7 +137,7 @@ func parseResultsFileDirectory() []string { // func closeDB() { log.Println("Closing datastore...") - naprrql.GetDB().Close() + naprrql.GetDB(true).Close() log.Println("Datastore closed.") } diff --git a/napcomp/napcomp.go b/napcomp/napcomp.go index 33bcf47..eacd5e1 100644 --- a/napcomp/napcomp.go +++ b/napcomp/napcomp.go @@ -57,7 +57,7 @@ func IngestData() { // func ingestResultsFile(resultsFilePath string) { - db := naprrql.GetDB() + db := naprrql.GetDB(true) ge := naprrql.GobEncoder{} // open the data file for streaming read @@ -128,7 +128,7 @@ func ingestResultsFile(resultsFilePath string) { // func ingestRegistrationFile(regFilePath string) { - db := naprrql.GetDB() + db := naprrql.GetDB(true) ge := naprrql.GobEncoder{} log.Printf("Reading data file [%s]", regFilePath) @@ -210,7 +210,7 @@ func makeComparisonKey(r *xml.RegistrationRecord) string { func WriteReports() { clearReportsDirectory() - db := naprrql.GetDB() + db := naprrql.GetDB(false) ge := naprrql.GobEncoder{} log.Println("generating difference reports...") @@ -329,7 +329,7 @@ func parseRegistrationFileDirectory() []string { // func CloseDB() { log.Println("Closing datastore...") - naprrql.GetDB().Close() + naprrql.GetDB(true).Close() log.Println("Datastore closed.") } diff --git a/naprrql/datastore.go b/naprrql/datastore.go index ade230f..fb97da4 100644 --- a/naprrql/datastore.go +++ b/naprrql/datastore.go @@ -17,10 +17,12 @@ var dbOpen bool = false var ge = GobEncoder{} -func GetDB() *leveldb.DB { +// if allow_empty is false, abort if database is empty: enforces +// requirement to run ingest first +func GetDB(allow_empty bool) *leveldb.DB { if !dbOpen { log.Println("DB not initialised. Opening...") - openDB() + openDB(allow_empty) } return db } @@ -28,7 +30,7 @@ func GetDB() *leveldb.DB { // // open the kv store, this must be called before any access is attempted // -func openDB() { +func openDB(allow_empty bool) { workingDir := "kvs" @@ -42,6 +44,15 @@ func openDB() { if dbErr != nil { log.Fatalln("DB Create error: ", dbErr) } + // if database is empty, abort + if !allow_empty { + iter := db.NewIterator(nil, nil) + iter.Next() + if len(iter.Key()) == 0 { + log.Fatalf("DB is Empty. Run naprrql --ingest with an extract file.") + } + } + dbOpen = true } @@ -52,7 +63,7 @@ func openDB() { // func getIdentifiers(keyPrefix string) []string { - db = GetDB() + db = GetDB(false) objIDs := make([]string, 0) searchKey := []byte(keyPrefix + ":") @@ -77,7 +88,7 @@ func getIdentifiers(keyPrefix string) []string { // func getObjects(objIDs []string) ([]interface{}, error) { - db = GetDB() + db = GetDB(false) objects := []interface{}{} for _, objID := range objIDs { diff --git a/naprrql/ingest.go b/naprrql/ingest.go index aef21be..aa8123f 100644 --- a/naprrql/ingest.go +++ b/naprrql/ingest.go @@ -18,7 +18,7 @@ var unfit bool func IngestResultsFile(resultsFilePath string) { fileGuids = make(map[string]bool) - db := GetDB() + db := GetDB(true) ge := GobEncoder{} // open the data file for streaming read From cbab4a62dd4377881678496e6aa6be138981fe88 Mon Sep 17 00:00:00 2001 From: Nick Nicholas Date: Wed, 27 Sep 2017 14:35:31 +1000 Subject: [PATCH 4/5] replace PSI with name in table displays, naprr --- app/naprrql/naplan_schema.graphql | 4 +++- app/naprrql/public/js/naprr_domainscores.js | 19 ++++++++++++++----- app/naprrql/public/js/naprr_participation.js | 15 ++++++++++----- .../public/js/naprr_studentpersonal.js | 9 ++++++--- naprrql/naprr_xmlhelpers.go | 1 + naprrql/report-resolvers.go | 7 ++++++- 6 files changed, 40 insertions(+), 15 deletions(-) diff --git a/app/naprrql/naplan_schema.graphql b/app/naprrql/naplan_schema.graphql index bc7b67a..3608709 100644 --- a/app/naprrql/naplan_schema.graphql +++ b/app/naprrql/naplan_schema.graphql @@ -173,6 +173,8 @@ type ParticipationSummary { ## Domain scores reporting object type ResponseDataSet { + ## The student who's participation is being reported + Student: RegistrationRecord ## the related test Test: NAPTest ## the response from which the domain score will be derived @@ -1015,4 +1017,4 @@ type SchoolDetails { - \ No newline at end of file + diff --git a/app/naprrql/public/js/naprr_domainscores.js b/app/naprrql/public/js/naprr_domainscores.js index 80cf7fa..75629e6 100644 --- a/app/naprrql/public/js/naprr_domainscores.js +++ b/app/naprrql/public/js/naprr_domainscores.js @@ -26,6 +26,10 @@ function domainScoresQuery() { return ` query DomainScoresData($acaraIDs: [String]) { domain_scores_report_by_school(acaraIDs: $acaraIDs) { + Student { + FamilyName + GivenName + } Test { TestID TestContent { @@ -156,7 +160,7 @@ function createDomainScoresReport() } // -// sort data - by year level & test domain for now +// sort data - by year level & test domain and student last then first name // function sortDomainScoresData(data) { @@ -170,11 +174,15 @@ function sortDomainScoresData(data) var compA = (a.Test.TestContent.TestLevel || '').toUpperCase() + (a.Test.TestContent.TestDomain || '').toUpperCase() + - (a.Response.DomainScore.StudentDomainBand || '').toUpperCase(); + (a.Response.DomainScore.StudentDomainBand || '').toUpperCase() + + (a.Student.FamilyName || '').toUpperCase() + + (a.Student.GivenName || '').toUpperCase() ; var compB = (b.Test.TestContent.TestLevel || '').toUpperCase() + (b.Test.TestContent.TestDomain || '').toUpperCase() + - (b.Response.DomainScore.StudentDomainBand || '').toUpperCase(); + (b.Response.DomainScore.StudentDomainBand || '').toUpperCase() + + (b.Student.FamilyName || '').toUpperCase() + + (b.Student.GivenName || '').toUpperCase() ; return (compA < compB) ? -1 : (compA > compB) ? 1 : 0; }); @@ -213,7 +221,7 @@ function createDomainScoresTableHeader() var hdr = $(""); var hdr_row = $("Level" + "Domain" + - "PSI" + + "Name" + "Raw Score" + "Scaled Score" + "Scaled Score Std. Error" + @@ -240,7 +248,8 @@ function createDomainScoresTableBody(data) var $row = $(""); $row.append("" + hideNull(rds.Test.TestContent.TestLevel) + "" + "" + hideNull(rds.Test.TestContent.TestDomain) + "" + - "" + hideNull(rds.Response.PSI) + "" + + //"" + hideNull(rds.Response.PSI) + "" + + "" + hideNull(rds.Student.GivenName) + " " + hideNull(rds.Student.FamilyName) + "" + "" + hideNull(rds.Response.DomainScore.RawScore) + "" + "" + hideNull(rds.Response.DomainScore.ScaledScoreValue) + "" + "" + hideNull(rds.Response.DomainScore.ScaledScoreStandardError) + "" + diff --git a/app/naprrql/public/js/naprr_participation.js b/app/naprrql/public/js/naprr_participation.js index 2d11d5d..444f709 100644 --- a/app/naprrql/public/js/naprr_participation.js +++ b/app/naprrql/public/js/naprr_participation.js @@ -132,8 +132,12 @@ function sortParticipationData(data) { data.sort(function(a, b) { - var compA = (a.EventInfos[0].Test.TestContent.TestLevel || '').toUpperCase(); - var compB = (b.EventInfos[0].Test.TestContent.TestLevel || '').toUpperCase(); + var compA = (a.EventInfos[0].Test.TestContent.TestLevel || '').toUpperCase() + + (a.Student.FamilyName || '').toUpperCase() + + (a.Student.GivenName || '').toUpperCase() ; + var compB = (b.EventInfos[0].Test.TestContent.TestLevel || '').toUpperCase() + + (b.Student.FamilyName || '').toUpperCase() + + (b.Student.GivenName || '').toUpperCase() ; return (compA < compB) ? -1 : (compA > compB) ? 1 : 0; }); @@ -168,7 +172,7 @@ function createParticipationTableHeader() { var hdr = $(""); var hdr_row = $("Level" + - "PSI" + + "Name" + "G and P" + "Numeracy" + "Reading" + @@ -205,7 +209,8 @@ function createParticipationTableBody(data) { var $row = $(""); $row.append("" + test.TestContent.TestLevel + "" + - "" + event.Event.PSI + "" + + // "" + event.Event.PSI + "" + + "" + pds.Student.GivenName + " " + pds.Student.FamilyName + "" + "" + hideNull(summary['Grammar and Punctuation']) + "" + "" + hideNull(summary['Numeracy']) + "" + "" + hideNull(summary['Reading']) + "" + @@ -384,4 +389,4 @@ function initParticipationDownloadLinkHandler() { }); -} \ No newline at end of file +} diff --git a/app/naprrql/public/js/naprr_studentpersonal.js b/app/naprrql/public/js/naprr_studentpersonal.js index 1347e82..e25a2e7 100644 --- a/app/naprrql/public/js/naprr_studentpersonal.js +++ b/app/naprrql/public/js/naprr_studentpersonal.js @@ -9,14 +9,17 @@ function getStudentInfoSummaryLine(psi) { // data can be found in participation info $.each(studentPersonalData, function(index, studentpersonal) { // ei = pds.EventInfos[0]; - sp = studentpersonal; + // sp = studentpersonal; $.each(studentpersonal.OtherIdList.OtherId, function(index, oid) { + if (oid.Type == "NAPPlatformStudentId") { if (oid.Value == psi) { // var student = pds.Student; // sp = student; + sp = studentpersonal; sp_psi = oid.Value; - return false; + //return false; } + } }); }); @@ -94,4 +97,4 @@ query NAPData($acaraIDs: [String]) { } ` -} \ No newline at end of file +} diff --git a/naprrql/naprr_xmlhelpers.go b/naprrql/naprr_xmlhelpers.go index ecfb314..9279626 100644 --- a/naprrql/naprr_xmlhelpers.go +++ b/naprrql/naprr_xmlhelpers.go @@ -26,6 +26,7 @@ func init() { // aggregating type used for reporting domain scores type ResponseDataSet struct { Test xml.NAPTest + Student xml.RegistrationRecord Response xml.NAPResponseSet } diff --git a/naprrql/report-resolvers.go b/naprrql/report-resolvers.go index 59e15f9..b592143 100644 --- a/naprrql/report-resolvers.go +++ b/naprrql/report-resolvers.go @@ -219,7 +219,12 @@ func buildReportResolvers() map[string]interface{} { if err != nil || !ok { return []interface{}{}, err } - rds := ResponseDataSet{Test: test, Response: resp} + students, err := getObjects([]string{resp.StudentID}) + student, ok := students[0].(xml.RegistrationRecord) + if err != nil || !ok { + return []interface{}{}, err + } + rds := ResponseDataSet{Test: test, Response: resp, Student: student} results = append(results, rds) } From 67ce002833777fad1742680a331dc30928433844 Mon Sep 17 00:00:00 2001 From: Nick Nicholas Date: Thu, 5 Oct 2017 11:05:55 +1100 Subject: [PATCH 5/5] ID check FTE across schools --- napval/idservice3.go | 76 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/napval/idservice3.go b/napval/idservice3.go index 13483cb..1b30a39 100644 --- a/napval/idservice3.go +++ b/napval/idservice3.go @@ -4,6 +4,7 @@ package napval import ( "fmt" "log" + "strconv" "github.com/nats-io/go-nats" "github.com/nsip/nias2/lib" @@ -55,6 +56,7 @@ type TransactionIDs struct { SimpleKeysPSI *set.Set ExtendedKeys *set.Set CrossSchoolMatches *set.Set + CrossSchoolFTEs cmap.ConcurrentMap } // create a new id service instance @@ -73,6 +75,41 @@ func (ids *IDService3) txMonitor() { log.Println("tx monitor is listening...") go ids.C.QueueSubscribe(lib.TRACK_TOPIC, "id3", func(txID string) { log.Println("Transaction complete message received for tx: ", txID) + /* + // and now issue report on FTEs of data matched students + tdata, ok := ids.Transactions.Get(txID) + tids, ok1 := tdata.(TransactionIDs) + if ok && ok1 { + tids.CrossSchoolMatches.Each(func(item interface{}) bool { + crossSchoolKey := item.(string) + fte, ok := tids.CrossSchoolFTEs.Get(crossSchoolKey) + if ok && fte != 1.0 { + loc, _ := tids.Locations.Get(crossSchoolKey) + ol, _ := loc.(string) + desc := fmt.Sprintf("Student with key %s at %s matched across schools has an FTE of %0.2f across all enrolments\n", + crossSchoolKey, ol, fte) + ve := ValidationError{ + Description: desc, + Field: "FTE", + OriginalLine: ol, + Vtype: "identity", + Severity: "warning", + } + r := lib.NiasMessage{} + r.TxID = txID + r.SeqNo = ol + r.Body = ve + r.Source = "id" + responses = append(responses, r) + log.Printf("%v\n", responses) + } + return true + }) + for _, r := range responses { + ids.C.Publish(lib.STORE_TOPIC, r) + } + } + */ transactionStore.Remove(txID) }) @@ -95,7 +132,9 @@ func (ids *IDService3) HandleMessage(req *lib.NiasMessage) ([]lib.NiasMessage, e SimpleKeysLocalId: set.New(), SimpleKeysPSI: set.New(), ExtendedKeys: set.New(), - CrossSchoolMatches: set.New()}) + CrossSchoolMatches: set.New(), + CrossSchoolFTEs: cmap.New(), + }) // retrieve the transaction dataset from the store tdata, ok := ids.Transactions.Get(req.TxID) @@ -133,6 +172,18 @@ func (ids *IDService3) HandleMessage(req *lib.NiasMessage) ([]lib.NiasMessage, e complexKey := fmt.Sprintf("%v", k2) crossSchoolKey := rr.FieldsKey(config.StudentMatch) + // record all FTEs, preemptively; we will report on those corresponding to cross-school collisions + fte, ok := tids.CrossSchoolFTEs.Get(crossSchoolKey) + fte_num := 0.0 + if ok { + fte_num = fte.(float64) + } + fte_new, err := strconv.ParseFloat(rr.FTE, 64) + if err != nil { + fte_new = 0.0 + } + tids.CrossSchoolFTEs.Set(crossSchoolKey, fte_num+fte_new) + //log.Printf("simplekey: %s\nsimplekey2: %s\ncompllexkey: %s", simpleKey1, simpleKey2, complexKey) var simpleRecordExists1, simpleRecordExists2, complexRecordExists, crossSchoolRecordExists bool @@ -179,6 +230,29 @@ func (ids *IDService3) HandleMessage(req *lib.NiasMessage) ([]lib.NiasMessage, e r.Body = ve responses = append(responses, r) log.Printf("%v\n", responses) + + if fte_num+fte_new != 1.0 { + // FTEs for colliding student do not add up to 1.0 + // This may be a false alarm if there is a third enrolment; but we can only address that if we have STAN queues, and the ability to rewind a queue, so that we can issue a check at the end of a transaction. + + desc1 := fmt.Sprintf("Student with key %s at %s matched across schools has an FTE of %0.2f across all enrolments so far\n", + crossSchoolKey, ol, fte_num+fte_new) + ve1 := ValidationError{ + Description: desc1, + Field: "FTE", + OriginalLine: req.SeqNo, + Vtype: "identity", + Severity: "warning", + } + r1 := lib.NiasMessage{} + r1.TxID = req.TxID + r1.SeqNo = req.SeqNo + r1.Body = ve1 + r1.Source = "id" + responses = append(responses, r1) + log.Printf("%v\n", r1) + } + } else if complexRecordExists { loc, _ := tids.Locations.Get(complexKey) ol, _ := loc.(string)