debug pearson ingest

app/naprr/naprr.go

@@ -37,10 +37,12 @@ func main() {
 
 		di := naprr.NewDataIngest()
 		di.Run()
-		di.RunCSV()
+		//di.RunYr3Writing()
 
-		log.Println("Generating report data...")
+		log.Println("Generating report data, Year 3 Writing...")
 		rb := naprr.NewReportBuilder()
+		//rb.RunYr3W(false)
+		log.Println("Generating report data...")
 		rb.Run()
 	}
 

naprr/contractor_csvdata_ingest.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 )
 
-func (di *DataIngest) RunCSV() {
+func (di *DataIngest) RunYr3Writing() {
 
 	uuid.Init()
 	csvFiles := parsePearsonCSVFileDirectory()
@@ -265,9 +265,9 @@ func (di *DataIngest) ingestPearsonResultsFile(resultsFilePath string, wg *sync.
 	if err != nil {
 		log.Println("Unable to gob-encode nap test: ", err)
 	}
-	di.sc.Publish("meta", gt)
+	di.sc.Publish("meta_yr3w", gt)
 
-	// we will set up 1 testlet and 1 item, which is assessed in the 10 rubrics
+	// we will set up 1 testlet and 10 items, which are assessed in 1 rubric each
 	testletRefId := uuid.NewV4().String()
 	naptestlet := nxml.NAPTestlet{TestletID: testletRefId,
 		NAPTestRefId: testRefId,
@@ -281,7 +281,7 @@ func (di *DataIngest) ingestPearsonResultsFile(resultsFilePath string, wg *sync.
 	if err != nil {
 		log.Println("Unable to gob-encode nap testlet: ", err)
 	}
-	di.sc.Publish("meta", gtl)
+	di.sc.Publish("meta_yr3w", gtl)
 
 	rubrics := [10]string{"audience", "text structure", "ideas", "character and setting", "vocabulary",
 		"cohesion", "paragraphing", "sentence structure", "punctuation", "spelling"}
@@ -295,7 +295,7 @@ func (di *DataIngest) ingestPearsonResultsFile(resultsFilePath string, wg *sync.
 		if err != nil {
 			log.Println("Unable to gob-encode nap test item: ", err)
 		}
-		di.sc.Publish("meta", gti)
+		di.sc.Publish("meta_yr3w", gti)
 	}
 
 	//defer file.Close()
@@ -336,7 +336,7 @@ func (di *DataIngest) ingestPearsonResultsFile(resultsFilePath string, wg *sync.
 		}
 		// store linkage locally
 		ss_link[regr.RefId] = regr.ASLSchoolId
-		di.sc.Publish(regr.ASLSchoolId, gsp)
+		di.sc.Publish("studentAndResults", gsp)
 		totalStudents++
 		event := nxml.NAPEvent{EventID: uuid.NewV4().String(),
 			SPRefID:           studentRefId,
@@ -362,7 +362,7 @@ func (di *DataIngest) ingestPearsonResultsFile(resultsFilePath string, wg *sync.
 		if err != nil {
 			log.Println("Unable to gob-encode nap event link: ", err)
 		}
-		di.sc.Publish(event.SchoolID, ge)
+		di.sc.Publish("studentAndResults", ge)
 
 		response := nxml.NAPResponseSet{ResponseID: uuid.NewV4().String(),
 			StudentID: studentRefId,
@@ -384,7 +384,7 @@ func (di *DataIngest) ingestPearsonResultsFile(resultsFilePath string, wg *sync.
 		if err != nil {
 			log.Println("Unable to gob-encode student response set: ", err)
 		}
-		di.sc.Publish("responses", gr)
+		di.sc.Publish("studentAndResults", gr)
 
 	}
 	log.Println("Finished reading data file...")
@@ -394,12 +394,12 @@ func (di *DataIngest) ingestPearsonResultsFile(resultsFilePath string, wg *sync.
 	if err != nil {
 		log.Println("Unable to gob-encode tx complete message: ", err)
 	}
-	di.sc.Publish("responses", geot)
-	di.sc.Publish("meta", geot)
+	di.sc.Publish("studentAndResults", geot)
+	di.sc.Publish("meta_yr3w", geot)
 
 	//di.assignResponsesToSchools(ss_link)
 
-	log.Println("response assignment complete")
+	//log.Println("response assignment complete")
 
 	log.Printf("ingestion complete for %s", resultsFilePath)
 

naprr/naprr_students_results.go

@@ -0,0 +1,26 @@
+// naprr_naplandata.go
+package naprr
+
+import (
+	"github.com/nsip/nias2/xml"
+	// "log"
+)
+
+// structs to hold student metadata around a test
+// assumes one test (for now)
+
+// each map indexed on Student RefID, so that RefID can be changed quickly
+type StudentAndResultsData struct {
+	Students     map[string]xml.RegistrationRecord
+	Events       map[string]xml.NAPEvent
+	ResponseSets map[string]xml.NAPResponseSet
+}
+
+func NewStudentAndResultsData() *StudentAndResultsData {
+	nd := StudentAndResultsData{
+		Students:     make(map[string]xml.RegistrationRecord),
+		Events:       make(map[string]xml.NAPEvent),
+		ResponseSets: make(map[string]xml.NAPResponseSet),
+	}
+	return &nd
+}

naprr/naprr_xmlhelpers.go

@@ -15,6 +15,7 @@ func init() {
 	gob.Register(ScoreSummaryDataSet{})
 	gob.Register(ResponseDataSet{})
 	gob.Register(CodeFrameDataSet{})
+	gob.Register(ResultsByStudent{})
 }
 
 // convenience types for aggregating response information sets
@@ -98,6 +99,13 @@ type CodeFrameDataSet struct {
 	Item    xml.NAPTestItem
 }
 
+// aggregate all objects referencing students for a single test
+type ResultsByStudent struct {
+	Student     xml.RegistrationRecord
+	Event       xml.NAPEvent
+	ResponseSet xml.NAPResponseSet
+}
+
 //
 // helper method that walks the structure to find location
 //

naprr/reportbuilder.go

@@ -22,7 +22,7 @@ func (rb *ReportBuilder) Run() {
 	var wg sync.WaitGroup
 
 	schools := rb.sr.GetSchoolDetails()
-	nd := rb.sr.GetNAPLANData()
+	nd := rb.sr.GetNAPLANData("meta")
 
 	for _, subslice := range schools {
 		for _, school := range subslice {
@@ -46,7 +46,10 @@ func (rb *ReportBuilder) RunYr3W(schools bool) {
 	var wg sync.WaitGroup
 
 	schoolslist := rb.sr.GetSchoolDetails()
-	nd := rb.sr.GetNAPLANData()
+	nd := rb.sr.GetNAPLANData("meta_yr3w")
+	log.Println("Getting student data")
+	sr := rb.sr.GetStudentAndResultsData()
+	log.Println("Gotten student data")
 
 	if schools {
 		for _, subslice := range schoolslist {
@@ -58,11 +61,11 @@ func (rb *ReportBuilder) RunYr3W(schools bool) {
 	}
 
 	wg.Add(1)
-	go rb.createYr3WReports(nd, &wg)
+	go rb.createYr3WReports(nd, sr, &wg)
 
 	// block until all reports generated
 	wg.Wait()
-	log.Println("All reports generated")
+	log.Println("All Year 3 Writing report data generated")
 
 }
 
@@ -87,8 +90,8 @@ func (rb *ReportBuilder) createTestReports(nd *NAPLANData, wg *sync.WaitGroup) {
 }
 
 // generate test-level reports
-func (rb *ReportBuilder) createYr3WReports(nd *NAPLANData, wg *sync.WaitGroup) {
-	rb.rg.GenerateYr3WData(nd)
+func (rb *ReportBuilder) createYr3WReports(nd *NAPLANData, sr *StudentAndResultsData, wg *sync.WaitGroup) {
+	rb.rg.GenerateYr3WData(nd, sr)
 	log.Println("Year 3 Writing XML data created.")
 	wg.Done()
 }

naprr/reportgenerator.go

@@ -24,12 +24,15 @@ func NewReportGenerator() *ReportGenerator {
 // routines to build the required reports
 //
 
-// generate XML representations of ingested data, used in first instance to deal with ingests of Year 3 Writing data
+// generate a student record, their event link, and their responses, for a single test
+// assumes a single test is represented in the ingested data (may yet refine this);
+// intended in the first instance for Yr3 Writing ingest
 // generated only once as represents structure of test not school-level data
-func (rg *ReportGenerator) GenerateYr3WData(nd *NAPLANData) {
+func (rg *ReportGenerator) GenerateYr3WData(nd *NAPLANData, sr *StudentAndResultsData) {
 
 	count := 0
 	cfds := make([]CodeFrameDataSet, 0)
+	rbs := make([]ResultsByStudent, 0)
 
 	for _, codeframe := range nd.Codeframes {
 		for _, cf_testlet := range codeframe.TestletList.Testlet {
@@ -61,15 +64,38 @@ func (rg *ReportGenerator) GenerateYr3WData(nd *NAPLANData) {
 		rg.sc.Publish("reports.xml", payload)
 	}
 
+	// assume 1 student 1 event 1 response set
+	for _, student := range sr.Students {
+		event := sr.Events[student.RefId]
+		responseset := sr.ResponseSets[student.RefId]
+		r := ResultsByStudent{
+			Student:     student,
+			Event:       event,
+			ResponseSet: responseset,
+		}
+		rbs = append(rbs, r)
+	}
+
+	count = count + len(rbs)
+
+	// publish the records
+	for _, r := range rbs {
+		payload, err := rg.ge.Encode(r)
+		if err != nil {
+			log.Println("unable to encode student results: ", err)
+		}
+		rg.sc.Publish("reports.xml", payload)
+	}
+
 	// finish the transaction - completion msg
 	txu := lib.TxStatusUpdate{TxComplete: true}
 	gtxu, err := rg.ge.Encode(txu)
 	if err != nil {
-		log.Println("unable to encode txu codeframe report: ", err)
+		log.Println("unable to encode txu Yr 3 Writing report: ", err)
 	}
-	rg.sc.Publish("reports.cframe", gtxu)
+	rg.sc.Publish("reports.xml", gtxu)
 
-	log.Printf("codeframe records %d: ", count)
+	log.Printf("Yr 3 Writing records %d: ", count)
 
 }
 

naprr/reportwriter.go

@@ -42,15 +42,16 @@ func (rw *ReportWriter) Run() {
 // create data reports from the test strucutre
 func (rw *ReportWriter) writeYr3WReports() {
 
-	log.Println("Creating test-level reports...")
+	log.Println("Creating Year 3 Writing XML...")
 
 	var wg sync.WaitGroup
 
 	cfds := rw.sr.GetCodeFrameData()
+	rbs := rw.sr.GetResultsByStudent()
 
 	wg.Add(2)
 
-	go rw.writeYr3WritingReport(cfds, &wg)
+	go rw.writeYr3WritingReport(cfds, rbs, &wg)
 
 	wg.Wait()
 
@@ -250,7 +251,7 @@ func (rw *ReportWriter) writeCodeFrameWritingReport(cfds []CodeFrameDataSet, wg
 
 // report of test structure for writing items only
 // with extended item information
-func (rw *ReportWriter) writeYr3WritingReport(cfds []CodeFrameDataSet, wg *sync.WaitGroup) {
+func (rw *ReportWriter) writeYr3WritingReport(cfds []CodeFrameDataSet, rbs []ResultsByStudent, wg *sync.WaitGroup) {
 
 	// create directory for the school
 	fpath := "yr3w/"
@@ -274,6 +275,10 @@ func (rw *ReportWriter) writeYr3WritingReport(cfds []CodeFrameDataSet, wg *sync.
 			e.Encode(cfd)
 		}
 	}
+	for _, r := range rbs {
+		e.Encode(r)
+	}
+
 	e.Flush()
 	f.WriteString("</NAPResulsReporting>\n")