diff --git a/app/naprrql/gql_schemas/naplan_schema.graphql b/app/naprrql/gql_schemas/naplan_schema.graphql index 4fe28b5..81f28ea 100644 --- a/app/naprrql/gql_schemas/naplan_schema.graphql +++ b/app/naprrql/gql_schemas/naplan_schema.graphql @@ -59,6 +59,8 @@ type NaplanData { item_results_report_by_school(acaraIDs: [String]): [ItemResponseDataSet] ## reporting (ui) object for writing item responses per student item_writing_results_report_by_school(acaraIDs: [String]): [ItemResponseDataSet] + ## reporting (ui) object for writing item responses per student, intended for export to marking systems, adds randomised ID to student (AnonymisedId in OtherIdList) + writing_item_for_marking_report_by_school(acaraIDs: [String]): [ItemResponseDataSet] ## report of all school summaries not linked to one of the acaraIDs orphan_school_summary_report(acaraIDs: [String]): [ScoreSummary] ## report of all events not linked to one of the acaraIDs diff --git a/app/naprrql/naprrql.go b/app/naprrql/naprrql.go index 978fbf1..9e7f7ce 100644 --- a/app/naprrql/naprrql.go +++ b/app/naprrql/naprrql.go @@ -23,6 +23,7 @@ var report = flag.Bool("report", false, "Creates .csv reports. Existing reports // var isrprint = flag.Bool("isrprint", false, "Creates .csv files for use in isr printing") // var itemprint = flag.Bool("itemprint", false, "Creates .csv files reporting item results for each student against items") +var writingextract = flag.Bool("writingextract", false, "Creates .csv file extract of all writing items, for input into marking systems") var qa = flag.Bool("qa", false, "Creates .csv files for QA checking of NAPLAN results") var vers = flag.Bool("version", false, "Reports version of NIAS distribution") @@ -83,6 +84,17 @@ func main() { closeDB() os.Exit(1) } + + // create the writing item report + if *writingextract { + // launch web-server + startWebServer(true) + writeWritingExtractReports() + // shut down + closeDB() + os.Exit(1) + } + /* // create the isr printing reports if *isrprint { @@ -181,6 +193,16 @@ func writeItemPrintingReports() { log.Println("item printing reports generated...") } +// +// create item printing reports +// +func writeWritingExtractReports() { + // clearReportsDirectory() // - check this + log.Println("generating Writing item extract reports...") + naprrql.GenerateWritingExtractReports() + log.Println("Writing item extract reports generated...") +} + // create QA reports // func writeQAPrintingReports() { diff --git a/app/naprrql/reporting_templates/writing_extract/itemWritingPrinting.gql b/app/naprrql/reporting_templates/writing_extract/itemWritingPrinting.gql new file mode 100644 index 0000000..fefa46c --- /dev/null +++ b/app/naprrql/reporting_templates/writing_extract/itemWritingPrinting.gql @@ -0,0 +1,85 @@ +query NAPItemResults($acaraIDs: [String]) { + writing_item_for_marking_report_by_school(acaraIDs: $acaraIDs) { + Test { + TestContent { + LocalId + TestName + TestLevel + TestDomain + TestYear + TestType + } + } + Testlet { + TestletContent { + LocalId + Node + LocationInStage + TestletName + LocalId + } + } + TestItem { + ItemID + TestItemContent { + NAPTestItemLocalId + ItemName + ItemType + Subdomain + WritingGenre + ItemSubstitutedForList { + SubstituteItem { + SubstituteItemRefId + LocalId + } + } + NAPWritingRubricList { + NAPWritingRubric { + RubricType + } + } + } + } + Student { + LocalId + BirthDate + Sex + YearLevel + ASLSchoolId + OtherIdList { + OtherId { + Type + Value + } + } + } + ParticipationCode + Response { + PathTakenForDomain + ParallelTest + PSI + TestletList { + Testlet { + NapTestletLocalId + TestletScore + ItemResponseList { + ItemResponse { + LocalID + Response + ResponseCorrectness + Score + LapsedTimeItem + SequenceNumber + SubscoreList { + Subscore { + SubscoreType + SubscoreValue + } + } + } + } + } + } + } + } +} diff --git a/app/naprrql/reporting_templates/writing_extract/itemWritingPrinting_map.csv b/app/naprrql/reporting_templates/writing_extract/itemWritingPrinting_map.csv new file mode 100644 index 0000000..139ee64 --- /dev/null +++ b/app/naprrql/reporting_templates/writing_extract/itemWritingPrinting_map.csv @@ -0,0 +1,2 @@ +Test Year,Test level,Jurisdiction Id,ACARA ID,PSI,Local school student ID,TAA student ID,Participation Code,Item Response,Anonymised Id +Test.TestContent.TestYear,Test.TestContent.TestLevel,Student.OtherIdList.OtherId.#[Type==JurisdictionId].Value,Student.ASLSchoolId,Response.PSI,Student.LocalId,Student.OtherIdList.OtherId.#[Type==TAAId].Value,ParticipationCode,Response.TestletList.Testlet.0.ItemResponseList.ItemResponse.0.Response,Student.OtherIdList.OtherId.#[Type==AnonymisedId].Value diff --git a/naprrql/item-report-pipeline.go b/naprrql/item-report-pipeline.go index f46a7fe..9e6aede 100644 --- a/naprrql/item-report-pipeline.go +++ b/naprrql/item-report-pipeline.go @@ -79,11 +79,59 @@ func runItemPipeline(schools []string) error { return WaitForPipeline(errcList...) } +// Slight variant of the foregoing +func runWritingExtractPipeline(schools []string) error { + + // setup pipeline cancellation + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + var errcList []<-chan error + + // input stage + varsc, errc, err := itemParametersSource(ctx, schools...) + if err != nil { + return err + } + errcList = append(errcList, errc) + + // transform stage + queryTemplates := getTemplates("./reporting_templates/writing_extract/") + var query string + for _, queryText := range queryTemplates { + query = queryText + } + jsonc, errc, err := itemQueryExecutor(ctx, query, DEF_GQL_URL, varsc) + if err != nil { + return err + } + errcList = append(errcList, errc) + + // sink stage + // create working directory if not there + outFileDir := "./out/writing_extract" + err = os.MkdirAll(outFileDir, os.ModePerm) + if err != nil { + return err + } + csvFileName := "writing_extract.csv" + outFileName := outFileDir + "/" + csvFileName + // we're assuming a single output report + mapFileName := "./reporting_templates/writing_extract/itemWritingPrinting_map.csv" + errc, err = csvFileSink(ctx, outFileName, mapFileName, jsonc) + if err != nil { + return err + } + errcList = append(errcList, errc) + + log.Println("Writing extract file writing... " + outFileName) + return WaitForPipeline(errcList...) +} + // // acts as input feed to the pipeline, sends parameters to retrieve data for // each school in turn // -func itemParametersSource(ctx context.Context, schools ...string) (<-chan itemQueryParams, <-chan error, error) { +func itemParametersSource(ctx context.Context, schools ...string) (<-chan systemQueryParams, <-chan error, error) { { //check input variables, handle errors before goroutine starts if len(schools) == 0 { @@ -92,7 +140,7 @@ func itemParametersSource(ctx context.Context, schools ...string) (<-chan itemQu } - out := make(chan itemQueryParams) + out := make(chan systemQueryParams) errc := make(chan error, 1) go func() { defer close(out) @@ -103,7 +151,8 @@ func itemParametersSource(ctx context.Context, schools ...string) (<-chan itemQu errc <- errors.Errorf("school %v is empty string", schoolIndex+1) return } - vars := itemQueryParams{school: school} + //vars := itemQueryParams{school: school} + vars := systemQueryParams{schoolAcaraID: school} // Send the data to the output channel but return early // if the context has been cancelled. select { @@ -120,7 +169,7 @@ func itemParametersSource(ctx context.Context, schools ...string) (<-chan itemQu // query executor transform stage takes query params in, excutes gql query // and writes results to output chaneel // -func itemQueryExecutor(ctx context.Context, query string, url string, in <-chan itemQueryParams) (<-chan gjson.Result, <-chan error, error) { +func itemQueryExecutor(ctx context.Context, query string, url string, in <-chan systemQueryParams) (<-chan gjson.Result, <-chan error, error) { out := make(chan gjson.Result) errc := make(chan error, 1) @@ -129,13 +178,16 @@ func itemQueryExecutor(ctx context.Context, query string, url string, in <-chan defer close(errc) gql := NewGqlClient() for params := range in { - vars := map[string]interface{}{"acaraID": params.school} + //vars := map[string]interface{}{"acaraID": params.school} + vars := map[string]interface{}{"acaraIDs": []string{params.schoolAcaraID}} + json, err := gql.DoQuery(url, query, vars) if err != nil { // Handle an error that occurs during the goroutine. errc <- err return } + // log.Printf("%+v\n", json) for _, result := range json.Array() { // Send the data to the output channel but return early // if the context has been cancelled. diff --git a/naprrql/napprr-rprt.go b/naprrql/napprr-rprt.go index c13d61c..8d4cba1 100644 --- a/naprrql/napprr-rprt.go +++ b/naprrql/napprr-rprt.go @@ -92,6 +92,31 @@ func GenerateItemPrintReports() { wg.Wait() } +// +// generates a specific 'report' which is the input +// file for item printing processes +// +func GenerateWritingExtractReports() { + + schools, err := getSchoolsList() + if err != nil { + log.Fatalln("Cannot connect to naprrql server: ", err) + } + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + err = runWritingExtractReports(schools) + if err != nil { + log.Println("Error creating isr printing reports: ", err) + } + }() + + wg.Wait() +} + // generates a specific 'report' which is the input // file for item printing processes // @@ -138,6 +163,14 @@ func runItemPrintReports(schools []string) error { } +func runWritingExtractReports(schools []string) error { + + var pipelineError error + pipelineError = runWritingExtractPipeline(schools) + return pipelineError + +} + func runQAReports(schools []string) error { var pipelineError error diff --git a/naprrql/report-resolvers.go b/naprrql/report-resolvers.go index 6c3c25a..321f6a6 100644 --- a/naprrql/report-resolvers.go +++ b/naprrql/report-resolvers.go @@ -9,6 +9,7 @@ import ( "errors" "log" + "github.com/nats-io/nuid" "github.com/nsip/nias2/xml" "github.com/playlyfe/go-graphql" ) @@ -813,6 +814,126 @@ func buildReportResolvers() map[string]interface{} { return results, nil } + // same as the above, just adds AnonymisedId + resolvers["NaplanData/writing_item_for_marking_report_by_school"] = func(params *graphql.ResolveParams) (interface{}, error) { + + reqErr := checkRequiredParams(params) + if reqErr != nil { + return nil, reqErr + } + // get the acara ids from the request params + acaraids := make([]string, 0) + for _, a_id := range params.Args["acaraIDs"].([]interface{}) { + acaraid, _ := a_id.(string) + acaraids = append(acaraids, acaraid) + } + + // get school names and student records + schoolnames := make(map[string]string) + studentids := make([]string, 0) + for _, acaraid := range acaraids { + key := "student_by_acaraid:" + acaraid + studentRefIds := getIdentifiers(key) + studentids = append(studentids, studentRefIds...) + + schoolrefid := getIdentifiers(acaraid + ":") + siObjects, err := getObjects(schoolrefid) + if err != nil { + return []interface{}{}, err + } + for _, sio := range siObjects { + si, _ := sio.(xml.SchoolInfo) + schoolnames[acaraid] = si.SchoolName + } + } + + // get responses for student + responseids := make([]string, 0) + for _, studentid := range studentids { + key := "responseset_by_student:" + studentid + responseRefId := getIdentifiers(key) + responseids = append(responseids, responseRefId...) + } + + // get responses + responses, err := getObjects(responseids) + if err != nil { + return []interface{}{}, err + } + + // convenience map to avoid revisiting db for tests + testLookup := make(map[string]xml.NAPTest) // key string = test refid + // get tests for yearLevel + for _, yrLvl := range []string{"3", "5", "7", "9"} { + tests, err := getTestsForYearLevel(yrLvl) + if err != nil { + return nil, err + } + for _, test := range tests { + t := test + testLookup[t.TestID] = t + } + } + + // construct RDS by including referenced test + results := make([]ItemResponseDataSet, 0) + for _, response := range responses { + resp, _ := response.(xml.NAPResponseSet) + + students, err := getObjects([]string{resp.StudentID}) + student, ok := students[0].(xml.RegistrationRecord) + if err != nil || !ok { + return []interface{}{}, err + } + student.Flatten() + // Only change + student.OtherIdList.OtherId = append(student.OtherIdList.OtherId, xml.XMLAttributeStruct{Type: "AnonymisedId", Value: nuid.New().Next()}) + + test := testLookup[resp.TestID] + + if test.TestContent.TestDomain != "Writing" { + continue + } + + eventsRefId := getIdentifiers("event_by_student_test:" + resp.StudentID + ":" + resp.TestID + ":") + events, err := getObjects(eventsRefId) + event, ok := events[0].(xml.NAPEvent) + if err != nil || !ok { + return []interface{}{}, err + } + + for _, testlet := range resp.TestletList.Testlet { + for _, item_response := range testlet.ItemResponseList.ItemResponse { + + resp1 := resp // pruned copy of response + resp1.TestletList.Testlet = make([]xml.NAPResponseSet_Testlet, 1) + resp1.TestletList.Testlet[0] = testlet + resp1.TestletList.Testlet[0].ItemResponseList.ItemResponse = make([]xml.NAPResponseSet_ItemResponse, 1) + resp1.TestletList.Testlet[0].ItemResponseList.ItemResponse[0] = item_response + + items, err := getObjects([]string{item_response.ItemRefID}) + item, ok := items[0].(xml.NAPTestItem) + if err != nil || !ok { + return []interface{}{}, err + } + + testlets, err := getObjects([]string{testlet.NapTestletRefId}) + tl := testlets[0].(xml.NAPTestlet) + if err != nil || !ok { + return []interface{}{}, err + } + + irds := ItemResponseDataSet{TestItem: item, Response: resp1, + Student: student, Test: test, Testlet: tl, + SchoolDetails: SchoolDetails{ACARAId: event.SchoolID, SchoolName: schoolnames[event.SchoolID]}, + ParticipationCode: event.ParticipationCode} + results = append(results, irds) + } + } + } + return results, nil + } + resolvers["NaplanData/orphan_school_summary_report"] = func(params *graphql.ResolveParams) (interface{}, error) { reqErr := checkRequiredParams(params) if reqErr != nil { diff --git a/version/version.go b/version/version.go index 0b5d1c6..4e93ef2 100644 --- a/version/version.go +++ b/version/version.go @@ -1,5 +1,5 @@ package version var( -Id = 10203800 +Id = 10203838 TagName = "0-9-22" )