RemoveBlanks() in CSV

nsip · Jun 30, 2016 · df9c4c3 · df9c4c3
1 parent f9bedc4
commit df9c4c3
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 3 deletions.
diff --git a/harness/schemas/core.json b/harness/schemas/core.json
@@ -288,6 +288,7 @@
             "id": "http://naplan.edu.au/PlatformId",
             "type": "string",
             "title": "PlatformId schema.",
+	    "pattern": "[RD][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][KMRASPDHEG]",
             "maxLength": 36,
             "description": "Description",
             "name": "PlatformId"
@@ -297,6 +298,7 @@
             "type": "string",
             "title": "Postcode schema.",
             "maxLength": 4,
+            "minLength": 4,
             "description": "Description",
             "name": "Postcode"
         },
@@ -354,6 +356,7 @@
             "title": "PreviousPlatformId schema.",
             "description": "Description",
             "maxLength": 36,
+	    "pattern": "[RD][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][KMRASPDHEG]",
             "name": "PreviousPlatformId"
         },
         "PreviousSectorId": {
@@ -474,7 +477,7 @@
             "id": "http://naplan.edu.au/YearLevel",
             "type": "string",
             "title": "YearLevel schema.",
-            "enum": ["F","P","0","1","2","3","4","5","6","7","8","9","10","11","12","UG"],
+            "enum": ["F","P","0","1","2","3","4","5","6","7","8","9","10","11","12","UG","11MINUS","12PLUS","CC","K","K3","K4","PS","UGJunSec","UGPri","UGSec","UGSnrSec"],
             "description": "Description",
             "name": "YearLevel"
         }

diff --git a/lib/webserver.go b/lib/webserver.go
@@ -40,6 +40,24 @@ type IngestResponse struct {
 	Records int
 }
 
+// truncate the record by removing items that have blank entries.
+// this prevents the validation from throwing validation exceptions
+// for fields that are not mandatory but included as empty in the
+// dataset
+func removeBlanks(m map[string]string) map[string]string {
+
+        log.Println("HELLO")
+        reducedmap := make(map[string]string)
+        for key, val := range m {
+                if val != "" {
+                        reducedmap[key] = val
+                }
+        }
+        log.Println(reducedmap)
+        return reducedmap
+}
+
+
 // read csv file as stream an post records onto processing queue
 func enqueueCSV(file multipart.File) (IngestResponse, error) {
 
@@ -57,9 +75,10 @@ func enqueueCSV(file multipart.File) (IngestResponse, error) {
 		// stripBlanks????
 
 		regr := RegistrationRecord{}
-		// r := removeBlanks(record.AsMap())
+		r := removeBlanks(record.AsMap())
 		// log.Printf("record is:\n%v\n", r)
-		decode_err := ms.Decode(record.AsMap(), &regr)
+		//decode_err := ms.Decode(record.AsMap(), &regr)
+		decode_err := ms.Decode(r, &regr)
 		if decode_err != nil {
 			return ir, decode_err
 		}
@@ -225,6 +244,7 @@ func (nws *NIASWebServer) Run() {
 		sprsnls := make([]map[string]string, 0)
 		for _, r := range records {
 			r := r.AsMap()
+			r = removeBlanks(r)
 			r["SIFuuid"] = uuid.NewV4().String()
 			sprsnls = append(sprsnls, r)
 		}