From 984845a58cf4e691d316ac738c32a0f2096a2fce Mon Sep 17 00:00:00 2001 From: stellanl Date: Wed, 14 Mar 2018 03:08:36 +0100 Subject: [PATCH 001/155] [#2536]Initial implementation. --- .../CheckTrimDevicesAndDeviceFiles.java | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java diff --git a/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java b/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java new file mode 100644 index 0000000000..f3608defa6 --- /dev/null +++ b/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2018 Stichting Akvo (Akvo Foundation) + * + * This file is part of Akvo FLOW. + * + * Akvo FLOW is free software: you can redistribute it and modify it under the terms of + * the GNU Affero General Public License (AGPL) as published by the Free Software Foundation, + * either version 3 of the License or any later version. + * + * Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License included below for more details. + * + * The full license text can also be seen at . + */ + +package org.akvo.gae.remoteapi; + +import static org.akvo.gae.remoteapi.DataUtils.batchSaveEntities; +import static org.akvo.gae.remoteapi.DataUtils.batchDelete; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.HashMap; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.FetchOptions; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.PreparedQuery; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Query.Filter; +import com.google.appengine.api.datastore.Query.FilterOperator; +import com.google.appengine.api.datastore.Query.FilterPredicate; + +/* + * - Checks that all surveys, groups, questions and options are consistent + */ +public class CheckTrimDevicesAndDeviceFiles implements Process { + + private int orphanSurveys = 0, allDevicesJobs = 0; + private Map devices = new HashMap<>(); + private Map deviceFiles = new HashMap<>(); + private Integer ageLimit; + private ListoldDevices = new ArrayList(); + + private boolean retireOld = false; // Make question survey pointer match the group's + private boolean deleteOrphans = false; + + Date now = new Date(); + Date then = new Date(); + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); + + @Override + public void execute(DatastoreService ds, String[] args) throws Exception { + + System.out.printf("#Arguments: [date [CULL]] to show/remove devices older than date.\n"); + for (int i = 0; i < args.length; i++) { + System.out.printf("#Argument %d: %s\n", i, args[i]); + } + if (args.length > 0) { + then = df.parse(args[0]); +// System.out.printf("#Cutoff: %tF\n", then); + } + if (args.length > 1 && args[1].equalsIgnoreCase("CULL")) { + retireOld = true; + } + + processDevices(ds); + processDeviceFiles(ds); + + System.out.printf("#Devices: %5d total, %4d older than %tF\n", devices.size(), orphanSurveys, then); + System.out.printf("#DeviceFileJobs: %5d total, %4d all-device\n", deviceFiles.size(), allDevicesJobs); + + } + + + + private void processDevices(DatastoreService ds) throws ParseException { + + System.out.println("#Processing Devices"); + + final Query group_q = new Query("Device"); + final PreparedQuery group_pq = ds.prepare(group_q); + + for (Entity g : group_pq.asIterable(FetchOptions.Builder.withChunkSize(500))) { + + Long deviceId = g.getKey().getId(); + String name = (String) g.getProperty("deviceIdentifier"); + Date lastBeacon = (Date) g.getProperty("lastLocationBeaconTime"); //TODO: if null + if (lastBeacon == null) lastBeacon = df.parse("2000-01-01"); + Date lastModified = (Date) g.getProperty("lastUpdateDateTime"); + //Long beaconDays = now-lastBeacon + System.out.printf("#INF %s Device beacon %tF, mod %tF '%s'\n", + then.after(lastBeacon)?"OLD":"NEW", + lastBeacon, + lastModified, + name + ); + if (then.after(lastBeacon)) { + orphanSurveys++; + oldDevices.add(g); + } + devices.put(deviceId, name); + } + } + + + private void processDeviceFiles(DatastoreService ds) { + + System.out.println("#Processing DeviceFileJobQueue"); + + //find jobs for "unknown" device (= for all devices) + final Filter f = new FilterPredicate("deviceId", FilterOperator.EQUAL, null); + final Query group_q = new Query("DeviceFileJobQueue").setFilter(f); + final PreparedQuery pq = ds.prepare(group_q); + + for (Entity g : pq.asIterable(FetchOptions.Builder.withChunkSize(500))) { + + Long dfjId = g.getKey().getId(); + String name = (String) g.getProperty("fileName"); + System.out.printf("#INF All-device job %d '%s'\n", + dfjId, + name + ); + allDevicesJobs++; + deviceFiles.put(dfjId, name); + } + } + +} From 8483b7d480a4ed33acb8fd5f7889990e889c52f7 Mon Sep 17 00:00:00 2001 From: stellanl Date: Thu, 15 Mar 2018 11:04:14 +0100 Subject: [PATCH 002/155] [#2536]Implement script for fixing DFJQ filenames. --- scripts/data/fix-device-file-job-queue.sh | 14 +++ .../gae/remoteapi/FixDeviceFileJobQueue.java | 100 ++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100755 scripts/data/fix-device-file-job-queue.sh create mode 100644 scripts/data/src/org/akvo/gae/remoteapi/FixDeviceFileJobQueue.java diff --git a/scripts/data/fix-device-file-job-queue.sh b/scripts/data/fix-device-file-job-queue.sh new file mode 100755 index 0000000000..fb71e4df36 --- /dev/null +++ b/scripts/data/fix-device-file-job-queue.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# USAGE: ./fix-device-file-job-queue.sh akvoflowsandbox + +APP_ID=$1 +SERVICE_ACCOUNT="sa-$APP_ID@$APP_ID.iam.gserviceaccount.com" +P12_FILE_PATH="/path/to/akvo-repositories/akvo-flow-server-config/$1/$1.p12" + +java -cp bin:"lib/*" \ + org.akvo.gae.remoteapi.RemoteAPI \ + FixDeviceFileJobQueue \ + $APP_ID \ + $SERVICE_ACCOUNT \ + $P12_FILE_PATH diff --git a/scripts/data/src/org/akvo/gae/remoteapi/FixDeviceFileJobQueue.java b/scripts/data/src/org/akvo/gae/remoteapi/FixDeviceFileJobQueue.java new file mode 100644 index 0000000000..9ee630f4ca --- /dev/null +++ b/scripts/data/src/org/akvo/gae/remoteapi/FixDeviceFileJobQueue.java @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2018 Stichting Akvo (Akvo Foundation) + * + * This file is part of Akvo FLOW. + * + * Akvo FLOW is free software: you can redistribute it and modify it under the terms of + * the GNU Affero General Public License (AGPL) as published by the Free Software Foundation, + * either version 3 of the License or any later version. + * + * Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License included below for more details. + * + * The full license text can also be seen at . + */ + +package org.akvo.gae.remoteapi; + +import static org.akvo.gae.remoteapi.DataUtils.batchSaveEntities; +import static org.akvo.gae.remoteapi.DataUtils.batchDelete; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.HashMap; + +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.FetchOptions; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.PreparedQuery; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Query.Filter; +import com.google.appengine.api.datastore.Query.FilterOperator; +import com.google.appengine.api.datastore.Query.FilterPredicate; + +/* + * - Fixes bad filenames in DeviceFileJobQueue entities + */ +public class FixDeviceFileJobQueue implements Process { + + private int allDevicesJobs = 0, jsonDeviceJobs = 0; + private Map devices = new HashMap<>(); + private ListfixupList = new ArrayList(); + private boolean doBatch = true; // any failure will mean nothing is updated + + @Override + public void execute(DatastoreService ds, String[] args) throws Exception { + + processDeviceFileJobQueue(ds); + + System.out.printf("#DeviceFileJobs: %d/%d with bad filenames\n", jsonDeviceJobs, allDevicesJobs); + + + } + + + private void processDeviceFileJobQueue(DatastoreService ds) throws InterruptedException { + + System.out.println("#Processing DeviceFileJobQueue"); + + //find all jobs made since we changed from wfpPhotonnnn format + final Filter f = new FilterPredicate("fileName", FilterOperator.LESS_THAN, "wfp"); + final Filter f2 = new FilterPredicate("fileName", FilterOperator.GREATER_THAN, "ff"); + final Query group_q = new Query("DeviceFileJobQueue");//.setFilter(f).setFilter(f2); + final PreparedQuery pq = ds.prepare(group_q); + + for (Entity job : pq.asIterable(FetchOptions.Builder.withChunkSize(500))) { + allDevicesJobs++; + Long dfjId = job.getKey().getId(); + String name = (String) job.getProperty("fileName"); + if (name.endsWith("}")) { //JSON - must fix up + jsonDeviceJobs++; + String name2 = name.substring(0,name.indexOf("\""));//strip rest of JSON "container" + System.out.println(name + " --> " + name2); + job.setProperty("fileName", name2); + fixupList.add(job); + if (!doBatch) { + ds.put(job); + Thread.sleep(100);//short delay to lessen server load + } else if (fixupList.size()>99){ + System.out.printf("#Fixing %d Jobs\n",fixupList.size()); + batchSaveEntities(ds, fixupList); + fixupList.clear(); + } + + } +// System.out.printf("#INF All-device job %d '%s'\n", dfjId, name); + } + } + +} From 5f35ab68fe09ecff72c5423804808796e3f7a977 Mon Sep 17 00:00:00 2001 From: stellanl Date: Mon, 19 Mar 2018 15:31:49 +0100 Subject: [PATCH 003/155] [#2536]Retire old DFJQ entries. --- .../CheckTrimDevicesAndDeviceFiles.java | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java b/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java index f3608defa6..7ac4353b30 100644 --- a/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java +++ b/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java @@ -43,14 +43,12 @@ */ public class CheckTrimDevicesAndDeviceFiles implements Process { - private int orphanSurveys = 0, allDevicesJobs = 0; + private int orphanSurveys = 0, jobs = 0, allDevicesJobs = 0, oldJobs = 0; private Map devices = new HashMap<>(); private Map deviceFiles = new HashMap<>(); - private Integer ageLimit; - private ListoldDevices = new ArrayList(); + private ListoldEntities = new ArrayList<>(); private boolean retireOld = false; // Make question survey pointer match the group's - private boolean deleteOrphans = false; Date now = new Date(); Date then = new Date(); @@ -59,15 +57,14 @@ public class CheckTrimDevicesAndDeviceFiles implements Process { @Override public void execute(DatastoreService ds, String[] args) throws Exception { - System.out.printf("#Arguments: [date [CULL]] to show/remove devices older than date.\n"); - for (int i = 0; i < args.length; i++) { - System.out.printf("#Argument %d: %s\n", i, args[i]); - } + System.out.printf("#Arguments: [date [--retire]] to show/remove deviceFiles older than date.\n"); +// for (int i = 0; i < args.length; i++) { +// System.out.printf("#Argument %d: %s\n", i, args[i]); +// } if (args.length > 0) { then = df.parse(args[0]); -// System.out.printf("#Cutoff: %tF\n", then); } - if (args.length > 1 && args[1].equalsIgnoreCase("CULL")) { + if (args.length > 1 && args[1].equalsIgnoreCase("--retire")) { retireOld = true; } @@ -75,7 +72,12 @@ public void execute(DatastoreService ds, String[] args) throws Exception { processDeviceFiles(ds); System.out.printf("#Devices: %5d total, %4d older than %tF\n", devices.size(), orphanSurveys, then); - System.out.printf("#DeviceFileJobs: %5d total, %4d all-device\n", deviceFiles.size(), allDevicesJobs); + System.out.printf("#DeviceFileJobs: %5d total, %4d all-device, %4d old \n", jobs, allDevicesJobs, oldJobs); + + if (retireOld) { + System.out.printf("#INF Deleting %d entites\n", oldEntities.size()); + batchDelete(ds, oldEntities); + } } @@ -104,7 +106,7 @@ private void processDevices(DatastoreService ds) throws ParseException { ); if (then.after(lastBeacon)) { orphanSurveys++; - oldDevices.add(g); +// oldEntities.add(g.getKey()); } devices.put(deviceId, name); } @@ -117,19 +119,35 @@ private void processDeviceFiles(DatastoreService ds) { //find jobs for "unknown" device (= for all devices) final Filter f = new FilterPredicate("deviceId", FilterOperator.EQUAL, null); - final Query group_q = new Query("DeviceFileJobQueue").setFilter(f); + final Query group_q = new Query("DeviceFileJobQueue");//.setFilter(f); final PreparedQuery pq = ds.prepare(group_q); for (Entity g : pq.asIterable(FetchOptions.Builder.withChunkSize(500))) { Long dfjId = g.getKey().getId(); + Long devId = (Long) g.getProperty("deviceId"); String name = (String) g.getProperty("fileName"); - System.out.printf("#INF All-device job %d '%s'\n", + Date lastModified = (Date) g.getProperty("lastUpdateDateTime"); + String state = then.after(lastModified)?"OLD":"NEW"; + if (devId != null && !devices.containsKey(devId)) { + state = "ORPHAN"; + } + /* + System.out.printf("#INF %s Device %d file job %d '%s'\n", + state, + devId, dfjId, name ); - allDevicesJobs++; - deviceFiles.put(dfjId, name); + */ + jobs++; + if (devId == null) { + allDevicesJobs++; + } + if (lastModified == null || then.after(lastModified)) { + oldJobs++; + oldEntities.add(g.getKey()); + } } } From 50068ff6c9dca40731a8fafad106b555dd7f4139 Mon Sep 17 00:00:00 2001 From: stellanl Date: Tue, 20 Mar 2018 14:37:44 +0100 Subject: [PATCH 004/155] [#2536]Delete in 1000-entity batches so a crash does not invalidate the entire leanup. --- .../CheckTrimDevicesAndDeviceFiles.java | 92 +++++++++++++++---- 1 file changed, 73 insertions(+), 19 deletions(-) diff --git a/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java b/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java index 7ac4353b30..0eecc35f68 100644 --- a/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java +++ b/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java @@ -19,6 +19,11 @@ import static org.akvo.gae.remoteapi.DataUtils.batchSaveEntities; import static org.akvo.gae.remoteapi.DataUtils.batchDelete; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.SocketTimeoutException; +import java.net.URL; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -43,7 +48,8 @@ */ public class CheckTrimDevicesAndDeviceFiles implements Process { - private int orphanSurveys = 0, jobs = 0, allDevicesJobs = 0, oldJobs = 0; + private int orphanSurveys = 0, jobs = 0, allDevicesJobs = 0, oldJobs = 0, badJobs = 0, orphanJobs = 0; + private int s3errors = 0, s3timeouts = 0; private Map devices = new HashMap<>(); private Map deviceFiles = new HashMap<>(); private ListoldEntities = new ArrayList<>(); @@ -53,26 +59,33 @@ public class CheckTrimDevicesAndDeviceFiles implements Process { Date now = new Date(); Date then = new Date(); SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); - + String baseUrl; + @Override public void execute(DatastoreService ds, String[] args) throws Exception { - System.out.printf("#Arguments: [date [--retire]] to show/remove deviceFiles older than date.\n"); + System.out.printf("#Arguments: [date [baseurl][--retire]] to show/remove deviceFiles older than date.\n"); // for (int i = 0; i < args.length; i++) { // System.out.printf("#Argument %d: %s\n", i, args[i]); // } if (args.length > 0) { then = df.parse(args[0]); } - if (args.length > 1 && args[1].equalsIgnoreCase("--retire")) { - retireOld = true; + if (args.length > 1) { + baseUrl = args[1]; + } + if (args.length > 1 && args[1].equalsIgnoreCase("--retire") + || args.length > 2 && args[2].equalsIgnoreCase("--retire")) { + retireOld = true; } processDevices(ds); processDeviceFiles(ds); - System.out.printf("#Devices: %5d total, %4d older than %tF\n", devices.size(), orphanSurveys, then); - System.out.printf("#DeviceFileJobs: %5d total, %4d all-device, %4d old \n", jobs, allDevicesJobs, oldJobs); + System.out.printf("#Devices: %5d total, %d older than %tF\n", devices.size(), orphanSurveys, then); + System.out.printf("#DeviceFileJobs: %5d total, %d all-device, %d old, %d bad, %d orphan\n", jobs, allDevicesJobs, oldJobs, badJobs, orphanJobs); + System.out.printf("#S3: %d timeouts, %d errors\n", s3timeouts, s3errors); + if (retireOld) { System.out.printf("#INF Deleting %d entites\n", oldEntities.size()); @@ -113,7 +126,7 @@ private void processDevices(DatastoreService ds) throws ParseException { } - private void processDeviceFiles(DatastoreService ds) { + private void processDeviceFiles(DatastoreService ds) throws MalformedURLException { System.out.println("#Processing DeviceFileJobQueue"); @@ -128,27 +141,68 @@ private void processDeviceFiles(DatastoreService ds) { Long devId = (Long) g.getProperty("deviceId"); String name = (String) g.getProperty("fileName"); Date lastModified = (Date) g.getProperty("lastUpdateDateTime"); + jobs++; + if (devId == null) { + allDevicesJobs++; + } + String state = then.after(lastModified)?"OLD":"NEW"; - if (devId != null && !devices.containsKey(devId)) { + if (baseUrl != null && presentInS3(name)) { + state = "BAD"; + badJobs++; + oldEntities.add(g.getKey()); + } else if (devId != null && !devices.containsKey(devId)) { state = "ORPHAN"; + orphanJobs++; + oldEntities.add(g.getKey()); } - /* - System.out.printf("#INF %s Device %d file job %d '%s'\n", + + if (lastModified == null || then.after(lastModified)) { + oldJobs++; + oldEntities.add(g.getKey()); + } + System.out.printf("#INF %s job (device %d) #%d filename '%s'\n", state, devId, dfjId, name ); - */ - jobs++; - if (devId == null) { - allDevicesJobs++; - } - if (lastModified == null || then.after(lastModified)) { - oldJobs++; - oldEntities.add(g.getKey()); + + //progressive delete so an error does not fail to delete anything + if (retireOld && oldEntities.size() >= 1000) { + System.out.printf("#DEL Deleting %d entites\n", oldEntities.size()); + batchDelete(ds, oldEntities); + oldEntities.clear(); } + } } + + private boolean presentInS3(String fn) throws MalformedURLException { + final String imageUrl = baseUrl + "/" + fn; + + // MalformedURLException exception caught by method signature + final URL url = new URL(imageUrl); + try { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setConnectTimeout(60000); //one minute + conn.setRequestMethod("HEAD"); + + if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) { + return true; + } else { + System.out.printf("Got %d while checking %s\n", conn.getResponseCode(), imageUrl); + } + } catch (SocketTimeoutException timeout) { + // reschedule the task without delay + // Possible a hiccup in GAE side + s3timeouts++; + } catch (IOException e) { + // IOException possible a http 403, reschedule the task + s3errors++; + } + return false; + + } } From 8ddf7676dd9b0151758368d4b2dff011979c1a3b Mon Sep 17 00:00:00 2001 From: stellanl Date: Fri, 6 Apr 2018 12:22:14 +0200 Subject: [PATCH 005/155] [#2536]Delete in 100-entity batches so a crash does not skip the entire cleanup. --- .../gae/remoteapi/FixDeviceFileJobQueue.java | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/scripts/data/src/org/akvo/gae/remoteapi/FixDeviceFileJobQueue.java b/scripts/data/src/org/akvo/gae/remoteapi/FixDeviceFileJobQueue.java index 9ee630f4ca..948c0717c2 100644 --- a/scripts/data/src/org/akvo/gae/remoteapi/FixDeviceFileJobQueue.java +++ b/scripts/data/src/org/akvo/gae/remoteapi/FixDeviceFileJobQueue.java @@ -17,25 +17,15 @@ package org.akvo.gae.remoteapi; import static org.akvo.gae.remoteapi.DataUtils.batchSaveEntities; -import static org.akvo.gae.remoteapi.DataUtils.batchDelete; -import java.io.IOException; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.HashMap; - -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.FetchOptions; -import com.google.appengine.api.datastore.Key; import com.google.appengine.api.datastore.PreparedQuery; import com.google.appengine.api.datastore.Query; import com.google.appengine.api.datastore.Query.Filter; @@ -50,7 +40,7 @@ public class FixDeviceFileJobQueue implements Process { private int allDevicesJobs = 0, jsonDeviceJobs = 0; private Map devices = new HashMap<>(); private ListfixupList = new ArrayList(); - private boolean doBatch = true; // any failure will mean nothing is updated + private boolean doBatch = true; @Override public void execute(DatastoreService ds, String[] args) throws Exception { @@ -58,9 +48,7 @@ public void execute(DatastoreService ds, String[] args) throws Exception { processDeviceFileJobQueue(ds); System.out.printf("#DeviceFileJobs: %d/%d with bad filenames\n", jsonDeviceJobs, allDevicesJobs); - - - } + } private void processDeviceFileJobQueue(DatastoreService ds) throws InterruptedException { @@ -68,7 +56,7 @@ private void processDeviceFileJobQueue(DatastoreService ds) throws InterruptedEx System.out.println("#Processing DeviceFileJobQueue"); //find all jobs made since we changed from wfpPhotonnnn format - final Filter f = new FilterPredicate("fileName", FilterOperator.LESS_THAN, "wfp"); + final Filter f = new FilterPredicate("fileName", FilterOperator.LESS_THAN, "wfp"); final Filter f2 = new FilterPredicate("fileName", FilterOperator.GREATER_THAN, "ff"); final Query group_q = new Query("DeviceFileJobQueue");//.setFilter(f).setFilter(f2); final PreparedQuery pq = ds.prepare(group_q); @@ -86,14 +74,19 @@ private void processDeviceFileJobQueue(DatastoreService ds) throws InterruptedEx if (!doBatch) { ds.put(job); Thread.sleep(100);//short delay to lessen server load - } else if (fixupList.size()>99){ + } else if (fixupList.size() > 99) { + // save once we have 100 victims, so that an occasional + // db failure will not ruin everything System.out.printf("#Fixing %d Jobs\n",fixupList.size()); batchSaveEntities(ds, fixupList); fixupList.clear(); } } -// System.out.printf("#INF All-device job %d '%s'\n", dfjId, name); + } + if (doBatch) { + System.out.printf("#Fixing final %d Jobs\n",fixupList.size()); + batchSaveEntities(ds, fixupList); } } From b60a2a9d8a46df47663d134a28501fa6da5a9d4e Mon Sep 17 00:00:00 2001 From: stellanl Date: Thu, 19 Apr 2018 15:28:27 +0200 Subject: [PATCH 006/155] [#2536]Remove unused stuff. --- .../CheckTrimDevicesAndDeviceFiles.java | 42 ++++++++----------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java b/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java index 0eecc35f68..697eaacb59 100644 --- a/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java +++ b/scripts/data/src/org/akvo/gae/remoteapi/CheckTrimDevicesAndDeviceFiles.java @@ -16,7 +16,6 @@ package org.akvo.gae.remoteapi; -import static org.akvo.gae.remoteapi.DataUtils.batchSaveEntities; import static org.akvo.gae.remoteapi.DataUtils.batchDelete; import java.io.IOException; @@ -24,7 +23,6 @@ import java.net.MalformedURLException; import java.net.SocketTimeoutException; import java.net.URL; -import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -39,9 +37,6 @@ import com.google.appengine.api.datastore.Key; import com.google.appengine.api.datastore.PreparedQuery; import com.google.appengine.api.datastore.Query; -import com.google.appengine.api.datastore.Query.Filter; -import com.google.appengine.api.datastore.Query.FilterOperator; -import com.google.appengine.api.datastore.Query.FilterPredicate; /* * - Checks that all surveys, groups, questions and options are consistent @@ -51,23 +46,19 @@ public class CheckTrimDevicesAndDeviceFiles implements Process { private int orphanSurveys = 0, jobs = 0, allDevicesJobs = 0, oldJobs = 0, badJobs = 0, orphanJobs = 0; private int s3errors = 0, s3timeouts = 0; private Map devices = new HashMap<>(); - private Map deviceFiles = new HashMap<>(); private ListoldEntities = new ArrayList<>(); private boolean retireOld = false; // Make question survey pointer match the group's - + Date now = new Date(); Date then = new Date(); SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); String baseUrl; - + @Override public void execute(DatastoreService ds, String[] args) throws Exception { System.out.printf("#Arguments: [date [baseurl][--retire]] to show/remove deviceFiles older than date.\n"); -// for (int i = 0; i < args.length; i++) { -// System.out.printf("#Argument %d: %s\n", i, args[i]); -// } if (args.length > 0) { then = df.parse(args[0]); } @@ -76,7 +67,7 @@ public void execute(DatastoreService ds, String[] args) throws Exception { } if (args.length > 1 && args[1].equalsIgnoreCase("--retire") || args.length > 2 && args[2].equalsIgnoreCase("--retire")) { - retireOld = true; + retireOld = true; } processDevices(ds); @@ -85,7 +76,7 @@ public void execute(DatastoreService ds, String[] args) throws Exception { System.out.printf("#Devices: %5d total, %d older than %tF\n", devices.size(), orphanSurveys, then); System.out.printf("#DeviceFileJobs: %5d total, %d all-device, %d old, %d bad, %d orphan\n", jobs, allDevicesJobs, oldJobs, badJobs, orphanJobs); System.out.printf("#S3: %d timeouts, %d errors\n", s3timeouts, s3errors); - + if (retireOld) { System.out.printf("#INF Deleting %d entites\n", oldEntities.size()); @@ -94,7 +85,7 @@ public void execute(DatastoreService ds, String[] args) throws Exception { } - + private void processDevices(DatastoreService ds) throws ParseException { @@ -107,8 +98,10 @@ private void processDevices(DatastoreService ds) throws ParseException { Long deviceId = g.getKey().getId(); String name = (String) g.getProperty("deviceIdentifier"); - Date lastBeacon = (Date) g.getProperty("lastLocationBeaconTime"); //TODO: if null - if (lastBeacon == null) lastBeacon = df.parse("2000-01-01"); + Date lastBeacon = (Date) g.getProperty("lastLocationBeaconTime"); + if (lastBeacon == null) { + lastBeacon = df.parse("2000-01-01"); + } Date lastModified = (Date) g.getProperty("lastUpdateDateTime"); //Long beaconDays = now-lastBeacon System.out.printf("#INF %s Device beacon %tF, mod %tF '%s'\n", @@ -125,14 +118,13 @@ private void processDevices(DatastoreService ds) throws ParseException { } } - + private void processDeviceFiles(DatastoreService ds) throws MalformedURLException { System.out.println("#Processing DeviceFileJobQueue"); - + //find jobs for "unknown" device (= for all devices) - final Filter f = new FilterPredicate("deviceId", FilterOperator.EQUAL, null); - final Query group_q = new Query("DeviceFileJobQueue");//.setFilter(f); + final Query group_q = new Query("DeviceFileJobQueue"); final PreparedQuery pq = ds.prepare(group_q); for (Entity g : pq.asIterable(FetchOptions.Builder.withChunkSize(500))) { @@ -145,7 +137,7 @@ private void processDeviceFiles(DatastoreService ds) throws MalformedURLExceptio if (devId == null) { allDevicesJobs++; } - + String state = then.after(lastModified)?"OLD":"NEW"; if (baseUrl != null && presentInS3(name)) { state = "BAD"; @@ -156,7 +148,7 @@ private void processDeviceFiles(DatastoreService ds) throws MalformedURLExceptio orphanJobs++; oldEntities.add(g.getKey()); } - + if (lastModified == null || then.after(lastModified)) { oldJobs++; oldEntities.add(g.getKey()); @@ -174,10 +166,10 @@ private void processDeviceFiles(DatastoreService ds) throws MalformedURLExceptio batchDelete(ds, oldEntities); oldEntities.clear(); } - + } } - + private boolean presentInS3(String fn) throws MalformedURLException { final String imageUrl = baseUrl + "/" + fn; @@ -203,6 +195,6 @@ private boolean presentInS3(String fn) throws MalformedURLException { s3errors++; } return false; - + } } From 9ee380a96dd06f420d61ff9aba0484fd41d2a3ea Mon Sep 17 00:00:00 2001 From: kymni Date: Tue, 9 Oct 2018 14:08:00 +0300 Subject: [PATCH 007/155] [#2802] remove redundant code --- .../js/lib/views/surveys/survey-details-views.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Dashboard/app/js/lib/views/surveys/survey-details-views.js b/Dashboard/app/js/lib/views/surveys/survey-details-views.js index ecaba7b314..4d6aa2ab2a 100644 --- a/Dashboard/app/js/lib/views/surveys/survey-details-views.js +++ b/Dashboard/app/js/lib/views/surveys/survey-details-views.js @@ -528,16 +528,14 @@ FLOW.QuestionGroupItemView = FLOW.View.extend({ qgQuery = setInterval(function () { // if the question group has a keyId, we can start polling it remotely if (self.content && self.content.get('keyId')) { - if (self.content.get('status') == "READY") { - self.set('qgCheckScheduled', false); - clearInterval(qgQuery); - } else { - // we have an id and can start polling remotely - self.ajaxCall(self.content.get('keyId')); - } + // we have an id and can start polling remotely + self.ajaxCall(self.content.get('keyId')); } - },5000); + }, 5000); } + } else { + this.set('qgCheckScheduled', false); + clearInterval(qgQuery); } }.observes('this.amCopying'), From fcaae94d6a969fff4bd6bc258e9163e44afe9803 Mon Sep 17 00:00:00 2001 From: Emmanuel Mulo Date: Mon, 26 Nov 2018 12:58:56 +0100 Subject: [PATCH 008/155] [#2896] Upgrade jackson library version * Replace the jackson library with a newer version * Use Flow wrappers for jackson JSON readers and writers to limit dependencies --- GAE/pom.xml | 17 +++++-- .../survey/dao/CaddisflyResourceDao.java | 25 +++------- GAE/src/org/akvo/flow/domain/DataUtils.java | 34 ++++++------- GAE/src/org/akvo/flow/events/EventLogger.java | 22 ++++----- GAE/src/org/akvo/flow/events/EventUtils.java | 11 ++--- .../org/akvo/flow/servlet/ReportServlet.java | 12 ++--- .../akvo/flow/util/FlowJsonObjectReader.java | 34 +++++++++++++ .../akvo/flow/util/FlowJsonObjectWriter.java | 49 +++++++++++++++++++ .../app/gwt/client/survey/SurveyGroupDto.java | 4 +- .../mapping/app/web/CurrentUserServlet.java | 16 ++---- .../app/web/SurveyInstanceServlet.java | 11 ++--- .../mapping/app/web/SurveyRestServlet.java | 8 +-- .../app/web/SurveyedLocaleServlet.java | 8 +-- .../mapping/app/web/dto/ApprovalGroupDTO.java | 4 +- .../mapping/app/web/dto/ApprovalStepDTO.java | 4 +- .../app/web/dto/DataPointApprovalDTO.java | 4 +- .../rest/dto/UserAuthorizationPayload.java | 4 +- .../app/web/rest/dto/UserRolePayload.java | 4 +- .../web/rest/security/RequestUriVoter.java | 10 ++-- .../GraphicalSurveySummaryExporter.java | 29 +++++------ .../RawDataSpreadsheetImporter.java | 9 ++-- .../dataexport/SurveySummaryExporter.java | 15 ++---- .../service/BulkDataServiceClient.java | 23 +++------ .../mapping/domain/CaddisflyResource.java | 4 +- .../mapping/domain/CaddisflyResult.java | 4 +- .../serialization/SurveyInstanceHandler.java | 9 ++-- .../serialization/response/MediaResponse.java | 15 +++--- 27 files changed, 217 insertions(+), 172 deletions(-) create mode 100644 GAE/src/org/akvo/flow/util/FlowJsonObjectReader.java create mode 100644 GAE/src/org/akvo/flow/util/FlowJsonObjectWriter.java diff --git a/GAE/pom.xml b/GAE/pom.xml index 48482f3ae9..c3be79b92f 100644 --- a/GAE/pom.xml +++ b/GAE/pom.xml @@ -42,6 +42,7 @@ 3.2.18.RELEASE 3.2.10.RELEASE 1.9.63 + 2.6.7 @@ -154,9 +155,19 @@ ${spring.version} - org.codehaus.jackson - jackson-mapper-asl - 1.8.5 + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} com.google.appengine diff --git a/GAE/src/com/gallatinsystems/survey/dao/CaddisflyResourceDao.java b/GAE/src/com/gallatinsystems/survey/dao/CaddisflyResourceDao.java index cbb1928f34..1a8b064ba0 100644 --- a/GAE/src/com/gallatinsystems/survey/dao/CaddisflyResourceDao.java +++ b/GAE/src/com/gallatinsystems/survey/dao/CaddisflyResourceDao.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016-2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2016-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -16,16 +16,14 @@ package com.gallatinsystems.survey.dao; -import java.io.InputStream; import java.net.URL; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; +import org.akvo.flow.util.FlowJsonObjectReader; import org.waterforpeople.mapping.domain.CaddisflyResource; /** @@ -33,7 +31,6 @@ * consists of only a single method. */ public class CaddisflyResourceDao { - private static ObjectMapper mapper = new ObjectMapper(); public static String DEFAULT_CADDISFLY_TESTS_FILE_URL = "https://akvoflow-public.s3.amazonaws.com/caddisfly-tests.json"; @@ -41,25 +38,19 @@ public class CaddisflyResourceDao { .getName()); public List listResources(String caddisflyTestsUrl) { - List result = null; + Map> testsMap = null; + FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); try { URL caddisflyFileUrl = new URL(caddisflyTestsUrl); - InputStream stream = caddisflyFileUrl.openStream(); - - // create a list of caddisflyResource objects - JsonNode rootNode = mapper.readTree(stream); - result = mapper.readValue(rootNode.get("tests"), - new TypeReference>() { - }); - + testsMap = jsonReader.readObject(caddisflyFileUrl); } catch (Exception e) { log.log(Level.SEVERE, "Error parsing Caddisfly resource: " + e.getMessage(), e); } - if (result != null) { - return result; + if (testsMap != null && testsMap.get("tests") != null) { + return testsMap.get("tests"); } else { return Collections.emptyList(); } diff --git a/GAE/src/org/akvo/flow/domain/DataUtils.java b/GAE/src/org/akvo/flow/domain/DataUtils.java index 23b8d356ef..a073e13b32 100644 --- a/GAE/src/org/akvo/flow/domain/DataUtils.java +++ b/GAE/src/org/akvo/flow/domain/DataUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2015,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -18,13 +18,11 @@ import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; +import org.akvo.flow.util.FlowJsonObjectReader; import org.apache.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; /** * Utilities class for performing transformations on survey data, i.e. response values @@ -33,8 +31,6 @@ public class DataUtils { private static final Logger log = Logger.getLogger(DataUtils.class); - public static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(); - public static String[] optionResponsesTextArray(String optionResponse) { String[] responseArray = null; @@ -84,7 +80,7 @@ public static String[] cascadeResponseValues(String data) { /** * Convert a JSON string response for OPTION type questions to the legacy pipe separated format * - * @param jsonResponse + * @param optionResponses * @return */ public static String jsonResponsesToPipeSeparated(String optionResponses) { @@ -106,15 +102,15 @@ public static String jsonResponsesToPipeSeparated(String optionResponses) { } /** - * Convert a JSON string response to a list containing corresponding maps + * Convert a JSON string response to a list of corresponding maps * * @param data String containing the JSON-formatted response * @return List of maps with response properties */ public static List> jsonStringToList(String data) { + FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); try { - return JSON_OBJECT_MAPPER.readValue(data, - new TypeReference>>() {}); + return jsonReader.readObject(data); } catch (IOException e) { // Data is not JSON-formatted } @@ -124,23 +120,21 @@ public static List> jsonStringToList(String data) { /** * Process the JSON formatted string value of a signature question and return the string - * representing the signatory. A blank string is returned + * representing the signatory. * * @param value * @return */ public static String parseSignatory(String value) { - String signatory = null; + FlowJsonObjectReader> jsonReader = new FlowJsonObjectReader<>(); Map signatureResponse = null; try { - signatureResponse = JSON_OBJECT_MAPPER.readValue(value, - new TypeReference>() { - }); - signatory = signatureResponse.get("name"); + signatureResponse = jsonReader.readObject(value); } catch (IOException e) { // ignore } - return signatory; + + return signatureResponse.get("name"); } /** @@ -151,9 +145,11 @@ public static String parseSignatory(String value) { */ @SuppressWarnings("unchecked") public static Map parseCaddisflyResponseValue(String caddisflyValue) { - Map caddisflyResponseMap = new HashMap<>(); + Map caddisflyResponseMap = null; + FlowJsonObjectReader> jsonReader = new FlowJsonObjectReader<>(); + try { - caddisflyResponseMap = JSON_OBJECT_MAPPER.readValue(caddisflyValue, Map.class); + caddisflyResponseMap = jsonReader.readObject(caddisflyValue); } catch (IOException e) { log.warn("Failed to parse the caddisfly response"); } diff --git a/GAE/src/org/akvo/flow/events/EventLogger.java b/GAE/src/org/akvo/flow/events/EventLogger.java index d37dcab84c..5fa11c239e 100644 --- a/GAE/src/org/akvo/flow/events/EventLogger.java +++ b/GAE/src/org/akvo/flow/events/EventLogger.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2015,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -25,8 +25,6 @@ import static org.akvo.flow.events.EventUtils.populateEntityProperties; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.StringWriter; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; @@ -42,7 +40,7 @@ import org.akvo.flow.events.EventUtils.EventTypes; import org.akvo.flow.events.EventUtils.Key; import org.akvo.flow.events.EventUtils.Prop; -import org.codehaus.jackson.map.ObjectMapper; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; @@ -83,14 +81,14 @@ private void sendNotification() { messageMap.put(Key.APP_ID, appId); messageMap.put(Key.URL, appId + ".appspot.com"); - ObjectMapper m = new ObjectMapper(); - OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream()); - m.writeValue(writer, messageMap); - writer.close(); + FlowJsonObjectWriter writer = new FlowJsonObjectWriter(); + writer.writeValue(connection.getOutputStream(), messageMap); + if (connection.getResponseCode() != HttpURLConnection.HTTP_NO_CONTENT) { logger.log(Level.SEVERE, "Unified log notification failed with status code: " + connection.getResponseCode()); } + connection.disconnect(); } catch (MalformedURLException e) { logger.log(Level.SEVERE, "Unified log notification failed with malformed URL exception", e); @@ -134,15 +132,13 @@ private void storeEvent(Map event, Date timestamp) { try { DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - ObjectMapper m = new ObjectMapper(); - StringWriter w = new StringWriter(); - m.writeValue(w, event); - Entity entity = new Entity("EventQueue"); entity.setProperty("createdDateTime", timestamp); entity.setProperty("lastUpdateDateTime", timestamp); - String payload = w.toString(); + FlowJsonObjectWriter writer = new FlowJsonObjectWriter(); + String payload = writer.writeValueAsString(event); + if (payload.length() > Constants.MAX_LENGTH) { entity.setProperty("payloadText", new Text(payload)); } else { diff --git a/GAE/src/org/akvo/flow/events/EventUtils.java b/GAE/src/org/akvo/flow/events/EventUtils.java index edfa4d2133..ad1978b7f2 100644 --- a/GAE/src/org/akvo/flow/events/EventUtils.java +++ b/GAE/src/org/akvo/flow/events/EventUtils.java @@ -17,7 +17,6 @@ package org.akvo.flow.events; import java.io.IOException; -import java.io.OutputStreamWriter; import java.net.HttpURLConnection; import java.net.URL; import java.util.Date; @@ -27,7 +26,7 @@ import java.util.logging.Level; import java.util.logging.Logger; -import org.codehaus.jackson.map.ObjectMapper; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.waterforpeople.mapping.app.web.rest.security.user.GaeUser; import com.gallatinsystems.survey.domain.SurveyGroup; @@ -38,7 +37,6 @@ public class EventUtils { private static Logger log = Logger.getLogger(EventUtils.class.getName()); - private static final ObjectMapper objectMapper = new ObjectMapper(); public enum EventSourceType { USER, DEVICE, SENSOR, WEBFORM, API, UNKNOWN, SYSTEM @@ -343,14 +341,11 @@ public static void sendEvents(String urlString, List> events connection.setRequestProperty("Content-Type", "application/json"); - OutputStreamWriter writer = new OutputStreamWriter( - connection.getOutputStream()); - objectMapper.writeValue(writer, events); + FlowJsonObjectWriter writer = new FlowJsonObjectWriter(); + writer.writeValue(connection.getOutputStream(), events); System.out.println(" " + connection.getResponseCode()); - writer.close(); connection.disconnect(); - } } diff --git a/GAE/src/org/akvo/flow/servlet/ReportServlet.java b/GAE/src/org/akvo/flow/servlet/ReportServlet.java index 91703dd7ca..df87ef9fd6 100644 --- a/GAE/src/org/akvo/flow/servlet/ReportServlet.java +++ b/GAE/src/org/akvo/flow/servlet/ReportServlet.java @@ -31,9 +31,7 @@ import org.akvo.flow.dao.ReportDao; import org.akvo.flow.domain.persistent.Report; import org.akvo.flow.rest.dto.ReportTaskRequest; -import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.waterforpeople.mapping.app.web.dto.TaskRequest; import com.gallatinsystems.common.util.PropertyUtil; @@ -197,8 +195,7 @@ private void requeueStart(ReportTaskRequest req, Report r, int err) { } } - private int startReportEngine(String baseUrl, Report r) - throws JsonGenerationException, JsonMappingException, IOException { + private int startReportEngine(String baseUrl, Report r) throws IOException { //look up user final String email = uDao.getByKey(r.getUser()).getEmailAddress(); @@ -226,8 +223,9 @@ private int startReportEngine(String baseUrl, Report r) criteria.opts.uploadUrl = PropertyUtil.getProperty("surveyuploadurl"); criteria.opts.flowServices = PropertyUtil.getProperty("flowServices"); criteria.opts.email = email; - ObjectMapper objectMapper = new ObjectMapper(); - String crit = java.net.URLEncoder.encode(objectMapper.writeValueAsString(criteria), "UTF-8"); + + FlowJsonObjectWriter writer = new FlowJsonObjectWriter(); + String crit = java.net.URLEncoder.encode(writer.writeValueAsString(criteria), "UTF-8"); URL url = new URL(PropertyUtil.getProperty("flowServices") + "/generate?criteria=" + crit); HttpURLConnection con = (HttpURLConnection) url.openConnection(); diff --git a/GAE/src/org/akvo/flow/util/FlowJsonObjectReader.java b/GAE/src/org/akvo/flow/util/FlowJsonObjectReader.java new file mode 100644 index 0000000000..1dafb5f92a --- /dev/null +++ b/GAE/src/org/akvo/flow/util/FlowJsonObjectReader.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2018 Stichting Akvo (Akvo Foundation) + * + * This file is part of Akvo FLOW. + * + * Akvo FLOW is free software: you can redistribute it and modify it under the terms of + * the GNU Affero General Public License (AGPL) as published by the Free Software Foundation, + * either version 3 of the License or any later version. + * + * Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License included below for more details. + * + * The full license text can also be seen at . + */ + +package org.akvo.flow.util; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.net.URL; + +public class FlowJsonObjectReader { + + public T readObject(String jsonString) throws IOException { + return new ObjectMapper().readValue(jsonString, new TypeReference(){}); + } + + public T readObject(URL url) throws IOException { + return new ObjectMapper().readValue(url, new TypeReference(){}); + } +} \ No newline at end of file diff --git a/GAE/src/org/akvo/flow/util/FlowJsonObjectWriter.java b/GAE/src/org/akvo/flow/util/FlowJsonObjectWriter.java new file mode 100644 index 0000000000..214c675647 --- /dev/null +++ b/GAE/src/org/akvo/flow/util/FlowJsonObjectWriter.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2018 Stichting Akvo (Akvo Foundation) + * + * This file is part of Akvo FLOW. + * + * Akvo FLOW is free software: you can redistribute it and modify it under the terms of + * the GNU Affero General Public License (AGPL) as published by the Free Software Foundation, + * either version 3 of the License or any later version. + * + * Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License included below for more details. + * + * The full license text can also be seen at . + */ + +package org.akvo.flow.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.io.OutputStream; + +public class FlowJsonObjectWriter { + private boolean excludeNullValues; + + public FlowJsonObjectWriter() {} + + public FlowJsonObjectWriter(boolean excludeNullValues) { + this.excludeNullValues = excludeNullValues; + } + + public ObjectMapper createObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + if (excludeNullValues) { + mapper.getSerializationConfig().withSerializationInclusion(JsonInclude.Include.NON_NULL); + } + return mapper; + } + + public void writeValue(OutputStream outStream, Object value) throws IOException { + createObjectMapper().writeValue(outStream, value); + } + + public String writeValueAsString(Object value) throws IOException { + return createObjectMapper().writeValueAsString(value); + } +} diff --git a/GAE/src/org/waterforpeople/mapping/app/gwt/client/survey/SurveyGroupDto.java b/GAE/src/org/waterforpeople/mapping/app/gwt/client/survey/SurveyGroupDto.java index b69d63ebba..390a3c62d4 100644 --- a/GAE/src/org/waterforpeople/mapping/app/gwt/client/survey/SurveyGroupDto.java +++ b/GAE/src/org/waterforpeople/mapping/app/gwt/client/survey/SurveyGroupDto.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2016 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2016,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -20,7 +20,7 @@ import java.util.Date; import java.util.List; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.gallatinsystems.framework.gwt.dto.client.BaseDto; import com.gallatinsystems.survey.domain.SurveyGroup; diff --git a/GAE/src/org/waterforpeople/mapping/app/web/CurrentUserServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/CurrentUserServlet.java index 2ee08a7e69..1ce4e80e15 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/CurrentUserServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/CurrentUserServlet.java @@ -33,12 +33,10 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.apache.velocity.Template; import org.apache.velocity.VelocityContext; import org.apache.velocity.app.VelocityEngine; -import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; import com.gallatinsystems.common.Constants; import com.gallatinsystems.user.dao.UserAuthorizationDAO; @@ -148,19 +146,15 @@ private String getPermissionsMap(User currentUser) { addSuperAdminPermissions(currentUser, permissions); - ObjectMapper jsonObjectMapper = new ObjectMapper(); - StringWriter writer = new StringWriter(); + FlowJsonObjectWriter writer = new FlowJsonObjectWriter(); + String permissionsString = null; try { - jsonObjectMapper.writeValue(writer, permissions); - } catch (JsonGenerationException e) { - // ignore - } catch (JsonMappingException e) { - // ignore + permissionsString = writer.writeValueAsString(permissions); } catch (IOException e) { // ignore } - return writer.toString(); + return permissionsString; } /** diff --git a/GAE/src/org/waterforpeople/mapping/app/web/SurveyInstanceServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/SurveyInstanceServlet.java index 9aaf52b691..ca3452f202 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/SurveyInstanceServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/SurveyInstanceServlet.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -21,8 +21,7 @@ import javax.servlet.http.HttpServletRequest; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.springframework.beans.BeanUtils; import org.waterforpeople.mapping.app.gwt.client.surveyinstance.SurveyInstanceDto; import org.waterforpeople.mapping.app.web.dto.InstanceDataDto; @@ -209,8 +208,8 @@ private String buildLatestApprovalStatus(ApprovalStep latestApprovalStep, @Override protected void writeOkResponse(RestResponse response) throws Exception { getResponse().setStatus(200); - ObjectMapper jsonMapper = new ObjectMapper(); - jsonMapper.getSerializationConfig().setSerializationInclusion(Inclusion.NON_NULL); - jsonMapper.writeValue(getResponse().getWriter(), response); + boolean excludeNullValues = true; + FlowJsonObjectWriter writer = new FlowJsonObjectWriter(excludeNullValues); + writer.writeValue(getResponse().getOutputStream(), response); } } diff --git a/GAE/src/org/waterforpeople/mapping/app/web/SurveyRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/SurveyRestServlet.java index a08c4a59a5..a762c356e8 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/SurveyRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/SurveyRestServlet.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -30,7 +30,7 @@ import org.akvo.flow.domain.mapper.QuestionDtoMapper; import org.akvo.flow.domain.mapper.QuestionOptionDtoMapper; -import org.codehaus.jackson.map.ObjectMapper; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.waterforpeople.mapping.analytics.dao.SurveyQuestionSummaryDao; import org.waterforpeople.mapping.analytics.domain.SurveyQuestionSummary; import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto; @@ -280,8 +280,8 @@ private SurveyDto getSurvey(Long surveyId) { @Override protected void writeOkResponse(RestResponse resp) throws Exception { getResponse().setStatus(200); - new ObjectMapper().writeValue(getResponse().getWriter(), resp); - + FlowJsonObjectWriter writer = new FlowJsonObjectWriter(); + writer.writeValue(getResponse().getOutputStream(), resp); } /** diff --git a/GAE/src/org/waterforpeople/mapping/app/web/SurveyedLocaleServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/SurveyedLocaleServlet.java index 48e1b1dd27..7a9bd4b9af 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/SurveyedLocaleServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/SurveyedLocaleServlet.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -29,7 +29,7 @@ import com.gallatinsystems.surveyal.domain.SurveyedLocale; import com.google.api.server.spi.config.Nullable; import org.akvo.flow.domain.DataUtils; -import org.codehaus.jackson.map.ObjectMapper; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.waterforpeople.mapping.app.web.dto.SurveyInstanceDto; import org.waterforpeople.mapping.app.web.dto.SurveyedLocaleDto; import org.waterforpeople.mapping.app.web.dto.SurveyedLocaleRequest; @@ -332,8 +332,8 @@ protected void writeOkResponse(RestResponse resp) throws Exception { } getResponse().setStatus(sc); if (sc == HttpServletResponse.SC_OK) { - ObjectMapper jsonMapper = new ObjectMapper(); - jsonMapper.writeValue(getResponse().getWriter(), resp); + FlowJsonObjectWriter writer = new FlowJsonObjectWriter(); + writer.writeValue(getResponse().getOutputStream(), resp); getResponse().getWriter().println(); } else { getResponse().getWriter().println(resp.getMessage()); diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/ApprovalGroupDTO.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/ApprovalGroupDTO.java index b06978e11f..cba2aa23ab 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/ApprovalGroupDTO.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/ApprovalGroupDTO.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2016,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo Flow. * @@ -16,7 +16,7 @@ package org.waterforpeople.mapping.app.web.dto; -import org.codehaus.jackson.annotate.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.gallatinsystems.framework.gwt.dto.client.BaseDto; import com.gallatinsystems.survey.domain.ApprovalGroup; diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/ApprovalStepDTO.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/ApprovalStepDTO.java index ce4b3b42d6..e6f8fa7a92 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/ApprovalStepDTO.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/ApprovalStepDTO.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2016,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo Flow. * @@ -18,7 +18,7 @@ import java.util.List; -import org.codehaus.jackson.annotate.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.gallatinsystems.framework.gwt.dto.client.BaseDto; import com.gallatinsystems.survey.domain.ApprovalStep; diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataPointApprovalDTO.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataPointApprovalDTO.java index 022c3a097c..a2e61c58d7 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataPointApprovalDTO.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataPointApprovalDTO.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2016,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo Flow. * @@ -18,7 +18,7 @@ import java.util.Date; -import org.codehaus.jackson.annotate.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.gallatinsystems.framework.gwt.dto.client.BaseDto; import com.gallatinsystems.survey.domain.DataPointApproval; diff --git a/GAE/src/org/waterforpeople/mapping/app/web/rest/dto/UserAuthorizationPayload.java b/GAE/src/org/waterforpeople/mapping/app/web/rest/dto/UserAuthorizationPayload.java index 00b364121d..4a86cfa200 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/rest/dto/UserAuthorizationPayload.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/rest/dto/UserAuthorizationPayload.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014-2015 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2014-2015,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -19,7 +19,7 @@ import java.util.Collection; import org.apache.commons.lang.StringUtils; -import org.codehaus.jackson.annotate.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.context.SecurityContextHolder; import org.waterforpeople.mapping.app.web.rest.security.AppRole; diff --git a/GAE/src/org/waterforpeople/mapping/app/web/rest/dto/UserRolePayload.java b/GAE/src/org/waterforpeople/mapping/app/web/rest/dto/UserRolePayload.java index fab7b2ed1c..2f9a7aaae9 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/rest/dto/UserRolePayload.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/rest/dto/UserRolePayload.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2014,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -18,7 +18,7 @@ import java.util.Set; -import org.codehaus.jackson.annotate.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.gallatinsystems.framework.gwt.dto.client.BaseDto; import com.gallatinsystems.user.domain.Permission; diff --git a/GAE/src/org/waterforpeople/mapping/app/web/rest/security/RequestUriVoter.java b/GAE/src/org/waterforpeople/mapping/app/web/rest/security/RequestUriVoter.java index cb8ce98b17..6168c6ab30 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/rest/security/RequestUriVoter.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/rest/security/RequestUriVoter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014-2015,2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2014-2015,2017-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -26,11 +26,11 @@ import javax.servlet.http.HttpServletRequest; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import org.akvo.flow.domain.RootFolder; import org.akvo.flow.domain.SecuredObject; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; import org.springframework.security.access.AccessDecisionVoter; import org.springframework.security.access.AccessDeniedException; import org.springframework.security.access.ConfigAttribute; @@ -199,7 +199,7 @@ private Long parsePayload(HttpServletRequest httpRequest) { try { JsonFactory f = new JsonFactory(); - JsonParser parser = f.createJsonParser(httpRequest.getInputStream()); + JsonParser parser = f.createParser(httpRequest.getInputStream()); boolean isSurvey = false; boolean isForm = false; diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java index e36d2b53d1..2a9b0eafac 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java @@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.akvo.flow.domain.DataUtils; +import org.akvo.flow.util.FlowJsonObjectReader; import org.akvo.flow.util.JFreechartChartUtil; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Level; @@ -59,8 +60,6 @@ import org.apache.poi.ss.util.WorkbookUtil; import org.apache.poi.xssf.streaming.SXSSFSheet; import org.apache.poi.xssf.streaming.SXSSFWorkbook; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; import org.waterforpeople.mapping.app.gwt.client.survey.OptionContainerDto; import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto; import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto.QuestionType; @@ -164,7 +163,6 @@ public class GraphicalSurveySummaryExporter extends SurveySummaryExporter { private Map questionsById; private SurveyGroupDto surveyGroupDto; private boolean lastCollection = false; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private CaddisflyResourceDao caddisflyResourceDao = new CaddisflyResourceDao(); private String caddisflyTestsFileUrl; private String selectionFrom = null; @@ -695,6 +693,8 @@ private synchronized int writeInstanceData( if (rollupOrder != null && rollupOrder.size() > 0) { rollups = formRollupStrings(responseMap); } + FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); + for (Entry entry : responseMap.entrySet()) { //OPTION, NUMBER and CASCADE summarizable now. if (!unsummarizable.contains(entry.getKey())) { @@ -706,10 +706,7 @@ private synchronized int writeInstanceData( String[] vals; if (entry.getValue().startsWith("[")) { //JSON try { - List> optionNodes = OBJECT_MAPPER.readValue( - entry.getValue(), - new TypeReference>>() {} - ); + List> optionNodes = jsonReader.readObject(entry.getValue()); List valsList = new ArrayList<>(); for (Map optionNode : optionNodes) { if (optionNode.containsKey("text")) { @@ -980,10 +977,9 @@ private Map> mapCaddisflyResultsById( List> cascadeNodes = new ArrayList<>(); if (value.startsWith("[")) { + FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); try { - cascadeNodes = OBJECT_MAPPER.readValue(value, - new TypeReference>>() { - }); + cascadeNodes = jsonReader.readObject(value); } catch (IOException e) { log.warn("Unable to parse CASCADE response - " + value, e); } @@ -1066,11 +1062,11 @@ private Map> mapCaddisflyResultsById( private List> getNodes(String value) { boolean isNewFormat = value.startsWith("["); List> optionNodes = new ArrayList<>(); + FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); + if (isNewFormat) { try { - optionNodes = OBJECT_MAPPER.readValue(value, - new TypeReference>>() { - }); + optionNodes = jsonReader.readObject(value); } catch (IOException e) { log.warn("Could not parse option response: " + value, e); } @@ -1654,12 +1650,9 @@ private void writeStatsAndGraphsSheet( } else { // Handle the json option question response type if (labelText.startsWith("[")) { + FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); try { - List> optionNodes = OBJECT_MAPPER - .readValue( - labelText, - new TypeReference>>() { - }); + List> optionNodes = jsonReader.readObject(labelText); StringBuilder labelTextBuilder = new StringBuilder(); for (Map optionNode : optionNodes) { diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java index ecd6efde69..71f8fb38a9 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java @@ -38,6 +38,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.apache.log4j.BasicConfigurator; import org.apache.log4j.Logger; import org.apache.poi.ss.usermodel.Cell; @@ -46,7 +47,6 @@ import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.WorkbookFactory; import org.apache.poi.ss.util.CellReference; -import org.codehaus.jackson.map.ObjectMapper; import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto; import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto.QuestionType; import org.waterforpeople.mapping.app.gwt.client.survey.QuestionOptionDto; @@ -67,8 +67,7 @@ public class RawDataSpreadsheetImporter implements DataImporter { private ThreadPoolExecutor threadPool; private BlockingQueue jobQueue; private List errorIds; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - + private static final FlowJsonObjectWriter JSON_OBJECT_WRITER = new FlowJsonObjectWriter(); private static final int LEGACY_MONITORING_FORMAT = 6; private static final int MONITORING_FORMAT_WITH_DEVICE_ID_COLUMN = 7; private static final int MONITORING_FORMAT_WITH_REPEAT_COLUMN = 8; @@ -637,7 +636,7 @@ private void getIterationResponse(Row iterationRow, cascadeList.add(cascadeMap); } try { - val = OBJECT_MAPPER.writeValueAsString(cascadeList); + val = JSON_OBJECT_WRITER.writeValueAsString(cascadeList); } catch (IOException e) { log.warn("Could not parse cascade string: " + cascadeString); } @@ -689,7 +688,7 @@ private void getIterationResponse(Row iterationRow, try { if (!optionList.isEmpty()) { - val = OBJECT_MAPPER.writeValueAsString(optionList); + val = JSON_OBJECT_WRITER.writeValueAsString(optionList); } } catch (IOException e) { log.warn("Could not parse option string: " + optionString, e); diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java index 5ce8d0fe85..5bc966f189 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -30,10 +30,8 @@ import java.util.Set; import org.akvo.flow.domain.DataUtils; +import org.akvo.flow.util.FlowJsonObjectReader; import org.apache.log4j.Logger; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; import org.json.JSONArray; import org.json.JSONObject; import org.waterforpeople.mapping.app.gwt.client.survey.OptionContainerDto; @@ -349,13 +347,10 @@ protected List parseQuestionGroups(String response) * @throws Exception */ protected List parseQuestions(String response) throws Exception { - final ObjectMapper JSON_RESPONSE_PARSER = new ObjectMapper(); + final FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); - final JsonNode questionListNode = - JSON_RESPONSE_PARSER.readTree(response).get("dtoList"); - final List qList = JSON_RESPONSE_PARSER.readValue( - questionListNode, new TypeReference>() { - }); + final Map> questionListMap = jsonReader.readObject(response); + final List qList = questionListMap.get("dtoList"); return qList; } diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java index 7a5cb39ab4..13be9e9838 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java @@ -44,13 +44,9 @@ import com.gallatinsystems.framework.rest.RestRequest; import com.gallatinsystems.survey.domain.SurveyGroup.PrivacyLevel; import com.gallatinsystems.survey.domain.SurveyGroup.ProjectType; +import org.akvo.flow.util.FlowJsonObjectReader; import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.JsonParseException; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -65,7 +61,6 @@ import org.waterforpeople.mapping.app.gwt.client.survey.TranslationDto; import org.waterforpeople.mapping.app.gwt.client.surveyinstance.SurveyInstanceDto; import org.waterforpeople.mapping.app.web.dto.DataBackoutRequest; -import org.waterforpeople.mapping.app.web.dto.DeviceFileRestRequest; import org.waterforpeople.mapping.app.web.dto.InstanceDataDto; import org.waterforpeople.mapping.app.web.dto.SurveyRestRequest; @@ -85,7 +80,6 @@ public class BulkDataServiceClient { private static final String SURVEY_SERVLET_PATH = "/surveyrestapi"; private static final String INSTANCE_DATA_SERVLET_PATH = "/instancedata"; private static final String DEVICE_FILES_SERVLET_PATH = "/devicefilesrestapi?action="; - private static final ObjectMapper JSON_RESPONSE_PARSER = new ObjectMapper(); /** * lists all responses from the server for a surveyInstance submission as a map of values keyed @@ -352,12 +346,10 @@ public static InstanceDataDto fetchInstanceData(Long surveyInstanceId, String se } private static InstanceDataDto parseInstanceData(String instanceDataResponse) { + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader<>(); try { - InstanceDataDto instanceData = JSON_RESPONSE_PARSER.readValue(instanceDataResponse, - InstanceDataDto.class); + InstanceDataDto instanceData = jsonReader.readObject(instanceDataResponse); return instanceData; - } catch (JsonParseException | JsonMappingException e) { - log.warn("Failed to parse the InstanceDataDto string: " + e); } catch (IOException e) { log.error("Error while parsing: ", e); } @@ -403,11 +395,10 @@ public static SurveyGroupDto fetchSurveyGroup(String surveyId, String serverBase log.debug("response: " + surveyGroupResponse); - final JsonNode surveyGroupListNode = JSON_RESPONSE_PARSER.readTree(surveyGroupResponse) - .get("dtoList"); - final List surveyGroupList = JSON_RESPONSE_PARSER.readValue( - surveyGroupListNode, new TypeReference>() { - }); + final FlowJsonObjectReader>> jsonDeserialiser = new FlowJsonObjectReader<>(); + final Map> surveyGroupListMap = jsonDeserialiser.readObject(surveyGroupResponse); + + final List surveyGroupList = surveyGroupListMap.get("dtoList"); if (surveyGroupList != null && !surveyGroupList.isEmpty()) { surveyGroupDto = surveyGroupList.get(0); } diff --git a/GAE/src/org/waterforpeople/mapping/domain/CaddisflyResource.java b/GAE/src/org/waterforpeople/mapping/domain/CaddisflyResource.java index 4fdfc6d9f2..9c91b78159 100644 --- a/GAE/src/org/waterforpeople/mapping/domain/CaddisflyResource.java +++ b/GAE/src/org/waterforpeople/mapping/domain/CaddisflyResource.java @@ -1,4 +1,4 @@ -/* Copyright (C) 2016-2017 Stichting Akvo (Akvo Foundation) +/* Copyright (C) 2016-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -15,7 +15,7 @@ package org.waterforpeople.mapping.domain; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.Serializable; import java.util.List; diff --git a/GAE/src/org/waterforpeople/mapping/domain/CaddisflyResult.java b/GAE/src/org/waterforpeople/mapping/domain/CaddisflyResult.java index aa01327e82..924e5d3bd1 100644 --- a/GAE/src/org/waterforpeople/mapping/domain/CaddisflyResult.java +++ b/GAE/src/org/waterforpeople/mapping/domain/CaddisflyResult.java @@ -1,4 +1,4 @@ -/* Copyright (C) 2016-2017 Stichting Akvo (Akvo Foundation) +/* Copyright (C) 2016-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -15,7 +15,7 @@ package org.waterforpeople.mapping.domain; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.Serializable; diff --git a/GAE/src/org/waterforpeople/mapping/serialization/SurveyInstanceHandler.java b/GAE/src/org/waterforpeople/mapping/serialization/SurveyInstanceHandler.java index fd1af5589d..3aa83f565d 100644 --- a/GAE/src/org/waterforpeople/mapping/serialization/SurveyInstanceHandler.java +++ b/GAE/src/org/waterforpeople/mapping/serialization/SurveyInstanceHandler.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015-2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2015-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -23,8 +23,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import org.akvo.flow.util.FlowJsonObjectReader; import org.apache.commons.lang.StringUtils; -import org.codehaus.jackson.map.ObjectMapper; import org.waterforpeople.mapping.domain.QuestionAnswerStore; import org.waterforpeople.mapping.domain.SurveyInstance; import org.waterforpeople.mapping.domain.response.FormInstance; @@ -52,9 +52,10 @@ public class SurveyInstanceHandler { public static SurveyInstance fromJSON(String data) { FormInstance formInstance = null; - ObjectMapper mapper = new ObjectMapper(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader<>(); + try { - formInstance = mapper.readValue(data, FormInstance.class); + formInstance = jsonReader.readObject(data); } catch (IOException e) { log.log(Level.SEVERE, "Error mapping JSON data: " + e.getMessage(), e); return null; diff --git a/GAE/src/org/waterforpeople/mapping/serialization/response/MediaResponse.java b/GAE/src/org/waterforpeople/mapping/serialization/response/MediaResponse.java index 568e93af1e..db3bc0ee87 100644 --- a/GAE/src/org/waterforpeople/mapping/serialization/response/MediaResponse.java +++ b/GAE/src/org/waterforpeople/mapping/serialization/response/MediaResponse.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2016,2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -19,12 +19,14 @@ import java.io.IOException; import java.util.logging.Logger; -import org.codehaus.jackson.map.ObjectMapper; +import org.akvo.flow.util.FlowJsonObjectReader; +import org.akvo.flow.util.FlowJsonObjectWriter; import org.waterforpeople.mapping.domain.response.value.Media; public class MediaResponse { private static final Logger log = Logger.getLogger(MediaResponse.class.getName()); - private static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(); + private static FlowJsonObjectReader jsonObjectReader = new FlowJsonObjectReader<>(); + private static FlowJsonObjectWriter jsonObjectWriter = new FlowJsonObjectWriter(); public static final int VERSION_STRING = 0; public static final int VERSION_GEOTAGGING = 1; @@ -36,8 +38,9 @@ public class MediaResponse { public static String format(String value, int version) { Media media; int savedVersion; + try { - media = JSON_OBJECT_MAPPER.readValue(value, Media.class); + media = jsonObjectReader.readObject(value); savedVersion = VERSION_GEOTAGGING; } catch (IOException e) { // Value is not JSON-formatted @@ -52,7 +55,7 @@ public static String format(String value, int version) { if (version == VERSION_GEOTAGGING) { try { - return JSON_OBJECT_MAPPER.writeValueAsString(media); + return jsonObjectWriter.writeValueAsString(media); } catch (IOException e) { log.warning(e.getMessage()); return ""; @@ -65,7 +68,7 @@ public static String format(String value, int version) { public static Media parse(String value) { try { - return JSON_OBJECT_MAPPER.readValue(value, Media.class); + return jsonObjectReader.readObject(value); } catch (IOException e) { } From d5a2cf6a568b1842d160eda8806ad72c79af21cb Mon Sep 17 00:00:00 2001 From: stellanl Date: Fri, 7 Dec 2018 17:20:17 +0100 Subject: [PATCH 009/155] [#2931]Run update on single-instance task queue. --- .../dao/SurveyQuestionSummaryDao.java | 6 +- .../app/web/DataProcessorRestServlet.java | 26 ++++++- .../mapping/app/web/TaskServlet.java | 3 +- .../app/web/dto/DataProcessorRequest.java | 7 +- .../mapping/dao/SurveyInstanceDAO.java | 77 ++++++++++++++++++- .../mapping/domain/SurveyInstance.java | 60 --------------- 6 files changed, 107 insertions(+), 72 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyQuestionSummaryDao.java b/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyQuestionSummaryDao.java index 0609e69c95..06d9e61759 100644 --- a/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyQuestionSummaryDao.java +++ b/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyQuestionSummaryDao.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2012, 2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2012, 2017-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -233,8 +233,8 @@ public SurveyQuestionSummary save(SurveyQuestionSummary summary) { * @param summary */ public List save(List summary) { - List savedSummaryList = (List) super - .save(summary); + List savedSummaryList = + (List) super.save(summary); cache(savedSummaryList); return savedSummaryList; } diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index eddab083da..f51ec78b0d 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -225,11 +225,14 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { if (dpReq.getSurveyInstanceId() != null) { deleteSurveyResponses(dpReq.getSurveyInstanceId()); } - } else if (DataProcessorRequest.SURVEY_RESPONSE_COUNT.equalsIgnoreCase(req - .getAction())) { + } else if (DataProcessorRequest.SURVEY_RESPONSE_COUNT.equalsIgnoreCase(req.getAction())) { if (dpReq.getSummaryCounterId() != null && dpReq.getDelta() != null) { updateSurveyResponseCounter(dpReq.getSummaryCounterId(), dpReq.getDelta()); } + } else if (DataProcessorRequest.UPDATE_SURVEY_INSTANCE_SUMMARIES.equalsIgnoreCase(req.getAction())) { + if (dpReq.getSurveyInstanceId() != null && dpReq.getDelta() != null) { + updateSurveyInstanceResponseCounters(dpReq.getSurveyInstanceId(), dpReq.getDelta()); + } } else if (DataProcessorRequest.DELETE_CASCADE_NODES.equalsIgnoreCase(req.getAction())) { deleteCascadeNodes(dpReq.getCascadeResourceId(), dpReq.getParentNodeId()); } else if (DataProcessorRequest.ASSEMBLE_DATAPOINT_NAME.equalsIgnoreCase(req.getAction())) { @@ -1274,7 +1277,7 @@ private void deleteSurveyResponses(Long surveyInstanceId) { } /** - * Update a survey response counter according to the provided delta. This method should be + * Update a single survey response counter according to the provided delta. This method should be * invoked though the task queue 'surveyResponseCount' to avoid concurrent updates * * @param summaryCounterId @@ -1291,6 +1294,23 @@ private void updateSurveyResponseCounter(long summaryCounterId, int delta) { summaryDao.save(summary); } + /** + * Update a single survey response counter according to the provided delta. This method should be + * invoked though the task queue 'surveyResponseCount' to avoid concurrent updates + * + * @param summaryCounterId + * @param delta +1 or -1 + */ + + private void updateSurveyInstanceResponseCounters(long surveyInstanceId, int delta) { + SurveyInstanceDAO siDao = new SurveyInstanceDAO(); + SurveyInstance si = siDao.getByKey(surveyInstanceId); + if (si != null && (delta == 1 || delta == -1)) { + siDao.updateSummaryCounts(si, delta > 0); + } + + } + private void deleteCascadeNodes(Long cascadeResourceId, Long parentNodeId) { final CascadeNodeDao dao = new CascadeNodeDao(); List nodes = dao.listCascadeNodesByResourceAndParentId(cascadeResourceId, diff --git a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java index 8e0dc274b3..7c3bced8b6 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2015 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2015, 2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -198,6 +198,7 @@ private List processFile(TaskRequest fileProcessTaskRequest) { for (SurveyInstance si : surveyInstances) { synchronized (LOCK) { // Synchronize datastore access. + // Only locked against access by other threads in same process. si = siDao.save(si, deviceFile); } // Fire a survey event diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java index 0c7275dc24..35d78d12d0 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2015 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2015, 2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -66,13 +66,14 @@ public class DataProcessorRequest extends RestRequest { public static final String ADD_CREATION_SURVEY_ID_TO_LOCALE = "addCreationSurveyIdToLocale"; public static final String ADD_TRANSLATION_FIELDS = "addTranslationFields"; public static final String RECREATE_LOCALES = "recreateLocales"; + public static final String CASCADE_RESOURCE_ID = "cascadeResourceId"; + public static final String PARENT_NODE_ID = "parentNodeId"; public static final String POP_QUESTION_ORDER_FIELDS_ACTION = "populateQuestionOrders"; public static final String POPULATE_MONITORING_FIELDS_LOCALE_ACTION = "populateMonitoringFieldsLocale"; public static final String CREATE_NEW_IDENTIFIERS_LOCALES_ACTION = "createNewIdentifiersLocales"; public static final String DELETE_SURVEY_INSTANCE_ACTION = "deleteSurveyInstance"; public static final String DELETE_CASCADE_NODES = "deleteCascadeNodes"; - public static final String CASCADE_RESOURCE_ID = "cascadeResourceId"; - public static final String PARENT_NODE_ID = "parentNodeId"; + public static final String UPDATE_SURVEY_INSTANCE_SUMMARIES = "updateSurveyInstanceSummaries"; public static final int MAX_TASK_RETRIES = 3; private String country; diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index a6a551a634..f37f50f3a4 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2015, 2017, 2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2015, 2017-2018 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -28,6 +28,7 @@ import javax.jdo.PersistenceManager; import javax.jdo.Query; +import org.akvo.flow.domain.DataUtils; import org.apache.commons.lang.StringUtils; import org.waterforpeople.mapping.analytics.dao.SurveyQuestionSummaryDao; import org.waterforpeople.mapping.analytics.domain.SurveyQuestionSummary; @@ -158,7 +159,7 @@ public SurveyInstance save(SurveyInstance si, DeviceFiles deviceFile) { } deviceFile.setSurveyInstanceId(si.getKey().getId()); - si.updateSummaryCounts(true); + queueSynchronizedSummaryUpdate(si, true); return si; } @@ -483,6 +484,78 @@ public List listQuestionAnswerStoreForQuestion( return (List) q.execute(questionId); } + /** + * Update counts of SurveyQuestionSummary entities related to responses from this survey + * instance. + * This does not yet execute on the surveyResponseCount task queue and so is not synchronized! + */ + public void updateSummaryCounts(SurveyInstance si, boolean increment) { + + // retrieve all summary objects + SurveyQuestionSummaryDao summaryDao = new SurveyQuestionSummaryDao(); + QuestionDao qDao = new QuestionDao(); + + List saveList = new ArrayList(); + List deleteList = new ArrayList(); + + for (QuestionAnswerStore response : si.getQuestionAnswersStore()) { + final Long questionId = Long.parseLong(response.getQuestionID()); + Question question = qDao.getByKey(questionId); + if (question == null || !question.canBeCharted()) { + continue; + } + + final String questionIdStr = response.getQuestionID(); + final String[] questionResponse = DataUtils.optionResponsesTextArray(response + .getValue()); + + for (int i = 0; i < questionResponse.length; i++) { + List questionSummaryList = summaryDao + .listByResponse(questionIdStr, questionResponse[i]); + SurveyQuestionSummary questionSummary = null; + if (questionSummaryList.isEmpty()) { + questionSummary = new SurveyQuestionSummary(); + questionSummary.setQuestionId(response.getQuestionID()); + questionSummary.setResponse(questionResponse[i]); + questionSummary.setCount(0L); + summaryDao.save(questionSummary);//New, so OK to save from here? NO it could be created from other threads simultaneously! + } else { + questionSummary = questionSummaryList.get(0); + } + + // update and save or delete + long count = questionSummary.getCount() == null ? 0 : questionSummary.getCount(); + count = increment ? ++count : --count; + questionSummary.setCount(count);//need to keep track of the delta instead! + + if (count > 0) { + saveList.add(questionSummary); + } else { + deleteList.add(questionSummary); + } + } + } + + summaryDao.save(saveList); + summaryDao.delete(deleteList); + + } + + private void queueSynchronizedSummaryUpdate(SurveyInstance si, boolean increment) { + Queue questionSummaryQueue = QueueFactory.getQueue("surveyResponseCount"); + TaskOptions to = TaskOptions.Builder + .withUrl("/app_worker/dataprocessor") + .param(DataProcessorRequest.ACTION_PARAM, + DataProcessorRequest.UPDATE_SURVEY_INSTANCE_SUMMARIES) + .param(DataProcessorRequest.SURVEY_INSTANCE_PARAM, + si.getKey().getId() + "") + .param(DataProcessorRequest.DELTA_PARAM, increment ? "1":"-1"); + questionSummaryQueue.add(to); + + } + + + /** * Deletes a surveyInstance and all its related objects * diff --git a/GAE/src/org/waterforpeople/mapping/domain/SurveyInstance.java b/GAE/src/org/waterforpeople/mapping/domain/SurveyInstance.java index a27ff94e49..1b1d7c6f95 100644 --- a/GAE/src/org/waterforpeople/mapping/domain/SurveyInstance.java +++ b/GAE/src/org/waterforpeople/mapping/domain/SurveyInstance.java @@ -19,21 +19,14 @@ import com.gallatinsystems.common.Constants; import com.gallatinsystems.device.domain.DeviceFiles; import com.gallatinsystems.framework.domain.BaseDomain; -import com.gallatinsystems.survey.dao.QuestionDao; import com.gallatinsystems.survey.dao.SurveyDAO; -import com.gallatinsystems.survey.domain.Question; -import org.akvo.flow.domain.DataUtils; import org.akvo.flow.domain.SecuredObject; import org.apache.commons.lang.StringUtils; -import org.waterforpeople.mapping.analytics.dao.SurveyQuestionSummaryDao; -import org.waterforpeople.mapping.analytics.domain.SurveyQuestionSummary; - import javax.jdo.annotations.IdentityType; import javax.jdo.annotations.NotPersistent; import javax.jdo.annotations.PersistenceCapable; import javax.jdo.annotations.Persistent; import java.lang.reflect.Field; -import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -276,59 +269,6 @@ public static Map retrieveGeoLocation( return geoLocationMap; } - /** - * Update counts of SurveyQuestionSummary entities related to responses from this survey - * instance. - */ - public void updateSummaryCounts(boolean increment) { - - // retrieve all summary objects - SurveyQuestionSummaryDao summaryDao = new SurveyQuestionSummaryDao(); - QuestionDao qDao = new QuestionDao(); - - List saveList = new ArrayList(); - List deleteList = new ArrayList(); - - for (QuestionAnswerStore response : questionAnswersStore) { - final Long questionId = Long.parseLong(response.getQuestionID()); - Question question = qDao.getByKey(questionId); - if (question == null || !question.canBeCharted()) { - continue; - } - - final String questionIdStr = response.getQuestionID(); - final String[] questionResponse = DataUtils.optionResponsesTextArray(response - .getValue()); - - for (int i = 0; i < questionResponse.length; i++) { - List questionSummaryList = summaryDao - .listByResponse(questionIdStr, questionResponse[i]); - SurveyQuestionSummary questionSummary = null; - if (questionSummaryList.isEmpty()) { - questionSummary = new SurveyQuestionSummary(); - questionSummary.setQuestionId(response.getQuestionID()); - questionSummary.setResponse(questionResponse[i]); - questionSummary.setCount(0L); - } else { - questionSummary = questionSummaryList.get(0); - } - - // update and save or delete - long count = questionSummary.getCount() == null ? 0 : questionSummary.getCount(); - count = increment ? ++count : --count; - questionSummary.setCount(count); - - if (count > 0) { - saveList.add(questionSummary); - } else { - deleteList.add(questionSummary); - } - } - } - - summaryDao.save(saveList); - summaryDao.delete(deleteList); - } @Override public SecuredObject getParentObject() { From 64bee3146aa06419bab625781fba678ca9637aee Mon Sep 17 00:00:00 2001 From: stellanl Date: Fri, 7 Dec 2018 19:44:27 +0100 Subject: [PATCH 010/155] [#2931]Fetch the QAS entities. Switch parameter. --- .../app/web/DataProcessorRestServlet.java | 9 ++++----- .../mapping/dao/SurveyInstanceDAO.java | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index f51ec78b0d..8fe0406d31 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -1295,18 +1295,17 @@ private void updateSurveyResponseCounter(long summaryCounterId, int delta) { } /** - * Update a single survey response counter according to the provided delta. This method should be + * Update all survey response counters for the provided surveyInstance. This method should be * invoked though the task queue 'surveyResponseCount' to avoid concurrent updates * - * @param summaryCounterId + * @param surveyInstanceId * @param delta +1 or -1 */ private void updateSurveyInstanceResponseCounters(long surveyInstanceId, int delta) { SurveyInstanceDAO siDao = new SurveyInstanceDAO(); - SurveyInstance si = siDao.getByKey(surveyInstanceId); - if (si != null && (delta == 1 || delta == -1)) { - siDao.updateSummaryCounts(si, delta > 0); + if (delta == 1 || delta == -1) { + siDao.updateSummaryCounts(surveyInstanceId, delta > 0); } } diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index f37f50f3a4..e804702a40 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -489,16 +489,20 @@ public List listQuestionAnswerStoreForQuestion( * instance. * This does not yet execute on the surveyResponseCount task queue and so is not synchronized! */ - public void updateSummaryCounts(SurveyInstance si, boolean increment) { - + public void updateSummaryCounts(long siId, boolean increment) { // retrieve all summary objects SurveyQuestionSummaryDao summaryDao = new SurveyQuestionSummaryDao(); + QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); QuestionDao qDao = new QuestionDao(); + List answerList = qasDao.listBySurveyInstance(siId); + if (answerList == null || answerList.isEmpty()) { + return; + } List saveList = new ArrayList(); List deleteList = new ArrayList(); - for (QuestionAnswerStore response : si.getQuestionAnswersStore()) { + for (QuestionAnswerStore response : answerList) { final Long questionId = Long.parseLong(response.getQuestionID()); Question question = qDao.getByKey(questionId); if (question == null || !question.canBeCharted()) { @@ -518,7 +522,7 @@ public void updateSummaryCounts(SurveyInstance si, boolean increment) { questionSummary.setQuestionId(response.getQuestionID()); questionSummary.setResponse(questionResponse[i]); questionSummary.setCount(0L); - summaryDao.save(questionSummary);//New, so OK to save from here? NO it could be created from other threads simultaneously! + summaryDao.save(questionSummary); } else { questionSummary = questionSummaryList.get(0); } @@ -526,7 +530,7 @@ public void updateSummaryCounts(SurveyInstance si, boolean increment) { // update and save or delete long count = questionSummary.getCount() == null ? 0 : questionSummary.getCount(); count = increment ? ++count : --count; - questionSummary.setCount(count);//need to keep track of the delta instead! + questionSummary.setCount(count); if (count > 0) { saveList.add(questionSummary); @@ -547,13 +551,11 @@ private void queueSynchronizedSummaryUpdate(SurveyInstance si, boolean increment .withUrl("/app_worker/dataprocessor") .param(DataProcessorRequest.ACTION_PARAM, DataProcessorRequest.UPDATE_SURVEY_INSTANCE_SUMMARIES) - .param(DataProcessorRequest.SURVEY_INSTANCE_PARAM, - si.getKey().getId() + "") + .param(DataProcessorRequest.SURVEY_INSTANCE_PARAM, si.getKey().getId() + "") .param(DataProcessorRequest.DELTA_PARAM, increment ? "1":"-1"); questionSummaryQueue.add(to); } - /** From f1c0f4b6fa8df68cdd372d480cf2d46efd6eb2b3 Mon Sep 17 00:00:00 2001 From: kymni Date: Mon, 28 Jan 2019 22:44:30 +0300 Subject: [PATCH 011/155] [#2842] add properties to CaddisflyTestDefinition object --- .../js/lib/controllers/survey-controllers.js | 5 +++++ Dashboard/app/js/lib/models/models.js | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/Dashboard/app/js/lib/controllers/survey-controllers.js b/Dashboard/app/js/lib/controllers/survey-controllers.js index e14e68c05a..6013c38199 100644 --- a/Dashboard/app/js/lib/controllers/survey-controllers.js +++ b/Dashboard/app/js/lib/controllers/survey-controllers.js @@ -1847,6 +1847,11 @@ FLOW.CaddisflyResourceController = Ember.ArrayController.extend({ "name": test.name, "brand": test.brand, "uuid": test.uuid, + "multiParameter": test.multiParameter, + "sample": test.sample, + "device": test.device, + "model": test.model, + "results": test.results })); }); diff --git a/Dashboard/app/js/lib/models/models.js b/Dashboard/app/js/lib/models/models.js index 0c8454c7cd..8f6d6dacbe 100644 --- a/Dashboard/app/js/lib/models/models.js +++ b/Dashboard/app/js/lib/models/models.js @@ -23,8 +23,30 @@ FLOW.BaseModel = DS.Model.extend({ FLOW.CaddisflyTestDefinition = Ember.Object.extend({ name: null, + multiParameter: null, + sample: DS.attr('string', { + defaultValue: '' + }), + device: null, brand: null, + model: null, uuid: null, + results: [], + + detailsDisplayName: function () { + var results = this.get('results'), displayName = ""; + for (var i in results) { + displayName += "name" in results[i] ? results[i].name : ""; + displayName += "chemical" in results[i] ? " "+results[i].chemical : ""; + displayName += "unit" in results[i] ? " "+results[i].unit : ""; + displayName += "range" in results[i] ? " "+results[i].range : ""; + displayName += i > 0 ? ", " : ""; + } + }, + + brandDisplayName: function () { + return this.get('brand') + " - " + this.get('model') + " - " + this.get('device') + }, displayName: function() { return this.get('name') + " (" + this.get('brand') +")"; From 864829124d6c1c7065700c4e34b8ef1712b122e3 Mon Sep 17 00:00:00 2001 From: kymni Date: Mon, 28 Jan 2019 22:45:17 +0300 Subject: [PATCH 012/155] [#2842] cascading caddisfly filters --- .../app/js/lib/views/surveys/question-view.js | 81 +++++++++++++++++++ .../navSurveys/question-view.handlebars | 34 ++++++++ 2 files changed, 115 insertions(+) diff --git a/Dashboard/app/js/lib/views/surveys/question-view.js b/Dashboard/app/js/lib/views/surveys/question-view.js index 99370fc67c..79661e5d94 100644 --- a/Dashboard/app/js/lib/views/surveys/question-view.js +++ b/Dashboard/app/js/lib/views/surveys/question-view.js @@ -800,6 +800,87 @@ FLOW.QuestionView = FLOW.View.extend({ var form = FLOW.selectedControl.get('selectedSurvey'); return FLOW.permControl.canEditForm(form); }.property('FLOW.selectedControl.selectedSurvey'), + + caddisflyTestSamples: function () { + var tests = FLOW.router.caddisflyResourceController.get('content'), unique = {}, distinct = []; + this.set('selectedCaddisflyTestSample', null); + tests.forEach(function (obj) { + if (typeof(unique[obj.sample]) == "undefined") { + distinct.push(obj); + } + unique[obj.sample] = 0; + }); + return distinct; + }.property('FLOW.router.caddisflyResourceController.content'), + + caddisflyTestNames: function () { + var tests = FLOW.router.caddisflyResourceController.get('content'), distinct = []; + this.set('selectedCaddisflyTestName', null); + if (this.get('selectedCaddisflyTestSample')) { + var sample = this.get('selectedCaddisflyTestSample'), unique = {}; + var names = tests.filter(function (item) { + return item['sample'] === sample['sample']; + }); + names.forEach(function (obj) { + if (typeof(unique[obj.name]) == "undefined") { + distinct.push(obj); + } + unique[obj.name] = 0; + }); + } + return distinct; + }.property('this.selectedCaddisflyTestSample'), + + caddisflyTestBrands: function () { + var tests = FLOW.router.caddisflyResourceController.get('content'), distinct = []; + this.set('selectedCaddisflyTestBrand', null); + if (this.get('selectedCaddisflyTestName')) { + var name = this.get('selectedCaddisflyTestName'), unique = {}; + var brands = tests.filter(function (item) { + return item['sample'] == name['sample'] && item['name'] === name['name']; + }); + brands.forEach(function (obj) { + var displayName = obj.brand + " - " + obj.model + " - " + obj.device; + if (typeof(unique[displayName]) == "undefined") { + obj['brandDisplayName'] = displayName; + distinct.push(obj); + } + unique[displayName] = 0; + }); + } else { + this.set('selectedCaddisflyTestBrand', null); + } + return distinct; + }.property('this.selectedCaddisflyTestName'), + + caddisflyTestDetails: function () { + var tests = FLOW.router.caddisflyResourceController.get('content'), distinct = []; + if (this.get('selectedCaddisflyTestBrand')) { + var brands = this.get('selectedCaddisflyTestBrand'), unique = {}; + var details = tests.filter(function (item) { + return item['sample'] == brands['sample'] && item['name'] === brands['name'] && + item['brand'] === brands['brand'] && item['model'] === brands['model'] && + item['device'] === brands['device']; + }); + details.forEach(function (obj) { + var results = obj.results, displayName = ""; + for (var i = 0; i < results.length; i++) { + displayName += "name" in results[i] ? results[i].name : ""; + displayName += "chemical" in results[i] ? " ("+results[i].chemical+")" : ""; + displayName += "range" in results[i] ? " "+results[i].range : ""; + displayName += "unit" in results[i] ? " "+results[i].unit : ""; + displayName += (i+1) < results.length ? ", " : ""; + } + + if (typeof(unique[displayName]) == "undefined") { + obj['detailsDisplayName'] = displayName; + distinct.push(obj); + } + unique[displayName] = 0; + }); + } + return distinct; + }.property('this.selectedCaddisflyTestBrand') }); /* diff --git a/Dashboard/app/js/templates/navSurveys/question-view.handlebars b/Dashboard/app/js/templates/navSurveys/question-view.handlebars index d1e446a169..c00a02f0e6 100644 --- a/Dashboard/app/js/templates/navSurveys/question-view.handlebars +++ b/Dashboard/app/js/templates/navSurveys/question-view.handlebars @@ -147,6 +147,40 @@ optionValuePath="content.keyId" prompt="" promptBinding="Ember.STRINGS._select_caddisfly_test"}} + {{view Ember.Select + contentBinding="view.caddisflyTestSamples" + selectionBinding="view.selectedCaddisflyTestSample" + optionLabelPath="content.sample" + optionValuePath="content.keyId" + prompt="" + promptBinding="Ember.STRINGS._select_caddisfly_test"}} + {{#if view.selectedCaddisflyTestSample}} + {{view Ember.Select + contentBinding="view.caddisflyTestNames" + selectionBinding="view.selectedCaddisflyTestName" + optionLabelPath="content.name" + optionValuePath="content.keyId" + prompt="" + promptBinding="Ember.STRINGS._select_caddisfly_test"}} + {{/if}} + {{#if view.selectedCaddisflyTestName}} + {{view Ember.Select + contentBinding="view.caddisflyTestBrands" + selectionBinding="view.selectedCaddisflyTestBrand" + optionLabelPath="content.brandDisplayName" + optionValuePath="content.keyId" + prompt="" + promptBinding="Ember.STRINGS._select_caddisfly_test"}} + {{/if}} + {{#if view.selectedCaddisflyTestBrand}} + {{view Ember.Select + contentBinding="view.caddisflyTestDetails" + selectionBinding="view.selectedCaddisflyTestDetail" + optionLabelPath="content.detailsDisplayName" + optionValuePath="content.keyId" + prompt="" + promptBinding="Ember.STRINGS._select_caddisfly_test"}} + {{/if}} {{else}}
{{t _failed_load_caddisfly_tests}}
{{/if}} From b18cab4ad91ded757ddd7729ab58a83b94759187 Mon Sep 17 00:00:00 2001 From: kymni Date: Tue, 29 Jan 2019 12:24:25 +0300 Subject: [PATCH 013/155] [#2842] pre-populate caddisfly filters when a test is already selected --- Dashboard/app/js/lib/views/surveys/question-view.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dashboard/app/js/lib/views/surveys/question-view.js b/Dashboard/app/js/lib/views/surveys/question-view.js index 79661e5d94..13519b8a89 100644 --- a/Dashboard/app/js/lib/views/surveys/question-view.js +++ b/Dashboard/app/js/lib/views/surveys/question-view.js @@ -205,6 +205,9 @@ FLOW.QuestionView = FLOW.View.extend({ if (!Ember.empty(FLOW.selectedControl.selectedQuestion.get('caddisflyResourceUuid'))) { var caddResource = FLOW.router.caddisflyResourceController.content.findProperty('uuid', FLOW.selectedControl.selectedQuestion.get('caddisflyResourceUuid')); if (!Ember.empty(caddResource)) { + this.set('selectedCaddisflyTestSample', this.get('caddisflyTestSamples').findProperty('sample', caddResource.get('sample'))); + this.set('selectedCaddisflyTestName', this.get('caddisflyTestNames').findProperty('name', caddResource.get('name'))); + //this.set('selectedCaddisflyTestBrand', this.get('caddisflyTestNames').findProperty('name', caddResource.get('name'))); FLOW.selectedControl.set('selectedCaddisflyResource',caddResource); } } From 36c9cc557a18890085a4729154a45f61985c8529 Mon Sep 17 00:00:00 2001 From: kymni Date: Tue, 29 Jan 2019 12:25:03 +0300 Subject: [PATCH 014/155] [#2842] remove unused properties in CaddisflyTestDefinition object --- Dashboard/app/js/lib/models/models.js | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/Dashboard/app/js/lib/models/models.js b/Dashboard/app/js/lib/models/models.js index 8f6d6dacbe..9d370dc21e 100644 --- a/Dashboard/app/js/lib/models/models.js +++ b/Dashboard/app/js/lib/models/models.js @@ -24,33 +24,12 @@ FLOW.BaseModel = DS.Model.extend({ FLOW.CaddisflyTestDefinition = Ember.Object.extend({ name: null, multiParameter: null, - sample: DS.attr('string', { - defaultValue: '' - }), + sample: null, device: null, brand: null, model: null, uuid: null, results: [], - - detailsDisplayName: function () { - var results = this.get('results'), displayName = ""; - for (var i in results) { - displayName += "name" in results[i] ? results[i].name : ""; - displayName += "chemical" in results[i] ? " "+results[i].chemical : ""; - displayName += "unit" in results[i] ? " "+results[i].unit : ""; - displayName += "range" in results[i] ? " "+results[i].range : ""; - displayName += i > 0 ? ", " : ""; - } - }, - - brandDisplayName: function () { - return this.get('brand') + " - " + this.get('model') + " - " + this.get('device') - }, - - displayName: function() { - return this.get('name') + " (" + this.get('brand') +")"; - }.property(''), }); FLOW.CascadeResource = FLOW.BaseModel.extend({ From 9c0716137728a1683d16876816d1ae26ea9cb97e Mon Sep 17 00:00:00 2001 From: kymni Date: Tue, 29 Jan 2019 15:39:05 +0300 Subject: [PATCH 015/155] [#1808] disable saving question when no test type is selected --- Dashboard/app/js/lib/views/surveys/question-view.js | 10 +++++++++- .../js/templates/navSurveys/question-view.handlebars | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Dashboard/app/js/lib/views/surveys/question-view.js b/Dashboard/app/js/lib/views/surveys/question-view.js index 13519b8a89..8b1fd5e873 100644 --- a/Dashboard/app/js/lib/views/surveys/question-view.js +++ b/Dashboard/app/js/lib/views/surveys/question-view.js @@ -806,6 +806,7 @@ FLOW.QuestionView = FLOW.View.extend({ caddisflyTestSamples: function () { var tests = FLOW.router.caddisflyResourceController.get('content'), unique = {}, distinct = []; + FLOW.selectedControl.set('selectedCaddisflyResource', null); this.set('selectedCaddisflyTestSample', null); tests.forEach(function (obj) { if (typeof(unique[obj.sample]) == "undefined") { @@ -818,6 +819,7 @@ FLOW.QuestionView = FLOW.View.extend({ caddisflyTestNames: function () { var tests = FLOW.router.caddisflyResourceController.get('content'), distinct = []; + FLOW.selectedControl.set('selectedCaddisflyResource', null); this.set('selectedCaddisflyTestName', null); if (this.get('selectedCaddisflyTestSample')) { var sample = this.get('selectedCaddisflyTestSample'), unique = {}; @@ -836,6 +838,7 @@ FLOW.QuestionView = FLOW.View.extend({ caddisflyTestBrands: function () { var tests = FLOW.router.caddisflyResourceController.get('content'), distinct = []; + FLOW.selectedControl.set('selectedCaddisflyResource', null); this.set('selectedCaddisflyTestBrand', null); if (this.get('selectedCaddisflyTestName')) { var name = this.get('selectedCaddisflyTestName'), unique = {}; @@ -883,7 +886,12 @@ FLOW.QuestionView = FLOW.View.extend({ }); } return distinct; - }.property('this.selectedCaddisflyTestBrand') + }.property('this.selectedCaddisflyTestBrand'), + + brandsObserver: function () { + //needed to disable save button when no resource brand is specified + FLOW.selectedControl.set('selectedCaddisflyResource', null); + }.observes('this.selectedCaddisflyTestBrand') }); /* diff --git a/Dashboard/app/js/templates/navSurveys/question-view.handlebars b/Dashboard/app/js/templates/navSurveys/question-view.handlebars index c00a02f0e6..cf54651dd4 100644 --- a/Dashboard/app/js/templates/navSurveys/question-view.handlebars +++ b/Dashboard/app/js/templates/navSurveys/question-view.handlebars @@ -255,8 +255,15 @@
  • {{t _save_question}}
    • {{/if}}{{/if}}{{/if}}{{/if}}{{/if}}{{/if}}{{/if}} {{else}} + {{#if view.amCaddisflyType}} + {{#unless FLOW.selectedControl.selectedCaddisflyResource}} +
  • {{t _save_question}}
    • + {{else}} +
  • {{t _save_question}}
    • + {{/unless}} + {{else}}
  • {{t _save_question}}
    • - {{/if}}{{/if}}{{/if}} + {{/if}}{{/if}}{{/if}}{{/if}}
  • {{t _cancel}}
    • From d8e44c98182963b1e3577947bf2a34fd0d95d02f Mon Sep 17 00:00:00 2001 From: kymni Date: Tue, 29 Jan 2019 15:40:51 +0300 Subject: [PATCH 016/155] [#2842] cascading caddisfly tests --- .../app/js/lib/views/surveys/question-view.js | 4 +++- .../navSurveys/question-view.handlebars | 19 ++++++------------- GAE/src/locale/en.properties | 4 ++++ GAE/src/locale/ui-strings.properties | 4 ++++ 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Dashboard/app/js/lib/views/surveys/question-view.js b/Dashboard/app/js/lib/views/surveys/question-view.js index 8b1fd5e873..e067fd1f09 100644 --- a/Dashboard/app/js/lib/views/surveys/question-view.js +++ b/Dashboard/app/js/lib/views/surveys/question-view.js @@ -207,7 +207,9 @@ FLOW.QuestionView = FLOW.View.extend({ if (!Ember.empty(caddResource)) { this.set('selectedCaddisflyTestSample', this.get('caddisflyTestSamples').findProperty('sample', caddResource.get('sample'))); this.set('selectedCaddisflyTestName', this.get('caddisflyTestNames').findProperty('name', caddResource.get('name'))); - //this.set('selectedCaddisflyTestBrand', this.get('caddisflyTestNames').findProperty('name', caddResource.get('name'))); + this.set('selectedCaddisflyTestBrand', this.get('caddisflyTestBrands').find(function (item) { + return item.brand === caddResource.get('brand') && item.model === caddResource.get('model') && item.device === caddResource.get('device') + })); FLOW.selectedControl.set('selectedCaddisflyResource',caddResource); } } diff --git a/Dashboard/app/js/templates/navSurveys/question-view.handlebars b/Dashboard/app/js/templates/navSurveys/question-view.handlebars index cf54651dd4..0ee6030db9 100644 --- a/Dashboard/app/js/templates/navSurveys/question-view.handlebars +++ b/Dashboard/app/js/templates/navSurveys/question-view.handlebars @@ -139,21 +139,14 @@

    {{t _settings}}:

    + + + org.junit.jupiter + junit-jupiter-api + ${junit.jupiter.version} + test + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.jupiter.version} + test + + org.jfree @@ -357,7 +373,7 @@ maven-surefire-plugin - 2.20 + 2.22.1 diff --git a/GAE/src/org/akvo/flow/util/FlowJsonObjectReader.java b/GAE/src/org/akvo/flow/util/FlowJsonObjectReader.java index 1dafb5f92a..c6fa594e1e 100644 --- a/GAE/src/org/akvo/flow/util/FlowJsonObjectReader.java +++ b/GAE/src/org/akvo/flow/util/FlowJsonObjectReader.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -17,18 +17,32 @@ package org.akvo.flow.util; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectReader; import java.io.IOException; import java.net.URL; +import java.util.List; -public class FlowJsonObjectReader { +public class FlowJsonObjectReader extends ObjectMapper { - public T readObject(String jsonString) throws IOException { - return new ObjectMapper().readValue(jsonString, new TypeReference(){}); + public T readObject(String jsonString, TypeReference typeReference) throws IOException { + return this.readValue(jsonString, typeReference); } - public T readObject(URL url) throws IOException { - return new ObjectMapper().readValue(url, new TypeReference(){}); + public List readDtoListObject(String dtoListJsonString, TypeReference listItemTypeReference) throws IOException { + + JavaType listItemType = this.getTypeFactory().constructType(listItemTypeReference); + JavaType type = this.getTypeFactory().constructParametrizedType(List.class,List.class, listItemType); + ObjectReader reader = this.readerFor(type); + + JsonNode dtoListNode = this.readTree(dtoListJsonString).get("dtoList"); + return reader.readValue(dtoListNode); + } + + public T readObject(URL url, TypeReference typeReference) throws IOException { + return this.readValue(url, typeReference); } -} \ No newline at end of file +} diff --git a/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java b/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java new file mode 100644 index 0000000000..4d562c77f3 --- /dev/null +++ b/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2019 Stichting Akvo (Akvo Foundation) + * + * This file is part of Akvo FLOW. + * + * Akvo FLOW is free software: you can redistribute it and modify it under the terms of + * the GNU Affero General Public License (AGPL) as published by the Free Software Foundation, + * either version 3 of the License or any later version. + * + * Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License included below for more details. + * + * The full license text can also be seen at . + */ + +package test.java.org.akvo.flow.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +import com.fasterxml.jackson.core.type.TypeReference; +import org.akvo.flow.util.FlowJsonObjectReader; +import org.junit.jupiter.api.Test; +import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto; + +import java.io.IOException; + +class FlowJsonObjectReaderTests { + + private String QUESTION_DTO_JSON_STRING = "{ \"type\": \"FREE_TEXT\",\n" + + "\"text\": \"How many toilets are present?\",\n" + + "\"dependentFlag\": false,\n" + + "\"questionGroupId\": 12345678,\n" + + "\"surveyId\": 910111213,\n" + + "\"order\": 0 }"; + + @Test + void testReadSimpleJsonObject() { + FlowJsonObjectReader reader = new FlowJsonObjectReader(); + TypeReference typeReference = new TypeReference() {}; + QuestionDto testQuestionDto = null; + try { + testQuestionDto = reader.readObject(QUESTION_DTO_JSON_STRING, typeReference); + } catch (IOException e) { + System.out.println("Reading error: " + e.getMessage()); + } + assertEquals(testQuestionDto.getType(), QuestionDto.QuestionType.FREE_TEXT); + assertEquals(testQuestionDto.getText(),"How many toilets are present?"); + assertFalse(testQuestionDto.getDependentFlag()); + assertEquals(testQuestionDto.getQuestionGroupId(), 12345678L); + assertEquals(testQuestionDto.getType(), QuestionDto.QuestionType.FREE_TEXT); + assertEquals(testQuestionDto.getOrder(), 0); + } +} From 3db68ed4f5aeb6c85acf2105d97aef75cbe3be01 Mon Sep 17 00:00:00 2001 From: Emmanuel Mulo Date: Mon, 18 Feb 2019 12:48:56 +0100 Subject: [PATCH 035/155] [#2896] WIP Add test case for complex JSON objects --- .../flow/util/FlowJsonObjectReaderTests.java | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java b/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java index 4d562c77f3..3eae0dc85b 100644 --- a/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java +++ b/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java @@ -18,14 +18,19 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import com.fasterxml.jackson.core.type.TypeReference; import org.akvo.flow.util.FlowJsonObjectReader; import org.junit.jupiter.api.Test; import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto; +import org.waterforpeople.mapping.domain.CaddisflyResource; import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; class FlowJsonObjectReaderTests { @@ -36,6 +41,53 @@ class FlowJsonObjectReaderTests { "\"surveyId\": 910111213,\n" + "\"order\": 0 }"; + private String COMPLEX_JSON_OBJECT = "{\n" + + " \"tests\": [\n" + + " {\n" + + " \"name\": \"Soil - Electrical Conductivity\",\n" + + " \"subtype\": \"sensor\",\n" + + " \"uuid\": \"80697cd1-acc9-4a15-8358-f32b4257dfaf\",\n" + + " \"deviceId\": \"SoilEC\",\n" + + " \"brand\": \"Caddisfly\",\n" + + " \"image\": \"Caddisfly-Soil-EC\",\n" + + " \"imageScale\": \"centerCrop\",\n" + + " \"ranges\": \"50,12800\",\n" + + " \"responseFormat\": \"$2,$1\",\n" + + " \"instructions\": [],\n" + + " \"results\": [\n" + + " {\n" + + " \"id\": 1,\n" + + " \"name\": \"Soil Electrical Conductivity\",\n" + + " \"unit\": \"μS/cm\"\n" + + " },\n" + + " {\n" + + " \"id\": 2,\n" + + " \"name\": \"Temperature\",\n" + + " \"unit\": \"°Celsius\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"name\": \"Soil - Moisture\",\n" + + " \"subtype\": \"sensor\",\n" + + " \"uuid\": \"0b4a0aaa-f556-4c11-a539-c4626582cca6\",\n" + + " \"deviceId\": \"Soil Moisture\",\n" + + " \"brand\": \"Caddisfly\",\n" + + " \"image\": \"Caddisfly-Soil-Moisture\",\n" + + " \"imageScale\": \"centerCrop\",\n" + + " \"ranges\": \"0,100\",\n" + + " \"responseFormat\": \"$1\",\n" + + " \"instructions\": [],\n" + + " \"results\": [\n" + + " {\n" + + " \"id\": 1,\n" + + " \"name\": \"Soil Moisture\",\n" + + " \"unit\": \"% VWC\"\n" + + " }\n" + + " ]\n" + + " }\n" + + "]}"; + @Test void testReadSimpleJsonObject() { FlowJsonObjectReader reader = new FlowJsonObjectReader(); @@ -44,7 +96,7 @@ void testReadSimpleJsonObject() { try { testQuestionDto = reader.readObject(QUESTION_DTO_JSON_STRING, typeReference); } catch (IOException e) { - System.out.println("Reading error: " + e.getMessage()); + // } assertEquals(testQuestionDto.getType(), QuestionDto.QuestionType.FREE_TEXT); assertEquals(testQuestionDto.getText(),"How many toilets are present?"); @@ -53,4 +105,25 @@ void testReadSimpleJsonObject() { assertEquals(testQuestionDto.getType(), QuestionDto.QuestionType.FREE_TEXT); assertEquals(testQuestionDto.getOrder(), 0); } + + @Test + void testReadComplexJsonObject() { + FlowJsonObjectReader reader = new FlowJsonObjectReader(); + TypeReference>> typeReference = new TypeReference>>() {}; + Map> resourcesMap = new HashMap<>(); + try { + resourcesMap = reader.readObject(COMPLEX_JSON_OBJECT, typeReference); + } catch (IOException e) { + // + } + + List resourcesList = resourcesMap.get("tests"); + assertNotEquals(resourcesList, null); + assertEquals(resourcesList.size(), 2); + assertEquals(resourcesList.get(0).getResults().size(), 2); + assertEquals(resourcesList.get(0).getName(), "Soil - Electrical Conductivity"); + assertEquals(resourcesList.get(1).getResults().size(), 1); + assertEquals(resourcesList.get(1).getUuid(), "0b4a0aaa-f556-4c11-a539-c4626582cca6"); + + } } From c18d54070d6e4376425a0148098bfa5de58d865d Mon Sep 17 00:00:00 2001 From: Emmanuel Mulo Date: Tue, 19 Feb 2019 12:58:56 +0100 Subject: [PATCH 036/155] [#2896] WIP Add test for deserialistion of DTO list responses --- .../flow/util/FlowJsonObjectReaderTests.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java b/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java index 3eae0dc85b..64fbcfaa18 100644 --- a/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java +++ b/GAE/src/test/java/org/akvo/flow/util/FlowJsonObjectReaderTests.java @@ -25,6 +25,7 @@ import org.akvo.flow.util.FlowJsonObjectReader; import org.junit.jupiter.api.Test; import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto; +import org.waterforpeople.mapping.app.gwt.client.survey.SurveyGroupDto; import org.waterforpeople.mapping.domain.CaddisflyResource; import java.io.IOException; @@ -88,6 +89,40 @@ class FlowJsonObjectReaderTests { " }\n" + "]}"; + private String DTO_LIST_JSON_OBJECT = "{\n" + + " \"code\": null, \n" + + " \"cursor\": null, \n" + + " \"dtoList\": [\n" + + " {\n" + + " \"ancestorIds\": [\n" + + " 0, \n" + + " 278889175415\n" + + " ], \n" + + " \"code\": \"1.10.36 all questions\", \n" + + " \"createdDateTime\": 1534846914945, \n" + + " \"dataApprovalGroupId\": null, \n" + + " \"defaultLanguageCode\": \"en\", \n" + + " \"description\": \"\", \n" + + " \"keyId\": 2989762914097, \n" + + " \"lastUpdateDateTime\": 1534846926804, \n" + + " \"monitoringGroup\": false, \n" + + " \"name\": \"1.10.36 all questions\", \n" + + " \"newLocaleSurveyId\": null, \n" + + " \"parentId\": 27888911545, \n" + + " \"path\": \"/_1.9.36 and 2.6.0/1.10.36 all questions\", \n" + + " \"privacyLevel\": \"PRIVATE\", \n" + + " \"projectType\": \"PROJECT\", \n" + + " \"published\": false, \n" + + " \"requireDataApproval\": false, \n" + + " \"surveyList\": null\n" + + " }\n" + + " ], \n" + + " \"message\": null, \n" + + " \"offset\": 0, \n" + + " \"resultCount\": 0, \n" + + " \"url\": null\n" + + "}\n"; + @Test void testReadSimpleJsonObject() { FlowJsonObjectReader reader = new FlowJsonObjectReader(); @@ -124,6 +159,22 @@ void testReadComplexJsonObject() { assertEquals(resourcesList.get(0).getName(), "Soil - Electrical Conductivity"); assertEquals(resourcesList.get(1).getResults().size(), 1); assertEquals(resourcesList.get(1).getUuid(), "0b4a0aaa-f556-4c11-a539-c4626582cca6"); + } + + @Test + void testDtoListResponses() { + FlowJsonObjectReader reader = new FlowJsonObjectReader(); + TypeReference typeReference = new TypeReference() {}; + List surveyList = null; + + try { + surveyList = reader.readDtoListObject(DTO_LIST_JSON_OBJECT, typeReference); + } catch (IOException e) { + // + } + assertNotEquals(surveyList, null); + assertEquals(surveyList.size(), 1); + assertEquals(surveyList.get(0).getName(),"1.10.36 all questions"); } } From d23eea15fcb7709ac6dba77a6ed5d7227e66c16c Mon Sep 17 00:00:00 2001 From: Emmanuel Mulo Date: Tue, 19 Feb 2019 14:03:38 +0100 Subject: [PATCH 037/155] [#2896] Setup JUnit dependencies only for testing * The dependencies will not be included in the packaged WAR --- GAE/pom.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/GAE/pom.xml b/GAE/pom.xml index fe585ea7a6..43b6fafefc 100644 --- a/GAE/pom.xml +++ b/GAE/pom.xml @@ -213,14 +213,14 @@ org.junit.jupiter junit-jupiter-api ${junit.jupiter.version} - test + provided org.junit.jupiter junit-jupiter-engine ${junit.jupiter.version} - test + provided @@ -251,7 +251,6 @@ 2.5.1 provided -     From f6b7ac11eb3ad88fdd324beb2bae94f6c386ccee Mon Sep 17 00:00:00 2001 From: stellanl Date: Tue, 19 Feb 2019 14:56:30 +0100 Subject: [PATCH 038/155] [#2978]Show a * if mandatory question. --- .../gae/remoteapi/PrintTreeStructure.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/data/src/org/akvo/gae/remoteapi/PrintTreeStructure.java b/scripts/data/src/org/akvo/gae/remoteapi/PrintTreeStructure.java index fc6ad5a647..cf1db11bc1 100644 --- a/scripts/data/src/org/akvo/gae/remoteapi/PrintTreeStructure.java +++ b/scripts/data/src/org/akvo/gae/remoteapi/PrintTreeStructure.java @@ -44,6 +44,7 @@ public class PrintTreeStructure implements Process { private Map qParents = new HashMap<>(); private Map qToSurvey = new HashMap<>(); private Map qOrder = new HashMap<>(); + private Map qMandatory = new HashMap<>(); private Map qgNames = new HashMap<>(); private Map qgOrder= new HashMap<>(); @@ -83,9 +84,9 @@ public void execute(DatastoreService ds, String[] args) throws Exception { if (showQuestions) { fetchQuestionGroups(ds); fetchQuestions(ds); - if (showOptions) { - fetchOptions(ds); - } + if (showOptions) { + fetchOptions(ds); + } } // System.out.printf("/ (root)\n"); @@ -141,18 +142,20 @@ private void drawGroupsIn(Long parent, int indent) { private void drawQuestionsIn(Long parent, int indent) { for (Long q : qParents.keySet()) { + String m = (qMandatory.get(q).equals(Boolean.TRUE)) ? "*":""; + if (qParents.get(q).equals(parent)) { String s = ""; for (int i = 0; i Date: Tue, 19 Feb 2019 14:59:02 +0100 Subject: [PATCH 039/155] [#2978]Update copright. Remove unused stuff. --- .../src/org/akvo/gae/remoteapi/PrintTreeStructure.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/data/src/org/akvo/gae/remoteapi/PrintTreeStructure.java b/scripts/data/src/org/akvo/gae/remoteapi/PrintTreeStructure.java index cf1db11bc1..49237b98bf 100644 --- a/scripts/data/src/org/akvo/gae/remoteapi/PrintTreeStructure.java +++ b/scripts/data/src/org/akvo/gae/remoteapi/PrintTreeStructure.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2017,2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -23,7 +23,6 @@ import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.FetchOptions; -import com.google.appengine.api.datastore.Key; import com.google.appengine.api.datastore.PreparedQuery; import com.google.appengine.api.datastore.Query; @@ -55,7 +54,6 @@ public class PrintTreeStructure implements Process { private Map oParents = new HashMap<>(); private boolean html = false; - private boolean gae = false; private boolean showQuestions = false; private boolean showOptions = false; @@ -68,9 +66,6 @@ public void execute(DatastoreService ds, String[] args) throws Exception { if (args[i].equalsIgnoreCase("--html")) { html = true; } - if (args[i].equalsIgnoreCase("--gae")) { - gae = true; - } if (args[i].equalsIgnoreCase("--questions")) { showQuestions = true; } From 37bd66425fd85adc43dddc1dde5cd61edf52142d Mon Sep 17 00:00:00 2001 From: Emmanuel Mulo Date: Tue, 19 Feb 2019 15:50:28 +0100 Subject: [PATCH 040/155] [#2896] WIP Remove generics from FlowJsonObjectReaderi * The type reference for the JSON to be deserialised is provided as a parameter to the class methods --- .../framework/gwt/dto/client/BaseDto.java | 5 +++- .../survey/dao/CaddisflyResourceDao.java | 7 ++--- GAE/src/org/akvo/flow/domain/DataUtils.java | 19 +++++++++----- .../GraphicalSurveySummaryExporter.java | 26 ++++++++++++------- .../dataexport/SurveySummaryExporter.java | 10 +++---- .../service/BulkDataServiceClient.java | 15 ++++++----- .../serialization/SurveyInstanceHandler.java | 9 ++++--- .../serialization/response/MediaResponse.java | 10 ++++--- 8 files changed, 63 insertions(+), 38 deletions(-) diff --git a/GAE/src/com/gallatinsystems/framework/gwt/dto/client/BaseDto.java b/GAE/src/com/gallatinsystems/framework/gwt/dto/client/BaseDto.java index 646e0a67d1..29821550c6 100644 --- a/GAE/src/com/gallatinsystems/framework/gwt/dto/client/BaseDto.java +++ b/GAE/src/com/gallatinsystems/framework/gwt/dto/client/BaseDto.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2014,2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -16,6 +16,8 @@ package com.gallatinsystems.framework.gwt.dto.client; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + import java.io.Serializable; /** @@ -23,6 +25,7 @@ * * @author Christopher Fagaini */ +@JsonIgnoreProperties(ignoreUnknown = true) public class BaseDto implements Serializable { private static final long serialVersionUID = -5905705837362187943L; diff --git a/GAE/src/com/gallatinsystems/survey/dao/CaddisflyResourceDao.java b/GAE/src/com/gallatinsystems/survey/dao/CaddisflyResourceDao.java index 1a8b064ba0..d9a2104b85 100644 --- a/GAE/src/com/gallatinsystems/survey/dao/CaddisflyResourceDao.java +++ b/GAE/src/com/gallatinsystems/survey/dao/CaddisflyResourceDao.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016-2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2016-2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -23,6 +23,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.fasterxml.jackson.core.type.TypeReference; import org.akvo.flow.util.FlowJsonObjectReader; import org.waterforpeople.mapping.domain.CaddisflyResource; @@ -39,11 +40,11 @@ public class CaddisflyResourceDao { public List listResources(String caddisflyTestsUrl) { Map> testsMap = null; - FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); try { URL caddisflyFileUrl = new URL(caddisflyTestsUrl); - testsMap = jsonReader.readObject(caddisflyFileUrl); + testsMap = jsonReader.readObject(caddisflyFileUrl, new TypeReference>>() {}); } catch (Exception e) { log.log(Level.SEVERE, "Error parsing Caddisfly resource: " + e.getMessage(), e); diff --git a/GAE/src/org/akvo/flow/domain/DataUtils.java b/GAE/src/org/akvo/flow/domain/DataUtils.java index a073e13b32..83d02b4651 100644 --- a/GAE/src/org/akvo/flow/domain/DataUtils.java +++ b/GAE/src/org/akvo/flow/domain/DataUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015,2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2015-2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; +import com.fasterxml.jackson.core.type.TypeReference; import org.akvo.flow.util.FlowJsonObjectReader; import org.apache.log4j.Logger; @@ -108,9 +109,10 @@ public static String jsonResponsesToPipeSeparated(String optionResponses) { * @return List of maps with response properties */ public static List> jsonStringToList(String data) { - FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + TypeReference>> typeReference = new TypeReference>>() {}; try { - return jsonReader.readObject(data); + return jsonReader.readObject(data, typeReference); } catch (IOException e) { // Data is not JSON-formatted } @@ -126,10 +128,12 @@ public static List> jsonStringToList(String data) { * @return */ public static String parseSignatory(String value) { - FlowJsonObjectReader> jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); Map signatureResponse = null; + TypeReference> typeReference = new TypeReference>() {}; + try { - signatureResponse = jsonReader.readObject(value); + signatureResponse = jsonReader.readObject(value, typeReference); } catch (IOException e) { // ignore } @@ -146,10 +150,11 @@ public static String parseSignatory(String value) { @SuppressWarnings("unchecked") public static Map parseCaddisflyResponseValue(String caddisflyValue) { Map caddisflyResponseMap = null; - FlowJsonObjectReader> jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + TypeReference> typeReference = new TypeReference>() {}; try { - caddisflyResponseMap = jsonReader.readObject(caddisflyValue); + caddisflyResponseMap = jsonReader.readObject(caddisflyValue, typeReference); } catch (IOException e) { log.warn("Failed to parse the caddisfly response"); } diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java index 2a9b0eafac..71e52a65c9 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -40,6 +40,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import com.fasterxml.jackson.core.type.TypeReference; import org.akvo.flow.domain.DataUtils; import org.akvo.flow.util.FlowJsonObjectReader; import org.akvo.flow.util.JFreechartChartUtil; @@ -693,7 +694,9 @@ private synchronized int writeInstanceData( if (rollupOrder != null && rollupOrder.size() > 0) { rollups = formRollupStrings(responseMap); } - FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + TypeReference>> typeReference = new TypeReference>>() {}; + for (Entry entry : responseMap.entrySet()) { //OPTION, NUMBER and CASCADE summarizable now. @@ -706,7 +709,7 @@ private synchronized int writeInstanceData( String[] vals; if (entry.getValue().startsWith("[")) { //JSON try { - List> optionNodes = jsonReader.readObject(entry.getValue()); + List> optionNodes = jsonReader.readObject(entry.getValue(), typeReference); List valsList = new ArrayList<>(); for (Map optionNode : optionNodes) { if (optionNode.containsKey("text")) { @@ -977,9 +980,11 @@ private Map> mapCaddisflyResultsById( List> cascadeNodes = new ArrayList<>(); if (value.startsWith("[")) { - FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + TypeReference>> typeReference = new TypeReference>>() {}; + try { - cascadeNodes = jsonReader.readObject(value); + cascadeNodes = jsonReader.readObject(value, typeReference); } catch (IOException e) { log.warn("Unable to parse CASCADE response - " + value, e); } @@ -1062,11 +1067,12 @@ private Map> mapCaddisflyResultsById( private List> getNodes(String value) { boolean isNewFormat = value.startsWith("["); List> optionNodes = new ArrayList<>(); - FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + TypeReference>> typeReference = new TypeReference>>() {}; if (isNewFormat) { try { - optionNodes = jsonReader.readObject(value); + optionNodes = jsonReader.readObject(value, typeReference); } catch (IOException e) { log.warn("Could not parse option response: " + value, e); } @@ -1650,9 +1656,11 @@ private void writeStatsAndGraphsSheet( } else { // Handle the json option question response type if (labelText.startsWith("[")) { - FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + TypeReference>> typeReference = new TypeReference>>() {}; + try { - List> optionNodes = jsonReader.readObject(labelText); + List> optionNodes = jsonReader.readObject(labelText, typeReference); StringBuilder labelTextBuilder = new StringBuilder(); for (Map optionNode : optionNodes) { diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java index 5bc966f189..711ec406ca 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -29,6 +29,7 @@ import java.util.Map.Entry; import java.util.Set; +import com.fasterxml.jackson.core.type.TypeReference; import org.akvo.flow.domain.DataUtils; import org.akvo.flow.util.FlowJsonObjectReader; import org.apache.log4j.Logger; @@ -347,10 +348,9 @@ protected List parseQuestionGroups(String response) * @throws Exception */ protected List parseQuestions(String response) throws Exception { - final FlowJsonObjectReader>> jsonReader = new FlowJsonObjectReader<>(); - - final Map> questionListMap = jsonReader.readObject(response); - final List qList = questionListMap.get("dtoList"); + final FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + final TypeReference listItemTypeReference = new TypeReference(){}; + final List qList = jsonReader.readDtoListObject(response, listItemTypeReference); return qList; } diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java index 13be9e9838..fddc16cb1f 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -44,6 +44,7 @@ import com.gallatinsystems.framework.rest.RestRequest; import com.gallatinsystems.survey.domain.SurveyGroup.PrivacyLevel; import com.gallatinsystems.survey.domain.SurveyGroup.ProjectType; +import com.fasterxml.jackson.core.type.TypeReference; import org.akvo.flow.util.FlowJsonObjectReader; import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; @@ -346,9 +347,11 @@ public static InstanceDataDto fetchInstanceData(Long surveyInstanceId, String se } private static InstanceDataDto parseInstanceData(String instanceDataResponse) { - FlowJsonObjectReader jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + TypeReference typeReference = new TypeReference() {}; + try { - InstanceDataDto instanceData = jsonReader.readObject(instanceDataResponse); + InstanceDataDto instanceData = jsonReader.readObject(instanceDataResponse, typeReference); return instanceData; } catch (IOException e) { log.error("Error while parsing: ", e); @@ -395,10 +398,10 @@ public static SurveyGroupDto fetchSurveyGroup(String surveyId, String serverBase log.debug("response: " + surveyGroupResponse); - final FlowJsonObjectReader>> jsonDeserialiser = new FlowJsonObjectReader<>(); - final Map> surveyGroupListMap = jsonDeserialiser.readObject(surveyGroupResponse); + final FlowJsonObjectReader jsonDeserialiser = new FlowJsonObjectReader(); + final TypeReference listItemTypeReference = new TypeReference(){}; + final List surveyGroupList = jsonDeserialiser.readDtoListObject(surveyGroupResponse, listItemTypeReference); - final List surveyGroupList = surveyGroupListMap.get("dtoList"); if (surveyGroupList != null && !surveyGroupList.isEmpty()) { surveyGroupDto = surveyGroupList.get(0); } diff --git a/GAE/src/org/waterforpeople/mapping/serialization/SurveyInstanceHandler.java b/GAE/src/org/waterforpeople/mapping/serialization/SurveyInstanceHandler.java index 3aa83f565d..bafdc8ac06 100644 --- a/GAE/src/org/waterforpeople/mapping/serialization/SurveyInstanceHandler.java +++ b/GAE/src/org/waterforpeople/mapping/serialization/SurveyInstanceHandler.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015-2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2015-2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -23,6 +23,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.fasterxml.jackson.core.type.TypeReference; import org.akvo.flow.util.FlowJsonObjectReader; import org.apache.commons.lang.StringUtils; import org.waterforpeople.mapping.domain.QuestionAnswerStore; @@ -52,10 +53,12 @@ public class SurveyInstanceHandler { public static SurveyInstance fromJSON(String data) { FormInstance formInstance = null; - FlowJsonObjectReader jsonReader = new FlowJsonObjectReader<>(); + FlowJsonObjectReader jsonReader = new FlowJsonObjectReader(); + TypeReference typeReference = new TypeReference() {}; + try { - formInstance = jsonReader.readObject(data); + formInstance = jsonReader.readObject(data, typeReference); } catch (IOException e) { log.log(Level.SEVERE, "Error mapping JSON data: " + e.getMessage(), e); return null; diff --git a/GAE/src/org/waterforpeople/mapping/serialization/response/MediaResponse.java b/GAE/src/org/waterforpeople/mapping/serialization/response/MediaResponse.java index db3bc0ee87..37aeae944c 100644 --- a/GAE/src/org/waterforpeople/mapping/serialization/response/MediaResponse.java +++ b/GAE/src/org/waterforpeople/mapping/serialization/response/MediaResponse.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016,2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2016,2018-2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -17,15 +17,17 @@ package org.waterforpeople.mapping.serialization.response; import java.io.IOException; +import java.lang.reflect.Type; import java.util.logging.Logger; +import com.fasterxml.jackson.core.type.TypeReference; import org.akvo.flow.util.FlowJsonObjectReader; import org.akvo.flow.util.FlowJsonObjectWriter; import org.waterforpeople.mapping.domain.response.value.Media; public class MediaResponse { private static final Logger log = Logger.getLogger(MediaResponse.class.getName()); - private static FlowJsonObjectReader jsonObjectReader = new FlowJsonObjectReader<>(); + private static FlowJsonObjectReader jsonObjectReader = new FlowJsonObjectReader(); private static FlowJsonObjectWriter jsonObjectWriter = new FlowJsonObjectWriter(); public static final int VERSION_STRING = 0; @@ -40,7 +42,7 @@ public static String format(String value, int version) { int savedVersion; try { - media = jsonObjectReader.readObject(value); + media = jsonObjectReader.readObject(value, new TypeReference() {}); savedVersion = VERSION_GEOTAGGING; } catch (IOException e) { // Value is not JSON-formatted @@ -68,7 +70,7 @@ public static String format(String value, int version) { public static Media parse(String value) { try { - return jsonObjectReader.readObject(value); + return jsonObjectReader.readObject(value, new TypeReference() {}); } catch (IOException e) { } From 32edd17ff66d119be7fefd2db775d2537b938522 Mon Sep 17 00:00:00 2001 From: kymni Date: Wed, 20 Feb 2019 16:27:45 +0300 Subject: [PATCH 041/155] [#2891] enable moving options up and down --- .../js/lib/controllers/survey-controllers.js | 34 +++++++++++++++++++ .../navSurveys/question-option.handlebars | 4 ++- GAE/src/locale/en.properties | 3 +- GAE/src/locale/ui-strings.properties | 2 ++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/Dashboard/app/js/lib/controllers/survey-controllers.js b/Dashboard/app/js/lib/controllers/survey-controllers.js index e14e68c05a..9dc7e9d518 100644 --- a/Dashboard/app/js/lib/controllers/survey-controllers.js +++ b/Dashboard/app/js/lib/controllers/survey-controllers.js @@ -1077,6 +1077,7 @@ FLOW.optionListControl = Ember.ArrayController.create({ * */ FLOW.questionOptionsControl = Ember.ArrayController.create({ + sortProperties: ["order"], content: null, questionId: null, emptyOptions: function () { @@ -1242,6 +1243,39 @@ FLOW.questionOptionsControl = Ember.ArrayController.create({ if (option.get('keyId')) { // clear persisted versions option.deleteRecord(); } + + //reorder all options + c.forEach(function (item, index) { + item.set('order', index + 1); + }); + }, + + moveOptionUp: function (event) { + var options = this.content, currentOption = event.view.content; + + if (currentOption && currentOption.get('order') > 0) { + var previousOption = options.find(function (option) { + return option.get('order') == (currentOption.get('order') - 1); + }); + var previousOptionOrder = previousOption.get('order'); + var currentOptionOrder = currentOption.get('order'); + previousOption.set('order', currentOptionOrder); + currentOption.set('order', previousOptionOrder); + } + }, + + moveOptionDown: function (event) { + var options = this.content, currentOption = event.view.content; + + if (currentOption && currentOption.get('order') < (options.get('length') - 1)) { + var nextOption = options.find(function (option) { + return option.get('order') == (currentOption.get('order') + 1); + }); + var nextOptionOrder = nextOption.get('order'); + var currentOptionOrder = currentOption.get('order'); + nextOption.set('order', currentOptionOrder); + currentOption.set('order', nextOptionOrder); + } } }); diff --git a/Dashboard/app/js/templates/navSurveys/question-option.handlebars b/Dashboard/app/js/templates/navSurveys/question-option.handlebars index 6004a2ee85..b998080d94 100644 --- a/Dashboard/app/js/templates/navSurveys/question-option.handlebars +++ b/Dashboard/app/js/templates/navSurveys/question-option.handlebars @@ -1,3 +1,5 @@ {{view Ember.TextField valueBinding="view.content.code" placeholderBinding="Ember.STRINGS._code" size=5}} {{view Ember.TextField valueBinding="view.content.text" placeholderBinding="Ember.STRINGS._text_placeholder" size=15}} -{{t _delete_option}} \ No newline at end of file +{{t _move_up}} +{{t _move_down}} +{{t _delete_option}} diff --git a/GAE/src/locale/en.properties b/GAE/src/locale/en.properties index a69bd8512e..94a9efbc1c 100644 --- a/GAE/src/locale/en.properties +++ b/GAE/src/locale/en.properties @@ -279,12 +279,13 @@ Missing\ date = Missing date Modified = Modified Monitoring = Monitoring Most\ recent\ submissions = Most recent submissions -Move = Move +Move\ down = Move down Move\ group\ here = Move group here Move\ here = Move here Move\ left = Move left Move\ question\ here = Move question here Move\ right = Move right +Move\ up = Move up Moving = Moving Name = Name New\ approval\ group = New approval group diff --git a/GAE/src/locale/ui-strings.properties b/GAE/src/locale/ui-strings.properties index 544947cee3..10b74cf90e 100644 --- a/GAE/src/locale/ui-strings.properties +++ b/GAE/src/locale/ui-strings.properties @@ -303,11 +303,13 @@ _missing_option_text = Options are not defined for these codes _modified = Modified _monitoring = Monitoring _move = Move +_move_down = Move down _move_group_here = Move group here _move_here = Move here _move_left = Move left _move_question_here = Move question here _move_right = Move right +_move_up = Move up _moving = Moving _name = Name _new_approval_group = New approval group From 168d6311684d66ae6f3b0d95eb08782ff20c2f48 Mon Sep 17 00:00:00 2001 From: kymni Date: Wed, 20 Feb 2019 16:30:01 +0300 Subject: [PATCH 042/155] [#2891] remove code causing reordering not to work --- Dashboard/app/js/lib/controllers/survey-controllers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dashboard/app/js/lib/controllers/survey-controllers.js b/Dashboard/app/js/lib/controllers/survey-controllers.js index 9dc7e9d518..e9fce332f3 100644 --- a/Dashboard/app/js/lib/controllers/survey-controllers.js +++ b/Dashboard/app/js/lib/controllers/survey-controllers.js @@ -1225,7 +1225,7 @@ FLOW.questionOptionsControl = Ember.ArrayController.create({ // trimmed whitespace option.set('text', text); - option.set('order', index); + if (!option.get('keyId')) { FLOW.store.createRecord(FLOW.QuestionOption, option); } From 63b53a9acb93ce13d5accae72d0d4345341764e6 Mon Sep 17 00:00:00 2001 From: kymni Date: Thu, 21 Feb 2019 11:42:10 +0300 Subject: [PATCH 043/155] [#2891] disable some links --- Dashboard/app/css/main.scss | 9 ++++++++- .../app/js/lib/controllers/survey-controllers.js | 2 +- .../app/js/lib/views/surveys/question-view.js | 14 +++++++++++++- .../navSurveys/question-option.handlebars | 8 ++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Dashboard/app/css/main.scss b/Dashboard/app/css/main.scss index 8c4f4f3ad3..ed9d3b73d4 100644 --- a/Dashboard/app/css/main.scss +++ b/Dashboard/app/css/main.scss @@ -6341,4 +6341,11 @@ svg { } } } -} \ No newline at end of file +} + +.not-active { + pointer-events: none; + cursor: default; + text-decoration: none; + color: black; +} diff --git a/Dashboard/app/js/lib/controllers/survey-controllers.js b/Dashboard/app/js/lib/controllers/survey-controllers.js index e9fce332f3..8126bd239c 100644 --- a/Dashboard/app/js/lib/controllers/survey-controllers.js +++ b/Dashboard/app/js/lib/controllers/survey-controllers.js @@ -1267,7 +1267,7 @@ FLOW.questionOptionsControl = Ember.ArrayController.create({ moveOptionDown: function (event) { var options = this.content, currentOption = event.view.content; - if (currentOption && currentOption.get('order') < (options.get('length') - 1)) { + if (currentOption && currentOption.get('order') < options.get('length')) { var nextOption = options.find(function (option) { return option.get('order') == (currentOption.get('order') + 1); }); diff --git a/Dashboard/app/js/lib/views/surveys/question-view.js b/Dashboard/app/js/lib/views/surveys/question-view.js index 99370fc67c..caa85635be 100644 --- a/Dashboard/app/js/lib/views/surveys/question-view.js +++ b/Dashboard/app/js/lib/views/surveys/question-view.js @@ -810,5 +810,17 @@ FLOW.OptionListView = Ember.CollectionView.extend({ content: null, itemViewClass: Ember.View.extend({ templateName: 'navSurveys/question-option', - }), + topOption: function () { + var option = this.get('content'); + if (option) { + return option.get('order') == 1; + } + }.property('content.order'), + bottomOption: function () { + var option = this.get('content'), options = FLOW.questionOptionsControl.get('content'); + if (option && options) { + return option.get('order') == options.get('length'); + } + }.property('content.order') + }) }); diff --git a/Dashboard/app/js/templates/navSurveys/question-option.handlebars b/Dashboard/app/js/templates/navSurveys/question-option.handlebars index b998080d94..24f249fce7 100644 --- a/Dashboard/app/js/templates/navSurveys/question-option.handlebars +++ b/Dashboard/app/js/templates/navSurveys/question-option.handlebars @@ -1,5 +1,13 @@ {{view Ember.TextField valueBinding="view.content.code" placeholderBinding="Ember.STRINGS._code" size=5}} {{view Ember.TextField valueBinding="view.content.text" placeholderBinding="Ember.STRINGS._text_placeholder" size=15}} +{{#if view.topOption}} +{{t _move_up}} +{{else}} {{t _move_up}} +{{/if}} +{{#if view.bottomOption}} +{{t _move_down}} +{{else}} {{t _move_down}} +{{/if}} {{t _delete_option}} From e954ddd500183d3e3d72195e13ba6973c83f64f8 Mon Sep 17 00:00:00 2001 From: kymni Date: Fri, 22 Feb 2019 11:12:25 +0300 Subject: [PATCH 044/155] [#2917] order forms alphabetically on surveys tab --- .../js/lib/controllers/survey-controllers.js | 41 ++++++++++++++++++- .../templates/navSurveys/project.handlebars | 2 +- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/Dashboard/app/js/lib/controllers/survey-controllers.js b/Dashboard/app/js/lib/controllers/survey-controllers.js index 6013c38199..42a550b272 100644 --- a/Dashboard/app/js/lib/controllers/survey-controllers.js +++ b/Dashboard/app/js/lib/controllers/survey-controllers.js @@ -596,8 +596,9 @@ FLOW.projectControl = Ember.ArrayController.create({ FLOW.surveyControl = Ember.ArrayController.create({ content: null, publishedContent: null, - sortProperties: ['createdDateTime'], + sortProperties: ['name'], sortAscending: true, + orderedForms: null, setPublishedContent: function () { var sgId; @@ -615,6 +616,44 @@ FLOW.surveyControl = Ember.ArrayController.create({ } }.observes('FLOW.selectedControl.selectedSurveyGroup'), + orderForms: function () { + if (FLOW.selectedControl.get('selectedSurveyGroup') && FLOW.selectedControl.selectedSurveyGroup.get('keyId') > 0) { + var sgId = FLOW.selectedControl.selectedSurveyGroup.get('keyId'), self = this; + self.orderedForms = []; + var forms = FLOW.store.filter(FLOW.Survey, function (item) { + return item.get('surveyGroupId') == sgId; + }); + + if (forms && FLOW.selectedControl.selectedSurveyGroup.get('monitoringGroup')) { + var regFormId = FLOW.selectedControl.selectedSurveyGroup.get('newLocaleSurveyId'); + + this.orderedForms.push(forms.find(function (form) { + return form.get('keyId') == regFormId; + })); + + forms.filter( function (form) { + return form.get('keyId') != regFormId; + }).sort(function (a, b) { + var nameA = a.get('name').toUpperCase(); + var nameB = b.get('name').toUpperCase(); + if (nameA < nameB) { + return -1; + } + if (nameA > nameB) { + return 1; + } + return 0; + }).forEach(function (form) { + self.orderedForms.push(form); + }); + } else { + self.orderedForms.push(forms.find(function (form) { + return form.get('surveyGroupId') == sgId; + })); + } + } + }.observes('FLOW.selectedControl.selectedSurveyGroup'), + populateAll: function () { FLOW.store.find(FLOW.Survey); }, diff --git a/Dashboard/app/js/templates/navSurveys/project.handlebars b/Dashboard/app/js/templates/navSurveys/project.handlebars index a79863af3e..bbb080b80e 100644 --- a/Dashboard/app/js/templates/navSurveys/project.handlebars +++ b/Dashboard/app/js/templates/navSurveys/project.handlebars @@ -156,7 +156,7 @@ {{/if}}