diff --git a/GAE/src/com/gallatinsystems/gis/map/MapUtils.java b/GAE/src/com/gallatinsystems/gis/map/MapUtils.java new file mode 100644 index 0000000000..49d1973388 --- /dev/null +++ b/GAE/src/com/gallatinsystems/gis/map/MapUtils.java @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2014 Stichting Akvo (Akvo Foundation) + * + * This file is part of Akvo FLOW. + * + * Akvo FLOW is free software: you can redistribute it and modify it under the terms of + * the GNU Affero General Public License (AGPL) as published by the Free Software Foundation, + * either version 3 of the License or any later version. + * + * Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License included below for more details. + * + * The full license text can also be seen at . + */ + +package com.gallatinsystems.gis.map; + +import java.util.HashMap; +import java.util.Map; + +import org.waterforpeople.mapping.dao.SurveyInstanceDAO; +import org.waterforpeople.mapping.domain.SurveyInstance; + +import net.sf.jsr107cache.Cache; + +import com.gallatinsystems.survey.dao.SurveyDAO; +import com.gallatinsystems.survey.domain.Survey; +import com.gallatinsystems.surveyal.dao.SurveyedLocaleClusterDao; +import com.gallatinsystems.surveyal.domain.SurveyedLocale; +import com.gallatinsystems.surveyal.domain.SurveyedLocaleCluster; + +public class MapUtils { + + // used to multiply latitude and longitude values, to fit them in a long + private static final int MULT = 1000000; + // used to divide long values by MULT, to go back to double values for + // latitude / longitude values + private static final double REVMULT = 0.000001; + + public static void recomputeCluster(Cache cache, SurveyedLocale locale) { + + final SurveyedLocaleClusterDao slcDao = new SurveyedLocaleClusterDao(); + final SurveyInstanceDAO siDao = new SurveyInstanceDAO(); + final SurveyDAO sDao = new SurveyDAO(); + + Long latTotal; + Long lonTotal; + Double latCenter; + Double lonCenter; + Boolean showOnPublicMap = false; + Long surveyId = null; + String surveyIdString = ""; + + if (locale.getLastSurveyalInstanceId() != null) { + SurveyInstance si = siDao.getByKey(locale.getLastSurveyalInstanceId()); + if (si != null) { + surveyId = si.getSurveyId(); + surveyIdString = surveyId.toString(); + + // get public status, first try from cache + String pubKey = surveyIdString + "-publicStatus"; + if (cache.containsKey(pubKey)) { + showOnPublicMap = (Boolean) cache.get(pubKey); + } else { + Survey s = sDao.getByKey(surveyId); + if (s != null) { + showOnPublicMap = showOnPublicMap || "Point".equals(s.getPointType()) + || "PublicInstitution".equals(s.getPointType()); + cache.put(pubKey, showOnPublicMap); + } + } + } + } + + for (int i = 1; i <= 4; i++) { + + String cell = locale.getGeocells().get(i) + "-" + showOnPublicMap.toString(); + + if (cache.containsKey(cell)) { + + @SuppressWarnings("unchecked") + final Map cellMap = (Map) cache.get(cell); + final Long count = (Long) cellMap.get("count"); + + latTotal = cellMap.get("lat") + Math.round(locale.getLatitude() * MULT); + lonTotal = cellMap.get("lon") + Math.round(locale.getLongitude() * MULT); + + addToCache(cache, cell, cellMap.get("id"), count + 1, latTotal, lonTotal); + + SurveyedLocaleCluster clusterInStore = slcDao.getByKey(cellMap.get("id")); + + if (clusterInStore != null) { + clusterInStore.setCount(cellMap.get("count").intValue() + 1); + clusterInStore.setLatCenter(REVMULT * latTotal / (count + 1)); + clusterInStore.setLonCenter(REVMULT * lonTotal / (count + 1)); + slcDao.save(clusterInStore); + } + + } else { + // try to get it in the datastore. This can happen when the + // cache has expired + final SurveyedLocaleCluster clusterInStore = slcDao.getExistingCluster(locale + .getGeocells().get(i), showOnPublicMap); + + if (clusterInStore != null) { + final Long count = clusterInStore.getCount().longValue(); + + latCenter = (clusterInStore.getLatCenter() * count + locale.getLatitude()) + / (count + 1); + + lonCenter = (clusterInStore.getLonCenter() * count + locale.getLongitude()) + / (count + 1); + + addToCache(cache, cell, clusterInStore.getKey().getId(), + clusterInStore.getCount() + 1, + Math.round(MULT * latCenter * (count + 1)), + Math.round(MULT * lonCenter * (count + 1))); + + clusterInStore.setCount(clusterInStore.getCount() + 1); + clusterInStore.setLatCenter(latCenter); + clusterInStore.setLonCenter(lonCenter); + slcDao.save(clusterInStore); + } else { + // create a new one + SurveyedLocaleCluster slcNew = new SurveyedLocaleCluster(locale.getLatitude(), + locale.getLongitude(), locale.getGeocells().subList(0, i), locale + .getGeocells().get(i), i + 1, locale.getKey().getId(), + showOnPublicMap, locale.getLastSurveyedDate()); + + slcDao.save(slcNew); + + addToCache(cache, cell, slcNew.getKey().getId(), 1, + Math.round(MULT * locale.getLatitude()), + Math.round(MULT * locale.getLongitude())); + } + } + } + + } + + private static void addToCache(Cache cache, String cell, Long id, long count, Long latTotal, + Long lonTotal) { + final Map v = new HashMap(); + v.put("count", count); + v.put("id", id); + // the cache stores lat/lon values as longs. We store the sums over the + // whole cluster. + v.put("lat", latTotal); + v.put("lon", lonTotal); + cache.put(cell, v); + } +} diff --git a/GAE/src/com/gallatinsystems/surveyal/app/web/SurveyalRestServlet.java b/GAE/src/com/gallatinsystems/surveyal/app/web/SurveyalRestServlet.java index 5b0a92a65d..02c7cee637 100644 --- a/GAE/src/com/gallatinsystems/surveyal/app/web/SurveyalRestServlet.java +++ b/GAE/src/com/gallatinsystems/surveyal/app/web/SurveyalRestServlet.java @@ -47,6 +47,7 @@ import com.gallatinsystems.gis.geography.domain.Country; import com.gallatinsystems.gis.location.GeoLocationServiceGeonamesImpl; import com.gallatinsystems.gis.location.GeoPlace; +import com.gallatinsystems.gis.map.MapUtils; import com.gallatinsystems.gis.map.domain.OGRFeature; import com.gallatinsystems.metric.dao.MetricDao; import com.gallatinsystems.metric.dao.SurveyMetricMappingDao; @@ -56,11 +57,9 @@ import com.gallatinsystems.survey.dao.SurveyDAO; import com.gallatinsystems.survey.domain.Question; import com.gallatinsystems.survey.domain.Survey; -import com.gallatinsystems.surveyal.dao.SurveyedLocaleClusterDao; import com.gallatinsystems.surveyal.dao.SurveyedLocaleDao; import com.gallatinsystems.surveyal.domain.SurveyalValue; import com.gallatinsystems.surveyal.domain.SurveyedLocale; -import com.gallatinsystems.surveyal.domain.SurveyedLocaleCluster; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.memcache.MemcacheService; import com.google.appengine.api.memcache.stdimpl.GCacheFactory; @@ -85,10 +84,6 @@ public class SurveyalRestServlet extends AbstractRestApiServlet { private static final double UNSET_VAL = -9999.9; private static final String DEFAULT = "DEFAULT"; private static final String DEFAULT_ORG_PROP = "defaultOrg"; - // used to multiply latitude and longitude values, to fit them in a long - private static final int MULT = 1000000; - // used to divide long values by MULT, to go back to double values for latitude / longitude values - private static final double REVMULT = 0.000001; private static final Logger log = Logger .getLogger(SurveyalRestServlet.class.getName()); @@ -486,20 +481,9 @@ private void ingestSurveyInstance(SurveyInstance instance) { // this method is synchronised, because we are changing counts. @SuppressWarnings({ "unchecked", "rawtypes" }) private synchronized void adaptClusterData(Long surveyedLocaleId) { - SurveyDAO sDao = new SurveyDAO(); - SurveyedLocaleClusterDao slcDao = new SurveyedLocaleClusterDao(); - SurveyedLocaleDao slDao = new SurveyedLocaleDao(); - SurveyInstanceDAO siDao = new SurveyInstanceDAO(); - Long surveyId = null; - String surveyIdString = ""; - Boolean showOnPublicMap = false; - Double latCenter; - Double lonCenter; - Long latTotal; - Long lonTotal; - Long count; - - SurveyedLocale locale = slDao.getById(surveyedLocaleId); + final SurveyedLocaleDao slDao = new SurveyedLocaleDao(); + final SurveyedLocale locale = slDao.getById(surveyedLocaleId); + if (locale == null){ log.log(Level.SEVERE, "Couldn't find surveyedLocale with id: " + surveyedLocaleId); @@ -533,77 +517,9 @@ private synchronized void adaptClusterData(Long surveyedLocaleId) { queue.add(to); return; } + + MapUtils.recomputeCluster(cache, locale); - if (locale.getLastSurveyalInstanceId() != null){ - SurveyInstance si = siDao.getByKey(locale.getLastSurveyalInstanceId()); - if (si != null) { - surveyId = si.getSurveyId(); - surveyIdString = surveyId.toString(); - - // get public status, first try from cache - String pubKey = surveyIdString + "-publicStatus"; - if (cache.containsKey(pubKey)){ - showOnPublicMap = (Boolean) cache.get(pubKey); - } else { - Survey s = sDao.getByKey(surveyId); - if (s != null){ - showOnPublicMap = showOnPublicMap || s.getPointType().equals("Point") || s.getPointType().equals("PublicInstitution"); - cache.put(pubKey, showOnPublicMap); - } - } - } - } - - Map cellMap; - for (int i = 1 ; i <= 4 ; i++){ - String cell = locale.getGeocells().get(i) + "-" + showOnPublicMap.toString(); - if (cache.containsKey(cell)){ - cellMap = (Map) cache.get(cell); - count = cellMap.get("count"); - latTotal = cellMap.get("lat") + Math.round(locale.getLatitude() * MULT); - lonTotal = cellMap.get("lon") + Math.round(locale.getLongitude() * MULT); - addToCache(cache, cell,cellMap.get("id"), count + 1, latTotal, lonTotal); - SurveyedLocaleCluster clusterInStore = slcDao.getByKey(cellMap.get("id")); - if (clusterInStore != null){ - clusterInStore.setCount(cellMap.get("count").intValue() + 1); - clusterInStore.setLatCenter(REVMULT * latTotal / (count + 1)); - clusterInStore.setLonCenter(REVMULT * lonTotal / (count + 1)); - slcDao.save(clusterInStore); - } - } else { - // try to get it in the datastore. This can happen when the cache - // has expired - SurveyedLocaleCluster clusterInStore = slcDao.getExistingCluster(locale.getGeocells().get(i), showOnPublicMap); - if (clusterInStore != null){ - count = clusterInStore.getCount().longValue(); - latCenter = (clusterInStore.getLatCenter() * count + locale.getLatitude()) / (count + 1); - lonCenter = (clusterInStore.getLonCenter() * count + locale.getLongitude()) / (count + 1); - addToCache(cache, cell, clusterInStore.getKey().getId(), - clusterInStore.getCount() + 1, Math.round(MULT * latCenter), Math.round(MULT * lonCenter)); - clusterInStore.setCount(clusterInStore.getCount() + 1); - clusterInStore.setLatCenter(latCenter); - clusterInStore.setLonCenter(lonCenter); - slcDao.save(clusterInStore); - } else { - // create a new one - SurveyedLocaleCluster slcNew = new SurveyedLocaleCluster(locale.getLatitude(), - locale.getLongitude(), locale.getGeocells().subList(0,i), - locale.getGeocells().get(i), i + 1, locale.getKey().getId(), showOnPublicMap,locale.getLastSurveyedDate()); - slcDao.save(slcNew); - addToCache(cache, cell, slcNew.getKey().getId(),1, Math.round(MULT * locale.getLatitude()), Math.round(MULT * locale.getLongitude())); - } - } - } - } - - private void addToCache(Cache cache, String cell, Long id, long count, Long latTotal, Long lonTotal){ - final Map v = new HashMap(); - v.put("count", count); - v.put("id", id); - // the cache stores lat/lon values as longs. We store the sums over the whole cluster. - v.put("lat",latTotal); - v.put("lon",lonTotal); - cache.put(cell, v); } private boolean isStatus(String name) { diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index ed138e2a96..d64cfba4ee 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -59,6 +59,7 @@ import com.gallatinsystems.gis.location.GeoLocationService; import com.gallatinsystems.gis.location.GeoLocationServiceGeonamesImpl; import com.gallatinsystems.gis.location.GeoPlace; +import com.gallatinsystems.gis.map.MapUtils; import com.gallatinsystems.messaging.dao.MessageDao; import com.gallatinsystems.messaging.domain.Message; import com.gallatinsystems.operations.dao.ProcessingStatusDao; @@ -73,11 +74,9 @@ import com.gallatinsystems.survey.domain.QuestionGroup; import com.gallatinsystems.survey.domain.QuestionOption; import com.gallatinsystems.survey.domain.Survey; -import com.gallatinsystems.surveyal.dao.SurveyedLocaleClusterDao; import com.gallatinsystems.survey.domain.Translation; import com.gallatinsystems.surveyal.dao.SurveyedLocaleDao; import com.gallatinsystems.surveyal.domain.SurveyedLocale; -import com.gallatinsystems.surveyal.domain.SurveyedLocaleCluster; import com.google.appengine.api.backends.BackendServiceFactory; import com.google.appengine.api.memcache.MemcacheService; import com.google.appengine.api.memcache.stdimpl.GCacheFactory; @@ -100,10 +99,6 @@ public class DataProcessorRestServlet extends AbstractRestApiServlet { private static final Integer LOCALE_PAGE_SIZE = 500; private static final Integer T_PAGE_SIZE = 300; private static final String QAS_TO_REMOVE = "QAStoRemove"; - // used to multiply latitude and longitude values, to fit them in a long - private static final int MULT = 1000000; - // used to divide long values by MULT, to go back to double values for latitude / longitude values - private static final double REVMULT = 0.000001; @Override protected RestRequest convertRequest() throws Exception { @@ -431,19 +426,9 @@ private void deleteDuplicatedQAS(Long offset) { */ @SuppressWarnings({ "unchecked", "rawtypes" }) private void recomputeLocaleClusters(String cursor) { - Long surveyId; - Boolean showOnPublicMap; - Long latTotal; - Long lonTotal; - Double latCenter; - Double lonCenter; - Long count; log.log(Level.INFO, "recomputing locale clusters [cursor: " + cursor + "]"); - SurveyDAO sDao = new SurveyDAO(); - SurveyInstanceDAO siDao = new SurveyInstanceDAO(); - SurveyedLocaleClusterDao slcDao = new SurveyedLocaleClusterDao(); SurveyedLocaleDao slDao = new SurveyedLocaleDao(); final List results = slDao.listAll(cursor, LOCALE_PAGE_SIZE); @@ -466,71 +451,9 @@ private void recomputeLocaleClusters(String cursor) { } for (SurveyedLocale locale : results) { - surveyId = null; - showOnPublicMap = false; - String surveyIdString = ""; - if (locale.getLastSurveyalInstanceId() != null){ - SurveyInstance si = siDao.getByKey(locale.getLastSurveyalInstanceId()); - if (si != null) { - surveyId = si.getSurveyId(); - surveyIdString = surveyId.toString(); - - // get public status, first try from cache - String pubKey = surveyIdString + "-publicStatus"; - if (cache.containsKey(pubKey)){ - showOnPublicMap = (Boolean) cache.get(pubKey); - } else { - Survey s = sDao.getByKey(surveyId); - if (s != null){ - showOnPublicMap = (showOnPublicMap != null && showOnPublicMap) || "Point".equals(s.getPointType()) || "PublicInstitution".equals(s.getPointType()); - cache.put(pubKey, showOnPublicMap); - } - } - } - } // adjust Geocell cluster data if (locale.getGeocells() != null && !locale.getGeocells().isEmpty()){ - - Map cellMap; - for (int i = 1 ; i <= 4 ; i++){ - String cell = locale.getGeocells().get(i) + "-" + showOnPublicMap.toString(); - if (cache.containsKey(cell)){ - cellMap = (Map) cache.get(cell); - count = cellMap.get("count"); - latTotal = cellMap.get("lat") + Math.round(locale.getLatitude() * MULT); - lonTotal = cellMap.get("lon") + Math.round(locale.getLongitude() * MULT); - addToCache(cache, cell,cellMap.get("id"), count + 1, latTotal, lonTotal); - SurveyedLocaleCluster clusterInStore = slcDao.getByKey(cellMap.get("id")); - if (clusterInStore != null){ - clusterInStore.setCount(cellMap.get("count").intValue() + 1); - clusterInStore.setLatCenter(REVMULT * latTotal / (count + 1)); - clusterInStore.setLonCenter(REVMULT * lonTotal / (count + 1)); - slcDao.save(clusterInStore); - } - } else { - // try to get it in the datastore. This can happen when the cache - // has expired - SurveyedLocaleCluster clusterInStore = slcDao.getExistingCluster(locale.getGeocells().get(i), showOnPublicMap); - if (clusterInStore != null){ - count = clusterInStore.getCount().longValue(); - latCenter = (clusterInStore.getLatCenter() * count + locale.getLatitude()) / (count + 1); - lonCenter = (clusterInStore.getLonCenter() * count + locale.getLongitude()) / (count + 1); - addToCache(cache, cell, clusterInStore.getKey().getId(), - clusterInStore.getCount() + 1, Math.round(MULT * latCenter), Math.round(MULT * lonCenter)); - clusterInStore.setCount(clusterInStore.getCount() + 1); - clusterInStore.setLatCenter(latCenter); - clusterInStore.setLonCenter(lonCenter); - slcDao.save(clusterInStore); - } else { - // create a new one - SurveyedLocaleCluster slcNew = new SurveyedLocaleCluster(locale.getLatitude(), - locale.getLongitude(), locale.getGeocells().subList(0,i), - locale.getGeocells().get(i), i + 1, locale.getKey().getId(), showOnPublicMap,locale.getLastSurveyedDate()); - slcDao.save(slcNew); - addToCache(cache, cell, slcNew.getKey().getId(),1, Math.round(MULT * locale.getLatitude()), Math.round(MULT * locale.getLongitude())); - } - } - } + MapUtils.recomputeCluster(cache, locale); } } @@ -548,16 +471,6 @@ private void recomputeLocaleClusters(String cursor) { } } - private void addToCache(Cache cache, String cell, Long id, long count, Long latTotal, Long lonTotal){ - final Map v = new HashMap(); - v.put("count", count); - v.put("id", id); - // the cache stores lat/lon values as longs. We store the sums over the whole cluster. - v.put("lat",latTotal); - v.put("lon",lonTotal); - cache.put(cell, v); - } - /** * this method re-runs scoring on all access points for a country * diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 56a08b159a..54ed4e041e 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,6 +5,11 @@ Read more about the [Akvo Platform](http://www.akvo.org/blog/?p=4822). Akvo FLOW Dashboard release notes ---- +#1.6.14.4 + +## Resolved issues +* Fixed wrong computation of cache value for cluster locations [#519] + #1.6.14.3 ## Resolved issues