Skip to content

Commit

Permalink
Merge pull request #3063 from akvo/issue/2772-add-form-version-column
Browse files Browse the repository at this point in the history 
Issue/2772 add form version column
  • Loading branch information
@stellanl
stellanl authored Apr 10, 2019
2 parents 6bc1275 + 3a3785c commit b6fd97c
Show file tree
Hide file tree
Showing 8 changed files with 252 additions and 159 deletions.
3 changes: 3 additions & 0 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,6 @@ script:

before_cache:
- find $HOME/.m2 -name resolver-status.properties -exec rm {} \;

notifications:
slack: akvo:ZLetmotGiT22QryK6pR5bnFS
11 changes: 10 additions & 1 deletion GAE/src/org/waterforpeople/mapping/app/gwt/client/surveyinstance/SurveyInstanceDto.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2010-2014 Stichting Akvo (Akvo Foundation)
* Copyright (C) 2010-2014, 2019 Stichting Akvo (Akvo Foundation)
*
* This file is part of Akvo FLOW.
*
Expand Down Expand Up @@ -40,13 +40,22 @@ public class SurveyInstanceDto extends BaseDto {
private Long surveyalTime = 0L;

private String submitterName;
private Double formVersion; //What form version was used to collect the data
private String deviceIdentifier;
private String surveyCode;
private String approvedFlag;
private Long surveyedLocaleId;
private String surveyedLocaleIdentifier;
private String surveyedLocaleDisplayName;

public Double getFormVersion() {
return formVersion;
}

public void setFormVersion(Double formVersion) {
this.formVersion = formVersion;
}

public String getApprovedFlag() {
return approvedFlag;
}
Expand Down
1 change: 1 addition & 0 deletions GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,7 @@ private SurveyInstance createInstance(RawDataImportRequest importReq) {
inst.setUuid(UUID.randomUUID().toString());
inst.setSubmitterName(importReq.getSubmitter());
inst.setSurveyalTime(importReq.getSurveyDuration());
inst.setFormVersion(importReq.getFormVersion());

// set the key so the subsequent logic can populate it in the
// QuestionAnswerStore objects
Expand Down
115 changes: 71 additions & 44 deletions GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2010-2015 Stichting Akvo (Akvo Foundation)
* Copyright (C) 2010-2015, 2019 Stichting Akvo (Akvo Foundation)
*
* This file is part of Akvo FLOW.
*
Expand All @@ -16,6 +16,7 @@

package org.waterforpeople.mapping.app.web.dto;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
Expand Down Expand Up @@ -51,6 +52,7 @@ protected DateFormat initialValue() {
public static final String FIXED_FIELD_VALUE_PARAM = "values";
public static final String LOCALE_ID_PARAM = "surveyedLocale";
public static final String DURATION_PARAM = "duration";
public static final String FORM_VER_PARAM = "formVersion";

public static final String SAVE_SURVEY_INSTANCE_ACTION = "saveSurveyInstance";
public static final String RESET_SURVEY_INSTANCE_ACTION = "resetSurveyInstance";
Expand All @@ -66,6 +68,7 @@ protected DateFormat initialValue() {
private Long duration = null;
private Date collectionDate = null;
private String submitter = null;
private Double formVersion = null;

// questionId -> iteration -> [response, type]
private Map<Long, Map<Integer, String[]>> responseMap = new HashMap<>();
Expand Down Expand Up @@ -166,49 +169,7 @@ protected void populateFields(HttpServletRequest req) throws Exception {
}
}
if (req.getParameter(QUESTION_ID_PARAM) != null) {
String[] answers = req.getParameterValues(QUESTION_ID_PARAM);
if (answers != null) {
for (String answer : answers) {
// answer: 242334|0=abc|1=def|2=ghi|type=VALUE
// The iteration responses are also URLEncoded in order to escape pipe
// characters
String[] parts = URLDecoder.decode(answer, "UTF-8").split("\\|");
Map<Integer, String> iterations = new HashMap<>();
Long questionId = Long.valueOf(parts[0]);
String type = null;
for (int i = 1; i < parts.length; i++) {
String part = parts[i];
String[] keyValue = part.split("=");
if (keyValue.length == 2) {
String key = keyValue[0];
String val = keyValue[1];

switch (key) {
case "type":
type = val;
break;
default:
// key is the iteration and value the response
iterations.put(Integer.valueOf(key),
URLDecoder.decode(val, "UTF-8"));
break;
}
}
}

if (questionId != null && type != null) {

for (Entry<Integer, String> iterationEntry : iterations.entrySet()) {
putResponse(questionId, iterationEntry.getKey(),
iterationEntry.getValue(),
type);
}
} else {
log.log(Level.WARNING, "Could not parse \"" + answer
+ "\" as RawDataImportRequest");
}
}
}
handleQuestionIdParam(req.getParameterValues(QUESTION_ID_PARAM));
} else {
log.warning("No question answers to import");
}
Expand Down Expand Up @@ -238,6 +199,64 @@ protected void populateFields(HttpServletRequest req) throws Exception {
setSurveyDuration(0L);
}
}
if (req.getParameter(FORM_VER_PARAM) != null) {
try {
setFormVersion(Double.valueOf(req.getParameter(FORM_VER_PARAM)));
} catch (NumberFormatException e) {
log.warning("Could not parse " + FORM_VER_PARAM + ": "
+ req.getParameter(FORM_VER_PARAM));
setFormVersion(0.0);
}
}
}

/**
* @param req
* @throws UnsupportedEncodingException
*/
private void handleQuestionIdParam(String[] answers) throws UnsupportedEncodingException {
if (answers != null) {
for (String answer : answers) {
// answer: 242334|0=abc|1=def|2=ghi|type=VALUE
// The iteration responses are also URLEncoded in order to escape pipe
// characters
String[] parts = URLDecoder.decode(answer, "UTF-8").split("\\|");
Map<Integer, String> iterations = new HashMap<>();
Long questionId = Long.valueOf(parts[0]);
String type = null;
for (int i = 1; i < parts.length; i++) {
String part = parts[i];
String[] keyValue = part.split("=");
if (keyValue.length == 2) {
String key = keyValue[0];
String val = keyValue[1];

switch (key) {
case "type":
type = val;
break;
default:
// key is the iteration and value the response
iterations.put(Integer.valueOf(key),
URLDecoder.decode(val, "UTF-8"));
break;
}
}
}

if (questionId != null && type != null) {

for (Entry<Integer, String> iterationEntry : iterations.entrySet()) {
putResponse(questionId, iterationEntry.getKey(),
iterationEntry.getValue(),
type);
}
} else {
log.log(Level.WARNING, "Could not parse \"" + answer
+ "\" as RawDataImportRequest");
}
}
}
}

public void setSubmitter(String submitter) {
Expand All @@ -264,4 +283,12 @@ public Long getSurveyDuration() {
return duration;
}

public Double getFormVersion() {
return formVersion;
}

public void setFormVersion(Double formVersion) {
this.formVersion = formVersion;
}

}
44 changes: 44 additions & 0 deletions GAE/src/org/waterforpeople/mapping/dataexport/ExportImportConstants.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright (C) 2019 Stichting Akvo (Akvo Foundation)
*
* This file is part of Akvo FLOW.
*
* Akvo FLOW is free software: you can redistribute it and modify it under the terms of
* the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
* either version 3 of the License or any later version.
*
* Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU Affero General Public License included below for more details.
*
* The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
*/
package org.waterforpeople.mapping.dataexport;

public final class ExportImportConstants {

//Output column group header, indicates modern data-cleaning format
protected static final String METADATA_LABEL = "Metadata";

//Metadata column headers, in canonical order
protected static final String IDENTIFIER_LABEL = "Identifier";
protected static final String DATA_APPROVAL_STATUS_LABEL = "Data approval status";
protected static final String REPEAT_LABEL = "Repeat no";
protected static final String DISPLAY_NAME_LABEL = "Display Name";
protected static final String DEVICE_IDENTIFIER_LABEL = "Device identifier";
protected static final String INSTANCE_LABEL = "Instance";
protected static final String SUB_DATE_LABEL = "Submission Date";
protected static final String SUBMITTER_LABEL = "Submitter";
protected static final String DURATION_LABEL = "Duration";
protected static final String FORM_VER_LABEL = "Form version"; //Good name?

//Data column headers
protected static final String LAT_LABEL = "Latitude";
protected static final String LON_LABEL = "Longitude";
protected static final String IMAGE_LABEL = "Image";
protected static final String ELEV_LABEL = "Elevation";
protected static final String ACC_LABEL = "Accuracy (m)";
protected static final String OTHER_SUFFIX = "--OTHER--";
protected static final String GEO_PREFIX = "--GEO";
protected static final String CADDISFLY_PREFIX = "--CADDISFLY";
}
36 changes: 15 additions & 21 deletions GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
import org.waterforpeople.mapping.domain.CaddisflyResult;
import org.waterforpeople.mapping.domain.response.value.Media;
import org.waterforpeople.mapping.serialization.response.MediaResponse;
import static org.waterforpeople.mapping.dataexport.ExportImportConstants.*;

import com.gallatinsystems.survey.dao.CaddisflyResourceDao;

Expand Down Expand Up @@ -107,17 +108,14 @@ public class GraphicalSurveySummaryExporter extends SurveySummaryExporter {

private static final String DIGEST_COLUMN = "NO_TITLE_DIGEST_COLUMN";

private static final String METADATA_LABEL = "Metadata";


//Statistics page header and labels
private static final String REPORT_HEADER = "Survey Summary Report";
private static final String FREQ_LABEL = "Frequency";
private static final String PCT_LABEL = "Percent";
private static final String SUMMARY_LABEL = "Summary";
private static final String RAW_DATA_LABEL = "Raw Data";
private static final String INSTANCE_LABEL = "Instance";
private static final String SUB_DATE_LABEL = "Submission Date";
private static final String SUBMITTER_LABEL = "Submitter";
private static final String DURATION_LABEL = "Duration";
private static final String REPEAT_LABEL = "Repeat no";
private static final String MEAN_LABEL = "Mean";
private static final String MEDIAN_LABEL = "Median";
private static final String MIN_LABEL = "Min";
Expand All @@ -127,16 +125,6 @@ public class GraphicalSurveySummaryExporter extends SurveySummaryExporter {
private static final String STD_D_LABEL = "Std Deviation";
private static final String TOTAL_LABEL = "Total";
private static final String RANGE_LABEL = "Range";
private static final String LAT_LABEL = "Latitude";
private static final String LON_LABEL = "Longitude";
private static final String IMAGE_LABEL = "Image";
private static final String ELEV_LABEL = "Elevation";
private static final String ACC_LABEL = "Accuracy (m)";
private static final String IDENTIFIER_LABEL = "Identifier";
private static final String DISPLAY_NAME_LABEL = "Display Name";
private static final String DEVICE_IDENTIFIER_LABEL = "Device identifier";
private static final String DATA_APPROVAL_STATUS_LABEL = "Data approval status";
private static final String OTHER_TAG = "--OTHER--";

// Maximum number of rows of a sheet kept in memory
// We must take care to never go back up longer than this
Expand Down Expand Up @@ -209,7 +197,7 @@ public void export(Map<String, String> criteria, File fileName,
final String surveyId = criteria.get(SurveyRestRequest.SURVEY_ID_PARAM).trim();
final String apiKey = criteria.get("apiKey").trim();

log.debug("===Export criteria=" + criteria.toString() +
log.debug("### Export criteria=" + criteria.toString() +
" filename=" + fileName.toString() +
" options=" + options.toString() );
if (!processOptions(options)) {
Expand Down Expand Up @@ -604,6 +592,11 @@ private void writeMetadata(Sheet sheet,
createCell(r, col++, sanitize(dto.getSubmitterName()));
String duration = getDurationText(dto.getSurveyalTime());
createCell(r, col++, duration);
Double v = dto.getFormVersion();
if (v != null) {
createCell(r, col++, v);
}
//TODO else a cell with 0.0??
}

}
Expand Down Expand Up @@ -1293,6 +1286,7 @@ private int addMetaDataHeaders(Sheet sheet, boolean showRepeatColumn) {
addMetaDataColumnHeader(SUB_DATE_LABEL, ++columnIdx, row);
addMetaDataColumnHeader(SUBMITTER_LABEL, ++columnIdx, row);
addMetaDataColumnHeader(DURATION_LABEL, ++columnIdx, row);
addMetaDataColumnHeader(FORM_VER_LABEL, ++columnIdx, row);
//Always put something in the top-left corner to identify the format
if (doGroupHeaders) {
row = getRow(0, sheet);
Expand Down Expand Up @@ -1524,7 +1518,7 @@ private int addGeoDataColumnHeaders(QuestionDto q, Row row,
createHeaderCell(row, offset++, q.getText() + " - " + LON_LABEL);
createHeaderCell(row, offset++, q.getText() + " - " + ELEV_LABEL);
}
} else { //Import currently relies on the --GEO headers
} else { //Import currently relies on the --GEO header prefix
createHeaderCell(row, offset++, q.getKeyId() + "|" + LAT_LABEL);
createHeaderCell(row, offset++, "--GEOLON--|" + LON_LABEL);
createHeaderCell(row, offset++, "--GEOELE--|" + ELEV_LABEL);
Expand Down Expand Up @@ -1557,7 +1551,7 @@ private int addExtraOptionColumnHeaders(String header,

// add 'other' column if needed
if (safeTrue(q.getAllowOtherFlag())){
createHeaderCell(row, offset++, header + OTHER_TAG);
createHeaderCell(row, offset++, header + OTHER_SUFFIX);
}

optionMap.put(q.getKeyId(), qoList);
Expand Down Expand Up @@ -2103,8 +2097,8 @@ public static void main(String[] args) {
GraphicalSurveySummaryExporter exporter = new GraphicalSurveySummaryExporter();
Map<String, String> criteria = new HashMap<String, String>();
Map<String, String> options = new HashMap<String, String>();
// options.put(TYPE_OPT, DATA_CLEANING_TYPE);
options.put(TYPE_OPT, DATA_ANALYSIS_TYPE);
options.put(TYPE_OPT, DATA_CLEANING_TYPE);
// options.put(TYPE_OPT, DATA_ANALYSIS_TYPE);
// options.put(TYPE_OPT, COMPREHENSIVE_TYPE);
options.put(LAST_COLLECTION_OPT, "false");
options.put(EMAIL_OPT, "email@example.com");
Expand Down
Loading

0 comments on commit b6fd97c

Please sign in to comment.