diff --git a/.travis.yml b/.travis.yml index 55cdef7192..15b3564693 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,3 +13,6 @@ script: before_cache: - find $HOME/.m2 -name resolver-status.properties -exec rm {} \; + +notifications: + slack: akvo:ZLetmotGiT22QryK6pR5bnFS \ No newline at end of file diff --git a/GAE/src/org/waterforpeople/mapping/app/gwt/client/surveyinstance/SurveyInstanceDto.java b/GAE/src/org/waterforpeople/mapping/app/gwt/client/surveyinstance/SurveyInstanceDto.java index ce8582d8c4..ecc93919db 100644 --- a/GAE/src/org/waterforpeople/mapping/app/gwt/client/surveyinstance/SurveyInstanceDto.java +++ b/GAE/src/org/waterforpeople/mapping/app/gwt/client/surveyinstance/SurveyInstanceDto.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2014, 2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -40,6 +40,7 @@ public class SurveyInstanceDto extends BaseDto { private Long surveyalTime = 0L; private String submitterName; + private Double formVersion; //What form version was used to collect the data private String deviceIdentifier; private String surveyCode; private String approvedFlag; @@ -47,6 +48,14 @@ public class SurveyInstanceDto extends BaseDto { private String surveyedLocaleIdentifier; private String surveyedLocaleDisplayName; + public Double getFormVersion() { + return formVersion; + } + + public void setFormVersion(Double formVersion) { + this.formVersion = formVersion; + } + public String getApprovedFlag() { return approvedFlag; } diff --git a/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java index 0f0200164f..17a8d3d09f 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java @@ -435,6 +435,7 @@ private SurveyInstance createInstance(RawDataImportRequest importReq) { inst.setUuid(UUID.randomUUID().toString()); inst.setSubmitterName(importReq.getSubmitter()); inst.setSurveyalTime(importReq.getSurveyDuration()); + inst.setFormVersion(importReq.getFormVersion()); // set the key so the subsequent logic can populate it in the // QuestionAnswerStore objects diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java index 19aee1b7d8..debaab4f0e 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2015 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2015, 2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -16,6 +16,7 @@ package org.waterforpeople.mapping.app.web.dto; +import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -51,6 +52,7 @@ protected DateFormat initialValue() { public static final String FIXED_FIELD_VALUE_PARAM = "values"; public static final String LOCALE_ID_PARAM = "surveyedLocale"; public static final String DURATION_PARAM = "duration"; + public static final String FORM_VER_PARAM = "formVersion"; public static final String SAVE_SURVEY_INSTANCE_ACTION = "saveSurveyInstance"; public static final String RESET_SURVEY_INSTANCE_ACTION = "resetSurveyInstance"; @@ -66,6 +68,7 @@ protected DateFormat initialValue() { private Long duration = null; private Date collectionDate = null; private String submitter = null; + private Double formVersion = null; // questionId -> iteration -> [response, type] private Map> responseMap = new HashMap<>(); @@ -166,49 +169,7 @@ protected void populateFields(HttpServletRequest req) throws Exception { } } if (req.getParameter(QUESTION_ID_PARAM) != null) { - String[] answers = req.getParameterValues(QUESTION_ID_PARAM); - if (answers != null) { - for (String answer : answers) { - // answer: 242334|0=abc|1=def|2=ghi|type=VALUE - // The iteration responses are also URLEncoded in order to escape pipe - // characters - String[] parts = URLDecoder.decode(answer, "UTF-8").split("\\|"); - Map iterations = new HashMap<>(); - Long questionId = Long.valueOf(parts[0]); - String type = null; - for (int i = 1; i < parts.length; i++) { - String part = parts[i]; - String[] keyValue = part.split("="); - if (keyValue.length == 2) { - String key = keyValue[0]; - String val = keyValue[1]; - - switch (key) { - case "type": - type = val; - break; - default: - // key is the iteration and value the response - iterations.put(Integer.valueOf(key), - URLDecoder.decode(val, "UTF-8")); - break; - } - } - } - - if (questionId != null && type != null) { - - for (Entry iterationEntry : iterations.entrySet()) { - putResponse(questionId, iterationEntry.getKey(), - iterationEntry.getValue(), - type); - } - } else { - log.log(Level.WARNING, "Could not parse \"" + answer - + "\" as RawDataImportRequest"); - } - } - } + handleQuestionIdParam(req.getParameterValues(QUESTION_ID_PARAM)); } else { log.warning("No question answers to import"); } @@ -238,6 +199,64 @@ protected void populateFields(HttpServletRequest req) throws Exception { setSurveyDuration(0L); } } + if (req.getParameter(FORM_VER_PARAM) != null) { + try { + setFormVersion(Double.valueOf(req.getParameter(FORM_VER_PARAM))); + } catch (NumberFormatException e) { + log.warning("Could not parse " + FORM_VER_PARAM + ": " + + req.getParameter(FORM_VER_PARAM)); + setFormVersion(0.0); + } + } + } + + /** + * @param req + * @throws UnsupportedEncodingException + */ + private void handleQuestionIdParam(String[] answers) throws UnsupportedEncodingException { + if (answers != null) { + for (String answer : answers) { + // answer: 242334|0=abc|1=def|2=ghi|type=VALUE + // The iteration responses are also URLEncoded in order to escape pipe + // characters + String[] parts = URLDecoder.decode(answer, "UTF-8").split("\\|"); + Map iterations = new HashMap<>(); + Long questionId = Long.valueOf(parts[0]); + String type = null; + for (int i = 1; i < parts.length; i++) { + String part = parts[i]; + String[] keyValue = part.split("="); + if (keyValue.length == 2) { + String key = keyValue[0]; + String val = keyValue[1]; + + switch (key) { + case "type": + type = val; + break; + default: + // key is the iteration and value the response + iterations.put(Integer.valueOf(key), + URLDecoder.decode(val, "UTF-8")); + break; + } + } + } + + if (questionId != null && type != null) { + + for (Entry iterationEntry : iterations.entrySet()) { + putResponse(questionId, iterationEntry.getKey(), + iterationEntry.getValue(), + type); + } + } else { + log.log(Level.WARNING, "Could not parse \"" + answer + + "\" as RawDataImportRequest"); + } + } + } } public void setSubmitter(String submitter) { @@ -264,4 +283,12 @@ public Long getSurveyDuration() { return duration; } + public Double getFormVersion() { + return formVersion; + } + + public void setFormVersion(Double formVersion) { + this.formVersion = formVersion; + } + } diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/ExportImportConstants.java b/GAE/src/org/waterforpeople/mapping/dataexport/ExportImportConstants.java new file mode 100644 index 0000000000..8d8ff6d03c --- /dev/null +++ b/GAE/src/org/waterforpeople/mapping/dataexport/ExportImportConstants.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2019 Stichting Akvo (Akvo Foundation) + * + * This file is part of Akvo FLOW. + * + * Akvo FLOW is free software: you can redistribute it and modify it under the terms of + * the GNU Affero General Public License (AGPL) as published by the Free Software Foundation, + * either version 3 of the License or any later version. + * + * Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License included below for more details. + * + * The full license text can also be seen at . + */ +package org.waterforpeople.mapping.dataexport; + +public final class ExportImportConstants { + + //Output column group header, indicates modern data-cleaning format + protected static final String METADATA_LABEL = "Metadata"; + + //Metadata column headers, in canonical order + protected static final String IDENTIFIER_LABEL = "Identifier"; + protected static final String DATA_APPROVAL_STATUS_LABEL = "Data approval status"; + protected static final String REPEAT_LABEL = "Repeat no"; + protected static final String DISPLAY_NAME_LABEL = "Display Name"; + protected static final String DEVICE_IDENTIFIER_LABEL = "Device identifier"; + protected static final String INSTANCE_LABEL = "Instance"; + protected static final String SUB_DATE_LABEL = "Submission Date"; + protected static final String SUBMITTER_LABEL = "Submitter"; + protected static final String DURATION_LABEL = "Duration"; + protected static final String FORM_VER_LABEL = "Form version"; //Good name? + + //Data column headers + protected static final String LAT_LABEL = "Latitude"; + protected static final String LON_LABEL = "Longitude"; + protected static final String IMAGE_LABEL = "Image"; + protected static final String ELEV_LABEL = "Elevation"; + protected static final String ACC_LABEL = "Accuracy (m)"; + protected static final String OTHER_SUFFIX = "--OTHER--"; + protected static final String GEO_PREFIX = "--GEO"; + protected static final String CADDISFLY_PREFIX = "--CADDISFLY"; +} diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java index c0bbfd63cc..f85652bd06 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java @@ -75,6 +75,7 @@ import org.waterforpeople.mapping.domain.CaddisflyResult; import org.waterforpeople.mapping.domain.response.value.Media; import org.waterforpeople.mapping.serialization.response.MediaResponse; +import static org.waterforpeople.mapping.dataexport.ExportImportConstants.*; import com.gallatinsystems.survey.dao.CaddisflyResourceDao; @@ -107,17 +108,14 @@ public class GraphicalSurveySummaryExporter extends SurveySummaryExporter { private static final String DIGEST_COLUMN = "NO_TITLE_DIGEST_COLUMN"; - private static final String METADATA_LABEL = "Metadata"; + + + //Statistics page header and labels private static final String REPORT_HEADER = "Survey Summary Report"; private static final String FREQ_LABEL = "Frequency"; private static final String PCT_LABEL = "Percent"; private static final String SUMMARY_LABEL = "Summary"; private static final String RAW_DATA_LABEL = "Raw Data"; - private static final String INSTANCE_LABEL = "Instance"; - private static final String SUB_DATE_LABEL = "Submission Date"; - private static final String SUBMITTER_LABEL = "Submitter"; - private static final String DURATION_LABEL = "Duration"; - private static final String REPEAT_LABEL = "Repeat no"; private static final String MEAN_LABEL = "Mean"; private static final String MEDIAN_LABEL = "Median"; private static final String MIN_LABEL = "Min"; @@ -127,16 +125,6 @@ public class GraphicalSurveySummaryExporter extends SurveySummaryExporter { private static final String STD_D_LABEL = "Std Deviation"; private static final String TOTAL_LABEL = "Total"; private static final String RANGE_LABEL = "Range"; - private static final String LAT_LABEL = "Latitude"; - private static final String LON_LABEL = "Longitude"; - private static final String IMAGE_LABEL = "Image"; - private static final String ELEV_LABEL = "Elevation"; - private static final String ACC_LABEL = "Accuracy (m)"; - private static final String IDENTIFIER_LABEL = "Identifier"; - private static final String DISPLAY_NAME_LABEL = "Display Name"; - private static final String DEVICE_IDENTIFIER_LABEL = "Device identifier"; - private static final String DATA_APPROVAL_STATUS_LABEL = "Data approval status"; - private static final String OTHER_TAG = "--OTHER--"; // Maximum number of rows of a sheet kept in memory // We must take care to never go back up longer than this @@ -209,7 +197,7 @@ public void export(Map criteria, File fileName, final String surveyId = criteria.get(SurveyRestRequest.SURVEY_ID_PARAM).trim(); final String apiKey = criteria.get("apiKey").trim(); - log.debug("===Export criteria=" + criteria.toString() + + log.debug("### Export criteria=" + criteria.toString() + " filename=" + fileName.toString() + " options=" + options.toString() ); if (!processOptions(options)) { @@ -604,6 +592,11 @@ private void writeMetadata(Sheet sheet, createCell(r, col++, sanitize(dto.getSubmitterName())); String duration = getDurationText(dto.getSurveyalTime()); createCell(r, col++, duration); + Double v = dto.getFormVersion(); + if (v != null) { + createCell(r, col++, v); + } + //TODO else a cell with 0.0?? } } @@ -1293,6 +1286,7 @@ private int addMetaDataHeaders(Sheet sheet, boolean showRepeatColumn) { addMetaDataColumnHeader(SUB_DATE_LABEL, ++columnIdx, row); addMetaDataColumnHeader(SUBMITTER_LABEL, ++columnIdx, row); addMetaDataColumnHeader(DURATION_LABEL, ++columnIdx, row); + addMetaDataColumnHeader(FORM_VER_LABEL, ++columnIdx, row); //Always put something in the top-left corner to identify the format if (doGroupHeaders) { row = getRow(0, sheet); @@ -1524,7 +1518,7 @@ private int addGeoDataColumnHeaders(QuestionDto q, Row row, createHeaderCell(row, offset++, q.getText() + " - " + LON_LABEL); createHeaderCell(row, offset++, q.getText() + " - " + ELEV_LABEL); } - } else { //Import currently relies on the --GEO headers + } else { //Import currently relies on the --GEO header prefix createHeaderCell(row, offset++, q.getKeyId() + "|" + LAT_LABEL); createHeaderCell(row, offset++, "--GEOLON--|" + LON_LABEL); createHeaderCell(row, offset++, "--GEOELE--|" + ELEV_LABEL); @@ -1557,7 +1551,7 @@ private int addExtraOptionColumnHeaders(String header, // add 'other' column if needed if (safeTrue(q.getAllowOtherFlag())){ - createHeaderCell(row, offset++, header + OTHER_TAG); + createHeaderCell(row, offset++, header + OTHER_SUFFIX); } optionMap.put(q.getKeyId(), qoList); @@ -2103,8 +2097,8 @@ public static void main(String[] args) { GraphicalSurveySummaryExporter exporter = new GraphicalSurveySummaryExporter(); Map criteria = new HashMap(); Map options = new HashMap(); -// options.put(TYPE_OPT, DATA_CLEANING_TYPE); - options.put(TYPE_OPT, DATA_ANALYSIS_TYPE); + options.put(TYPE_OPT, DATA_CLEANING_TYPE); +// options.put(TYPE_OPT, DATA_ANALYSIS_TYPE); // options.put(TYPE_OPT, COMPREHENSIVE_TYPE); options.put(LAST_COLLECTION_OPT, "false"); options.put(EMAIL_OPT, "email@example.com"); diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java index 4c3279aecc..0b70833a6b 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2018 Stichting Akvo (Akvo Foundation) + * Copyright (C) 2010-2019 Stichting Akvo (Akvo Foundation) * * This file is part of Akvo FLOW. * @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -53,6 +52,7 @@ import org.waterforpeople.mapping.app.gwt.client.surveyinstance.SurveyInstanceDto; import org.waterforpeople.mapping.app.web.dto.RawDataImportRequest; import org.waterforpeople.mapping.dataexport.service.BulkDataServiceClient; +import static org.waterforpeople.mapping.dataexport.ExportImportConstants.*; import com.gallatinsystems.framework.dataexport.applet.DataImporter; @@ -72,6 +72,7 @@ public class RawDataSpreadsheetImporter implements DataImporter { private static final int MONITORING_FORMAT_WITH_DEVICE_ID_COLUMN = 7; private static final int MONITORING_FORMAT_WITH_REPEAT_COLUMN = 8; private static final int MONITORING_FORMAT_WITH_APPROVAL_COLUMN = 9; + private static final int MONITORING_FORMAT_WITH_FORM_VERSION = 10; private boolean otherValuesInSeparateColumns = false; //until we find one @@ -84,9 +85,8 @@ public class RawDataSpreadsheetImporter implements DataImporter { public static final String COLLECTION_DATE_COLUMN_KEY = "collectionDate"; public static final String SUBMITTER_COLUMN_KEY = "submitterName"; public static final String DURATION_COLUMN_KEY = "surveyalTime"; + public static final String FORM_VER_COLUMN_KEY = "formVersion"; - public static final String METADATA_HEADER = "Metadata"; - public static final String OTHER_SUFFIX = "--OTHER--"; public static final String NEW_DATA_PATTERN = "^[Nn]ew-\\d+"; // new- or New- followed by one or more digits @@ -241,7 +241,7 @@ public List parseSplitSheets(Sheet baseSheet, //these are all for the base sheet Map columnIndexToQuestionId = sheetMap.get(baseSheet); int firstQuestionColumnIndex = Collections.min(columnIndexToQuestionId.keySet()); - Map metadataColumnHeaderIndex = calculateMetadataColumnIndex(firstQuestionColumnIndex, false); + Map metadataColumnHeaderIndex = getMetadataColumnIndex(baseSheet, firstQuestionColumnIndex, headerRowIndex, false); Map repMetadataIndex = null; //lazy calc, done if needed; all rep sheets should be the same! int row = headerRowIndex + 1; //where the data starts @@ -262,8 +262,8 @@ public List parseSplitSheets(Sheet baseSheet, if (repSheet != baseSheet) { Map repQMap = sheetMap.get(repSheet); int repFirstQIdx = Collections.min(repQMap.keySet()); - if (repMetadataIndex == null) { //do this only once - repMetadataIndex = calculateMetadataColumnIndex(repFirstQIdx, true); + if (repMetadataIndex == null) { //do this only once?? + repMetadataIndex = getMetadataColumnIndex(repSheet, repFirstQIdx, headerRowIndex, true); } Integer pos = sheetPosition.get(repSheet); if (pos == null) { //never scanned this one before; start at top @@ -300,59 +300,53 @@ public List parseSplitSheets(Sheet baseSheet, } + /** - * creates a map of where the metadata columns are + * returns a map of where the metadata columns are on this sheet * @param firstQuestionColumnIndex * @return */ - private static Map calculateMetadataColumnIndex(int firstQuestionColumnIndex, boolean singleOrRepSheet) { - Map metadataColumnIndex = new HashMap<>(); - - int currentColumnIndex = -1; - - metadataColumnIndex.put(DATAPOINT_IDENTIFIER_COLUMN_KEY, ++currentColumnIndex); - - if (hasApprovalColumn(firstQuestionColumnIndex, singleOrRepSheet)) { - metadataColumnIndex.put(DATAPOINT_APPROVAL_COLUMN_KEY, ++currentColumnIndex); - } - - if (hasRepeatIterationColumn(firstQuestionColumnIndex, singleOrRepSheet)) { - metadataColumnIndex.put(REPEAT_COLUMN_KEY, ++currentColumnIndex); - } - - metadataColumnIndex.put(DATAPOINT_NAME_COLUMN_KEY, ++currentColumnIndex); - - if (hasDeviceIdentifierColumn(firstQuestionColumnIndex, singleOrRepSheet)) { - metadataColumnIndex.put(DEVICE_IDENTIFIER_COLUMN_KEY, ++currentColumnIndex); + private Map getMetadataColumnIndex(Sheet sheet, int firstQuestionColumnIndex, int headerRow, boolean singleOrRepSheet) { + Map index = new HashMap<>(); + + Row row = sheet.getRow(headerRow); + for (int i = 0; i < firstQuestionColumnIndex; i++) { + String header = row.getCell(i).getStringCellValue(); + if (header.equalsIgnoreCase(IDENTIFIER_LABEL)) { + index.put(DATAPOINT_IDENTIFIER_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(DATA_APPROVAL_STATUS_LABEL)) { + index.put(DATAPOINT_APPROVAL_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(REPEAT_LABEL)) { + index.put(REPEAT_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(DISPLAY_NAME_LABEL)) { + index.put(DATAPOINT_NAME_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(DEVICE_IDENTIFIER_LABEL)) { + index.put(DEVICE_IDENTIFIER_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(INSTANCE_LABEL)) { + index.put(SURVEY_INSTANCE_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(SUB_DATE_LABEL)) { + index.put(COLLECTION_DATE_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(SUBMITTER_LABEL)) { + index.put(SUBMITTER_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(DURATION_LABEL)) { + index.put(DURATION_COLUMN_KEY, i); + } else if (header.equalsIgnoreCase(FORM_VER_LABEL)) { + index.put(FORM_VER_COLUMN_KEY, i); + } else { + log.warn("Unknown column header '" + header + "'"); + } } - metadataColumnIndex.put(SURVEY_INSTANCE_COLUMN_KEY, ++currentColumnIndex); - metadataColumnIndex.put(COLLECTION_DATE_COLUMN_KEY, ++currentColumnIndex); - metadataColumnIndex.put(SUBMITTER_COLUMN_KEY, ++currentColumnIndex); - metadataColumnIndex.put(DURATION_COLUMN_KEY, ++currentColumnIndex); - - return metadataColumnIndex; + return index; } - // This is based solely on the number of columns, which was good - // when the headers might be localized. - // Since we are dropping that, we should consider *reading* the headers in the future. - - private static boolean hasApprovalColumn(int firstQuestionColumnIndex, boolean repSheet) { - return (repSheet && firstQuestionColumnIndex == MONITORING_FORMAT_WITH_APPROVAL_COLUMN) - || (!repSheet && firstQuestionColumnIndex == MONITORING_FORMAT_WITH_REPEAT_COLUMN); - } - - private static boolean hasRepeatIterationColumn(int firstQuestionColumnIndex, boolean repSheet) { - return repSheet || hasApprovalColumn(firstQuestionColumnIndex, repSheet) - || firstQuestionColumnIndex == MONITORING_FORMAT_WITH_REPEAT_COLUMN; - } - - private static boolean hasDeviceIdentifierColumn(int firstQuestionColumnIndex, boolean repSheet) { - return repSheet || hasRepeatIterationColumn(firstQuestionColumnIndex, repSheet) - || firstQuestionColumnIndex == MONITORING_FORMAT_WITH_DEVICE_ID_COLUMN; + private boolean checkCol(Map index, String name) { + if (!index.containsKey(name)) { + log.warn("Required column '" + name + "' not found!"); + return false; + } + return true; } - /** * @return */ @@ -493,7 +487,7 @@ public InstanceData parseInstance(Sheet sheet, metadataColumnHeaderIndex, DATAPOINT_NAME_COLUMN_KEY); String deviceIdentifier = ""; - if (hasDeviceIdentifierColumn(firstQuestionColumnIndex, singleOrRepSheet)) { + if (metadataColumnHeaderIndex.containsKey(DEVICE_IDENTIFIER_COLUMN_KEY)) { deviceIdentifier = getMetadataCellContent(baseRow, metadataColumnHeaderIndex, DEVICE_IDENTIFIER_COLUMN_KEY); } @@ -507,11 +501,23 @@ public InstanceData parseInstance(Sheet sheet, String surveyalTime = getMetadataCellContent(baseRow, metadataColumnHeaderIndex, DURATION_COLUMN_KEY); + Double formVer = null; + if (metadataColumnHeaderIndex.containsKey(FORM_VER_COLUMN_KEY)) { + String fvStr = getMetadataCellContent(baseRow, metadataColumnHeaderIndex, + FORM_VER_COLUMN_KEY); + try + { + formVer = Double.valueOf(fvStr); + } + catch (NumberFormatException e) { /*ignore*/ } + } + + int iterations = 1; int repeatIterationColumnIndex = -1; // Count the maximum number of iterations for this instance - if (hasRepeatIterationColumn(firstQuestionColumnIndex, singleOrRepSheet)) { + if (singleOrRepSheet) { repeatIterationColumnIndex = metadataColumnHeaderIndex.get(REPEAT_COLUMN_KEY);//unsafe assignment while (true) { Row row = sheet.getRow(startRow + iterations); @@ -544,7 +550,7 @@ public InstanceData parseInstance(Sheet sheet, Row iterationRow = sheet.getRow(startRow + iter); long iteration = 1; - if (hasRepeatIterationColumn(firstQuestionColumnIndex, singleOrRepSheet)) { + if (singleOrRepSheet) { Cell cell = iterationRow.getCell(repeatIterationColumnIndex); if (cell != null) { iteration = (long) iterationRow.getCell(repeatIterationColumnIndex) @@ -567,6 +573,7 @@ public InstanceData parseInstance(Sheet sheet, surveyInstanceDto.setCollectionDate(collectionDate); surveyInstanceDto.setSubmitterName(submitterName); surveyInstanceDto.setSurveyalTime((long) durationToSeconds(surveyalTime)); + surveyInstanceDto.setFormVersion(formVer); InstanceData instanceData = new InstanceData(surveyInstanceDto, responseMap); //Copies and sorts the responseMap instanceData.maxIterationsCount = iterations; @@ -600,9 +607,9 @@ private void getIterationResponse(Row iterationRow, Cell cell = iterationRow.getCell(columnIndex); if (cell != null //misses empty-but-has-other - || (questionType == questionType.OPTION - && Boolean.TRUE.equals(questionDto.getAllowOtherFlag() - && otherValuesInSeparateColumns))) { + || (questionType == QuestionType.OPTION + && Boolean.TRUE.equals(questionDto.getAllowOtherFlag() + && otherValuesInSeparateColumns))) { switch (questionType) { case GEO: String latitude = ExportImportUtils.parseCellAsString(cell); @@ -785,9 +792,9 @@ private static Map processHeader(Sheet sheet, int headerRowIndex) for (Cell cell : headerRow) { String cellValue = cell.getStringCellValue(); if (cell.getStringCellValue().indexOf("|") > -1 - && !cellValue.startsWith("--GEO") + && !cellValue.startsWith(GEO_PREFIX) && !cellValue.endsWith(OTHER_SUFFIX) - && !cellValue.startsWith("--CADDISFLY")) { + && !cellValue.startsWith(CADDISFLY_PREFIX)) { String[] parts = cell.getStringCellValue().split("\\|"); if (parts[0].trim().length() > 0) { columnIndexToQuestionId.put(cell.getColumnIndex(), @@ -868,13 +875,18 @@ private static String buildImportURL(InstanceData instanceData, String surveyId, String dateString = ExportImportUtils.formatDateTime(dto.getCollectionDate()); sb.append(RawDataImportRequest.COLLECTION_DATE_PARAM + "=" - + URLEncoder.encode(dateString, "UTF-8") + "&"); + + URLEncoder.encode(dateString, "UTF-8")); // Submitter - sb.append("submitter=" + URLEncoder.encode(dto.getSubmitterName(), "UTF-8") + "&"); + sb.append("&submitter=" + URLEncoder.encode(dto.getSubmitterName(), "UTF-8")); // Duration - sb.append("duration=" + dto.getSurveyalTime()); + sb.append("&duration=" + dto.getSurveyalTime()); + + // Form version + if (dto.getFormVersion() != null) { + sb.append("&formVersion=" + dto.getFormVersion()); + } // questionId=123|0=sfijd|2=fjsoi|type=GEO&questionId=... for (Entry> entry : instanceData.responseMap @@ -1006,9 +1018,9 @@ public Map validate(File file) { Sheet sheet = getDataSheet(file); //Find out if this is a 2017-style report w group headers and rqg's on separate sheets - boolean splitSheets = safeCellCompare(sheet, 0, 0, METADATA_HEADER); + boolean splitSheets = safeCellCompare(sheet, 0, 0, METADATA_LABEL); if (!splitSheets) { - errorMap.put(0, "First header cell must contain '" + METADATA_HEADER + "'"); + errorMap.put(0, "First header cell must contain '" + METADATA_LABEL + "'"); return errorMap; } @@ -1050,37 +1062,35 @@ public Map validate(File file) { errorMap.put(-1, "A question could not be found"); } - //verify that the repeat column is all number cells - if (firstQuestionFound && hasRepeatIterationColumn(firstQuestionColumnIndex, !splitSheets)) { - Iterator iter = sheet.iterator(); - iter.next(); // Skip the header row. - if (splitSheets) { // just in case we change surrounding logic - iter.next(); // Skip the second header row. + Workbook wb = sheet.getWorkbook(); + + //check that all mandatory columns exist on all sheets + for (int i = 0; i < wb.getNumberOfSheets(); i++) { + sheet = wb.getSheetAt(i); + Map index = getMetadataColumnIndex(sheet, firstQuestionColumnIndex, headerRowIndex, + false); + if (!checkCol(index, DATAPOINT_IDENTIFIER_COLUMN_KEY)) { + errorMap.put(-1, "Column header '" + IDENTIFIER_LABEL + "' missing on sheet " + i); } - int repeatIterationColumnIndex = -1; - if (hasApprovalColumn(firstQuestionColumnIndex, !splitSheets)) { - repeatIterationColumnIndex = 2; - } else { - repeatIterationColumnIndex = 1; + if (!checkCol(index, DATAPOINT_NAME_COLUMN_KEY)) { + errorMap.put(-1, "Column header '" + DISPLAY_NAME_LABEL + "' missing on sheet " + i); } - - while (iter.hasNext()) { // gets "phantom" rows, too - Row row = iter.next(); - if (isEmptyRow(row)) { - break; // phantom row - just stop - } - Cell cell = row.getCell(repeatIterationColumnIndex); - if (cell == null) { - // include 1-based row number in error log - errorMap.put(-1, "Repeat column is empty in row: " + row.getRowNum() + 1); - break; - } - if (cell.getCellType() != Cell.CELL_TYPE_NUMERIC) { - errorMap.put(-1, "Repeat column must contain a numeric value in row: " - + row.getRowNum() + 1); - break; - } + if (!checkCol(index, SURVEY_INSTANCE_COLUMN_KEY)) { + errorMap.put(-1, "Column header '" + INSTANCE_LABEL + "' missing on sheet " + i); + } + if (!checkCol(index, COLLECTION_DATE_COLUMN_KEY)) { + errorMap.put(-1, "Column header '" + SUB_DATE_LABEL + "' missing on sheet " + i); + } + if (!checkCol(index, SUBMITTER_COLUMN_KEY)) { + errorMap.put(-1, "Column header '" + SUBMITTER_LABEL + "' missing on sheet " + i); + } + if (!checkCol(index, DURATION_COLUMN_KEY)) { + errorMap.put(-1, "Column header '" + DURATION_LABEL + "' missing on sheet " + i); + } + if (i > 0 && !checkCol(index, REPEAT_LABEL)) { + errorMap.put(-1, "Column header '" + REPEAT_LABEL + "' missing on sheet " + i); } + i++; } } catch (Exception e) { @@ -1094,7 +1104,8 @@ private boolean isSupportedReportFormat(int firstQuestionColumnIndex) { return firstQuestionColumnIndex == LEGACY_MONITORING_FORMAT || firstQuestionColumnIndex == MONITORING_FORMAT_WITH_DEVICE_ID_COLUMN || firstQuestionColumnIndex == MONITORING_FORMAT_WITH_REPEAT_COLUMN - || firstQuestionColumnIndex == MONITORING_FORMAT_WITH_APPROVAL_COLUMN; + || firstQuestionColumnIndex == MONITORING_FORMAT_WITH_APPROVAL_COLUMN + || firstQuestionColumnIndex == MONITORING_FORMAT_WITH_FORM_VERSION; } /** diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java index fddc16cb1f..5730247839 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java @@ -533,6 +533,10 @@ private static SurveyInstanceDto parseSurveyInstance(String response) if (json.has("collectionDate")) { dto.setCollectionDate(new Date(json.getLong("collectionDate"))); } + if (!json.has("formVersion")) { + dto.setFormVersion(json.getDouble("formVersion")); + } + } } }