Skip to content

Commit

Permalink
Loop in external data
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexisszmundy
alexisszmundy committed Jan 9, 2025
1 parent aff055c commit 1319be6
Show file tree
Hide file tree
Showing 6 changed files with 5,584 additions and 102 deletions.
91 changes: 47 additions & 44 deletions src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import fr.insee.bpm.metadata.model.VariablesMap;
import fr.insee.genesis.controller.sources.xml.LunaticXmlCollectedData;
import fr.insee.genesis.controller.sources.xml.LunaticXmlOtherData;
import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
import fr.insee.genesis.controller.sources.xml.ValueType;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
Expand All @@ -14,23 +15,22 @@
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

@UtilityClass
public class LunaticXmlAdapter {

/**
* Convert a Lunatic XML survey unit into a genesis survey unit DTO
* Convert a Lunatic XML survey unit into a genesis survey unit model
* @param su Lunatic XML survey unit to convert
* @param variablesMap variable definitions (used for loops)
* @param idCampaign survey id
* @return a genesis survey unit DTO
* @return Genesis SurveyUnitModels for each data state
*/
public static List<SurveyUnitModel> convert(LunaticXmlSurveyUnit su, VariablesMap variablesMap, String idCampaign, Mode mode){
//Get COLLECTED Data and external variables
List<SurveyUnitModel> surveyUnitModelList = new ArrayList<>();
SurveyUnitModel surveyUnitModel = getStateDataFromSurveyUnit(su, variablesMap, idCampaign, DataState.COLLECTED, mode);
getExternalDataFromSurveyUnit(su, surveyUnitModel);
getExternalDataFromSurveyUnit(su, surveyUnitModel, variablesMap);

surveyUnitModelList.add(surveyUnitModel);

Expand Down Expand Up @@ -64,8 +64,8 @@ public static List<SurveyUnitModel> convert(LunaticXmlSurveyUnit su, VariablesMa
* @param su source XML Survey Unit
* @param variablesMap variable definitions (used for loops)
* @param idCampaign survey id
* @param dataState state of the DTO to generate
* @return Survey Unit DTO with a specific state
* @param dataState state of the SurveyUnitModel to generate
* @return SurveyUnitModel with a specific state
*/
private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit su, VariablesMap variablesMap, String idCampaign, DataState dataState, Mode mode) {
SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
Expand All @@ -83,15 +83,15 @@ private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit s


/**
* Gets data from a specific state and put it into DTO's data
* Gets data from a specific state and put it into Model's data
* @param su XML survey unit to extract from
* @param surveyUnitModel DTO to aliment
* @param surveyUnitModel Model to aliment
* @param variablesMap variables definitions (used for loops)
* @param dataState data state from XML
* @return the DTO containing data, null if no data and not COLLECTED
* @return the SurveyUnitModel containing data, null if no data and not COLLECTED
*/
private static SurveyUnitModel getCollectedDataFromSurveyUnit(LunaticXmlSurveyUnit su, SurveyUnitModel surveyUnitModel, VariablesMap variablesMap, DataState dataState) {
List<VariableModel> variablesUpdate = new ArrayList<>();
List<VariableModel> variableModels = new ArrayList<>();

int dataCount = 0;
for (LunaticXmlCollectedData lunaticXmlCollectedData : su.getData().getCollected()){
Expand All @@ -115,23 +115,24 @@ private static SurveyUnitModel getCollectedDataFromSurveyUnit(LunaticXmlSurveyUn
default:
return null;
}
if(valueTypeList != null) {
for (int i = 1; i <= valueTypeList.size(); i++) {
List<String> variableValues = new ArrayList<>();
if (valueTypeList.get(i-1).getValue()!=null) {
variableValues.add(valueTypeList.get(i-1).getValue());
variablesUpdate.add(VariableModel.builder()
.idVar(lunaticXmlCollectedData.getVariableName())
.values(variableValues)
.idLoop(LoopIdentifier.getLoopIdentifier(lunaticXmlCollectedData.getVariableName(), variablesMap, i))
.idParent(LoopIdentifier.getRelatedVariableName(lunaticXmlCollectedData.getVariableName(), variablesMap))
.build());
dataCount++;
}
if(valueTypeList == null) {
continue; //Go to next data
}
for (int i = 1; i <= valueTypeList.size(); i++) {
List<String> variableValues = new ArrayList<>();
if (valueTypeList.get(i-1).getValue()!=null) {
variableValues.add(valueTypeList.get(i-1).getValue());
variableModels.add(VariableModel.builder()
.idVar(lunaticXmlCollectedData.getVariableName())
.values(variableValues)
.idLoop(LoopIdentifier.getLoopIdentifier(lunaticXmlCollectedData.getVariableName(), variablesMap, i))
.idParent(LoopIdentifier.getRelatedVariableName(lunaticXmlCollectedData.getVariableName(), variablesMap))
.build());
dataCount++;
}
}
}
surveyUnitModel.setCollectedVariables(variablesUpdate);
surveyUnitModel.setCollectedVariables(variableModels);

//Return null if no data and not COLLECTED
if(dataCount > 0 || dataState.equals(DataState.COLLECTED)){
Expand All @@ -143,29 +144,31 @@ private static SurveyUnitModel getCollectedDataFromSurveyUnit(LunaticXmlSurveyUn


/**
* Extract external data from XML survey unit and put it into DTO
* Extract external data from XML survey unit and put it into Model
* @param su XML survey unit
* @param surveyUnitModel DTO to aliment
* @param surveyUnitModel Model to aliment
*/
private static void getExternalDataFromSurveyUnit(LunaticXmlSurveyUnit su, SurveyUnitModel surveyUnitModel) {
List<VariableModel> externalVariables = new ArrayList<>();
su.getData().getExternal().forEach(lunaticXmlExternalData ->
externalVariables.add(VariableModel.builder()
.idVar(lunaticXmlExternalData.getVariableName())
.values(getValuesFromValueTypeList(lunaticXmlExternalData.getValues()))
.build())
);
surveyUnitModel.setExternalVariables(externalVariables);
}
private static void getExternalDataFromSurveyUnit(LunaticXmlSurveyUnit su, SurveyUnitModel surveyUnitModel, VariablesMap variablesMap) {
List<VariableModel> variableModels = new ArrayList<>();

private static List<String> getValuesFromValueTypeList(List<ValueType> valueTypeList) {
if (!valueTypeList.isEmpty()){
return valueTypeList.stream()
.map(ValueType::getValue)
.filter(Objects::nonNull)
.toList();
for(LunaticXmlOtherData lunaticXmlExternalData : su.getData().getExternal()){
List<ValueType> valueTypeList = lunaticXmlExternalData.getValues();
if(valueTypeList == null) {
continue; //Go to next data
}
for (int i = 1; i <= valueTypeList.size(); i++) {
List<String> variableValues = new ArrayList<>();
if (valueTypeList.get(i-1).getValue()!=null) {
variableValues.add(valueTypeList.get(i-1).getValue());
variableModels.add(VariableModel.builder()
.idVar(lunaticXmlExternalData.getVariableName())
.values(variableValues)
.idLoop(LoopIdentifier.getLoopIdentifier(lunaticXmlExternalData.getVariableName(), variablesMap, i))
.idParent(LoopIdentifier.getRelatedVariableName(lunaticXmlExternalData.getVariableName(), variablesMap))
.build());
}
}
}
return List.of();
surveyUnitModel.setExternalVariables(variableModels);
}

}
97 changes: 47 additions & 50 deletions src/test/java/cucumber/functional_tests/MainDefinitions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
import io.cucumber.java.After;
import io.cucumber.java.Before;
import io.cucumber.java.en.And;
import io.cucumber.java.en.Given;
import io.cucumber.java.en.Then;
import io.cucumber.java.en.When;
Expand Down Expand Up @@ -77,6 +78,13 @@ public void init_folder(String directory) throws IOException {
Files.createDirectories(inDirectory);
}

@Given("We copy data file {string} to that directory")
public void copy_data_file(String dataFile) throws IOException {
Path dataFilePath = Paths.get(TestConstants.TEST_RESOURCES_DIRECTORY).resolve(dataFile);
Files.copy(dataFilePath, this.inDirectory.resolve(dataFilePath.getFileName().toString()),
StandardCopyOption.REPLACE_EXISTING);
}

@When("We create DTOs from file {string} with DDI {string}")
public void get_dtos(String fileName, String ddiName) throws IOException, ParserConfigurationException,
SAXException, GenesisException, MetadataParserException {
Expand All @@ -100,13 +108,6 @@ public void get_dtos(String fileName, String ddiName) throws IOException, Parser
}
}

@When("We copy data file {string} to that directory")
public void copy_data_file(String dataFile) throws IOException {
Path dataFilePath = Paths.get(TestConstants.TEST_RESOURCES_DIRECTORY).resolve(dataFile);
Files.copy(dataFilePath, this.inDirectory.resolve(dataFilePath.getFileName().toString()),
StandardCopyOption.REPLACE_EXISTING);
}

@When("We save data from that directory")
public void get_dtos_from_folder() throws Exception {
responseController.saveResponsesFromXmlCampaignFolder(this.inDirectory.getFileName().toString(), null, true);
Expand Down Expand Up @@ -178,27 +179,24 @@ public void check_survey_unit_dto_content(String dataState, String surveyUnitId,
}

@Then("We should have {int} values for external variable {string} for survey unit {string}")
public void external_variable_volumetric_check(int expectedNumberOfValues, String externalVariableName, String surveyUnitId) {
//Get DTO
Assertions.assertThat(this.surveyUnitModels).filteredOn(surveyUnitDto ->
surveyUnitDto.getState().equals(DataState.COLLECTED)
&& surveyUnitDto.getIdUE().equals(surveyUnitId)
).isNotEmpty();

Optional<SurveyUnitModel> concernedSurveyUnitModelOptional = this.surveyUnitModels.stream().filter(dto ->
dto.getState().equals(DataState.COLLECTED)
&& dto.getIdUE().equals(surveyUnitId)
).findFirst();

Assertions.assertThat(concernedSurveyUnitModelOptional).isPresent();
public void external_variable_volumetric_check(int expectedNumberOfValues, String externalVariableName,
String interrogationId) {
//Get SurveyUnitModel
Assertions.assertThat(surveyUnitPersistence.getMongoStub()).filteredOn(surveyUnitModel ->
surveyUnitModel.getState().equals(DataState.COLLECTED)
&& surveyUnitModel.getIdUE().equals(interrogationId)
).isNotEmpty().hasSize(1);

SurveyUnitModel concernedSurveyUnitModel = concernedSurveyUnitModelOptional.get();
SurveyUnitModel surveyUnitModel = surveyUnitPersistence.getMongoStub().stream().filter(surveyUnitModel1 ->
surveyUnitModel1.getState().equals(DataState.COLLECTED)
&& surveyUnitModel1.getIdUE().equals(interrogationId)
).toList().getFirst();

//Get Variable
Assertions.assertThat(concernedSurveyUnitModel.getExternalVariables()).filteredOn(variableModel ->
Assertions.assertThat(surveyUnitModel.getExternalVariables()).filteredOn(variableModel ->
variableModel.idVar().equals(externalVariableName)).isNotEmpty();

Optional<VariableModel> concernedVariableOptional = concernedSurveyUnitModel.getExternalVariables().stream().filter(variable ->
Optional<VariableModel> concernedVariableOptional = surveyUnitModel.getExternalVariables().stream().filter(variable ->
variable.idVar().equals(externalVariableName)
).findFirst();

Expand All @@ -211,43 +209,42 @@ public void external_variable_volumetric_check(int expectedNumberOfValues, Strin

}

@Then("For external variable {string} in survey unit {string} we should have {string} as value number {int}")
public void external_variable_content_check(String externalVariableName, String surveyUnitId, String expectedValue, int expectedValueIndex) {
@And("For external variable {string} in survey unit {string} we should have {string} and loopId {string}")
public void forExternalVariableInSurveyUnitWeShouldHaveForLoop(String externalVariableName,
String interrogationId,
String expectedValue,
String expectedLoopId) {
//Get SurveyUnitModel
Assertions.assertThat(this.surveyUnitModels).filteredOn(surveyUnitDto ->
Assertions.assertThat(surveyUnitPersistence.getMongoStub()).filteredOn(surveyUnitDto ->
surveyUnitDto.getState().equals(DataState.COLLECTED)
&& surveyUnitDto.getIdUE().equals(surveyUnitId)
).isNotEmpty();
&& surveyUnitDto.getIdUE().equals(interrogationId)
).isNotEmpty().hasSize(1);

Optional<SurveyUnitModel> concernedSurveyUnitModelOptional = this.surveyUnitModels.stream().filter(dto ->
SurveyUnitModel surveyUnitModel = surveyUnitPersistence.getMongoStub().stream().filter(dto ->
dto.getState().equals(DataState.COLLECTED)
&& dto.getIdUE().equals(surveyUnitId)
).findFirst();

Assertions.assertThat(concernedSurveyUnitModelOptional).isPresent();

SurveyUnitModel concernedSurveyUnitModel = concernedSurveyUnitModelOptional.get();
&& dto.getIdUE().equals(interrogationId)
).toList().getFirst();

//Get Variable
Assertions.assertThat(concernedSurveyUnitModel.getExternalVariables()).filteredOn(variableModel ->
variableModel.idVar().equals(externalVariableName)).isNotEmpty();

Optional<VariableModel> concernedExternalVariableOptional = concernedSurveyUnitModel.getExternalVariables().stream().filter(variable ->
variable.idVar().equals(externalVariableName)
).findFirst();

Assertions.assertThat(concernedExternalVariableOptional).isPresent();
Assertions.assertThat(surveyUnitModel.getExternalVariables()).filteredOn(variableModel ->
variableModel.idVar().equals(externalVariableName)
&& Objects.equals(variableModel.idLoop(), expectedLoopId)
).isNotEmpty().hasSize(1);

VariableModel concernedExternalVariable = concernedExternalVariableOptional.get();
VariableModel concernedExternalVariable = surveyUnitModel.getExternalVariables().stream().filter(variableModel ->
variableModel.idVar().equals(externalVariableName)
&& Objects.equals(variableModel.idLoop(), expectedLoopId)
).toList().getFirst();

//Value content assertion
Assertions.assertThat(concernedExternalVariable.values()).hasSizeGreaterThan(expectedValueIndex);
Assertions.assertThat(concernedExternalVariable.values().get(expectedValueIndex)).isEqualTo(expectedValue);
Assertions.assertThat(concernedExternalVariable.values()).hasSize(1);
Assertions.assertThat(concernedExternalVariable.values().getFirst()).isEqualTo(expectedValue);
Assertions.assertThat(concernedExternalVariable.idLoop()).isNotNull().isEqualTo(expectedLoopId);
}

@Then("For external variable {string} in survey unit {string} we should have {string} as idLoop and {string} as first " +
"value")
public void check_idLoop_and_value(String externalVariableName, String interrogationId, String expectedIdLoop,
public void check_idLoop_and_value(String externalVariableName, String interrogationId, String expectedLoopId,
String expectedValue) {
//Get SurveyUnitModel
Assertions.assertThat(surveyUnitPersistence.getMongoStub()).filteredOn(surveyUnitDto ->
Expand All @@ -263,16 +260,16 @@ public void check_idLoop_and_value(String externalVariableName, String interroga
//Get Variable
Assertions.assertThat(surveyUnitModel.getExternalVariables()).filteredOn(variableModel ->
variableModel.idVar().equals(externalVariableName)
&& Objects.equals(variableModel.idLoop(), expectedIdLoop)
&& Objects.equals(variableModel.idLoop(), expectedLoopId)
).isNotEmpty().hasSize(1);

VariableModel concernedExternalVariable = surveyUnitModel.getExternalVariables().stream().filter(variableModel ->
variableModel.idVar().equals(externalVariableName)
&& Objects.equals(variableModel.idLoop(), expectedIdLoop)
&& Objects.equals(variableModel.idLoop(), expectedLoopId)
).toList().getFirst();

//Value content assertion
Assertions.assertThat(concernedExternalVariable.idLoop()).isNotNull().isEqualTo(expectedIdLoop);
Assertions.assertThat(concernedExternalVariable.idLoop()).isNotNull().isEqualTo(expectedLoopId);
Assertions.assertThat(concernedExternalVariable.values()).hasSize(1);
Assertions.assertThat(concernedExternalVariable.values().getFirst()).isEqualTo(expectedValue);
}
Expand Down
Loading

0 comments on commit 1319be6

Please sign in to comment.