From 63167b28fbbc14d7e96c29eccde07e9dadc3fa4a Mon Sep 17 00:00:00 2001 From: Emanuela Epure <67077116+emanuelaepure10@users.noreply.github.com> Date: Fri, 1 Sep 2023 17:00:29 +0200 Subject: [PATCH] fixup: feat: add multiple selection to CSV writer Same approach as for XSL has been implemented for the CSV writer. In the wizard of selecting a feature type has been implemented the multi selection table and the same number of .csv files are exported as the selection. ING-3987 --- .../io/csv/writer/CSVInstanceWriterTest.java | 33 ++- .../ui/InstanceExportConfigurationPage.java | 200 ++++++++++++------ .../hale/io/csv/InstanceTableIOConstants.java | 15 ++ .../writer/internal/CSVInstanceWriter.java | 73 +++++-- 4 files changed, 229 insertions(+), 92 deletions(-) diff --git a/io/plugins/eu.esdihumboldt.hale.io.csv.test/src/eu/esdihumboldt/hale/io/csv/writer/CSVInstanceWriterTest.java b/io/plugins/eu.esdihumboldt.hale.io.csv.test/src/eu/esdihumboldt/hale/io/csv/writer/CSVInstanceWriterTest.java index d8a0dbff61..9e1901da95 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.csv.test/src/eu/esdihumboldt/hale/io/csv/writer/CSVInstanceWriterTest.java +++ b/io/plugins/eu.esdihumboldt.hale.io.csv.test/src/eu/esdihumboldt/hale/io/csv/writer/CSVInstanceWriterTest.java @@ -23,6 +23,8 @@ import java.util.Iterator; import java.util.List; +import javax.xml.namespace.QName; + import org.eclipse.core.runtime.content.IContentType; import org.junit.BeforeClass; import org.junit.Rule; @@ -39,6 +41,7 @@ import eu.esdihumboldt.hale.common.instance.io.InstanceWriter; import eu.esdihumboldt.hale.common.instance.model.InstanceCollection; import eu.esdihumboldt.hale.common.schema.model.Schema; +import eu.esdihumboldt.hale.common.schema.model.impl.DefaultSchemaSpace; import eu.esdihumboldt.hale.common.test.TestUtil; import eu.esdihumboldt.hale.io.csv.InstanceTableIOConstants; import eu.esdihumboldt.hale.io.csv.reader.internal.CSVSchemaReader; @@ -74,6 +77,7 @@ public static void waitForServices() { @Test public void testWriteSimpleSchema() throws Exception { + Schema schema = CSVInstanceWriterTestUtil.createExampleSchema(); TransformationExample example = TransformationExamples .getExample(TransformationExamples.SIMPLE_ASSIGN); // alternative the data could be generated by iterating through the @@ -86,10 +90,11 @@ public void testWriteSimpleSchema() throws Exception { int numberOfRows = 3; char sep = ','; + QName qname = new QName("http://www.opengis.net/om/2.0", "OM_ObservationType"); File tmpFile = tmpFolder.newFile("csvTestWriteSimpleSchema.csv"); assertTrue("Csv Export was not successful.", writeCsvToFile(tmpFile, true, false, - Value.of(sep), null, null, example.getSourceInstances())); + Value.of(sep), null, null, example.getSourceInstances(), schema, qname)); CSVReader reader = new CSVReader(new FileReader(tmpFile), sep); List rows = reader.readAll(); @@ -146,10 +151,11 @@ public void testWriteSimpleSchemaColOrder() throws Exception { int numberOfRows = 3; // counting also the header char sep = ','; + QName qname = new QName("http://www.opengis.net/om/2.0", "OM_ObservationType"); File tmpFile = tmpFolder.newFile("csvTestWriteSimpleSchema.csv"); - assertTrue("Csv Export was not successful.", - writeCsvToFile(tmpFile, true, true, Value.of(sep), null, null, instance)); + assertTrue("Csv Export was not successful.", writeCsvToFile(tmpFile, true, true, + Value.of(sep), null, null, instance, schema, qname)); CSVReader reader = new CSVReader(new FileReader(tmpFile), sep); List rows = reader.readAll(); @@ -195,6 +201,7 @@ public void testWriteSimpleSchemaColOrder() throws Exception { @Test public void testWriteSimpleSchemaDelimiter() throws Exception { + Schema schema = CSVInstanceWriterTestUtil.createExampleSchema(); TransformationExample example = TransformationExamples .getExample(TransformationExamples.SIMPLE_ASSIGN); @@ -209,10 +216,12 @@ public void testWriteSimpleSchemaDelimiter() throws Exception { char quo = '\''; char esc = '"'; + QName qname = new QName("http://www.opengis.net/om/2.0", "OM_ObservationType"); File tmpFile = tmpFolder.newFile("csvTestWriteSimpleSchemaDelimiter.csv"); - assertTrue("Csv Export was not successful.", writeCsvToFile(tmpFile, true, false, - Value.of(sep), Value.of(quo), Value.of(esc), example.getSourceInstances())); + assertTrue("Csv Export was not successful.", + writeCsvToFile(tmpFile, true, false, Value.of(sep), Value.of(quo), Value.of(esc), + example.getSourceInstances(), schema, qname)); CSVReader reader = new CSVReader(new FileReader(tmpFile), sep, quo, esc); List rows = reader.readAll(); @@ -248,7 +257,7 @@ public void testWriteSimpleSchemaDelimiter() throws Exception { */ @Test public void testWriteComplexSchema() throws Exception { - + Schema schema = CSVInstanceWriterTestUtil.createExampleSchema(); TransformationExample example = TransformationExamples .getExample(TransformationExamples.SIMPLE_COMPLEX); // alternative the data could be generated by iterating through the @@ -262,9 +271,10 @@ public void testWriteComplexSchema() throws Exception { char sep = ','; File tmpFile = tmpFolder.newFile("csvTestWriteComplexSchema.csv"); + QName qname = new QName("http://www.opengis.net/om/2.0", "OM_ObservationType"); assertTrue("Csv Export was not successful.", writeCsvToFile(tmpFile, true, false, - Value.of(sep), null, null, example.getSourceInstances())); + Value.of(sep), null, null, example.getSourceInstances(), schema, qname)); CSVReader reader = new CSVReader(new FileReader(tmpFile), sep); List rows = reader.readAll(); @@ -298,7 +308,8 @@ public void testWriteComplexSchema() throws Exception { } private boolean writeCsvToFile(File tmpFile, boolean solveNestedProperties, boolean useSchema, - Value sep, Value quo, Value esc, InstanceCollection instances) throws Exception { + Value sep, Value quo, Value esc, InstanceCollection instances, Schema schema, + QName qname) throws Exception { // set instances to xls instance writer InstanceWriter writer = new CSVInstanceWriter(); IContentType contentType = HalePlatform.getContentTypeManager() @@ -309,7 +320,13 @@ private boolean writeCsvToFile(File tmpFile, boolean solveNestedProperties, bool writer.setParameter(CSVSchemaReader.PARAM_SEPARATOR, sep); writer.setParameter(CSVSchemaReader.PARAM_QUOTE, quo); writer.setParameter(CSVSchemaReader.PARAM_ESCAPE, esc); + writer.setParameter(InstanceTableIOConstants.EXPORT_TYPE, Value.of(qname.getLocalPart())); writer.setInstances(instances); + + DefaultSchemaSpace ss = new DefaultSchemaSpace(); + ss.addSchema(schema); + writer.setTargetSchema(ss); + // write instances to a temporary CSV file writer.setTarget(new FileIOSupplier(tmpFile)); writer.setContentType(contentType); diff --git a/io/plugins/eu.esdihumboldt.hale.io.csv.ui/src/eu/esdihumboldt/hale/io/csv/ui/InstanceExportConfigurationPage.java b/io/plugins/eu.esdihumboldt.hale.io.csv.ui/src/eu/esdihumboldt/hale/io/csv/ui/InstanceExportConfigurationPage.java index 00cd28575e..c3fde50b8e 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.csv.ui/src/eu/esdihumboldt/hale/io/csv/ui/InstanceExportConfigurationPage.java +++ b/io/plugins/eu.esdihumboldt.hale.io.csv.ui/src/eu/esdihumboldt/hale/io/csv/ui/InstanceExportConfigurationPage.java @@ -18,14 +18,23 @@ import java.util.Set; import org.eclipse.jface.layout.GridDataFactory; -import org.eclipse.jface.viewers.ISelectionChangedListener; -import org.eclipse.jface.viewers.SelectionChangedEvent; -import org.eclipse.jface.viewers.Viewer; -import org.eclipse.jface.viewers.ViewerFilter; +import org.eclipse.jface.viewers.ArrayContentProvider; +import org.eclipse.jface.viewers.CheckStateChangedEvent; +import org.eclipse.jface.viewers.CheckboxTableViewer; +import org.eclipse.jface.viewers.ICheckStateListener; +import org.eclipse.jface.viewers.ICheckStateProvider; +import org.eclipse.jface.viewers.LabelProvider; import org.eclipse.swt.SWT; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; import org.eclipse.swt.graphics.Color; +import org.eclipse.swt.layout.GridData; +import org.eclipse.swt.layout.GridLayout; +import org.eclipse.swt.widgets.Button; import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Group; import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Table; import org.eclipse.ui.PlatformUI; import eu.esdihumboldt.hale.common.core.io.Value; @@ -33,7 +42,6 @@ import eu.esdihumboldt.hale.common.instance.model.DataSet; import eu.esdihumboldt.hale.common.schema.model.TypeDefinition; import eu.esdihumboldt.hale.io.csv.InstanceTableIOConstants; -import eu.esdihumboldt.hale.ui.common.definition.selector.TypeDefinitionSelector; import eu.esdihumboldt.hale.ui.service.instance.InstanceService; /** @@ -43,33 +51,10 @@ */ public class InstanceExportConfigurationPage extends CommonInstanceExportConfigurationPage { - private TypeDefinitionSelector typeSelector; - - private final ViewerFilter validTypesToSelect = new ViewerFilter() { - - @Override - public boolean select(Viewer viewer, Object parentElement, Object element) { - if (!(element instanceof TypeDefinition)) - return false; - - InstanceService instanceService = PlatformUI.getWorkbench() - .getService(InstanceService.class); - - Set instanceSourceTypes = instanceService - .getInstanceTypes(DataSet.SOURCE); - if (instanceSourceTypes.contains(element)) { - return true; - } - - Set instanceTransformedTypes = instanceService - .getInstanceTypes(DataSet.TRANSFORMED); - if (instanceTransformedTypes.contains(element)) { - return true; - } - - return false; - } - }; + private CheckboxTableViewer featureTypeTable; + private Button selectAll = null; + private Group chooseFeatureTypes; + private Table table; /** * @@ -96,17 +81,6 @@ public void disable() { // not required } - /** - * @see eu.esdihumboldt.hale.ui.io.IOWizardPage#updateConfiguration(eu.esdihumboldt.hale.common.core.io.IOProvider) - */ - @Override - public boolean updateConfiguration(InstanceWriter provider) { - super.updateConfiguration(provider); - provider.setParameter(InstanceTableIOConstants.EXPORT_TYPE, - Value.of(typeSelector.getSelectedObject().getName().toString())); - return true; - } - /** * @see eu.esdihumboldt.hale.ui.HaleWizardPage#createContent(org.eclipse.swt.widgets.Composite) */ @@ -114,16 +88,11 @@ public boolean updateConfiguration(InstanceWriter provider) { protected void createContent(Composite page) { super.createContent(page); - final Label label = new Label(page, SWT.NONE); - label.setText("Choose your Type you want to export:"); - - Label separatorLabel = new Label(page, SWT.NONE); - separatorLabel.setText("Warning! Feature types with no data are not selectable"); - - // Set the text colour of the label to yellow - Color greyLabel = PlatformUI.getWorkbench().getDisplay() - .getSystemColor(SWT.COLOR_DARK_GRAY); - separatorLabel.setForeground(greyLabel); + GridDataFactory groupData = GridDataFactory.fillDefaults().grab(true, false); + chooseFeatureTypes = new Group(page, SWT.NONE); + chooseFeatureTypes.setLayout(new GridLayout(1, false)); + chooseFeatureTypes.setText("Choose your Type you want to export"); + groupData.applyTo(chooseFeatureTypes); page.pack(); @@ -134,26 +103,129 @@ protected void createContent(Composite page) { @Override protected void onShowPage(boolean firstShow) { if (firstShow) { - ViewerFilter[] filters = { validTypesToSelect }; - typeSelector = new TypeDefinitionSelector(page, "Select the corresponding schema type", - getWizard().getProvider().getTargetSchema(), filters); + selectAll = new Button(chooseFeatureTypes, SWT.CHECK); + selectAll.setText("Select all"); + selectAll.setLayoutData(new GridData(SWT.FILL, SWT.FILL, true, false, 2, 1)); + + selectAll.addSelectionListener(new SelectionAdapter() { + + @Override + public void widgetSelected(SelectionEvent e) { + featureTypeTable.setAllChecked(((Button) e.getSource()).getSelection()); + page.layout(); + page.pack(); + setPageComplete(validate()); + } + }); + + Label separatorLabel = new Label(page, SWT.NONE); + separatorLabel.setText("Warning! Feature types with no data are not selectable"); + + // Set the text colour of the label to yellow + Color greyLabel = PlatformUI.getWorkbench().getDisplay() + .getSystemColor(SWT.COLOR_DARK_GRAY); + separatorLabel.setForeground(greyLabel); + + table = new Table(chooseFeatureTypes, SWT.CHECK | SWT.MULTI | SWT.SCROLL_PAGE); + table.setHeaderVisible(false); + table.setLinesVisible(false); + table.setBackground(PlatformUI.getWorkbench().getDisplay() + .getSystemColor(SWT.COLOR_LIST_BACKGROUND)); + GridDataFactory groupData = GridDataFactory.fillDefaults().grab(true, false); + groupData.applyTo(table); + + featureTypeTable = new CheckboxTableViewer(table); + featureTypeTable.setLabelProvider(new LabelProvider() { + + @Override + public String getText(Object element) { + return ((TypeDefinition) element).getDisplayName(); + } + + }); + featureTypeTable.setContentProvider(ArrayContentProvider.getInstance()); + + featureTypeTable.setInput( + getWizard().getProvider().getTargetSchema().getMappingRelevantTypes()); + + featureTypeTable.addCheckStateListener(new ICheckStateListener() { + + @Override + public void checkStateChanged(CheckStateChangedEvent event) { + // Programmatic action to toggle the state + selectAll.setSelection( + featureTypeTable.getCheckedElements().length == featureTypeTable + .getTable().getItemCount()); + + page.layout(); + page.pack(); + setPageComplete(validate()); + } + }); - typeSelector.getControl().setLayoutData( - GridDataFactory.fillDefaults().grab(true, false).span(1, 1).create()); - typeSelector.addSelectionChangedListener(new ISelectionChangedListener() { + featureTypeTable.setCheckStateProvider(new ICheckStateProvider() { @Override - public void selectionChanged(SelectionChangedEvent event) { - setPageComplete(!(event.getSelection().isEmpty())); - if (typeSelector.getSelectedObject() != null) { - page.layout(); - page.pack(); + public boolean isChecked(Object element) { + if (!(element instanceof TypeDefinition)) + return false; + return checkboxState(element); + } + + @Override + public boolean isGrayed(Object element) { + if (!(element instanceof TypeDefinition)) + return false; + return checkboxState(element); + } + + /** + * @param element + * @return true if the button cannot be selected + */ + private boolean checkboxState(Object element) { + InstanceService instanceService = PlatformUI.getWorkbench() + .getService(InstanceService.class); + + Set instanceSourceTypes = instanceService + .getInstanceTypes(DataSet.SOURCE); + if (instanceSourceTypes.contains(element)) { + return false; + } + + Set instanceTransformedTypes = instanceService + .getInstanceTypes(DataSet.TRANSFORMED); + if (instanceTransformedTypes.contains(element)) { + return false; } + return true; } }); } page.layout(); page.pack(); } + + private boolean validate() { + return (featureTypeTable.getCheckedElements().length > 0); + } + + /** + * @see eu.esdihumboldt.hale.ui.io.IOWizardPage#updateConfiguration(eu.esdihumboldt.hale.common.core.io.IOProvider) + */ + @Override + public boolean updateConfiguration(InstanceWriter provider) { + super.updateConfiguration(provider); + + Object[] elements = featureTypeTable.getCheckedElements(); + String param = ""; + for (Object el : elements) { + param = param + ((TypeDefinition) el).getName().toString() + ","; + } + provider.setParameter(InstanceTableIOConstants.EXPORT_TYPE, Value.of(param)); + + return true; + } + } diff --git a/io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/InstanceTableIOConstants.java b/io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/InstanceTableIOConstants.java index 0b547d15b7..b1b3c75ea8 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/InstanceTableIOConstants.java +++ b/io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/InstanceTableIOConstants.java @@ -45,4 +45,19 @@ public class InstanceTableIOConstants { */ public static final String EXPORT_TYPE = "selectedExportType"; + /** + * Constant for underscore. + */ + public static final String UNDERSCORE = "_"; + + /** + * Constant for the shape file extension. + */ + public static final String CSV_EXTENSION = ".csv"; + + /** + * Constant for the url. Used to create schema for shape file writer. + */ + public static final String URL_STRING = "url"; + } diff --git a/io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/writer/internal/CSVInstanceWriter.java b/io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/writer/internal/CSVInstanceWriter.java index 0c6c492fba..4a9928d40f 100644 --- a/io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/writer/internal/CSVInstanceWriter.java +++ b/io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/writer/internal/CSVInstanceWriter.java @@ -18,12 +18,16 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -40,7 +44,6 @@ import eu.esdihumboldt.hale.common.core.io.ProgressIndicator; import eu.esdihumboldt.hale.common.core.io.report.IOReport; import eu.esdihumboldt.hale.common.core.io.report.IOReporter; -import eu.esdihumboldt.hale.common.core.io.report.impl.IOMessageImpl; import eu.esdihumboldt.hale.common.instance.model.Instance; import eu.esdihumboldt.hale.common.instance.model.InstanceCollection; import eu.esdihumboldt.hale.common.instance.model.ResourceIterator; @@ -76,27 +79,61 @@ public boolean isCancelable() { protected IOReport execute(ProgressIndicator progress, IOReporter reporter) throws IOProviderConfigurationException, IOException { - boolean solveNestedProperties = getParameter( - InstanceTableIOConstants.SOLVE_NESTED_PROPERTIES).as(Boolean.class, false); - // XXX what does "solve nested properties" mean? - // get separation, quote and escape sign sep = CSVUtil.getSep(this); quote = CSVUtil.getQuote(this); esc = CSVUtil.getEscape(this); - List headerRow = new ArrayList(); // empty list - - // get the parameter to get the type definition String exportType = getParameter(InstanceTableIOConstants.EXPORT_TYPE).as(String.class); - QName selectedTypeName = null; + ArrayList selectedFeatureTypes = new ArrayList<>(); + + if (exportType != null) { + String[] splitExportType = exportType.split(","); + for (String featureType : splitExportType) { + selectedFeatureTypes.add(QName.valueOf(featureType)); + } - if (exportType != null && !exportType.equals("") && !exportType.equals(" ")) { - selectedTypeName = QName.valueOf(exportType); + for (QName selectedTypeName : selectedFeatureTypes) { + // get all instances of the selected Type + InstanceCollection instances = getInstanceCollection(selectedTypeName); + + if (selectedFeatureTypes.size() != 1 && getTarget().getLocation() != null) { + URI location = getTarget().getLocation(); + + String filePath = Paths.get(location).getParent().toString(); + String baseFilename = Paths.get(location).getFileName().toString(); + baseFilename = baseFilename.substring(0, baseFilename.lastIndexOf(".")); + String filenameWithType = filePath + FileSystems.getDefault().getSeparator() + + baseFilename + InstanceTableIOConstants.UNDERSCORE + + selectedTypeName.getLocalPart() + + InstanceTableIOConstants.CSV_EXTENSION; + + addCSVFileByQName(reporter, new FileOutputStream(filenameWithType), instances); + } + else { + addCSVFileByQName(reporter, getTarget().getOutput(), instances); + } + } } - // get all instances of the selected Type - InstanceCollection instances = getInstanceCollection(selectedTypeName); + reporter.setSuccess(true); + return reporter; + } + + /** + * @param reporter + * @param selectedTypeName + * @param instances InstanceCollection available + * @throws IOException + * @throws FileNotFoundException + */ + private void addCSVFileByQName(IOReporter reporter, OutputStream outputStream, + InstanceCollection instances) throws IOException, FileNotFoundException { + + boolean solveNestedProperties = getParameter( + InstanceTableIOConstants.SOLVE_NESTED_PROPERTIES).as(Boolean.class, false); + + List headerRow = new ArrayList(); // empty list // use ResourceIterator in a try block because is closable - // avoid infinite // cleaning project after exporting data @@ -105,9 +142,7 @@ protected IOReport execute(ProgressIndicator progress, IOReporter reporter) try { instance = instanceIterator.next(); } catch (NoSuchElementException e) { - reporter.error( - new IOMessageImpl("There are no instances for the selected type.", e)); - return reporter; + return; } // get definition of current instance (only this properties with @@ -119,7 +154,7 @@ protected IOReport execute(ProgressIndicator progress, IOReporter reporter) // write // it to a temp directory File tempDir = Files.createTempDir(); - File tempFile = new File(tempDir, "tempInstances.csv"); + File tempFile = new File(tempDir, "_tempInstances.csv"); // write instances to csv file (without header) // try to be sure resources are all closed after try-block @@ -138,12 +173,10 @@ protected IOReport execute(ProgressIndicator progress, IOReporter reporter) // header is only finished if all properties have been processed // insert header to temp file and write it to output - insertHeader(tempFile, getTarget().getOutput(), headerRow); + insertHeader(tempFile, outputStream, headerRow); FileUtils.deleteDirectory(tempDir); } - reporter.setSuccess(true); - return reporter; } @Override