Skip to content

Commit

Permalink
fixup! fix: download linked objects in a WFS using "resolvedepth"
Browse files Browse the repository at this point in the history
  • Loading branch information
@emanuelaepure10
emanuelaepure10 committed Aug 14, 2024
1 parent 86b21a3 commit 6c526f7
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 125 deletions.
13 changes: 11 additions & 2 deletions io/plugins/eu.esdihumboldt.hale.io.gml.test/src/data/filter_doubles/filter_with_doubles.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
</xplan:BP_Plan>
</wfs:member>
<wfs:member>
<xplan:BP_Plan gml:id="GML_5969D76A-3674-4D22-2BA7-DA2F04FAB74E">
<xplan:BP_Plan gml:id="GML_D4BF1E6F-133B-4F58-64A8-7F1C3652A5CB">
<gml:boundedBy>
<gml:Envelope srsName="http://www.opengis.net/def/crs/EPSG/0/4258">
<gml:lowerCorner>48.878401 10.043646</gml:lowerCorner>
Expand Down Expand Up @@ -167,7 +167,16 @@
<xplan:name>66-02/8</xplan:name>
<xplan:versionBauNVODatum>1993-04-22</xplan:versionBauNVODatum>
<xplan:versionBauNVOText>BauNVO 1990 zul. geändert 22.04.1993</xplan:versionBauNVOText>
<xplan:gehoertZuPlan xlink:href="#GML_5969D76A-3674-4D22-2BA7-DA2F04FAB74E"/>
<xplan:gehoertZuPlan xlink:href="#GML_D4BF1E6F-133B-4F58-64A8-7F1C3652A5CB"/>
</xplan:BP_Bereich>
</wfs:member>
<wfs:member>
<xplan:BP_Bereich gml:id="GML_6730AEF5-C86C-42E9-EF80-064BFD472CB1">
<xplan:nummer>0</xplan:nummer>
<xplan:name>66-02/8</xplan:name>
<xplan:versionBauNVODatum>1993-04-22</xplan:versionBauNVODatum>
<xplan:versionBauNVOText>BauNVO 1990 zul. geändert 22.04.1993</xplan:versionBauNVOText>
<xplan:gehoertZuPlan xlink:href="#GML_D4BF1E6F-133B-4F58-64A8-7F1C3652A5CB"/>
</xplan:BP_Bereich>
</wfs:member>
</wfs:SimpleFeatureCollection>
Expand Down
34 changes: 13 additions & 21 deletions ...o.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/DuplicateIDsFilterIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,10 @@
import eu.esdihumboldt.hale.io.gml.geometry.GMLConstants;

/**
* Filter the IDs
* Filter the instances by gml:id, storing only those with unique IDs.
*/
public class DuplicateIDsFilterIterator implements InstanceIterator {

// store the "main" instances of the GML
private final HashSet<String> uniqueIDInstances = new HashSet<String>();
private final InstanceIterator iterator;
private Instance nextInstance;
Expand Down Expand Up @@ -93,24 +92,23 @@ public void skip() {
public boolean hasNext() {
if (nextInstance == null || instranceAlreadyReturned) {

if (!iterator.hasNext()) {
return false;
}
else {
Instance instance = iterator.next();
if (!isTheInstancePresent(instance)) {
nextInstance = instance;
instranceAlreadyReturned = false;
return true;
while (true) {
if (iterator != null && !iterator.hasNext()) {
return false;
}
else {
nextInstance = null;
instranceAlreadyReturned = true;
return false;
Instance instance = iterator.next();
if (!isTheInstancePresent(instance)) {
nextInstance = instance;
instranceAlreadyReturned = false;
return true;
}
}
}
}
return true;
else {
return true;
}
}

/**
Expand Down Expand Up @@ -138,16 +136,10 @@ private boolean isTheInstancePresent(Instance instance) {
String gmlIDToCheck = (String) gmlID[0];

if (!uniqueIDInstances.contains(gmlIDToCheck)) {
System.out.println("ADD gmlIDToCheck:" + gmlIDToCheck + " "
+ uniqueIDInstances.contains(gmlIDToCheck) + ", size: "
+ uniqueIDInstances.size());
uniqueIDInstances.add(gmlIDToCheck);
return false;
}
else {
System.out.println("gmlIDToCheck:" + gmlIDToCheck + " "
+ uniqueIDInstances.contains(gmlIDToCheck) + ", size: "
+ uniqueIDInstances.size());
return true;
}

Expand Down
125 changes: 23 additions & 102 deletions ...l/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/WfsBackedGmlInstanceCollection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,10 @@
import java.net.URL;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;

import javax.xml.namespace.QName;

import org.apache.commons.io.FileUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URIBuilder;
Expand All @@ -50,8 +47,8 @@
import eu.esdihumboldt.hale.common.instance.model.impl.IndexInstanceReference;
import eu.esdihumboldt.hale.common.schema.model.TypeDefinition;
import eu.esdihumboldt.hale.common.schema.model.TypeIndex;
import eu.esdihumboldt.hale.io.gml.geometry.GMLConstants;
import eu.esdihumboldt.hale.io.gml.reader.internal.GmlInstanceCollection;
import eu.esdihumboldt.hale.io.gml.reader.internal.GmlInstanceCollection.GmlInstanceIterator;
import eu.esdihumboldt.hale.io.gml.reader.internal.instance.StreamGmlInstance;

/**
Expand Down Expand Up @@ -309,6 +306,9 @@ public int size() {
*/
@Override
public InstanceIterator iterator() {
if (primordialQueryParams.containsKey("RESOLVEDEPTH")) {
return new DuplicateIDsFilterIterator(new WfsBackedGmlInstanceIterator());
}
return new WfsBackedGmlInstanceIterator();
}

Expand Down Expand Up @@ -420,12 +420,8 @@ private String getMaxFeaturesParameterName(String version) {
public class WfsBackedGmlInstanceIterator implements InstanceIterator {

private GmlInstanceCollection currentCollection;
private InstanceIterator iterator;
private GmlInstanceIterator iterator;
private int totalFeaturesProcessed;
// store the additional objects
private final HashSet<String> uniqueIDInstancesAdditionalObjects = new HashSet<String>();
// store the "main" features of the GML
private final HashSet<String> uniqueIDMainInstances = new HashSet<String>();

/**
* Create the iterator
Expand All @@ -440,15 +436,17 @@ public WfsBackedGmlInstanceIterator() {
* no result, this {@link WfsBackedGmlInstanceIterator} is closed.
*/
private void proceedOrClose() {
iterator.close();
if (iterator != null) {
iterator.close();
}

if (!isPaged() || isFeatureLimitReached()) {
close();
}
else {
createNextIterator();

if (!iterator.hasNext()) {
if (iterator != null && !iterator.hasNext()) {
close();
}
}
Expand Down Expand Up @@ -491,9 +489,6 @@ private void createNextIterator() {
sourceSchema, restrictToFeatures, ignoreRoot, strict, ignoreNamespaces,
crsProvider, ioProvider);
iterator = currentCollection.iterator();
if (primordialQueryParams.containsKey("RESOLVEDEPTH")) {
iterator = new DuplicateIDsFilterIterator(iterator);
}

// Make sure root element is processed by the iterator
iterator.hasNext();
Expand Down Expand Up @@ -566,12 +561,12 @@ public boolean hasNext() {
* @return true if the number of features processed is equal to (or
* exceeds) the maximum number of features to processed or the
* number of results reported by the WFS.
*
* The !iterator.hasNext() condition is necessary to ensure that
* instances from additionalObjects are processed after the
* final 'main' instance.
*/
protected boolean isFeatureLimitReached() {
// the condition (totalFeaturesProcessed >= size &&
// !iterator.hasNext()) should be there in order to process the
// instances coming from the additionalObjects after the last "main"
// instance
return (maxNumberOfFeatures != UNLIMITED
&& totalFeaturesProcessed >= maxNumberOfFeatures)
|| (size != UNKNOWN_SIZE && totalFeaturesProcessed >= size
Expand All @@ -580,6 +575,10 @@ protected boolean isFeatureLimitReached() {

/**
* @see java.util.Iterator#next()
*
* Condition: if the instance is part of additional objects then
* that instance is it added but is not counted for the
* totalFeaturesProcessed
*/
@Override
public Instance next() {
Expand All @@ -588,96 +587,18 @@ public Instance next() {
}

Instance instance = iterator.next();
return new StreamGmlInstance(instance, totalFeaturesProcessed++);
}

/**
* @param instance Instance
* @return Instance
*/
private Instance processInstanceWithResolveDepth(Instance instance) {
for (QName propertyName : instance.getPropertyNames()) {
if (isGmlIdProperty(propertyName)) {
Object[] gmlID = instance.getProperty(propertyName);
if (gmlID[0] != null) {
String gmlIDToCheck = (String) gmlID[0];

if (instance.getMetaData(GmlInstanceCollection.ADDITIONAL_OBJECTS) != null
&& !instance.getMetaData(GmlInstanceCollection.ADDITIONAL_OBJECTS)
.isEmpty()) {
if (!uniqueIDInstancesAdditionalObjects.contains(gmlIDToCheck)) {
if (uniqueIDMainInstances.contains(gmlIDToCheck)) {
if (iterator.hasNext()) {
return next();
}
}
uniqueIDInstancesAdditionalObjects.add(gmlIDToCheck);
System.out
.println("totalFeaturesProcessed:" + totalFeaturesProcessed
+ " - uniqueIDInstancesAdditionalObjects:"
+ uniqueIDInstancesAdditionalObjects.size()
+ " ADD to additional");
return new StreamGmlInstance(instance, totalFeaturesProcessed);
}
}
else {
if (!uniqueIDMainInstances.contains(gmlIDToCheck)) {
uniqueIDMainInstances.add(gmlIDToCheck);
totalFeaturesProcessed++;
if (uniqueIDInstancesAdditionalObjects.contains(gmlIDToCheck)) {
uniqueIDInstancesAdditionalObjects.remove(gmlIDToCheck);
System.out.println(
"totalFeaturesProcessed:" + totalFeaturesProcessed
+ " - uniqueIDInstancesAdditionalObjects:"
+ uniqueIDInstancesAdditionalObjects.size()
+ " Exists in ADDITIONAL SKIP");
if (iterator.hasNext()) {
return next();
}
}
System.out
.println("totalFeaturesProcessed:" + totalFeaturesProcessed
+ " - uniqueIDInstancesAdditionalObjects:"
+ uniqueIDInstancesAdditionalObjects.size()
+ " ADD to main");
return new StreamGmlInstance(instance, totalFeaturesProcessed);
}
}
}
}
}
return processRemainingInstances();

}
boolean hasAdditionalObjects = instance
.getMetaData(GmlInstanceCollection.ADDITIONAL_OBJECTS) != null
&& !instance.getMetaData(GmlInstanceCollection.ADDITIONAL_OBJECTS).isEmpty();

private Instance processRemainingInstances() {
if (iterator != null && iterator.hasNext()) {
return next();
if (primordialQueryParams.containsKey("RESOLVEDEPTH") && hasAdditionalObjects) {
return new StreamGmlInstance(instance, totalFeaturesProcessed);
}
else {
closeAndRecreateIterator();
if (iterator != null && iterator.hasNext()) {
return next();
}
else {
// return iterator.next();
throw new NoSuchElementException();
}
return new StreamGmlInstance(instance, totalFeaturesProcessed++);
}
}

private void closeAndRecreateIterator() {
close();
createNextIterator();
}

private boolean isGmlIdProperty(QName propertyName) {
return (propertyName.getNamespaceURI().startsWith(GMLConstants.NS_WFS)
|| propertyName.getNamespaceURI().startsWith(GMLConstants.GML_NAMESPACE_CORE))
&& "id".equals(propertyName.getLocalPart())
&& "gml".equals(propertyName.getPrefix());
}

/**
* @see eu.esdihumboldt.hale.common.instance.model.ResourceIterator#close()
*/
Expand Down

0 comments on commit 6c526f7

Please sign in to comment.