Skip to content

Commit

Permalink
Merge pull request #1087 from ebocher/postgis_geojson
Browse files Browse the repository at this point in the history 
Fix all drivers with postgis
  • Loading branch information
@SPalominos
SPalominos authored Jun 11, 2020
2 parents daccb5e + a691737 commit 189a181
Show file tree
Hide file tree
Showing 16 changed files with 441 additions and 137 deletions.
2 changes: 1 addition & 1 deletion h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/DBFDriverFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ public void importFile(Connection connection, String tableReference, File fileNa
@Override
public void importFile(Connection connection, String tableReference, File fileName,
boolean deleteTables,ProgressVisitor progress) throws SQLException, IOException {
importFile(connection, tableReference, fileName,null, false, progress);
importFile(connection, tableReference, fileName,null, deleteTables, progress);
}

@Override
Expand Down
80 changes: 36 additions & 44 deletions h2gis-functions/src/main/java/org/h2gis/functions/io/geojson/GeoJsonReaderDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,19 @@
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import org.h2.value.ValueGeometry;
import org.h2.value.ValueNull;
import org.h2gis.api.EmptyProgressVisitor;
import org.h2gis.api.ProgressVisitor;
import org.h2gis.utilities.JDBCUtilities;
import org.h2gis.utilities.TableLocation;
import org.locationtech.jts.geom.*;
import org.locationtech.jts.io.WKBWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.math.BigDecimal;
import java.sql.*;
import java.util.*;
import java.util.zip.GZIPInputStream;
Expand All @@ -51,7 +55,6 @@
*/
public class GeoJsonReaderDriver {

private final static ArrayList<String> geomTypes;
private final File fileName;
private final Connection connection;
private static GeometryFactory GF;
Expand All @@ -73,18 +76,9 @@ public class GeoJsonReaderDriver {
private LinkedHashMap<String, Integer> cachedColumnIndex;
private static final int BATCH_MAX_SIZE = 100;

static {
geomTypes = new ArrayList<String>();
geomTypes.add(GeoJsonField.POINT);
geomTypes.add(GeoJsonField.MULTIPOINT);
geomTypes.add(GeoJsonField.LINESTRING);
geomTypes.add(GeoJsonField.MULTILINESTRING);
geomTypes.add(GeoJsonField.POLYGON);
geomTypes.add(GeoJsonField.MULTIPOLYGON);

}
private Set finalGeometryTypes;
private JsonEncoding jsonEncoding;
private boolean hasZ =false;

/**
* Driver to import a GeoJSON file into a spatial table.
Expand Down Expand Up @@ -132,7 +126,7 @@ public void read(ProgressVisitor progress, String tableReference) throws SQLExce
this.tableLocation = TableLocation.parse(tableReference, isH2);
if (deleteTable) {
Statement stmt = connection.createStatement();
stmt.execute("DROP TABLE IF EXISTS " + tableLocation);
stmt.execute("DROP TABLE IF EXISTS " + tableLocation.toString(isH2));
stmt.close();
}

Expand All @@ -145,13 +139,12 @@ public void read(ProgressVisitor progress, String tableReference) throws SQLExce
GF = new GeometryFactory(new PrecisionModel(), parsedSRID);
fis = new FileInputStream(fileName);
parseData(new GZIPInputStream(fis));
setGeometryTypeConstraints();
connection.setAutoCommit(true);
} else {
throw new SQLException("Cannot create the table " + tableLocation + " to import the GeoJSON data");
}
} else {
JDBCUtilities.createEmptyTable(connection, tableLocation.toString());
JDBCUtilities.createEmptyTable(connection, tableLocation.toString(isH2));
}
} else {
throw new SQLException("The geojson read driver supports only geojson or gz extensions");
Expand Down Expand Up @@ -185,12 +178,11 @@ public void read(ProgressVisitor progress, String tableReference) throws SQLExce
*/
private void parseGeoJson(ProgressVisitor progress) throws SQLException, IOException {
this.progress = progress.subProcess(100);
init();;
init();
if (parseMetadata(new FileInputStream(fileName))) {
connection.setAutoCommit(false);
GF = new GeometryFactory(new PrecisionModel(), parsedSRID);
parseData(new FileInputStream(fileName));
setGeometryTypeConstraints();
connection.setAutoCommit(true);

} else {
Expand Down Expand Up @@ -237,15 +229,20 @@ private boolean parseMetadata(InputStream is) throws SQLException, IOException {
if (hasGeometryField) {
StringBuilder createTable = new StringBuilder();
createTable.append("CREATE TABLE ");
createTable.append(tableLocation);
createTable.append(tableLocation.toString(isH2));
createTable.append(" (");

//Add the geometry column
createTable.append("THE_GEOM GEOMETRY(geometry,").append(parsedSRID).append(")");

String finalGeometryType = GeoJsonField.GEOMETRY;
if (finalGeometryTypes.size() == 1) {
finalGeometryType = (String) finalGeometryTypes.iterator().next();
createTable.append("THE_GEOM GEOMETRY(").append(hasZ?finalGeometryType+"Z":finalGeometryType).append(",").append(parsedSRID).append(")");
}
else{
createTable.append("THE_GEOM GEOMETRY(GEOMETRY,").append(parsedSRID).append(")");
}
cachedColumnIndex = new LinkedHashMap<>();
StringBuilder insertTable = new StringBuilder("INSERT INTO ");
insertTable.append(tableLocation).append(" VALUES(?");
insertTable.append(tableLocation.toString(isH2)).append(" VALUES(ST_GeomFromWKB(?, ").append(parsedSRID).append(")");
int i = 1;
for (Map.Entry<String, Integer> columns : cachedColumnNames.entrySet()) {
String columnName = columns.getKey();
Expand Down Expand Up @@ -655,6 +652,7 @@ private void parseCoordinateMetadata(JsonParser jp) throws IOException {
jp.nextToken();
if (jp.getCurrentToken() != JsonToken.END_ARRAY) {
jp.nextToken(); // exit array
hasZ=true;
}
jp.nextToken();
}
Expand Down Expand Up @@ -842,7 +840,8 @@ private void setGeometry(JsonParser jp, Object[] values) throws IOException, SQL
jp.nextToken(); // FIELD_NAME type
jp.nextToken(); //VALUE_STRING Point
String geometryType = jp.getText();
values[0] = parseGeometry(jp, geometryType);
Geometry geom = parseGeometry(jp, geometryType);
values[0] = ValueGeometry.getFromGeometry(geom).getBytesNoCopy();
}
}

Expand Down Expand Up @@ -900,7 +899,8 @@ private void parseProperties(JsonParser jp, Object[] values) throws IOException,
} else if (value == JsonToken.VALUE_NUMBER_FLOAT) {
values[cachedColumnIndex.get(fieldName)] = jp.getValueAsDouble();
} else if (value == JsonToken.VALUE_NUMBER_INT) {
values[cachedColumnIndex.get(fieldName)] = jp.getBigIntegerValue();
BigDecimal bigDecId = new BigDecimal(jp.getBigIntegerValue());
values[cachedColumnIndex.get(fieldName)] = bigDecId;
} else if (value == JsonToken.START_ARRAY) {
ArrayList<Object> arrayList = parseArray(jp);
values[cachedColumnIndex.get(fieldName)] = arrayList.toArray();
Expand Down Expand Up @@ -954,7 +954,6 @@ private void parseFeatures(JsonParser jp) throws IOException, SQLException {
for (int i = 0; i < values.length; i++) {
preparedStatement.setObject(i + 1, values[i]);
}

preparedStatement.addBatch();
batchSize++;
if (batchSize >= BATCH_MAX_SIZE) {
Expand All @@ -968,16 +967,20 @@ private void parseFeatures(JsonParser jp) throws IOException, SQLException {
featureCounter++;
progress.setStep((int) ((featureCounter / nbFeature) * 100));
if (batchSize > 0) {
preparedStatement.executeBatch();
connection.commit();
preparedStatement.clearBatch();
try {
preparedStatement.executeBatch();
connection.commit();
preparedStatement.clearBatch();
}catch (SQLException ex){
throw new SQLException(ex.getNextException());
}
}
} else {
throw new SQLException("Malformed GeoJSON file. Expected 'Feature', found '" + geomType + "'");
}
}
//LOOP END_ARRAY ]
log.info(featureCounter + " geojson features have been imported.");
log.info(featureCounter-1 + " geojson features have been imported.");
} else {
throw new SQLException("Malformed GeoJSON file. Expected 'features', found '" + firstParam + "'");
}
Expand Down Expand Up @@ -1262,7 +1265,11 @@ private Coordinate parseCoordinate(JsonParser jp) throws IOException {
//We look for a z value
jp.nextToken();
if (jp.getCurrentToken() == JsonToken.END_ARRAY) {
coord = new Coordinate(x, y);
if(hasZ) {
coord = new Coordinate(x, y, 0);
}else {
coord = new Coordinate(x, y);
}
} else {
double z = jp.getDoubleValue();
jp.nextToken(); // exit array
Expand Down Expand Up @@ -1373,21 +1380,6 @@ private String skipCRS(JsonParser jp) throws IOException {
return jp.getText();
}

/**
* Adds the geometry type constraint and the SRID
*/
private void setGeometryTypeConstraints() throws SQLException {
String finalGeometryType = GeoJsonField.GEOMETRY;
if (finalGeometryTypes.size() == 1) {
finalGeometryType = (String) finalGeometryTypes.iterator().next();
}
if (isH2) {
finalGeometryType = GeoJsonField.GEOMETRY;//workaround for H2
connection.createStatement().execute(String.format("ALTER TABLE %s ALTER COLUMN the_geom %s", tableLocation.toString(), finalGeometryType));
} else {
connection.createStatement().execute(String.format("ALTER TABLE %s ALTER COLUMN the_geom SET DATA TYPE geometry(%s,%d)", tableLocation.toString(), finalGeometryType, parsedSRID));
}
}

/**
* Parses Json Array. Syntax: Json Array: {"member1": value1}, value2,
Expand Down
2 changes: 1 addition & 1 deletion h2gis-functions/src/main/java/org/h2gis/functions/io/gpx/model/AbstractGpxParserDefault.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ public boolean read(String tableName, ProgressVisitor progress) throws SQLExcept
GPXTablesFactory.dropOSMTables(connection, isH2, requestedTable);
}
if (fileName.length() == 0) {
JDBCUtilities.createEmptyTable(connection, requestedTable.toString());
JDBCUtilities.createEmptyTable(connection, requestedTable.toString(isH2));
}
else {
String table = requestedTable.getTable();
Expand Down
2 changes: 1 addition & 1 deletion h2gis-functions/src/main/java/org/h2gis/functions/io/gpx/model/GPXTablesFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ public static PreparedStatement createTrackPointsTable(Connection connection, St
* @throws SQLException
*/
public static void dropOSMTables(Connection connection, boolean isH2, TableLocation tablePrefix) throws SQLException {
String gpxTableName = tablePrefix.toString();
String gpxTableName = tablePrefix.toString(isH2);
String[] gpxTables = new String[]{WAYPOINT,ROUTE,ROUTEPOINT, TRACK, TRACKPOINT, TRACKSEGMENT};
StringBuilder sb = new StringBuilder("drop table if exists ");
String gpxTableSuffix = gpxTables[0];
Expand Down
4 changes: 2 additions & 2 deletions h2gis-functions/src/main/java/org/h2gis/functions/io/shp/SHPDriverFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,8 +278,8 @@ public void importFile(Connection connection, String tableReference, File fileNa
final TableLocation parse;
int srid;
try (
// Build CREATE TABLE sql request
Statement st = connection.createStatement()) {
// Build CREATE TABLE sql request
Statement st = connection.createStatement()) {
String types = DBFDriverFunction.getSQLColumnTypes(dbfHeader, isH2);
if (!types.isEmpty()) {
types = ", " + types;
Expand Down
1 change: 0 additions & 1 deletion h2gis-functions/src/main/java/org/h2gis/functions/io/shp/SHPRead.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.h2gis.api.AbstractFunction;
import org.h2gis.api.EmptyProgressVisitor;
import org.h2gis.api.ScalarFunction;
import org.h2gis.functions.io.utility.FileUtil;
import org.h2gis.utilities.TableLocation;
import org.h2gis.utilities.URIUtilities;

Expand Down
8 changes: 4 additions & 4 deletions h2gis-functions/src/main/java/org/h2gis/functions/io/tsv/TSVDriverFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ public void exportTable(Connection connection, String tableReference, File fileN
new ZipOutputStream(new FileOutputStream(fileName))))) {
try (Statement st = connection.createStatement()) {
JDBCUtilities.attachCancelResultSet(st, progress);
exportFromResultSet(connection, st.executeQuery(tableReference), bw, encoding, new EmptyProgressVisitor());
exportFromResultSet(connection, st.executeQuery(location.toString(isH2)), bw, encoding, new EmptyProgressVisitor());
}
}
} else {
Expand All @@ -161,7 +161,7 @@ public void exportTable(Connection connection, String tableReference, File fileN
try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName)))) {
try (Statement st = connection.createStatement()) {
JDBCUtilities.attachCancelResultSet(st, progress);
exportFromResultSet(connection, st.executeQuery("SELECT * FROM " + location.toString()), bw, encoding, new EmptyProgressVisitor());
exportFromResultSet(connection, st.executeQuery("SELECT * FROM " + location.toString(isH2)), bw, encoding, new EmptyProgressVisitor());
}
}
} else if (FileUtil.isExtensionWellFormated(fileName, "gz")) {
Expand All @@ -174,7 +174,7 @@ public void exportTable(Connection connection, String tableReference, File fileN
new GZIPOutputStream(new FileOutputStream(fileName))))) {
try (Statement st = connection.createStatement()) {
JDBCUtilities.attachCancelResultSet(st, progress);
exportFromResultSet(connection, st.executeQuery("SELECT * FROM " + location.toString()), bw, encoding, new EmptyProgressVisitor());
exportFromResultSet(connection, st.executeQuery("SELECT * FROM " + location.toString(isH2)), bw, encoding, new EmptyProgressVisitor());
}
}
} else if (FileUtil.isExtensionWellFormated(fileName, "zip")) {
Expand All @@ -187,7 +187,7 @@ public void exportTable(Connection connection, String tableReference, File fileN
new ZipOutputStream(new FileOutputStream(fileName))))) {
try (Statement st = connection.createStatement()) {
JDBCUtilities.attachCancelResultSet(st, progress);
exportFromResultSet(connection, st.executeQuery("SELECT * FROM " + location.toString()), bw, encoding, new EmptyProgressVisitor());
exportFromResultSet(connection, st.executeQuery("SELECT * FROM " + location.toString(isH2)), bw, encoding, new EmptyProgressVisitor());
}
}
} else {
Expand Down
3 changes: 0 additions & 3 deletions h2gis-functions/src/main/java/org/h2gis/functions/spatial/convert/ST_PolyFromWKB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,7 @@
import org.h2.value.ValueGeometry;
import org.h2gis.api.DeterministicScalarFunction;
import org.h2gis.utilities.GeometryTypeCodes;
import org.h2gis.utilities.GeometryMetaData;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.io.ParseException;
import org.locationtech.jts.io.WKBReader;

import java.io.IOException;
import java.sql.SQLException;
Expand Down
Loading

0 comments on commit 189a181

Please sign in to comment.