Skip to content

Commit

Permalink
Add generic unit tests to check data formatting for BDTopo 2 and 3 ve…
Browse files Browse the repository at this point in the history 
…rsions
  • Loading branch information
ebocher committed Oct 26, 2023
1 parent eb6c6d2 commit 97b7253
Show file tree
Hide file tree
Showing 8 changed files with 341 additions and 83 deletions.
4 changes: 2 additions & 2 deletions bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ Integer loadDataFromPostGIS(Object input_database_properties, Object code, Objec

if (inputTables.surface_eau) {
//Extract surface_eau
def inputTableName = "(SELECT ID, st_setsrid(the_geom, $commune_srid) as the_geom FROM ${inputTables.surface_eau} WHERE st_setsrid(the_geom, $commune_srid) && 'SRID=$commune_srid;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(st_setsrid(the_geom, $commune_srid), 'SRID=$commune_srid;$geomToExtract'::GEOMETRY))"
def inputTableName = "(SELECT ID, st_setsrid(the_geom, $commune_srid) as the_geom , NATURE FROM ${inputTables.surface_eau} WHERE st_setsrid(the_geom, $commune_srid) && 'SRID=$commune_srid;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(st_setsrid(the_geom, $commune_srid), 'SRID=$commune_srid;$geomToExtract'::GEOMETRY))"
outputTableName = "SURFACE_EAU"
logger.debug "Loading in the H2GIS database $outputTableName"
IOMethods.exportToDataBase(sourceConnection, inputTableName, h2gis_datasource.getConnection(), outputTableName, -1, 1000)
Expand Down Expand Up @@ -368,7 +368,7 @@ def filterLinkedShapeFiles(def location, float distance, LinkedHashMap inputTabl
//Extract surface_eau
logger.debug "Loading in the H2GIS database $outputTableName"
outputTableName = "SURFACE_EAU"
h2gis_datasource.execute("DROP TABLE IF EXISTS $outputTableName ; CREATE TABLE $outputTableName as SELECT ID, $formatting_geom FROM ${inputTables.surface_eau} WHERE the_geom && 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(the_geom, 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY)".toString())
h2gis_datasource.execute("DROP TABLE IF EXISTS $outputTableName ; CREATE TABLE $outputTableName as SELECT ID, $formatting_geom , NATURE FROM ${inputTables.surface_eau} WHERE the_geom && 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(the_geom, 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY)".toString())

}

Expand Down
141 changes: 98 additions & 43 deletions bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@
package org.orbisgis.geoclimate.bdtopo

import groovy.transform.BaseScript
import net.postgis.jdbc.geometry.LineString
import org.locationtech.jts.geom.Geometry
import org.locationtech.jts.geom.Polygon
import org.orbisgis.data.H2GIS
import org.orbisgis.data.api.dataset.ISpatialTable
import org.orbisgis.data.jdbc.JdbcDataSource
import org.orbisgis.geoclimate.Geoindicators


@BaseScript BDTopo bdTopo
Expand Down Expand Up @@ -168,11 +170,12 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo
def zIndex = 0
if (formatedHeight.nbLevels > 0) {
Geometry geom = values.the_geom
def srid = geom.getSRID()
for (int i = 0; i < geom.getNumGeometries(); i++) {
Geometry subGeom = geom.getGeometryN(i)
if (subGeom instanceof Polygon && subGeom.getArea() > 1) {
stmt.addBatch """
if (!geom.isEmpty()) {
def srid = geom.getSRID()
for (int i = 0; i < geom.getNumGeometries(); i++) {
Geometry subGeom = geom.getGeometryN(i)
if (subGeom instanceof Polygon && subGeom.getArea() > 1) {
stmt.addBatch """
INSERT INTO ${outputTableName} values(
ST_GEOMFROMTEXT('${subGeom}',$srid),
$id_build,
Expand All @@ -185,7 +188,8 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo
${zIndex}, null)
""".toString()

id_build++
id_build++
}
}
}
}
Expand Down Expand Up @@ -322,12 +326,10 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "")

def queryMapper = "SELECT "
if (datasource.hasTable(road)) {
def columnNames = datasource.getColumnNames(road)
columnNames.remove("THE_GEOM")
queryMapper += columnNames.join(",")
queryMapper += Geoindicators.DataUtils.aliasColumns(datasource, road, "a", ["THE_GEOM"])
if (zone) {
datasource.createSpatialIndex(road, "the_geom")
queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " +
queryMapper += ", ST_CollectionExtract(st_intersection(a.the_geom, b.the_geom), 2) as the_geom " +
"FROM " +
"$road AS a, $zone AS b " +
"WHERE " +
Expand Down Expand Up @@ -437,14 +439,16 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "")
}
def road_crossing = row.CROSSING
if (road_crossing == 'Gué ou radier') {
qualified_crossing = 'crossing'
qualified_road_zindex = 0
} else if (road_crossing == 'Pont') {
qualified_crossing = 'bridge'
if (!qualified_road_zindex) {
qualified_road_zindex = 1
}
}else if(road_crossing=='NC'){
qualified_crossing=null
}

def road_sens = row.DIRECTION

if (road_sens == "Double") {
Expand All @@ -458,25 +462,29 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "")
}

def ID_SOURCE = row.ID_SOURCE

if (qualified_road_zindex >= 0 && qualified_road_type != 'path') {
Geometry geom = row.the_geom
def epsg = geom.getSRID()
for (int i = 0; i < geom.getNumGeometries(); i++) {
stmt.addBatch """
if (!geom.isEmpty()) {
def epsg = geom.getSRID()
for (int i = 0; i < geom.getNumGeometries(); i++) {
Geometry subGeom = geom.getGeometryN(i)
if (!subGeom.isEmpty()) {
stmt.addBatch """
INSERT INTO $outputTableName VALUES(ST_GEOMFROMTEXT(
'${geom.getGeometryN(i)}',$epsg),
'${subGeom}',$epsg),
${rowcount++},
'${ID_SOURCE}',
${qualified_road_width},
'${qualified_road_type}',
'${qualified_crossing}',
${qualified_crossing ? "'${qualified_crossing}'" : qualified_crossing},
'${qualified_road_surface}',
'${qualified_sidewalk}',
${qualified_road_maxspeed},
${road_sens},
${qualified_road_zindex})
""".toString()
}
}
}
}
}
Expand All @@ -498,15 +506,15 @@ String formatHydroLayer(JdbcDataSource datasource, String water, String zone = "
debug('Hydro transformation starts')
def outputTableName = postfix("HYDRO")
datasource.execute """Drop table if exists $outputTableName;
CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, id serial, ID_SOURCE VARCHAR, TYPE VARCHAR, ZINDEX INTEGER);""".toString()
CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, ID_WATER serial, ID_SOURCE VARCHAR, TYPE VARCHAR, ZINDEX INTEGER);""".toString()

if (water) {
if (datasource.hasTable(water)) {
String query
if (zone) {
datasource.createSpatialIndex(water, "the_geom")
query = "select st_intersection(a.the_geom, b.the_geom) as the_geom " +
", a.ZINDEX, a.ID_SOURCE" +
", a.ZINDEX, a.ID_SOURCE, a.TYPE " +
" FROM " +
"$water AS a, $zone AS b " +
"WHERE " +
Expand All @@ -515,17 +523,49 @@ String formatHydroLayer(JdbcDataSource datasource, String water, String zone = "
query = "select * FROM $water "

}
def water_types =
["Aqueduc" : "aqueduct",
"Canal" : "canal",
"Delta" : "bay",
"Ecoulement canalisé" : "canal",
"Ecoulement endoréique" : "water",
"Ecoulement hyporhéique": "water",
"Ecoulement karstique" : "water",
"Ecoulement naturel" : "water",
"Ecoulement phréatique" : "water",
"Estuaire" : "bay",
"Inconnue" : "water",
"Lac" : "lake",
"Lagune" : "lagoon", "Mangrove": "mangrove",
"Mare" : "pond",
"Plan d'eau de gravière": "pond",
"Plan d'eau de mine": "basin", "Ravine": "water",
"Réservoir-bassin" : "basin",
"Réservoir-bassin d'orage": "basin",
"Réservoir-bassin piscicole": "basin",
"Retenue" : "basin",
"Retenuebarrage": "basin",
"Retenue-bassin portuaire": "basin",
"Retenue-digue": "basin",
"Surface d'eau" :"water",
"Bassin" :"basin"
]

int rowcount = 1
datasource.withBatch(100) { stmt ->
datasource.eachRow(query) { row ->
def water_type = 'water'
def water_type = water_types.get(row.TYPE)
def water_zindex = 0
Geometry geom = row.the_geom
def epsg = geom.getSRID()
for (int i = 0; i < geom.getNumGeometries(); i++) {
Geometry subGeom = geom.getGeometryN(i)
if (subGeom instanceof Polygon && subGeom.getArea() > 1) {
stmt.addBatch "insert into $outputTableName values(ST_GEOMFROMTEXT('${subGeom}',$epsg), ${rowcount++}, '${row.ID_SOURCE}', '${water_type}', ${water_zindex})".toString()
if (water_type) {
Geometry geom = row.the_geom
if (!geom.isEmpty()) {
def epsg = geom.getSRID()
for (int i = 0; i < geom.getNumGeometries(); i++) {
Geometry subGeom = geom.getGeometryN(i)
if (subGeom instanceof Polygon && subGeom.getArea() > 1) {
stmt.addBatch "insert into $outputTableName values(ST_GEOMFROMTEXT('${subGeom}',$epsg), ${rowcount++}, '${row.ID_SOURCE}', '${water_type}', ${water_zindex})".toString()
}
}
}
}
}
Expand Down Expand Up @@ -553,9 +593,7 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = ""
if (rail) {
def queryMapper = "SELECT "
if (datasource.hasTable(rail)) {
def columnNames = datasource.getColumnNames(rail)
columnNames.remove("THE_GEOM")
queryMapper += columnNames.join(",")
queryMapper += Geoindicators.DataUtils.aliasColumns(datasource, rail, "a", ["THE_GEOM"])
if (zone) {
datasource.createSpatialIndex(rail, "the_geom")
queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " +
Expand Down Expand Up @@ -584,14 +622,14 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = ""
'NC' : null]
int rowcount = 1
datasource.withBatch(100) { stmt ->
datasource.eachRow(queryMapper) { row ->
datasource.eachRow(queryMapper.toString()) { row ->
def rail_type = row.TYPE
if (rail_type) {
rail_type = rail_types.get(rail_type)
} else {
rail_type = "unclassified"
}
def rail_usage = ""
def rail_usage = null
if (rail_type in ["highspeed", "rail", "tram", "bridge"]) {
rail_usage = "main"
}
Expand All @@ -613,19 +651,24 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = ""
}
if (rail_zindex >= 0 && rail_type) {
Geometry geom = row.the_geom
def epsg = geom.getSRID()
for (int i = 0; i < geom.getNumGeometries(); i++) {
stmt.addBatch """
if (!geom.isEmpty()) {
def epsg = geom.getSRID()
for (int i = 0; i < geom.getNumGeometries(); i++) {
Geometry subGeom = geom.getGeometryN(i)
if (!subGeom.isEmpty()) {
stmt.addBatch """
INSERT INTO $outputTableName values(ST_GEOMFROMTEXT(
'${geom.getGeometryN(i)}',$epsg),
'${subGeom}',$epsg),
${rowcount++},
'${row.ID_SOURCE}',
'${rail_type}',
'${rail_crossing}',
${rail_crossing ? "'${rail_crossing}'" : rail_crossing},
${rail_zindex},
${rail_width},
'${rail_usage}')
${rail_usage ? "'${rail_usage}'" : rail_usage})
"""
}
}
}
}
}
Expand All @@ -649,13 +692,11 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin
def outputTableName = postfix "VEGET"
datasource """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, id serial, ID_SOURCE VARCHAR, TYPE VARCHAR, HEIGHT_CLASS VARCHAR(4), ZINDEX INTEGER);""".toString()
CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, id_veget serial, ID_SOURCE VARCHAR, TYPE VARCHAR, HEIGHT_CLASS VARCHAR(4), ZINDEX INTEGER);""".toString()
if (vegetation) {
def queryMapper = "SELECT "
if (datasource.hasTable(vegetation)) {
def columnNames = datasource.getColumnNames(vegetation)
columnNames.remove("THE_GEOM")
queryMapper += columnNames.join(",")
queryMapper += Geoindicators.DataUtils.aliasColumns(datasource, vegetation, "a", ["THE_GEOM"])
if (zone) {
datasource.createSpatialIndex(vegetation, "the_geom")
queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " +
Expand Down Expand Up @@ -684,7 +725,8 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin
'Rizière' : 'rice_field',
'Piste en herbe' : 'grass',
'Terrain de football' : 'grass',
'Terrain de rugby' : 'grass']
'Terrain de rugby' : 'grass',
'Marais' : 'marsh']

def vegetation_classes = [
'tree' : 'high',
Expand All @@ -703,7 +745,8 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin
'unclassified' : 'low',
'hops' : 'low',
'rice_field' : 'low',
'grass' : 'low'
'grass' : 'low',
'marsh' : 'low'
]

int rowcount = 1
Expand Down Expand Up @@ -857,3 +900,15 @@ String formatImperviousLayer(H2GIS datasource, String impervious) {
debug('Impervious areas transformation finishes')
return outputTableName
}

/**
*
* @return
*/
String setAliasOnColumns(String tableName){
return columnNames.inject([]) { result, iter ->
result += "a.$iter"
}.join(",")
}


Loading

0 comments on commit 97b7253

Please sign in to comment.