From 97b7253bda8426c3956aeff30c578aaf3ea4958f Mon Sep 17 00:00:00 2001 From: ebocher Date: Thu, 26 Oct 2023 17:14:18 +0200 Subject: [PATCH] Add generic unit tests to check data formatting for BDTopo 2 and 3 versions --- .../geoclimate/bdtopo/BDTopoV2Workflow.groovy | 4 +- .../bdtopo/InputDataFormatting.groovy | 141 ++++++++++++------ .../geoclimate/bdtopo/InputDataLoading.groovy | 28 ++-- .../bdtopo/WorkflowAbstractTest.groovy | 28 +--- .../bdtopo/v2/WorkflowBDTopoV2Test.groovy | 108 +++++++++++++- .../bdtopo/v3/WorkflowBDTopoV3Test.groovy | 83 +++++++++++ .../geoclimate/geoindicators/DataUtils.groovy | 30 ++++ .../geoclimate/osm/InputDataFormatting.groovy | 2 +- 8 files changed, 341 insertions(+), 83 deletions(-) diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy index caaeffafd6..9178cb44eb 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy @@ -148,7 +148,7 @@ Integer loadDataFromPostGIS(Object input_database_properties, Object code, Objec if (inputTables.surface_eau) { //Extract surface_eau - def inputTableName = "(SELECT ID, st_setsrid(the_geom, $commune_srid) as the_geom FROM ${inputTables.surface_eau} WHERE st_setsrid(the_geom, $commune_srid) && 'SRID=$commune_srid;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(st_setsrid(the_geom, $commune_srid), 'SRID=$commune_srid;$geomToExtract'::GEOMETRY))" + def inputTableName = "(SELECT ID, st_setsrid(the_geom, $commune_srid) as the_geom , NATURE FROM ${inputTables.surface_eau} WHERE st_setsrid(the_geom, $commune_srid) && 'SRID=$commune_srid;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(st_setsrid(the_geom, $commune_srid), 'SRID=$commune_srid;$geomToExtract'::GEOMETRY))" outputTableName = "SURFACE_EAU" logger.debug "Loading in the H2GIS database $outputTableName" IOMethods.exportToDataBase(sourceConnection, inputTableName, h2gis_datasource.getConnection(), outputTableName, -1, 1000) @@ -368,7 +368,7 @@ def filterLinkedShapeFiles(def location, float distance, LinkedHashMap inputTabl //Extract surface_eau logger.debug "Loading in the H2GIS database $outputTableName" outputTableName = "SURFACE_EAU" - h2gis_datasource.execute("DROP TABLE IF EXISTS $outputTableName ; CREATE TABLE $outputTableName as SELECT ID, $formatting_geom FROM ${inputTables.surface_eau} WHERE the_geom && 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(the_geom, 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY)".toString()) + h2gis_datasource.execute("DROP TABLE IF EXISTS $outputTableName ; CREATE TABLE $outputTableName as SELECT ID, $formatting_geom , NATURE FROM ${inputTables.surface_eau} WHERE the_geom && 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(the_geom, 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY)".toString()) } diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy index 5fb41c4fcb..7e54624257 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy @@ -20,11 +20,13 @@ package org.orbisgis.geoclimate.bdtopo import groovy.transform.BaseScript +import net.postgis.jdbc.geometry.LineString import org.locationtech.jts.geom.Geometry import org.locationtech.jts.geom.Polygon import org.orbisgis.data.H2GIS import org.orbisgis.data.api.dataset.ISpatialTable import org.orbisgis.data.jdbc.JdbcDataSource +import org.orbisgis.geoclimate.Geoindicators @BaseScript BDTopo bdTopo @@ -168,11 +170,12 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo def zIndex = 0 if (formatedHeight.nbLevels > 0) { Geometry geom = values.the_geom - def srid = geom.getSRID() - for (int i = 0; i < geom.getNumGeometries(); i++) { - Geometry subGeom = geom.getGeometryN(i) - if (subGeom instanceof Polygon && subGeom.getArea() > 1) { - stmt.addBatch """ + if (!geom.isEmpty()) { + def srid = geom.getSRID() + for (int i = 0; i < geom.getNumGeometries(); i++) { + Geometry subGeom = geom.getGeometryN(i) + if (subGeom instanceof Polygon && subGeom.getArea() > 1) { + stmt.addBatch """ INSERT INTO ${outputTableName} values( ST_GEOMFROMTEXT('${subGeom}',$srid), $id_build, @@ -185,7 +188,8 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo ${zIndex}, null) """.toString() - id_build++ + id_build++ + } } } } @@ -322,12 +326,10 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") def queryMapper = "SELECT " if (datasource.hasTable(road)) { - def columnNames = datasource.getColumnNames(road) - columnNames.remove("THE_GEOM") - queryMapper += columnNames.join(",") + queryMapper += Geoindicators.DataUtils.aliasColumns(datasource, road, "a", ["THE_GEOM"]) if (zone) { datasource.createSpatialIndex(road, "the_geom") - queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " + + queryMapper += ", ST_CollectionExtract(st_intersection(a.the_geom, b.the_geom), 2) as the_geom " + "FROM " + "$road AS a, $zone AS b " + "WHERE " + @@ -437,14 +439,16 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") } def road_crossing = row.CROSSING if (road_crossing == 'Gué ou radier') { + qualified_crossing = 'crossing' qualified_road_zindex = 0 } else if (road_crossing == 'Pont') { qualified_crossing = 'bridge' if (!qualified_road_zindex) { qualified_road_zindex = 1 } + }else if(road_crossing=='NC'){ + qualified_crossing=null } - def road_sens = row.DIRECTION if (road_sens == "Double") { @@ -458,25 +462,29 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") } def ID_SOURCE = row.ID_SOURCE - if (qualified_road_zindex >= 0 && qualified_road_type != 'path') { Geometry geom = row.the_geom - def epsg = geom.getSRID() - for (int i = 0; i < geom.getNumGeometries(); i++) { - stmt.addBatch """ + if (!geom.isEmpty()) { + def epsg = geom.getSRID() + for (int i = 0; i < geom.getNumGeometries(); i++) { + Geometry subGeom = geom.getGeometryN(i) + if (!subGeom.isEmpty()) { + stmt.addBatch """ INSERT INTO $outputTableName VALUES(ST_GEOMFROMTEXT( - '${geom.getGeometryN(i)}',$epsg), + '${subGeom}',$epsg), ${rowcount++}, '${ID_SOURCE}', ${qualified_road_width}, '${qualified_road_type}', - '${qualified_crossing}', + ${qualified_crossing ? "'${qualified_crossing}'" : qualified_crossing}, '${qualified_road_surface}', '${qualified_sidewalk}', ${qualified_road_maxspeed}, ${road_sens}, ${qualified_road_zindex}) """.toString() + } + } } } } @@ -498,7 +506,7 @@ String formatHydroLayer(JdbcDataSource datasource, String water, String zone = " debug('Hydro transformation starts') def outputTableName = postfix("HYDRO") datasource.execute """Drop table if exists $outputTableName; - CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, id serial, ID_SOURCE VARCHAR, TYPE VARCHAR, ZINDEX INTEGER);""".toString() + CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, ID_WATER serial, ID_SOURCE VARCHAR, TYPE VARCHAR, ZINDEX INTEGER);""".toString() if (water) { if (datasource.hasTable(water)) { @@ -506,7 +514,7 @@ String formatHydroLayer(JdbcDataSource datasource, String water, String zone = " if (zone) { datasource.createSpatialIndex(water, "the_geom") query = "select st_intersection(a.the_geom, b.the_geom) as the_geom " + - ", a.ZINDEX, a.ID_SOURCE" + + ", a.ZINDEX, a.ID_SOURCE, a.TYPE " + " FROM " + "$water AS a, $zone AS b " + "WHERE " + @@ -515,17 +523,49 @@ String formatHydroLayer(JdbcDataSource datasource, String water, String zone = " query = "select * FROM $water " } + def water_types = + ["Aqueduc" : "aqueduct", + "Canal" : "canal", + "Delta" : "bay", + "Ecoulement canalisé" : "canal", + "Ecoulement endoréique" : "water", + "Ecoulement hyporhéique": "water", + "Ecoulement karstique" : "water", + "Ecoulement naturel" : "water", + "Ecoulement phréatique" : "water", + "Estuaire" : "bay", + "Inconnue" : "water", + "Lac" : "lake", + "Lagune" : "lagoon", "Mangrove": "mangrove", + "Mare" : "pond", + "Plan d'eau de gravière": "pond", + "Plan d'eau de mine": "basin", "Ravine": "water", + "Réservoir-bassin" : "basin", + "Réservoir-bassin d'orage": "basin", + "Réservoir-bassin piscicole": "basin", + "Retenue" : "basin", + "Retenuebarrage": "basin", + "Retenue-bassin portuaire": "basin", + "Retenue-digue": "basin", + "Surface d'eau" :"water", + "Bassin" :"basin" + ] + int rowcount = 1 datasource.withBatch(100) { stmt -> datasource.eachRow(query) { row -> - def water_type = 'water' + def water_type = water_types.get(row.TYPE) def water_zindex = 0 - Geometry geom = row.the_geom - def epsg = geom.getSRID() - for (int i = 0; i < geom.getNumGeometries(); i++) { - Geometry subGeom = geom.getGeometryN(i) - if (subGeom instanceof Polygon && subGeom.getArea() > 1) { - stmt.addBatch "insert into $outputTableName values(ST_GEOMFROMTEXT('${subGeom}',$epsg), ${rowcount++}, '${row.ID_SOURCE}', '${water_type}', ${water_zindex})".toString() + if (water_type) { + Geometry geom = row.the_geom + if (!geom.isEmpty()) { + def epsg = geom.getSRID() + for (int i = 0; i < geom.getNumGeometries(); i++) { + Geometry subGeom = geom.getGeometryN(i) + if (subGeom instanceof Polygon && subGeom.getArea() > 1) { + stmt.addBatch "insert into $outputTableName values(ST_GEOMFROMTEXT('${subGeom}',$epsg), ${rowcount++}, '${row.ID_SOURCE}', '${water_type}', ${water_zindex})".toString() + } + } } } } @@ -553,9 +593,7 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = "" if (rail) { def queryMapper = "SELECT " if (datasource.hasTable(rail)) { - def columnNames = datasource.getColumnNames(rail) - columnNames.remove("THE_GEOM") - queryMapper += columnNames.join(",") + queryMapper += Geoindicators.DataUtils.aliasColumns(datasource, rail, "a", ["THE_GEOM"]) if (zone) { datasource.createSpatialIndex(rail, "the_geom") queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " + @@ -584,14 +622,14 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = "" 'NC' : null] int rowcount = 1 datasource.withBatch(100) { stmt -> - datasource.eachRow(queryMapper) { row -> + datasource.eachRow(queryMapper.toString()) { row -> def rail_type = row.TYPE if (rail_type) { rail_type = rail_types.get(rail_type) } else { rail_type = "unclassified" } - def rail_usage = "" + def rail_usage = null if (rail_type in ["highspeed", "rail", "tram", "bridge"]) { rail_usage = "main" } @@ -613,19 +651,24 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = "" } if (rail_zindex >= 0 && rail_type) { Geometry geom = row.the_geom - def epsg = geom.getSRID() - for (int i = 0; i < geom.getNumGeometries(); i++) { - stmt.addBatch """ + if (!geom.isEmpty()) { + def epsg = geom.getSRID() + for (int i = 0; i < geom.getNumGeometries(); i++) { + Geometry subGeom = geom.getGeometryN(i) + if (!subGeom.isEmpty()) { + stmt.addBatch """ INSERT INTO $outputTableName values(ST_GEOMFROMTEXT( - '${geom.getGeometryN(i)}',$epsg), + '${subGeom}',$epsg), ${rowcount++}, '${row.ID_SOURCE}', '${rail_type}', - '${rail_crossing}', + ${rail_crossing ? "'${rail_crossing}'" : rail_crossing}, ${rail_zindex}, ${rail_width}, - '${rail_usage}') + ${rail_usage ? "'${rail_usage}'" : rail_usage}) """ + } + } } } } @@ -649,13 +692,11 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin def outputTableName = postfix "VEGET" datasource """ DROP TABLE IF EXISTS $outputTableName; - CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, id serial, ID_SOURCE VARCHAR, TYPE VARCHAR, HEIGHT_CLASS VARCHAR(4), ZINDEX INTEGER);""".toString() + CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, id_veget serial, ID_SOURCE VARCHAR, TYPE VARCHAR, HEIGHT_CLASS VARCHAR(4), ZINDEX INTEGER);""".toString() if (vegetation) { def queryMapper = "SELECT " if (datasource.hasTable(vegetation)) { - def columnNames = datasource.getColumnNames(vegetation) - columnNames.remove("THE_GEOM") - queryMapper += columnNames.join(",") + queryMapper += Geoindicators.DataUtils.aliasColumns(datasource, vegetation, "a", ["THE_GEOM"]) if (zone) { datasource.createSpatialIndex(vegetation, "the_geom") queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " + @@ -684,7 +725,8 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin 'Rizière' : 'rice_field', 'Piste en herbe' : 'grass', 'Terrain de football' : 'grass', - 'Terrain de rugby' : 'grass'] + 'Terrain de rugby' : 'grass', + 'Marais' : 'marsh'] def vegetation_classes = [ 'tree' : 'high', @@ -703,7 +745,8 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin 'unclassified' : 'low', 'hops' : 'low', 'rice_field' : 'low', - 'grass' : 'low' + 'grass' : 'low', + 'marsh' : 'low' ] int rowcount = 1 @@ -857,3 +900,15 @@ String formatImperviousLayer(H2GIS datasource, String impervious) { debug('Impervious areas transformation finishes') return outputTableName } + +/** + * + * @return + */ +String setAliasOnColumns(String tableName){ + return columnNames.inject([]) { result, iter -> + result += "a.$iter" + }.join(",") +} + + diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy index 29dcfc12c1..3a608a0eee 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy @@ -147,7 +147,7 @@ def loadV2( String surface_eau = tablesExist.get("surface_eau") if (!surface_eau) { surface_eau = "surface_eau" - datasource.execute("DROP TABLE IF EXISTS $surface_eau; CREATE TABLE $surface_eau (THE_GEOM geometry(polygon, $srid), ID varchar);".toString()) + datasource.execute("DROP TABLE IF EXISTS $surface_eau; CREATE TABLE $surface_eau (THE_GEOM geometry(polygon, $srid), ID varchar, NATURE VARCHAR);".toString()) } String zone_vegetation = tablesExist.get("zone_vegetation") if (!zone_vegetation) { @@ -239,9 +239,9 @@ def loadV2( datasource.execute(""" DROP TABLE IF EXISTS INPUT_ROAD; CREATE TABLE INPUT_ROAD (THE_GEOM geometry, ID_SOURCE varchar(24), WIDTH DOUBLE PRECISION, TYPE varchar, ZINDEX integer, CROSSING varchar, - DIRECTION varchar, MAXSPEED INTEGER, RANK INTEGER, ADMIN_SCALE VARCHAR) + DIRECTION varchar, RANK INTEGER, ADMIN_SCALE VARCHAR) AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.LARGEUR, a.NATURE, - a.POS_SOL, a.FRANCHISST, a.SENS ,NULL, + a.POS_SOL, a.FRANCHISST, a.SENS , CASE WHEN a.IMPORTANCE IN ('1', '2', '3', '4', '5') THEN CAST (a.IMPORTANCE AS INTEGER) ELSE NULL END , a.CL_ADMIN FROM $route a, ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0; @@ -259,8 +259,8 @@ def loadV2( //6. Prepare the Hydrography table (from the layer "SURFACE_EAU") that are in the study area (ZONE_EXTENDED) datasource.execute(""" DROP TABLE IF EXISTS INPUT_HYDRO; - CREATE TABLE INPUT_HYDRO (THE_GEOM geometry, ID_SOURCE varchar(24), ZINDEX integer) - AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, 0 FROM $surface_eau a, ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom); + CREATE TABLE INPUT_HYDRO (THE_GEOM geometry, ID_SOURCE varchar(24), ZINDEX integer, TYPE VARCHAR) + AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, 0, a.NATURE FROM $surface_eau a, ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom); """.toString()) //7. Prepare the Vegetation table (from the layer "ZONE_VEGETATION") that are in the study area (ZONE_EXTENDED) @@ -272,7 +272,6 @@ def loadV2( SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.NATURE, 0 FROM $piste_aerodrome a, $zoneTable b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.NATURE = 'Piste en herbe'; - ; """.toString()) //8. Prepare the Urban areas @@ -573,9 +572,12 @@ Map loadV3(JdbcDataSource datasource, CREATE TABLE INPUT_ROAD (THE_GEOM geometry, ID_SOURCE varchar(24), WIDTH DOUBLE PRECISION, TYPE varchar, ZINDEX integer, CROSSING varchar, DIRECTION varchar, RANK INTEGER, ADMIN_SCALE VARCHAR) - AS SELECT ST_FORCE2D(a.THE_GEOM) as the_geom, a.ID, a.LARGEUR, CASE WHEN a.NAT_RESTR = 'Piste cyclable' then a.NAT_RESTR else a.NATURE end, + AS SELECT ST_FORCE2D(a.THE_GEOM) as the_geom, a.ID, a.LARGEUR, + CASE WHEN a.NAT_RESTR = 'Piste cyclable' then a.NAT_RESTR else a.NATURE end, CASE WHEN a.POS_SOL='Gué ou radier' THEN 0 ELSE CAST(a.POS_SOL AS INT ) END AS POS_SOL, - CASE WHEN a.POS_SOL='1' OR a.POS_SOL='Gué ou radier' THEN a.POS_SOL else null end , + CASE WHEN a.POS_SOL in ('1', '2', '3', '4') THEN 'Pont' + WHEN a.POS_SOL='Gué ou radier' THEN a.POS_SOL + else null end , CASE WHEN a.SENS='Double sens' THEN 'Double' WHEN a.SENS='Sens direct' THEN 'Direct' WHEN a.SENS='Sens inverse' THEN 'Inverse' @@ -592,17 +594,18 @@ Map loadV3(JdbcDataSource datasource, DROP TABLE IF EXISTS INPUT_RAIL; CREATE TABLE INPUT_RAIL (THE_GEOM geometry, ID_SOURCE varchar(24), TYPE varchar, ZINDEX integer, CROSSING varchar, WIDTH DOUBLE PRECISION) AS SELECT ST_FORCE2D(a.THE_GEOM) as the_geom, a.ID, a.NATURE, - a.POS_SOL, CASE WHEN a.POS_SOL=1 THEN 'Pont' else '' end, + a.POS_SOL, + CASE WHEN a.POS_SOL in ('1', '2', '3', '4') THEN 'Pont' else null end, CASE WHEN a.NB_VOIES = 0 THEN 1.435 ELSE 1.435 * a.NB_VOIES END FROM $troncon_de_voie_ferree a, $zoneTable b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0; """.toString()) - //6. Prepare the Hydrography table (from the layer "surface_hydrographique") that are in the study area (ZONE_EXTENDED) datasource.execute(""" DROP TABLE IF EXISTS INPUT_HYDRO; CREATE TABLE INPUT_HYDRO (THE_GEOM geometry, ID_SOURCE varchar(24), ZINDEX integer, TYPE varchar) AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, 0, a.NATURE FROM $surface_hydrographique a, ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0 + and a.NATURE not in ('Conduit buse', 'Conduit forcé', 'Marais', 'Glacier névé') union all SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, 0, a.NATURE FROM $terrain_de_sport a, ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) @@ -623,7 +626,10 @@ Map loadV3(JdbcDataSource datasource, SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.NAT_DETAIL, 0 FROM $terrain_de_sport a, $zoneTable b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.NAT_DETAIL in ('Terrain de football', 'Terrain de rugby') - ; + UNION ALL + SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.NATURE , 0 FROM $surface_hydrographique a, ZONE_EXTENDED + b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0 + and a.NATURE in ('Marais') ; """.toString()) //8. Prepare the Urban areas diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy index 2f1f0d45d5..569bcd8ec3 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy @@ -52,6 +52,8 @@ abstract class WorkflowAbstractTest { */ abstract String getInseeCode() + abstract void checkFormatData(); + /** * The names of file used for the test @@ -385,31 +387,7 @@ abstract class WorkflowAbstractTest { Map process = BDTopo.workflow(bdTopoParameters, getVersion()) assertNotNull(process) - - Map resultFiles = getResultFiles(folder.absolutePath) - H2GIS h2GIS = H2GIS.open(folder.getAbsolutePath()+File.separator+"bdtopo_${getVersion()}_format") - resultFiles.each { - h2GIS.load( it.value, it.key,true) - } - //Check the data - //Building - int count = h2GIS.getRowCount("building") - List building_cols = [ "ID_BUILD", "ID_SOURCE", "HEIGHT_WALL", "HEIGHT_ROOF", "NB_LEV", "TYPE", "MAIN_USE", "ROOF_SHAPE","ZINDEX", "THE_GEOM"] - assertTrue h2GIS.getColumnNames("building").intersect(building_cols).size()==building_cols.size() - assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where HEIGHT_WALL = 0 OR HEIGHT_ROOF = 0 OR NB_LEV = 0").count) - assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where TYPE IS NOT NULL OR MAIN_USE IS NOT NULL").count) - assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where ZINDEX BETWEEN -4 AND 4").count) - - //Road - count = h2GIS.getRowCount("road") - List road_cols = [ "ID_ROAD", "ID_SOURCE", "WIDTH","TYPE", "SURFACE", "SIDEWALK", "CROSSING","MAXSPEED", "DIRECTION", "ZINDEX", "THE_GEOM"] - assertTrue h2GIS.getColumnNames("road").intersect(road_cols).size()==road_cols.size() - assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where WIDTH = 0 ").count) - assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where TYPE IS NOT NULL OR SIDEWALK is not null").count) - assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where MAXSPEED !=0 OR MAXSPEED>= -1").count) - assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where DIRECTION !=0 OR DIRECTION>= -1").count) - assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where ZINDEX BETWEEN -4 AND 4").count) - + checkFormatData() } } \ No newline at end of file diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy index 4d2656f1bc..f37b7630f7 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy @@ -19,9 +19,34 @@ */ package org.orbisgis.geoclimate.bdtopo.v2 - +import org.orbisgis.data.H2GIS import org.orbisgis.geoclimate.bdtopo.WorkflowAbstractTest +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertTrue +import static org.junit.jupiter.api.Assertions.assertTrue +import static org.junit.jupiter.api.Assertions.assertTrue +import static org.junit.jupiter.api.Assertions.assertTrue +import static org.junit.jupiter.api.Assertions.assertTrue + class WorkflowBDTopoV2Test extends WorkflowAbstractTest { @@ -48,4 +73,85 @@ class WorkflowBDTopoV2Test extends WorkflowAbstractTest { String getInseeCode() { return "12174" } + + @Override + void checkFormatData() { + Map resultFiles = getResultFiles(folder.absolutePath) + H2GIS h2GIS = H2GIS.open(folder.getAbsolutePath()+File.separator+"bdtopo_${getVersion()}_format") + resultFiles.each { + h2GIS.load( it.value, it.key,true) + } + //Check the data + //Building + int count = h2GIS.getRowCount("building") + List cols = [ "ID_BUILD", "ID_SOURCE", "HEIGHT_WALL", "HEIGHT_ROOF", "NB_LEV", "TYPE", "MAIN_USE", "ROOF_SHAPE","ZINDEX", "THE_GEOM"] + assertTrue h2GIS.getColumnNames("building").intersect(cols).size()==cols.size() + assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where HEIGHT_WALL = 0 OR HEIGHT_ROOF = 0 OR NB_LEV = 0").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where TYPE IS NOT NULL OR MAIN_USE IS NOT NULL").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where ZINDEX BETWEEN -4 AND 4").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + //Road + count = h2GIS.getRowCount("road") + cols = [ "ID_ROAD", "ID_SOURCE", "WIDTH","TYPE", "SURFACE", "SIDEWALK", "CROSSING","MAXSPEED", "DIRECTION", "ZINDEX", "THE_GEOM"] + assertTrue h2GIS.getColumnNames("road").intersect(cols).size()==cols.size() + assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where WIDTH = 0 ").count) + assertEquals(22, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where crossing in ('bridge', 'crossing')").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where TYPE IS NOT NULL OR SIDEWALK is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where MAXSPEED !=0 OR MAXSPEED>= -1").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where DIRECTION !=0 OR DIRECTION>= -1").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where ZINDEX BETWEEN -4 AND 4").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Rail + count = h2GIS.getRowCount("rail") + cols = ["THE_GEOM", "ID_RAIL", + "ID_SOURCE" , "TYPE" ,"CROSSING", "ZINDEX" , "WIDTH", "USAGE"] + assertTrue h2GIS.getColumnNames("rail").intersect(cols).size()==cols.size() + assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where WIDTH = 0 ").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where type is not null").count) + assertEquals(2, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where crossing is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where ZINDEX BETWEEN -4 AND 4").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Vegetation + count = h2GIS.getRowCount("vegetation") + cols = ["THE_GEOM", "ID_VEGET", "ID_SOURCE", "TYPE", "HEIGHT_CLASS", "ZINDEX"] + assertTrue h2GIS.getColumnNames("vegetation").intersect(cols).size()==cols.size() + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where type is not null").count) + assertEquals(670, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where height_class ='high'").count) + assertEquals(2, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where height_class = 'low'").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where ZINDEX BETWEEN 0 AND 1 ").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Water + count = h2GIS.getRowCount("water") + cols = ["THE_GEOM", "ID_WATER", "ID_SOURCE", "TYPE", "ZINDEX"] + assertTrue h2GIS.getColumnNames("water").intersect(cols).size()==cols.size() + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM water where type is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM water where ZINDEX BETWEEN 0 AND 1 ").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM water where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Impervious areas + count = h2GIS.getRowCount("impervious") + cols = ["THE_GEOM", "ID_IMPERVIOUS", "TYPE"] + assertTrue h2GIS.getColumnNames("impervious").intersect(cols).size()==cols.size() + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM impervious where type is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM impervious where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Urban areas + count = h2GIS.getRowCount("urban_areas") + cols = ["THE_GEOM", "ID_URBAN", "ID_SOURCE", "TYPE"] + assertTrue h2GIS.getColumnNames("urban_areas").intersect(cols).size()==cols.size() + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM urban_areas where type is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM urban_areas where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + + } } \ No newline at end of file diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy index 8f7f82b389..2bdc37c10f 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy @@ -49,6 +49,89 @@ class WorkflowBDTopoV3Test extends WorkflowAbstractTest { return "12174" } + @Override + void checkFormatData() { + Map resultFiles = getResultFiles(folder.absolutePath) + H2GIS h2GIS = H2GIS.open(folder.getAbsolutePath()+File.separator+"bdtopo_${getVersion()}_format") + resultFiles.each { + h2GIS.load( it.value, it.key,true) + } + //Check the data + //Building + int count = h2GIS.getRowCount("building") + List cols = [ "ID_BUILD", "ID_SOURCE", "HEIGHT_WALL", "HEIGHT_ROOF", "NB_LEV", "TYPE", "MAIN_USE", "ROOF_SHAPE","ZINDEX", "THE_GEOM"] + assertTrue h2GIS.getColumnNames("building").intersect(cols).size()==cols.size() + assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where HEIGHT_WALL = 0 OR HEIGHT_ROOF = 0 OR NB_LEV = 0").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where TYPE IS NOT NULL OR MAIN_USE IS NOT NULL").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where ZINDEX BETWEEN -4 AND 4").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Road + count = h2GIS.getRowCount("road") + cols = [ "ID_ROAD", "ID_SOURCE", "WIDTH","TYPE", "SURFACE", "SIDEWALK", "CROSSING","MAXSPEED", "DIRECTION", "ZINDEX", "THE_GEOM"] + assertTrue h2GIS.getColumnNames("road").intersect(cols).size()==cols.size() + assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where WIDTH = 0 ").count) + assertEquals(22, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where crossing is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where TYPE IS NOT NULL OR SIDEWALK is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where MAXSPEED !=0 OR MAXSPEED>= -1").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where DIRECTION !=0 OR DIRECTION>= -1").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where ZINDEX BETWEEN -4 AND 4").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Rail + count = h2GIS.getRowCount("rail") + + cols = ["THE_GEOM", "ID_RAIL", + "ID_SOURCE" , "TYPE" ,"CROSSING", "ZINDEX" , "WIDTH", "USAGE"] + assertTrue h2GIS.getColumnNames("rail").intersect(cols).size()==cols.size() + assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where WIDTH = 0 ").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where type is not null").count) + assertEquals(2, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where crossing is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where ZINDEX BETWEEN -4 AND 4").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM rail where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Vegetation + count = h2GIS.getRowCount("vegetation") + cols = ["THE_GEOM", "ID_VEGET", "ID_SOURCE", "TYPE", "HEIGHT_CLASS", "ZINDEX"] + assertTrue h2GIS.getColumnNames("vegetation").intersect(cols).size()==cols.size() + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where type is not null").count) + assertEquals(670, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where height_class ='high'").count) + assertEquals(2, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where height_class = 'low'").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where ZINDEX BETWEEN 0 AND 1 ").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM vegetation where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Water + count = h2GIS.getRowCount("water") + cols = ["THE_GEOM", "ID_WATER", "ID_SOURCE", "TYPE", "ZINDEX"] + assertTrue h2GIS.getColumnNames("water").intersect(cols).size()==cols.size() + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM water where type is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM water where ZINDEX BETWEEN 0 AND 1 ").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM water where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Impervious areas + count = h2GIS.getRowCount("impervious") + cols = ["THE_GEOM", "ID_IMPERVIOUS", "TYPE"] + assertTrue h2GIS.getColumnNames("impervious").intersect(cols).size()==cols.size() + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM impervious where type is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM impervious where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + //Urban areas + count = h2GIS.getRowCount("urban_areas") + cols = ["THE_GEOM", "ID_URBAN", "ID_SOURCE", "TYPE"] + assertTrue h2GIS.getColumnNames("urban_areas").intersect(cols).size()==cols.size() + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM urban_areas where type is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM urban_areas where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) + + + } + + @Override ArrayList getFileNames() { return ["COMMUNE", "BATIMENT", "ZONE_D_ACTIVITE_OU_D_INTERET", "TERRAIN_DE_SPORT", "CIMETIERE", diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/DataUtils.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/DataUtils.groovy index 66b73dcbf8..f1a1678179 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/DataUtils.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/DataUtils.groovy @@ -147,3 +147,33 @@ static Map parametersMapping(def file, def altResourceStream) { } return jsonSlurper.parse(paramStream) } + +/** + * Create the select projection and alias all columns + * @param datasource + * @param tableName + * @param alias + * @return + */ +static String aliasColumns(JdbcDataSource datasource, String tableName, String alias){ + Collection columnNames = datasource.getColumnNames(tableName) + return columnNames.inject([]) { result, iter -> + result += "$alias.$iter" + }.join(",") +} + +/** + * Create the select projection and alias all columns + * @param datasource + * @param tableName + * @param alias + * @param exceptColumns + * @return + */ +static String aliasColumns(JdbcDataSource datasource, String tableName, String alias, Collection exceptColumns){ + Collection columnNames = datasource.getColumnNames(tableName) + columnNames.removeAll(exceptColumns) + return columnNames.inject([]) { result, iter -> + result += "$alias.$iter" + }.join(",") +} diff --git a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy index 90abd4a345..43aaa4cb36 100644 --- a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy +++ b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy @@ -366,7 +366,7 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = "" } int rowcount = 1 datasource.withBatch(100) { stmt -> - datasource.eachRow(queryMapper) { row -> + datasource.eachRow(queryMapper.toString()) { row -> def type = getTypeValue(row, columnNames, mappingType) def zIndex = getZIndex(row.'layer') //special treatment if type is subway