diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebb3d55 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/bin/ +geodata +suomi +target +update_bats +.idea diff --git a/USAGE.md b/USAGE.md index 12860a3..fc32d63 100644 --- a/USAGE.md +++ b/USAGE.md @@ -3,17 +3,27 @@ * osmconvert64 (https://wiki.openstreetmap.org/wiki/Osmconvert) and osmfilter (http://wiki.openstreetmap.org/wiki/Osmfilter) needs to be found from PATH * mgkmap and splitter (http://www.mkgmap.org.uk/) needs to be found from PATH * Osmosis for Mapsforge conversion (https://wiki.openstreetmap.org/wiki/Osmosis) +* wget +* Python 2 (http://www.python.org) + * requests (`pip install requests`) + * urllib3 (`pip install urllib3`) +* IntelliJ (CE works fine) - this is the easiest way, but you can probably use maven from command line. +* Not necessary, but can be helpful: + * WFS client, e.g. QGIS * Finnish national grid dataset (only 12km x 12km grid needed) * Can be obtained from https://tiedostopalvelu.maanmittauslaitos.fi/geoserver/ows/?service=wfs (License unknown, assumed CC 4.0 like other NLS data) * Layer "Grid" + (`wget --no-check-certificate -O grid.gml "https://tiedostopalvelu.maanmittauslaitos.fi/geoserver/ows/?service=wfs&request=GetFeature&typeName=Grid"`) + * National topographic database from NLS (about 9.5 GB) - * Loaded using helper script `update_data.py`. Simply running `python update_data.py ` creates necessary directory structure and creates 8 .bat files to load the whole (or just changed grid tiles) topographic database using wget (assumes wget can be found from PATH) + * Loaded using helper script `update_data.py`. Simply running `python update_data.py ` creates necessary directory structure and creates 8 .bat files to load the whole (or just changed grid tiles) topographic database using wget (assumes wget can be found from PATH). * Requires personal API-key to ATOM-feed (http://www.maanmittauslaitos.fi/aineistot-palvelut/latauspalvelut/avoimien-aineistojen-tiedostopalvelu/muutostietopalvelu) * Whole process is based on NLS specific GML format because it is the only format containing all of the data and attributes. * OpenStreetMap * Only features with specific tags are used * Load Finland OSM PBF dump from http://download.geofabrik.de/europe/finland-latest.osm.pbf - * `join_osm_mtk.bat` creates new o5m file with desired filters applied, and actually merges filtered o5m file to output from the actual topographic database conversion process + * `join_osm_mtk.bat` creates new o5m file with desired filters applied, and actually merges filtered o5m file to the output from the actual topographic database conversion process. + * Finnish Transport Agency (Nautical chart data & depth data) * Depth data from https://extranet.liikennevirasto.fi/inspirepalvelu/rajoitettu/wfs * Whole dataset can be loaded from wfs and is used by the process as a zipped shape @@ -29,7 +39,12 @@ * pienriista (not included in styles / tag mapping) * National/official hunting areas are not included in the maps because valid concerns about data validity and timeliness. +### OSX / Linux notes +(So far) The batch files can be run as is provided you mark them executable (e.g. `chmod u+x update_bats/loaddata_0.bat`) + ## MTKtoGarminConverter +`mvn package` to build ? + This is the actual process doing the conversion from national topographic database GML-files to OSM PBF format and combining most of the former datasets with the topograhic data. It does some necessary data transformations and simplifications on the way (defined in `*TagHandler`classes). All of the work is done in ETRS-TM35FIN (3067) coordinate system and coordinates are transformed to WGS84 only for PBF output. @@ -38,7 +53,7 @@ Because of this combination/connection process, the whole conversion process mus Process is especially memory optimized and runs consistently well within 2 GB of memory, there is much room for more optimization and for example threaded processing is completely possible. Process assumes that all data is located in `c:\geodata`. The process works one grid cell at a time and loads auxillary data based on grid cell bounding box and removes unnecessary data from memory after processing. Technically every OGR compatible format should be good for the auxillary data. -`C:\geodata\mtkgml\` directory is read using 2 level directory structure (`eg. C:\geodata\mtkgml\L4\L44\*.zip`). Processing of the whole topographic database is not necessary, if the directory structure matches. +`C:\geodata\mtkgml\` directory is read using 2 level directory structure (`eg. C:\geodata\mtkgml\L4\L44\*.zip`). Processing of the whole topographic database is not necessary, if the directory structure matches. This step takes about 8 hours with a (slow) SSD, Intel Core i7 920 and 12 GB RAM (and output to HD). @@ -58,7 +73,7 @@ Resulting gmapsupp.img is good to be transferred to GPS or it can be splitted to ## Creating Mapsforge map in AWS EC2 Consult also https://github.com/mapsforge/mapsforge/blob/master/docs/Getting-Started-Map-Writer.md -This steps requires large amounts of memory so EC2 instance is used, r3.2xlarge seems to have enough memory. Tag mapping affects memory usage significantly. +This steps requires large amounts of memory so EC2 instance is used, r3.2xlarge seems to have enough memory. Tag mapping affects memory usage significantly. Styles should be defined at least on some level before the conversion proces because conversion includes only features defined in tag mapping. @@ -73,4 +88,3 @@ The actual process is as follows: 2. Run Osmosis with writer plugin and tag-mapping and copyright notices with `osmosis --rbf all_osm.osm.pbf --mapfile-writer file=all.map bbox=59.4507573,19.0714057,70.1120744,31.6133108 tag-conf-file=mml_tag-mapping_tidy.xml type=hd comment="(c) NLS, Metsahallitus, Liikennevirasto, OpenStreetMap contributors 2016"` And out comes all.map containing all of the data in Mapsforge format - diff --git a/join_osm_mtk.sh b/join_osm_mtk.sh new file mode 100755 index 0000000..9fee66e --- /dev/null +++ b/join_osm_mtk.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +DATA_FOLDER=./geodata/ + +osmconvert ${DATA_FOLDER}finland-latest.osm.pbf -o=${DATA_FOLDER}finland-latest.o5m +osmfilter ${DATA_FOLDER}finland-latest.o5m --keep="highway=track =path =footway amenity=shelter =toilets tourism=fireplace =lean_to =wilderness man_made=tower and tower:type=observation" --drop-author --drop-version --verbose --out-o5m >${DATA_FOLDER}finland-filtered.o5m +osmconvert ${DATA_FOLDER}finland-filtered.o5m --out-o5m | osmconvert - suomi/all.osm.pbf -o=suomi/all_osm.osm.pbf diff --git a/liikennevirasto.sh b/liikennevirasto.sh new file mode 100755 index 0000000..98b464a --- /dev/null +++ b/liikennevirasto.sh @@ -0,0 +1,3 @@ +#!/bin/bash +wget -O geodata/syvyyskayrat.zip "https://extranet.liikennevirasto.fi/inspirepalvelu/rajoitettu/wfs?request=GetFeature&typeName=rajoitettu:syvyyskayra_v&outputFormat=shape-zip" +wget -O geodata/syvyyspiste_p.zip "https://extranet.liikennevirasto.fi/inspirepalvelu/rajoitettu/wfs?request=GetFeature&typeName=rajoitettu:syvyyspiste_p&outputFormat=shape-zip" diff --git a/retkikartta.sh b/retkikartta.sh new file mode 100755 index 0000000..fac2c24 --- /dev/null +++ b/retkikartta.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +wget -O geodata/kesaretkeilyreitit.zip "http://www.retkikartta.fi/wfs/a9e9a1840ee69e32d59af86dd1ffeb44/?request=GetFeature&typeName=retkikartta_euref:kesaretkeilyreitit&outputFormat=shape-zip" +wget -O geodata/ulkoilureitit.zip "http://www.retkikartta.fi/wfs/a9e9a1840ee69e32d59af86dd1ffeb44/?request=GetFeature&typeName=retkikartta_euref:ulkoilureitit&outputFormat=shape-zip" +wget -O geodata/luontopolut.zip "http://www.retkikartta.fi/wfs/a9e9a1840ee69e32d59af86dd1ffeb44/?request=GetFeature&typeName=retkikartta_euref:luontopolut&outputFormat=shape-zip" +wget -O geodata/point_dump.zip "http://www.retkikartta.fi/wfs/a9e9a1840ee69e32d59af86dd1ffeb44/?request=GetFeature&typeName=retkikartta_euref:point_dump&outputFormat=shape-zip" +wget -O geodata/hirvialueet.zip "http://www.retkikartta.fi/wfs/a9e9a1840ee69e32d59af86dd1ffeb44/?request=GetFeature&typeName=retkikartta_euref:hirvialueet&outputFormat=shape-zip" +wget -O geodata/pienriista.zip "http://www.retkikartta.fi/wfs/a9e9a1840ee69e32d59af86dd1ffeb44/?request=GetFeature&typeName=retkikartta_euref:pienriista&outputFormat=shape-zip" diff --git a/src/main/java/org/hylly/mtk2garmin/MTKToGarminConverter.java b/src/main/java/org/hylly/mtk2garmin/MTKToGarminConverter.java index 5eafc5b..76fee46 100644 --- a/src/main/java/org/hylly/mtk2garmin/MTKToGarminConverter.java +++ b/src/main/java/org/hylly/mtk2garmin/MTKToGarminConverter.java @@ -1,10 +1,9 @@ package org.hylly.mtk2garmin; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -790,7 +789,26 @@ private void trackCounts() { System.out.println("max_nodes " + max_nodes + ", max_ways " + max_ways + ", max_relations " + max_relations); } - + + // An attempt to get the absolute paths into one place ... + private static String getDataFile(String file) throws IOException { + String DATA_DIR = null; + if (Files.exists(Paths.get("C:\\geodata", file))) { + DATA_DIR = "C:\\geodata"; + } else if (Files.exists(Paths.get("R:\\syvyys", file))) { + DATA_DIR = "R:\\syvyys"; + } else if (Files.exists(Paths.get("C:\\geodata\\retkikartta", file))) { + DATA_DIR = "C:\\geodata\\retkikartta"; + } else if (Files.exists(Paths.get("./geodata", file))) { + DATA_DIR = Paths.get("").resolve("geodata").toAbsolutePath().toString(); + } + + Path fullPath = Paths.get(DATA_DIR, file); + String path = fullPath.toString(); + if (!Files.exists(fullPath)) { throw new FileNotFoundException("Data file '" + path + "' not found.");} + return path; + } + public static void main(String[] args) throws IOException, InterruptedException { System.out.println("Starting conversion"); ogr.UseExceptions(); @@ -801,7 +819,7 @@ public static void main(String[] args) throws IOException, InterruptedException // wget --no-check-certificate -O grid.gml // "https://tiedostopalvelu.maanmittauslaitos.fi/geoserver/ows/?service=wfs&request=GetFeature&typeName=Grid" - DataSource gridds = ogr.Open("C:\\geodata\\12x12grid.shp"); + DataSource gridds = ogr.Open(getDataFile("grid.gml")); Layer lyr = gridds.GetLayer(0); lyr.SetAttributeFilter("gridSize = '12x12'"); @@ -818,7 +836,7 @@ public static void main(String[] args) throws IOException, InterruptedException gridds.delete(); HashMap> areas = new HashMap>(); - File srcdir = new File("C:\\geodata\\mtkgml"); + File srcdir = new File(getDataFile("mtkgml")); for (File g1 : srcdir.listFiles()) { for (File g2 : g1.listFiles()) { for (File g3 : g2.listFiles()) { @@ -849,13 +867,13 @@ public static void main(String[] args) throws IOException, InterruptedException ShapeFeaturePreprocess shapePreprocessor = new ShapeFeaturePreprocess(); double[] mml_extent; - + int currentArea = 0; for (Object area : areassorted) { + System.out.printf("Processing area (%s) %d/%d%n", area, ++currentArea, areassorted.length); ArrayList files = areas.get(area); String[] filenames = new String[files.size()]; int i = 0; for (File f : files) { - filenames[i] = f.getAbsolutePath(); i++; } @@ -863,7 +881,7 @@ public static void main(String[] args) throws IOException, InterruptedException Arrays.sort(filenames); for (String fn : filenames) { - String cell = fn.substring(fn.lastIndexOf("\\") + 1, fn.lastIndexOf("\\") + 7); + String cell = fn.substring(fn.lastIndexOf(File.separator) + 1, fn.lastIndexOf(File.separator) + 7); stringtable = new StringTable(); tyyppi_string_id = stringtable.getStringId("tyyppi"); @@ -872,7 +890,8 @@ public static void main(String[] args) throws IOException, InterruptedException syvyysTagHandler = new ShapeSyvyysTagHandler(stringtable); mtk2g.startWritingOSMPBF( - String.format("K:\\koodi\\mtk2garmin3\\mtk2garminjava\\suomi\\%s.osm.pbf", cell)); + // "K:\koodi\mtk2garmin3\mtk2garminjava\ + Paths.get("suomi", String.format("%s.osm.pbf", cell)).toString()); System.out.println(fn + " (" + cell + ")"); @@ -886,38 +905,39 @@ public static void main(String[] args) throws IOException, InterruptedException System.out.println(Arrays.toString(mml_extent)); st = System.nanoTime(); - mtk2g.readOGRsource(stringtable,"/vsizip/R:\\syvyys\\syvyyskayrat.zip\\syvyyskayrat.shp", shapePreprocessor, + // WAS: syvyyskayrat.shp + mtk2g.readOGRsource(stringtable,"/vsizip/" + getDataFile("syvyyskayrat.zip") + File.separator +"syvyyskayra_v.shp", shapePreprocessor, syvyysTagHandler, false, mml_extent); mtk2g.printCounts(); System.out.println("Syvyyskayrat read " + (System.nanoTime() - st) / 1000000000.0); st = System.nanoTime(); - mtk2g.readOGRsource(stringtable,"/vsizip/R:\\syvyys\\syvyyspiste_p.zip\\syvyyspiste_p.shp", shapePreprocessor, + mtk2g.readOGRsource(stringtable,"/vsizip/" + getDataFile("syvyyspiste_p.zip") + File.separator + "syvyyspiste_p.shp", shapePreprocessor, syvyysTagHandler, false, mml_extent); mtk2g.printCounts(); System.out.println("Syvyyspisteet read " + (System.nanoTime() - st) / 1000000000.0); st = System.nanoTime(); mtk2g.readOGRsource(stringtable, - "/vsizip/C:\\geodata\\retkikartta\\kesaretkeilyreitit.zip\\kesaretkeilyreititLine.shp", + "/vsizip/" + getDataFile("kesaretkeilyreitit.zip") + File.separator + "kesaretkeilyreititLine.shp", shapePreprocessor, retkeilyTagHandler, false, mml_extent); mtk2g.printCounts(); System.out.println("Kesaretkeilyreitit read " + (System.nanoTime() - st) / 1000000000.0); st = System.nanoTime(); - mtk2g.readOGRsource(stringtable,"/vsizip/C:\\geodata\\retkikartta\\ulkoilureitit.zip\\ulkoilureititLine.shp", + mtk2g.readOGRsource(stringtable,"/vsizip/" + getDataFile("ulkoilureitit.zip") + File.separator + "ulkoilureititLine.shp", shapePreprocessor, retkeilyTagHandler, false, mml_extent); mtk2g.printCounts(); System.out.println("Ulkoilureitit read " + (System.nanoTime() - st) / 1000000000.0); st = System.nanoTime(); - mtk2g.readOGRsource(stringtable,"/vsizip/C:\\geodata\\retkikartta\\luontopolut.zip\\luontopolut.shp", + mtk2g.readOGRsource(stringtable,"/vsizip/" + getDataFile("luontopolut.zip") + File.separator + "luontopolutLine.shp", shapePreprocessor, retkeilyTagHandler, false, mml_extent); mtk2g.printCounts(); System.out.println("Luontopolut read " + (System.nanoTime() - st) / 1000000000.0); st = System.nanoTime(); - mtk2g.readOGRsource(stringtable,"/vsizip/C:\\geodata\\retkikartta\\point_dump.zip\\point_dumpPoint.shp", + mtk2g.readOGRsource(stringtable,"/vsizip/" + getDataFile("point_dump.zip") + File.separator + "point_dumpPoint.shp", shapePreprocessor, retkeilyTagHandler, false, mml_extent); mtk2g.printCounts(); System.out.println("Point_dump read " + (System.nanoTime() - st) / 1000000000.0); @@ -945,8 +965,6 @@ public static void main(String[] args) throws IOException, InterruptedException System.out.println("pbf " + (System.nanoTime() - st) / 1000000000.0); } - - if (mtk2g.nodepos.size() > 0) { System.out.println(area + " done!"); diff --git a/update_data.py b/update_data.py index 89403cb..4fd65fd 100644 --- a/update_data.py +++ b/update_data.py @@ -5,6 +5,9 @@ import datetime import StringIO +if len(sys.argv) < 2: + print "Usage: python update_data.py API_KEY" + sys.exit(1) api_key = sys.argv[1] class MMLGrid(handler.ContentHandler): @@ -17,9 +20,14 @@ def __init__(self): self.files = [] self.nfiles = 8 - + BATCH_FILE_DIR = 'update_bats' + try: + os.makedirs(BATCH_FILE_DIR) + except OSError: + if not os.path.isdir(BATCH_FILE_DIR): + raise for i in xrange(self.nfiles): - self.files.append(open('update_bats/loaddata_%d.bat' % i,'wb')) + self.files.append(open(BATCH_FILE_DIR + '/loaddata_%d.bat' % i,'wb')) self.gridnum = 0 self.next = None