From 8720aa674f1f6601646f286eab6b054f7e189a0d Mon Sep 17 00:00:00 2001 From: Anson Stewart Date: Fri, 4 Oct 2024 14:00:33 -0400 Subject: [PATCH] Add custom walking traversal permissions via TransportNetworkConfig (#943) * Add SidewalkTraversalPermissionLabeler activated via TransportNetworkConfig when building a network from a config JSON * Remove deprecated buildNetworkFromBundleZip Building from .zip files on S3 has not been supported since 2016. * Enable TransportNetworkConfig in non-cluster use such as PointToPointRouterServer * Make sidewalk PermissionLabeler more restrictive Disallow walking anywhere driving is allowed * Set permissionLabeler more cleanly --- .../cluster/TransportNetworkConfig.java | 11 +-- .../SidewalkTraversalPermissionLabeler.java | 35 ++++++++++ .../com/conveyal/r5/streets/StreetLayer.java | 24 ++++++- .../conveyal/r5/transit/TransportNetwork.java | 58 +++++++++++---- .../r5/transit/TransportNetworkCache.java | 70 ++----------------- 5 files changed, 113 insertions(+), 85 deletions(-) create mode 100644 src/main/java/com/conveyal/r5/labeling/SidewalkTraversalPermissionLabeler.java diff --git a/src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java b/src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java index 012e3204a..387742634 100644 --- a/src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java +++ b/src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java @@ -2,12 +2,8 @@ import com.conveyal.r5.analyst.fare.InRoutingFareCalculator; import com.conveyal.r5.analyst.scenario.Modification; -import com.conveyal.r5.analyst.scenario.RasterCost; -import com.conveyal.r5.analyst.scenario.ShapefileLts; import com.conveyal.r5.profile.StreetMode; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.List; import java.util.Set; @@ -54,4 +50,11 @@ public class TransportNetworkConfig { */ public Set buildGridsForModes; + /** + * Specifies which "labeler" to use when setting traversal mode permissions from OSM tags. For now, only + * implemented with "sidewalk" to use the SidewalkTraversalPermissionLayer. This should eventually be cleaned up + * (specifying different labelers, using enums). + */ + public String traversalPermissionLabeler; + } diff --git a/src/main/java/com/conveyal/r5/labeling/SidewalkTraversalPermissionLabeler.java b/src/main/java/com/conveyal/r5/labeling/SidewalkTraversalPermissionLabeler.java new file mode 100644 index 000000000..34d2c8465 --- /dev/null +++ b/src/main/java/com/conveyal/r5/labeling/SidewalkTraversalPermissionLabeler.java @@ -0,0 +1,35 @@ +package com.conveyal.r5.labeling; + +import com.conveyal.osmlib.Way; +import com.conveyal.r5.streets.EdgeStore; + +/** + * Traversal permission labeler that restricts walking on most driving ways (useful for networks with complete + * sidewalks). Also includes permissions for the United States (see USTraversalPermissionLabeler). + */ +public class SidewalkTraversalPermissionLabeler extends TraversalPermissionLabeler { + static { + addPermissions("pedestrian", "bicycle=yes"); + addPermissions("bridleway", "bicycle=yes;foot=yes"); //horse=yes but we don't support horse + addPermissions("cycleway", "bicycle=yes;foot=yes"); + addPermissions("trunk|primary|secondary|tertiary|unclassified|residential|living_street|road|service|track", + "access=yes"); + } + + @Override + public RoadPermission getPermissions(Way way) { + RoadPermission rp = super.getPermissions(way); + if (rp.forward.contains(EdgeStore.EdgeFlag.ALLOWS_CAR) || + rp.forward.contains(EdgeStore.EdgeFlag.NO_THRU_TRAFFIC_CAR) || + rp.backward.contains(EdgeStore.EdgeFlag.ALLOWS_CAR) || + rp.backward.contains(EdgeStore.EdgeFlag.NO_THRU_TRAFFIC_CAR) + ) { + rp.forward.remove(EdgeStore.EdgeFlag.ALLOWS_PEDESTRIAN); + rp.forward.remove(EdgeStore.EdgeFlag.NO_THRU_TRAFFIC_PEDESTRIAN); + rp.backward.remove(EdgeStore.EdgeFlag.ALLOWS_PEDESTRIAN); + rp.backward.remove(EdgeStore.EdgeFlag.NO_THRU_TRAFFIC_PEDESTRIAN); + } + return rp; + } + +} diff --git a/src/main/java/com/conveyal/r5/streets/StreetLayer.java b/src/main/java/com/conveyal/r5/streets/StreetLayer.java index 3d781b41b..4e31481ee 100644 --- a/src/main/java/com/conveyal/r5/streets/StreetLayer.java +++ b/src/main/java/com/conveyal/r5/streets/StreetLayer.java @@ -6,12 +6,14 @@ import com.conveyal.osmlib.OSMEntity; import com.conveyal.osmlib.Relation; import com.conveyal.osmlib.Way; +import com.conveyal.r5.analyst.cluster.TransportNetworkConfig; import com.conveyal.r5.analyst.scenario.PickupWaitTimes; import com.conveyal.r5.api.util.BikeRentalStation; import com.conveyal.r5.api.util.ParkRideParking; import com.conveyal.r5.common.GeometryUtils; import com.conveyal.r5.labeling.LevelOfTrafficStressLabeler; import com.conveyal.r5.labeling.RoadPermission; +import com.conveyal.r5.labeling.SidewalkTraversalPermissionLabeler; import com.conveyal.r5.labeling.SpeedLabeler; import com.conveyal.r5.labeling.StreetClass; import com.conveyal.r5.labeling.TraversalPermissionLabeler; @@ -132,9 +134,9 @@ public class StreetLayer implements Serializable, Cloneable { public TIntObjectMap parkRideLocationsMap; // TODO these are only needed when building the network, should we really be keeping them here in the layer? - // We should instead have a network builder that holds references to this transient state. - // TODO don't hardwire to US - private transient TraversalPermissionLabeler permissionLabeler = new USTraversalPermissionLabeler(); + // We should instead have a network builder that holds references to this transient state. Note initial + // approach of specifying a TraversalPermissionLabeler in TransportNetworkConfig. + private transient TraversalPermissionLabeler permissionLabeler; private transient LevelOfTrafficStressLabeler stressLabeler = new LevelOfTrafficStressLabeler(); private transient TypeOfEdgeLabeler typeOfEdgeLabeler = new TypeOfEdgeLabeler(); private transient SpeedLabeler speedLabeler; @@ -207,6 +209,22 @@ public class StreetLayer implements Serializable, Cloneable { public StreetLayer() { speedLabeler = new SpeedLabeler(SpeedConfig.defaultConfig()); + permissionLabeler = new USTraversalPermissionLabeler(); + } + + public StreetLayer(TransportNetworkConfig config) { + this(); + if (config != null) { + permissionLabeler = switch (config.traversalPermissionLabeler) { + case "sidewalk" -> new SidewalkTraversalPermissionLabeler(); + case null -> new USTraversalPermissionLabeler(); + default -> throw new IllegalArgumentException( + "Unknown traversal permission labeler: " + config.traversalPermissionLabeler + ); + }; + } else { + permissionLabeler = new USTraversalPermissionLabeler(); + } } /** Load street layer from an OSM-lib OSM DB */ diff --git a/src/main/java/com/conveyal/r5/transit/TransportNetwork.java b/src/main/java/com/conveyal/r5/transit/TransportNetwork.java index bbf4f8f70..4d351eedf 100644 --- a/src/main/java/com/conveyal/r5/transit/TransportNetwork.java +++ b/src/main/java/com/conveyal/r5/transit/TransportNetwork.java @@ -4,9 +4,11 @@ import com.conveyal.osmlib.OSM; import com.conveyal.r5.analyst.LinkageCache; import com.conveyal.r5.analyst.WebMercatorGridPointSet; +import com.conveyal.r5.analyst.cluster.TransportNetworkConfig; import com.conveyal.r5.analyst.error.TaskError; import com.conveyal.r5.analyst.fare.InRoutingFareCalculator; import com.conveyal.r5.analyst.scenario.Scenario; +import com.conveyal.r5.common.JsonUtilities; import com.conveyal.r5.kryo.KryoNetworkSerializer; import com.conveyal.r5.profile.StreetMode; import com.conveyal.r5.streets.StreetLayer; @@ -77,8 +79,6 @@ public class TransportNetwork implements Serializable { */ public String scenarioId = null; - public static final String BUILDER_CONFIG_FILENAME = "build-config.json"; - public InRoutingFareCalculator fareCalculator; /** Non-fatal warnings encountered when applying the scenario, null on a base network */ @@ -96,7 +96,9 @@ public void rebuildTransientIndexes() { streetLayer.indexStreets(); transitLayer.rebuildTransientIndexes(); } - + public static TransportNetwork fromFiles (String osmSourceFile, List gtfsSourceFiles) { + return fromFiles(osmSourceFile, gtfsSourceFiles, null); + } /** * OSM PBF files are fragments of a single global database with a single namespace. Therefore it is valid to load * more than one PBF file into a single OSM storage object. However they might be from different points in time, so @@ -110,9 +112,11 @@ public void rebuildTransientIndexes() { * NOTE the feedId of the gtfs feeds loaded here will be the ones declared by the feeds or based on their filenames. * This method makes no effort to impose the more unique feed IDs created by the Analysis backend. */ + public static TransportNetwork fromFiles ( String osmSourceFile, - List gtfsSourceFiles + List gtfsSourceFiles, + String configFile ) throws DuplicateFeedException { // Load OSM data into MapDB to pass into network builder. OSM osm = new OSM(osmSourceFile + ".mapdb"); @@ -120,21 +124,37 @@ public static TransportNetwork fromFiles ( osm.readFromFile(osmSourceFile); // Supply feeds with a stream so they do not sit open in memory while other feeds are being processed. Stream feeds = gtfsSourceFiles.stream().map(GTFSFeed::readOnlyTempFileFromGtfs); - return fromInputs(osm, feeds); + if (configFile == null) { + return fromInputs(osm, feeds); + } else { + try { + // Use lenient mapper to mimic behavior in objectFromRequestBody. + TransportNetworkConfig config = JsonUtilities.lenientObjectMapper.readValue(configFile, + TransportNetworkConfig.class); + return fromInputs(osm, feeds, config); + } catch (IOException e) { + throw new RuntimeException("Error reading TransportNetworkConfig. Does it contain new unrecognized fields?", e); + } + } + } + + public static TransportNetwork fromInputs (OSM osm, Stream gtfsFeeds) { + return fromInputs(osm, gtfsFeeds, null); } /** - * This is the core method for building a street and transit network. It takes osm-lib and gtfs-lib objects as - * parameters. It is wrapped in various other methods that create those OSM and GTFS objects from filenames, input - * directories etc. The supplied OSM object must have intersections already detected. - * The GTFS feeds are supplied as a stream so that they can be loaded one by one on demand. + * This is the method for building a street and transit network locally (as opposed to + * TransportNetworkCache#buildNetworkfromConfig, which is used in cluster builds). This method takes osm-lib, + * gtfs-lib, and config objects as parameters. It is wrapped in various other methods that create those OSM and + * GTFS objects from filenames, input directories etc. The supplied OSM object must have intersections already + * detected. The GTFS feeds are supplied as a stream so that they can be loaded one by one on demand. */ - public static TransportNetwork fromInputs (OSM osm, Stream gtfsFeeds) { + public static TransportNetwork fromInputs (OSM osm, Stream gtfsFeeds, TransportNetworkConfig config) { // Create a transport network to hold the street and transit layers TransportNetwork transportNetwork = new TransportNetwork(); // Make street layer from OSM data in MapDB - StreetLayer streetLayer = new StreetLayer(); + StreetLayer streetLayer = new StreetLayer(config); transportNetwork.streetLayer = streetLayer; streetLayer.parentNetwork = transportNetwork; streetLayer.loadFromOsm(osm); @@ -176,14 +196,16 @@ public static TransportNetwork fromInputs (OSM osm, Stream gtfsFeeds) } /** - * Scan a directory detecting all the files that are network inputs, then build a network from those files. + * Scan a directory detecting all the files that are network inputs, then build a network from those files. This + * method is used in the PointToPointRouterServer, not the cluster-based analysis backend. * - * NOTE the feedId of the gtfs feeds laoded here will be the ones declared by the feeds or based on their filenames. + * NOTE the feedId of the gtfs feeds loaded here will be the ones declared by the feeds or based on their filenames. * This method makes no effort to impose the more unique feed IDs created by the Analysis backend. */ public static TransportNetwork fromDirectory (File directory) throws DuplicateFeedException { File osmFile = null; List gtfsFiles = new ArrayList<>(); + File configFile = null; for (File file : directory.listFiles()) { switch (InputFileType.forFile(file)) { case GTFS: @@ -198,6 +220,9 @@ public static TransportNetwork fromDirectory (File directory) throws DuplicateFe LOG.warn("Can only load one OSM file at a time."); } break; + case CONFIG: + LOG.info("Found config file {}", file); + configFile = file; case DEM: LOG.warn("DEM file '{}' not yet supported.", file); break; @@ -209,7 +234,11 @@ public static TransportNetwork fromDirectory (File directory) throws DuplicateFe LOG.error("An OSM PBF file is required to build a network."); return null; } else { - return fromFiles(osmFile.getAbsolutePath(), gtfsFiles); + if (configFile == null) { + return fromFiles(osmFile.getAbsolutePath(), gtfsFiles); + } else { + return fromFiles(osmFile.getAbsolutePath(), gtfsFiles, configFile.getAbsolutePath()); + } } } @@ -255,6 +284,7 @@ public static InputFileType forFile(File file) { if (name.endsWith(".pbf") || name.endsWith(".vex")) return OSM; if (name.endsWith(".tif") || name.endsWith(".tiff")) return DEM; // Digital elevation model (elevation raster) if (name.endsWith("network.dat")) return OUTPUT; + if (name.endsWith(".json")) return CONFIG; return OTHER; } } diff --git a/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java b/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java index 98db69a97..c67fe9d24 100644 --- a/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java +++ b/src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java @@ -25,14 +25,9 @@ import javax.annotation.Nonnull; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.util.List; import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import static com.conveyal.file.FileCategory.BUNDLES; import static com.conveyal.file.FileCategory.DATASOURCES; @@ -207,10 +202,8 @@ private TransportNetworkConfig loadNetworkConfig (String networkId) { TransportNetworkConfig networkConfig = loadNetworkConfig(networkId); if (networkConfig == null) { // The switch to use JSON manifests instead of zips occurred in 32a1aebe in July 2016. - // Over six years have passed, buildNetworkFromBundleZip is deprecated and could probably be removed. - LOG.warn("No network config (aka manifest) found. Assuming old-format network inputs bundle stored as a single ZIP file."); - // FIXME Bundle ZIP building to reduce duplicate code. - network = buildNetworkFromBundleZip(networkId); + // buildNetworkFromBundleZip was deprecated for years then removed in 2024. + throw new RuntimeException("No network config (aka manifest) found."); } else { network = buildNetworkFromConfig(networkConfig); } @@ -243,70 +236,19 @@ private TransportNetworkConfig loadNetworkConfig (String networkId) { return network; } - /** Build a transport network given a network ID, using a zip of all bundle files in S3. */ - @Deprecated - private TransportNetwork buildNetworkFromBundleZip (String networkId) { - // The location of the inputs that will be used to build this graph - File dataDirectory = FileUtils.createScratchDirectory(); - FileStorageKey zipKey = new FileStorageKey(BUNDLES, networkId + ".zip"); - File zipFile = fileStorage.getFile(zipKey); - - try { - ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile)); - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - File entryDestination = new File(dataDirectory, entry.getName()); - if (!entryDestination.toPath().normalize().startsWith(dataDirectory.toPath())) { - throw new Exception("Bad zip entry"); - } - - // Are both these mkdirs calls necessary? - entryDestination.getParentFile().mkdirs(); - if (entry.isDirectory()) - entryDestination.mkdirs(); - else { - OutputStream entryFileOut = new FileOutputStream(entryDestination); - zis.transferTo(entryFileOut); - entryFileOut.close(); - } - } - zis.close(); - } catch (Exception e) { - // TODO delete cache dir which is probably corrupted. - LOG.warn("Error retrieving transportation network input files", e); - return null; - } - - // Now we have a local copy of these graph inputs. Make a graph out of them. - TransportNetwork network; - try { - network = TransportNetwork.fromDirectory(dataDirectory); - } catch (DuplicateFeedException e) { - LOG.error("Duplicate feeds in transport network {}", networkId, e); - throw new RuntimeException(e); - } - - // Set the ID on the network and its layers to allow caching linkages and analysis results. - network.scenarioId = networkId; - - return network; - } - /** * Build a network from a JSON TransportNetworkConfig in file storage. * This describes the locations of files used to create a bundle, as well as options applied at network build time. * It contains the unique IDs of the GTFS feeds and OSM extract. */ private TransportNetwork buildNetworkFromConfig (TransportNetworkConfig config) { - // FIXME duplicate code. All internal building logic should be encapsulated in a method like - // TransportNetwork.build(osm, gtfs1, gtfs2...) - // We currently have multiple copies of it, in buildNetworkFromConfig and buildNetworkFromBundleZip so you've - // got to remember to do certain things like set the network ID of the network in multiple places in the code. - // Maybe we should just completely deprecate bundle ZIPs and remove those code paths. + // FIXME All internal building logic should be encapsulated in a method like TransportNetwork.build(osm, + // gtfs1, gtfs2...) (see various methods in TransportNetwork). TransportNetwork network = new TransportNetwork(); - network.streetLayer = new StreetLayer(); + network.streetLayer = new StreetLayer(config); + network.streetLayer.loadFromOsm(osmCache.get(config.osmId)); network.streetLayer.parentNetwork = network;