Add custom walking traversal permissions via TransportNetworkConfig (#…

…943)

* Add SidewalkTraversalPermissionLabeler

activated via TransportNetworkConfig when building a network from a config JSON

* Remove deprecated buildNetworkFromBundleZip

Building from .zip files on S3 has not been supported since 2016.

* Enable TransportNetworkConfig in non-cluster use

such as PointToPointRouterServer

* Make sidewalk PermissionLabeler more restrictive

Disallow walking anywhere driving is allowed

* Set permissionLabeler more cleanly

src/main/java/com/conveyal/r5/analyst/cluster/TransportNetworkConfig.java

@@ -2,12 +2,8 @@
 
 import com.conveyal.r5.analyst.fare.InRoutingFareCalculator;
 import com.conveyal.r5.analyst.scenario.Modification;
-import com.conveyal.r5.analyst.scenario.RasterCost;
-import com.conveyal.r5.analyst.scenario.ShapefileLts;
 import com.conveyal.r5.profile.StreetMode;
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 
 import java.util.List;
 import java.util.Set;
@@ -54,4 +50,11 @@ public class TransportNetworkConfig {
      */
     public Set<StreetMode> buildGridsForModes;
 
+    /**
+     * Specifies which "labeler" to use when setting traversal mode permissions from OSM tags. For now, only
+     * implemented with "sidewalk" to use the SidewalkTraversalPermissionLayer. This should eventually be cleaned up
+     * (specifying different labelers, using enums).
+     */
+    public String traversalPermissionLabeler;
+
 }

src/main/java/com/conveyal/r5/labeling/SidewalkTraversalPermissionLabeler.java

@@ -0,0 +1,35 @@
+package com.conveyal.r5.labeling;
+
+import com.conveyal.osmlib.Way;
+import com.conveyal.r5.streets.EdgeStore;
+
+/**
+ * Traversal permission labeler that restricts walking on most driving ways (useful for networks with complete
+ * sidewalks). Also includes permissions for the United States (see USTraversalPermissionLabeler).
+ */
+public class SidewalkTraversalPermissionLabeler extends TraversalPermissionLabeler {
+    static {
+        addPermissions("pedestrian", "bicycle=yes");
+        addPermissions("bridleway", "bicycle=yes;foot=yes"); //horse=yes but we don't support horse
+        addPermissions("cycleway", "bicycle=yes;foot=yes");
+        addPermissions("trunk|primary|secondary|tertiary|unclassified|residential|living_street|road|service|track",
+                "access=yes");
+    }
+
+    @Override
+    public RoadPermission getPermissions(Way way) {
+        RoadPermission rp = super.getPermissions(way);
+        if (rp.forward.contains(EdgeStore.EdgeFlag.ALLOWS_CAR) ||
+            rp.forward.contains(EdgeStore.EdgeFlag.NO_THRU_TRAFFIC_CAR) ||
+            rp.backward.contains(EdgeStore.EdgeFlag.ALLOWS_CAR) ||
+            rp.backward.contains(EdgeStore.EdgeFlag.NO_THRU_TRAFFIC_CAR)
+        ) {
+            rp.forward.remove(EdgeStore.EdgeFlag.ALLOWS_PEDESTRIAN);
+            rp.forward.remove(EdgeStore.EdgeFlag.NO_THRU_TRAFFIC_PEDESTRIAN);
+            rp.backward.remove(EdgeStore.EdgeFlag.ALLOWS_PEDESTRIAN);
+            rp.backward.remove(EdgeStore.EdgeFlag.NO_THRU_TRAFFIC_PEDESTRIAN);
+        }
+        return rp;
+    }
+
+}

src/main/java/com/conveyal/r5/streets/StreetLayer.java

@@ -6,12 +6,14 @@
 import com.conveyal.osmlib.OSMEntity;
 import com.conveyal.osmlib.Relation;
 import com.conveyal.osmlib.Way;
+import com.conveyal.r5.analyst.cluster.TransportNetworkConfig;
 import com.conveyal.r5.analyst.scenario.PickupWaitTimes;
 import com.conveyal.r5.api.util.BikeRentalStation;
 import com.conveyal.r5.api.util.ParkRideParking;
 import com.conveyal.r5.common.GeometryUtils;
 import com.conveyal.r5.labeling.LevelOfTrafficStressLabeler;
 import com.conveyal.r5.labeling.RoadPermission;
+import com.conveyal.r5.labeling.SidewalkTraversalPermissionLabeler;
 import com.conveyal.r5.labeling.SpeedLabeler;
 import com.conveyal.r5.labeling.StreetClass;
 import com.conveyal.r5.labeling.TraversalPermissionLabeler;
@@ -132,9 +134,9 @@ public class StreetLayer implements Serializable, Cloneable {
     public TIntObjectMap<ParkRideParking> parkRideLocationsMap;
 
     // TODO these are only needed when building the network, should we really be keeping them here in the layer?
-    //      We should instead have a network builder that holds references to this transient state.
-    // TODO don't hardwire to US
-    private transient TraversalPermissionLabeler permissionLabeler = new USTraversalPermissionLabeler();
+    //      We should instead have a network builder that holds references to this transient state. Note initial
+    //      approach of specifying a TraversalPermissionLabeler in TransportNetworkConfig.
+    private transient TraversalPermissionLabeler permissionLabeler;
     private transient LevelOfTrafficStressLabeler stressLabeler = new LevelOfTrafficStressLabeler();
     private transient TypeOfEdgeLabeler typeOfEdgeLabeler = new TypeOfEdgeLabeler();
     private transient SpeedLabeler speedLabeler;
@@ -207,6 +209,22 @@ public class StreetLayer implements Serializable, Cloneable {
 
     public StreetLayer() {
         speedLabeler = new SpeedLabeler(SpeedConfig.defaultConfig());
+        permissionLabeler = new USTraversalPermissionLabeler();
+    }
+
+    public StreetLayer(TransportNetworkConfig config) {
+        this();
+        if (config != null) {
+            permissionLabeler = switch (config.traversalPermissionLabeler) {
+                case "sidewalk" -> new SidewalkTraversalPermissionLabeler();
+                case null -> new USTraversalPermissionLabeler();
+                default -> throw new IllegalArgumentException(
+                        "Unknown traversal permission labeler: " + config.traversalPermissionLabeler
+                );
+            };
+        } else {
+            permissionLabeler = new USTraversalPermissionLabeler();
+        }
     }
 
     /** Load street layer from an OSM-lib OSM DB */

src/main/java/com/conveyal/r5/transit/TransportNetwork.java

@@ -4,9 +4,11 @@
 import com.conveyal.osmlib.OSM;
 import com.conveyal.r5.analyst.LinkageCache;
 import com.conveyal.r5.analyst.WebMercatorGridPointSet;
+import com.conveyal.r5.analyst.cluster.TransportNetworkConfig;
 import com.conveyal.r5.analyst.error.TaskError;
 import com.conveyal.r5.analyst.fare.InRoutingFareCalculator;
 import com.conveyal.r5.analyst.scenario.Scenario;
+import com.conveyal.r5.common.JsonUtilities;
 import com.conveyal.r5.kryo.KryoNetworkSerializer;
 import com.conveyal.r5.profile.StreetMode;
 import com.conveyal.r5.streets.StreetLayer;
@@ -77,8 +79,6 @@ public class TransportNetwork implements Serializable {
      */
     public String scenarioId = null;
 
-    public static final String BUILDER_CONFIG_FILENAME = "build-config.json";
-
     public InRoutingFareCalculator fareCalculator;
 
     /** Non-fatal warnings encountered when applying the scenario, null on a base network */
@@ -96,7 +96,9 @@ public void rebuildTransientIndexes() {
         streetLayer.indexStreets();
         transitLayer.rebuildTransientIndexes();
     }
-
+    public static TransportNetwork fromFiles (String osmSourceFile, List<String> gtfsSourceFiles) {
+        return fromFiles(osmSourceFile, gtfsSourceFiles, null);
+    }
     /**
      * OSM PBF files are fragments of a single global database with a single namespace. Therefore it is valid to load
      * more than one PBF file into a single OSM storage object. However they might be from different points in time, so
@@ -110,31 +112,49 @@ public void rebuildTransientIndexes() {
      * NOTE the feedId of the gtfs feeds loaded here will be the ones declared by the feeds or based on their filenames.
      * This method makes no effort to impose the more unique feed IDs created by the Analysis backend.
      */
+
     public static TransportNetwork fromFiles (
             String osmSourceFile,
-            List<String> gtfsSourceFiles
+            List<String> gtfsSourceFiles,
+            String configFile
     ) throws DuplicateFeedException {
         // Load OSM data into MapDB to pass into network builder.
         OSM osm = new OSM(osmSourceFile + ".mapdb");
         osm.intersectionDetection = true;
         osm.readFromFile(osmSourceFile);
         // Supply feeds with a stream so they do not sit open in memory while other feeds are being processed.
         Stream<GTFSFeed> feeds = gtfsSourceFiles.stream().map(GTFSFeed::readOnlyTempFileFromGtfs);
-        return fromInputs(osm, feeds);
+        if (configFile == null) {
+            return fromInputs(osm, feeds);
+        } else {
+            try {
+                // Use lenient mapper to mimic behavior in objectFromRequestBody.
+                TransportNetworkConfig config = JsonUtilities.lenientObjectMapper.readValue(configFile,
+                        TransportNetworkConfig.class);
+                return fromInputs(osm, feeds, config);
+            } catch (IOException e) {
+                throw new RuntimeException("Error reading TransportNetworkConfig. Does it contain new unrecognized fields?", e);
+            }
+        }
+    }
+
+    public static TransportNetwork fromInputs (OSM osm, Stream<GTFSFeed> gtfsFeeds) {
+        return fromInputs(osm, gtfsFeeds, null);
     }
 
     /**
-     * This is the core method for building a street and transit network. It takes osm-lib and gtfs-lib objects as
-     * parameters. It is wrapped in various other methods that create those OSM and GTFS objects from filenames, input
-     * directories etc. The supplied OSM object must have intersections already detected.
-     * The GTFS feeds are supplied as a stream so that they can be loaded one by one on demand.
+     * This is the method for building a street and transit network locally (as opposed to
+     * TransportNetworkCache#buildNetworkfromConfig, which is used in cluster builds). This method takes osm-lib,
+     * gtfs-lib, and config objects as parameters. It is wrapped in various other methods that create those OSM and
+     * GTFS objects from filenames, input directories etc. The supplied OSM object must have intersections already
+     * detected. The GTFS feeds are supplied as a stream so that they can be loaded one by one on demand.
      */
-    public static TransportNetwork fromInputs (OSM osm, Stream<GTFSFeed> gtfsFeeds) {
+    public static TransportNetwork fromInputs (OSM osm, Stream<GTFSFeed> gtfsFeeds, TransportNetworkConfig config) {
         // Create a transport network to hold the street and transit layers
         TransportNetwork transportNetwork = new TransportNetwork();
 
         // Make street layer from OSM data in MapDB
-        StreetLayer streetLayer = new StreetLayer();
+        StreetLayer streetLayer = new StreetLayer(config);
         transportNetwork.streetLayer = streetLayer;
         streetLayer.parentNetwork = transportNetwork;
         streetLayer.loadFromOsm(osm);
@@ -176,14 +196,16 @@ public static TransportNetwork fromInputs (OSM osm, Stream<GTFSFeed> gtfsFeeds)
     }
 
     /**
-     * Scan a directory detecting all the files that are network inputs, then build a network from those files.
+     * Scan a directory detecting all the files that are network inputs, then build a network from those files. This
+     * method is used in the PointToPointRouterServer, not the cluster-based analysis backend.
      *
-     * NOTE the feedId of the gtfs feeds laoded here will be the ones declared by the feeds or based on their filenames.
+     * NOTE the feedId of the gtfs feeds loaded here will be the ones declared by the feeds or based on their filenames.
      * This method makes no effort to impose the more unique feed IDs created by the Analysis backend.
      */
     public static TransportNetwork fromDirectory (File directory) throws DuplicateFeedException {
         File osmFile = null;
         List<String> gtfsFiles = new ArrayList<>();
+        File configFile = null;
         for (File file : directory.listFiles()) {
             switch (InputFileType.forFile(file)) {
                 case GTFS:
@@ -198,6 +220,9 @@ public static TransportNetwork fromDirectory (File directory) throws DuplicateFe
                         LOG.warn("Can only load one OSM file at a time.");
                     }
                     break;
+                case CONFIG:
+                    LOG.info("Found config file {}", file);
+                    configFile = file;
                 case DEM:
                     LOG.warn("DEM file '{}' not yet supported.", file);
                     break;
@@ -209,7 +234,11 @@ public static TransportNetwork fromDirectory (File directory) throws DuplicateFe
             LOG.error("An OSM PBF file is required to build a network.");
             return null;
         } else {
-            return fromFiles(osmFile.getAbsolutePath(), gtfsFiles);
+            if (configFile == null) {
+                return fromFiles(osmFile.getAbsolutePath(), gtfsFiles);
+            } else {
+                return fromFiles(osmFile.getAbsolutePath(), gtfsFiles, configFile.getAbsolutePath());
+            }
         }
     }
 
@@ -255,6 +284,7 @@ public static InputFileType forFile(File file) {
             if (name.endsWith(".pbf") || name.endsWith(".vex")) return OSM;
             if (name.endsWith(".tif") || name.endsWith(".tiff")) return DEM; // Digital elevation model (elevation raster)
             if (name.endsWith("network.dat")) return OUTPUT;
+            if (name.endsWith(".json")) return CONFIG;
             return OTHER;
         }
     }

src/main/java/com/conveyal/r5/transit/TransportNetworkCache.java

@@ -25,14 +25,9 @@
 
 import javax.annotation.Nonnull;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.util.List;
 import java.util.Set;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
 
 import static com.conveyal.file.FileCategory.BUNDLES;
 import static com.conveyal.file.FileCategory.DATASOURCES;
@@ -207,10 +202,8 @@ private TransportNetworkConfig loadNetworkConfig (String networkId) {
         TransportNetworkConfig networkConfig = loadNetworkConfig(networkId);
         if (networkConfig == null) {
             // The switch to use JSON manifests instead of zips occurred in 32a1aebe in July 2016.
-            // Over six years have passed, buildNetworkFromBundleZip is deprecated and could probably be removed.
-            LOG.warn("No network config (aka manifest) found. Assuming old-format network inputs bundle stored as a single ZIP file.");
-            // FIXME Bundle ZIP building to reduce duplicate code.
-            network = buildNetworkFromBundleZip(networkId);
+            // buildNetworkFromBundleZip was deprecated for years then removed in 2024.
+            throw new RuntimeException("No network config (aka manifest) found.");
         } else {
             network = buildNetworkFromConfig(networkConfig);
         }
@@ -243,70 +236,19 @@ private TransportNetworkConfig loadNetworkConfig (String networkId) {
         return network;
     }
 
-    /** Build a transport network given a network ID, using a zip of all bundle files in S3. */
-    @Deprecated
-    private TransportNetwork buildNetworkFromBundleZip (String networkId) {
-        // The location of the inputs that will be used to build this graph
-        File dataDirectory = FileUtils.createScratchDirectory();
-        FileStorageKey zipKey = new FileStorageKey(BUNDLES, networkId + ".zip");
-        File zipFile = fileStorage.getFile(zipKey);
-
-        try {
-            ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile));
-            ZipEntry entry;
-            while ((entry = zis.getNextEntry()) != null) {
-                File entryDestination = new File(dataDirectory, entry.getName());
-                if (!entryDestination.toPath().normalize().startsWith(dataDirectory.toPath())) {
-                    throw new Exception("Bad zip entry");
-                }
-
-                // Are both these mkdirs calls necessary?
-                entryDestination.getParentFile().mkdirs();
-                if (entry.isDirectory())
-                    entryDestination.mkdirs();
-                else {
-                    OutputStream entryFileOut = new FileOutputStream(entryDestination);
-                    zis.transferTo(entryFileOut);
-                    entryFileOut.close();
-                }
-            }
-            zis.close();
-        } catch (Exception e) {
-            // TODO delete cache dir which is probably corrupted.
-            LOG.warn("Error retrieving transportation network input files", e);
-            return null;
-        }
-
-        // Now we have a local copy of these graph inputs. Make a graph out of them.
-        TransportNetwork network;
-        try {
-            network = TransportNetwork.fromDirectory(dataDirectory);
-        } catch (DuplicateFeedException e) {
-            LOG.error("Duplicate feeds in transport network {}", networkId, e);
-            throw new RuntimeException(e);
-        }
-
-        // Set the ID on the network and its layers to allow caching linkages and analysis results.
-        network.scenarioId = networkId;
-
-        return network;
-    }
-
     /**
      * Build a network from a JSON TransportNetworkConfig in file storage.
      * This describes the locations of files used to create a bundle, as well as options applied at network build time.
      * It contains the unique IDs of the GTFS feeds and OSM extract.
      */
     private TransportNetwork buildNetworkFromConfig (TransportNetworkConfig config) {
-        // FIXME duplicate code. All internal building logic should be encapsulated in a method like
-        //  TransportNetwork.build(osm, gtfs1, gtfs2...)
-        // We currently have multiple copies of it, in buildNetworkFromConfig and buildNetworkFromBundleZip so you've
-        // got to remember to do certain things like set the network ID of the network in multiple places in the code.
-        // Maybe we should just completely deprecate bundle ZIPs and remove those code paths.
+        // FIXME All internal building logic should be encapsulated in a method like TransportNetwork.build(osm,
+        //  gtfs1, gtfs2...) (see various methods in TransportNetwork).
 
         TransportNetwork network = new TransportNetwork();
 
-        network.streetLayer = new StreetLayer();
+        network.streetLayer = new StreetLayer(config);
+
         network.streetLayer.loadFromOsm(osmCache.get(config.osmId));
 
         network.streetLayer.parentNetwork = network;