Skip to content
This repository has been archived by the owner on Nov 11, 2020. It is now read-only.

Commit

Permalink
Merge pull request #261 from conveyal/worker-accessibility
Browse files Browse the repository at this point in the history
Worker-side single point accessibility
  • Loading branch information
abyrd authored Oct 1, 2020
2 parents a2b9299 + 6b70ab5 commit 7ed8274
Show file tree
Hide file tree
Showing 15 changed files with 346 additions and 84 deletions.
10 changes: 7 additions & 3 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,15 @@ jar {
// For Java 11 Modules, specify a module name.
// Do not create module-info.java until all our dependencies specify a module name.
// Main-Class R5Main will start a worker, BackendMain must be specified on JVM command line to start backend.
// Build-Jdk-Spec mimics a Maven manifest entry that helps us automatically install the right JVM.
// Implementation-X attributes are needed for ImageIO (used by Geotools) to initialize in some environments.
manifest {
attributes 'Automatic-Module-Name': 'com.conveyal.analysis',
'Main-Class': 'com.conveyal.r5.R5Main',
// Mimic Maven manifest entry that helps automatically install the right JVM
'Build-Jdk-Spec': targetCompatibility.getMajorVersion()
'Main-Class': 'com.conveyal.r5.R5Main',
'Build-Jdk-Spec': targetCompatibility.getMajorVersion(),
'Implementation-Title': 'Conveyal Analysis Backend',
'Implementation-Vendor': 'Conveyal LLC',
'Implementation-Version': project.version
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.conveyal.analysis.components.eventbus.SinglePointEvent;
import com.conveyal.analysis.models.AnalysisRequest;
import com.conveyal.analysis.models.Bundle;
import com.conveyal.analysis.models.OpportunityDataset;
import com.conveyal.analysis.models.Project;
import com.conveyal.analysis.persistence.Persistence;
import com.conveyal.analysis.util.HttpStatus;
Expand Down Expand Up @@ -50,6 +51,9 @@
import java.util.List;
import java.util.Map;

import static com.conveyal.r5.common.Util.notNullOrEmpty;
import static com.google.common.base.Preconditions.checkNotNull;

/**
* This is a Spark HTTP controller to handle connections from workers reporting their status and requesting work.
* It also handles connections from the front end for single-point requests.
Expand Down Expand Up @@ -133,6 +137,28 @@ private Object singlePoint(Request request, Response response) {
Project project = Persistence.projects.findByIdIfPermitted(analysisRequest.projectId, accessGroup);
// Transform the analysis UI/backend task format into a slightly different type for R5 workers.
TravelTimeSurfaceTask task = (TravelTimeSurfaceTask) analysisRequest.populateTask(new TravelTimeSurfaceTask(), project);
// If destination opportunities are supplied, prepare to calculate accessibility worker-side
if (notNullOrEmpty(analysisRequest.destinationPointSetIds)){
// Look up all destination opportunity data sets from the database and derive their storage keys.
// This is mostly copypasted from the code to create a regional analysis.
// Ideally we'd reuse the same code in both places.
// We should refactor the populateTask method (and move it off the request) to take care of all this.
List<OpportunityDataset> opportunityDatasets = new ArrayList<>();
for (String destinationPointSetId : analysisRequest.destinationPointSetIds) {
OpportunityDataset opportunityDataset = Persistence.opportunityDatasets.findByIdIfPermitted(
destinationPointSetId,
accessGroup
);
checkNotNull(opportunityDataset, "Opportunity dataset could not be found in database.");
opportunityDatasets.add(opportunityDataset);
}
task.destinationPointSetKeys = opportunityDatasets.stream()
.map(OpportunityDataset::storageLocation)
.toArray(String[]::new);
// Also do a preflight validation of the cutoffs and percentiles arrays for all non-TAUI regional tasks.
// Don't validate cutoffs because those are implied to be [0...120) and generated by the worker itself.
task.validatePercentiles();
}
if (request.headers("Accept").equals("image/tiff")) {
// If the client requested a Geotiff using HTTP headers (for exporting results to GIS),
// signal this using a field on the request sent to the worker.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ public class AnalysisRequest {
* The IDs of pointsets to be used as destinations in accessibility or travel time calculations. This can be
* one or more grids with identical extents, or a single freeform pointset.
* This replaces the deprecated singular opportunityDatasetId.
* This field is required for regional analyses, which always compute accessibility to destinations.
* On the other hand, in a single point request this may be null, in which case the worker will report only
* travel times to destinations and not accessibility figures.
*/
public String[] destinationPointSetIds;

Expand Down
94 changes: 94 additions & 0 deletions src/main/java/com/conveyal/r5/analyst/GridTransformWrapper.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package com.conveyal.r5.analyst;

import org.locationtech.jts.geom.Envelope;

import static com.google.common.base.Preconditions.checkArgument;

/**
* This wraps a gridded destination pointset (the "source"), remapping its point indexes to match those of another grid.
* This can be used to stack pointsets of varying dimensions, or to calculate accessibility to pointsets of
* different dimensions than a travel time surface grid.
*
* These wrappers should not be used for linking as they don't have a BasePointSet. Another WebMercatorGridPointSet will
* be linked, and these will just serve as opportunity grids of identical dimensions during accessibility calculations.
*/
public class GridTransformWrapper extends PointSet {

/** Defer to this PointSet for everything but opportunity counts, including grid dimensions and lat/lon. */
private WebMercatorGridPointSet targetGrid;

/** Defer to this PointSet to get opportunity counts (transforming indexes to those of targetPointSet). */
private Grid sourceGrid;

/**
* Wraps the sourceGrid such that the opportunity count is read from the geographic locations of indexes in the
* targetGrid. For the time being, both pointsets must be at the same zoom level. Any opportunities outside the
* targetGrid cannot be indexed so are effectively zero for the purpose of accessibility calculations.
*/
public GridTransformWrapper (WebMercatorExtents targetGridExtents, Grid sourceGrid) {
checkArgument(targetGridExtents.zoom == sourceGrid.zoom, "Zoom levels must be identical.");
// Make a pointset for these extents so we can defer to its methods for lat/lon lookup, size, etc.
this.targetGrid = new WebMercatorGridPointSet(targetGridExtents);
this.sourceGrid = sourceGrid;
}

// Given an index in the targetPointSet, return the corresponding 1D index into the sourceGrid or -1 if the target
// index is for a point outside the source grid.
// This could certainly be made more efficient (but complex) by forcing sequential iteration over opportunity counts
// and disallowing random access, using a new PointSetIterator class that allows reading lat, lon, and counts.
private int transformIndex (int i) {
final int x = (i % targetGrid.width) + targetGrid.west - sourceGrid.west;
final int y = (i / targetGrid.width) + targetGrid.north - sourceGrid.north;
if (x < 0 || x >= sourceGrid.width || y < 0 || y >= sourceGrid.height) {
// Point in target grid lies outside source grid, there is no valid index. Return special value.
return -1;
}
return y * sourceGrid.width + x;
}

@Override
public double getLat (int i) {
return targetGrid.getLat(i);
}

@Override
public double getLon (int i) {
return targetGrid.getLon(i);
}

@Override
public int featureCount () {
return targetGrid.featureCount();
}

@Override
public double sumTotalOpportunities() {
// Very inefficient compared to the other implementations as it does a lot of index math, but it should work.
double totalOpportunities = 0;
for (int i = 0; i < featureCount(); i++) {
totalOpportunities += getOpportunityCount(i);
}
return totalOpportunities;
}

@Override
public double getOpportunityCount (int targetIndex) {
int sourceindex = transformIndex(targetIndex);
if (sourceindex < 0) {
return 0;
} else {
return sourceGrid.getOpportunityCount(sourceindex);
}
}

@Override
public Envelope getWgsEnvelope () {
return targetGrid.getWgsEnvelope();
}

@Override
public WebMercatorExtents getWebMercatorExtents () {
return targetGrid.getWebMercatorExtents();
}

}
42 changes: 20 additions & 22 deletions src/main/java/com/conveyal/r5/analyst/TravelTimeReducer.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

import java.util.Arrays;

import static com.conveyal.r5.common.Util.notNullOrEmpty;
import static com.conveyal.r5.profile.FastRaptorWorker.UNREACHED;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
Expand All @@ -38,16 +39,16 @@ public class TravelTimeReducer {
/** If we are calculating accessibility, the PointSets containing opportunities. */
private PointSet[] destinationPointSets;

/** TODO Explain. */
/** The array indexes at which we'll find each percentile in a sorted list of length timesPerDestination. */
private final int[] percentileIndexes;

/** TODO Explain. */
/** The number of different percentiles that were requested. */
private final int nPercentiles;

/** TODO explain. */
/** The travel time cutoffs supplied in the request, validated and converted to seconds. */
private int[] cutoffsSeconds;

/** TODO Explain. */
/** The length of the cutoffs array, just for convenience and code clarity. */
private int nCutoffs;

/**
Expand Down Expand Up @@ -118,18 +119,16 @@ public TravelTimeReducer (AnalysisWorkerTask task) {
// back to the broker in JSON.

// Decide which elements we'll be calculating, retaining, and returning.
calculateAccessibility = calculateTravelTimes = false;
// Always copy this field, the array in the task may be null or empty but we detect that case.
this.destinationPointSets = task.destinationPointSets;
if (task instanceof TravelTimeSurfaceTask) {
calculateTravelTimes = true;
calculateAccessibility = notNullOrEmpty(task.destinationPointSets);
} else {
// Maybe we should define recordAccessibility and recordTimes on the common superclass AnalysisWorkerTask.
RegionalTask regionalTask = (RegionalTask) task;
if (regionalTask.recordAccessibility) {
calculateAccessibility = true;
this.destinationPointSets = regionalTask.destinationPointSets;
}
if (regionalTask.recordTimes || regionalTask.makeTauiSite) {
calculateTravelTimes = true;
}
calculateAccessibility = regionalTask.recordAccessibility;
calculateTravelTimes = regionalTask.recordTimes || regionalTask.makeTauiSite;
}

// Instantiate and initialize objects to accumulate the kinds of results we expect to produce.
Expand All @@ -141,13 +140,13 @@ public TravelTimeReducer (AnalysisWorkerTask task) {
travelTimeResult = new TravelTimeResult(task);
}

// Validate and copy the travel time cutoffs, which only makes sense when calculating accessibility.
// Validation should probably happen earlier when making or handling incoming tasks.
// Validate and copy the travel time cutoffs, converting them to seconds to avoid repeated multiplication
// in tight loops. Also find the points where the decay function reaches zero for these cutoffs.
// This is only relevant when calculating accessibility.
this.decayFunction = task.decayFunction;
if (calculateAccessibility) {
task.validateCutoffsMinutes();
this.nCutoffs = task.cutoffsMinutes.length;
// Convert cutoffs to seconds, to avoid repeated multiplication in tight loops.
this.cutoffsSeconds = new int[nCutoffs];
this.zeroPointsForCutoffs = new int[nCutoffs];
for (int c = 0; c < nCutoffs; c++) {
Expand Down Expand Up @@ -235,10 +234,7 @@ private void recordTravelTimePercentilesForTarget (int target, int[] travelTimeP
}
if (calculateAccessibility) {
// This can handle multiple opportunity grids as long as they have exactly the same extents.
// That should cover common use cases but will eventually need to be adapted to handle multiple different
// grid extents. This will require transforming indexes between multiple grids possibly of different sizes
// (a GridIndexTransform class?). If the transform is only for reading into a single super-grid, they will
// only need to add a single number to the width and y.
// Grids of different extents are handled by using GridTransformWrapper to give them all the same extents.
for (int d = 0; d < destinationPointSets.length; d++) {
final double opportunityCountAtTarget = destinationPointSets[d].getOpportunityCount(target);
if (!(opportunityCountAtTarget > 0)) {
Expand Down Expand Up @@ -296,9 +292,11 @@ private int convertToMinutes (int timeSeconds) {
}

/**
* If no travel times to destinations have been streamed in by calling recordTravelTimesForTarget, the
* TimeGrid will have a buffer full of UNREACHED. This allows shortcutting around
* routing and propagation when the origin point is not connected to the street network.
* This is the primary way to create a OneOriginResult and end the processing.
* Some alternate code paths exist for TAUI site generation and testing, but this handles all other cases.
* For example, if no travel times to destinations have been streamed in by calling recordTravelTimesForTarget, the
* TimeGrid will have a buffer full of UNREACHED. This allows shortcutting around routing and propagation when the
* origin point is not connected to the street network.
*/
public OneOriginResult finish () {
return new OneOriginResult(travelTimeResult, accessibilityResult);
Expand Down
22 changes: 19 additions & 3 deletions src/main/java/com/conveyal/r5/analyst/WebMercatorExtents.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkElementIndex;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;

/**
* Really we should be embedding one of these in the tasks, grids, etc. to factor out all the common fields.
Expand Down Expand Up @@ -39,15 +40,14 @@ public static WebMercatorExtents forTask (AnalysisWorkerTask task) {
return new WebMercatorExtents(task.west, task.north, task.width, task.height, task.zoom);
}

/** If pointSets are all gridded, return the minimum-sized WebMercatorExtents containing them all. */
public static WebMercatorExtents forPointsets (PointSet[] pointSets) {
checkNotNull(pointSets);
checkElementIndex(0, pointSets.length, "You must supply at least one destination PointSet.");
if (pointSets[0] instanceof Grid) {
WebMercatorExtents extents = pointSets[0].getWebMercatorExtents();
// TODO handle case of pointsets with different extents; for now just validate that they are identical.
for (PointSet pointSet : pointSets) {
checkArgument(pointSet instanceof Grid, "All destination PointSets must be of the same type.");
checkArgument(extents.equals(pointSet.getWebMercatorExtents()));
extents = extents.expandToInclude(pointSet.getWebMercatorExtents());
}
return extents;
} else {
Expand All @@ -60,6 +60,22 @@ public static WebMercatorExtents forPointsets (PointSet[] pointSets) {
}
}

/** Create the minimum-size immutable WebMercatorExtents containing both this one and the other one. */
public WebMercatorExtents expandToInclude (WebMercatorExtents other) {
checkState(this.zoom == other.zoom, "All grids supplied must be at the same zoom level.");
final int thisEast = this.west + this.width;
final int otherEast = other.west + other.width;
final int thisSouth = this.north + other.height;
final int otherSouth = other.north + other.height;
final int outWest = Math.min(other.west, this.west);
final int outEast = Math.max(otherEast, thisEast);
final int outNorth = Math.min(other.north, this.north);
final int outSouth = Math.max(otherSouth, thisSouth);
final int outWidth = outEast - outWest;
final int outHeight = outSouth - outNorth;
return new WebMercatorExtents(outWest, outNorth, outWidth, outHeight, this.zoom);
}

public static WebMercatorExtents forWgsEnvelope (Envelope wgsEnvelope, int zoom) {
/*
The grid extent is computed from the points. If the cell number for the right edge of the grid is rounded
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

/**
* A pointset that represents a grid of pixels from the web mercator projection.
* This PointSet subclass does not yet support opportunity counts.
* TODO merge this class with Grid, which does have opportunity counts.
*/
public class WebMercatorGridPointSet extends PointSet implements Serializable {

Expand Down Expand Up @@ -79,6 +81,11 @@ public WebMercatorGridPointSet (Envelope wgsEnvelope) {
this.basePointSet = null;
}

/** The resulting PointSet will not have a null basePointSet, so should generally not be used for linking. */
public WebMercatorGridPointSet (WebMercatorExtents extents) {
this(extents.zoom, extents.west, extents.north, extents.width, extents.height, null);
}

@Override
public int featureCount() {
return height * width;
Expand All @@ -87,7 +94,6 @@ public int featureCount() {
@Override
public double sumTotalOpportunities () {
// For now we are counting each point as 1 opportunity because this class does not have opportunity counts.
// TODO merge this class with Grid, which does have opportunity counts.
return featureCount();
}

Expand Down
Loading

0 comments on commit 7ed8274

Please sign in to comment.