-
Notifications
You must be signed in to change notification settings - Fork 37
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add WrappedProcessor to help reading MVTs (#222)
My ultimate aim is to read an mbtiles file with a single zoom level and extract features from it. It contains compressed MVT data. To that end, I wrote https://github.com/acteng/will-it-fit/blob/main/data_prep/fix_osmm/src/main.rs using geozero and some other crates. It works like this: 1) Use the `mbtiles` crate to open the file 2) Calculate all tiles in the file 3) For each tile, read the raw data 4) Use `flate2` to gunzip it 5) Decode into the `geozero::mvt::Tile` proto 6) Use the existing mvt geozero reader to process all layers. 7) But because the geometry in each tile is scaled to fit that tile, I need to transform the coordinates back into WGS84. IIUC the geozero approach correctly, doing that translation as I go is more performant than collecting into something like an FGB writer first and then trying to go back and transform everything. So, I wrote `WrappedProcessor` to delegate to another `FeatureProcessor` (an `FgbWriter` in this example), but call a function on every coordinate first This PR adds the `WrappedProcessor`, in case it might be helpful for other use cases. But I'm not sure this approach is the nicest one -- maybe the [mvt::process](https://github.com/georust/geozero/blob/c8a5f9103fc5ecc0ae9c7fcd2663b094e620da38/geozero/src/mvt/mvt_reader.rs#L17) function should instead take optional info about the current tile (the tile x and y, the zoom level, and the extent), plumb it through the private methods in that file, and apply in [process_coord](https://github.com/georust/geozero/blob/c8a5f9103fc5ecc0ae9c7fcd2663b094e620da38/geozero/src/mvt/mvt_reader.rs#L104)? I'm not opinionated about the best way to read an mvt tile doing this coordinate transformation, so happy to implement the other approach or something else entirely. Thanks! --------- Co-authored-by: Michael Kirk <michael.code@endoftheworl.de>
- Loading branch information
1 parent
73902dd
commit c30f80e
Showing
4 changed files
with
312 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,268 @@ | ||
use crate::{ | ||
error::Result, ColumnValue, CoordDimensions, FeatureProcessor, GeomProcessor, PropertyProcessor, | ||
}; | ||
|
||
/// Wraps another [`FeatureProcessor`], first transforming coordinates. | ||
pub struct WrappedXYProcessor<T, F: Fn(&mut f64, &mut f64)> { | ||
/// The underlying FeatureProcessor | ||
pub inner: T, | ||
pre_process_xy: F, | ||
} | ||
|
||
impl<T, F: Fn(&mut f64, &mut f64)> WrappedXYProcessor<T, F> { | ||
/// Wraps an inner [`FeatureProcessor`], calling `transform_coordinates` on [GeomProcessor::xy] | ||
/// and [GeomProcessor::coordinate] first. The function takes and returns `(x, y)`. | ||
pub fn new(inner: T, pre_process_xy: F) -> Self { | ||
Self { | ||
inner, | ||
pre_process_xy, | ||
} | ||
} | ||
|
||
pub fn into_inner(self) -> T { | ||
self.inner | ||
} | ||
} | ||
|
||
// The trait has many default implementations, but every single call must be specified here to | ||
// delegate | ||
impl<T: GeomProcessor, F: Fn(&mut f64, &mut f64)> GeomProcessor for WrappedXYProcessor<T, F> { | ||
fn dimensions(&self) -> CoordDimensions { | ||
self.inner.dimensions() | ||
} | ||
fn multi_dim(&self) -> bool { | ||
self.inner.multi_dim() | ||
} | ||
fn srid(&mut self, srid: Option<i32>) -> Result<()> { | ||
self.inner.srid(srid) | ||
} | ||
fn xy(&mut self, mut x: f64, mut y: f64, idx: usize) -> Result<()> { | ||
(self.pre_process_xy)(&mut x, &mut y); | ||
self.inner.xy(x, y, idx) | ||
} | ||
fn coordinate( | ||
&mut self, | ||
mut x: f64, | ||
mut y: f64, | ||
z: Option<f64>, | ||
m: Option<f64>, | ||
t: Option<f64>, | ||
tm: Option<u64>, | ||
idx: usize, | ||
) -> Result<()> { | ||
(self.pre_process_xy)(&mut x, &mut y); | ||
self.inner.coordinate(x, y, z, m, t, tm, idx) | ||
} | ||
fn empty_point(&mut self, idx: usize) -> Result<()> { | ||
self.inner.empty_point(idx) | ||
} | ||
fn point_begin(&mut self, idx: usize) -> Result<()> { | ||
self.inner.point_begin(idx) | ||
} | ||
fn point_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.point_end(idx) | ||
} | ||
fn multipoint_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.multipoint_begin(size, idx) | ||
} | ||
fn multipoint_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.multipoint_end(idx) | ||
} | ||
fn linestring_begin(&mut self, tagged: bool, size: usize, idx: usize) -> Result<()> { | ||
self.inner.linestring_begin(tagged, size, idx) | ||
} | ||
fn linestring_end(&mut self, tagged: bool, idx: usize) -> Result<()> { | ||
self.inner.linestring_end(tagged, idx) | ||
} | ||
fn multilinestring_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.multilinestring_begin(size, idx) | ||
} | ||
fn multilinestring_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.multilinestring_end(idx) | ||
} | ||
fn polygon_begin(&mut self, tagged: bool, size: usize, idx: usize) -> Result<()> { | ||
self.inner.polygon_begin(tagged, size, idx) | ||
} | ||
fn polygon_end(&mut self, tagged: bool, idx: usize) -> Result<()> { | ||
self.inner.polygon_end(tagged, idx) | ||
} | ||
fn multipolygon_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.multipolygon_begin(size, idx) | ||
} | ||
fn multipolygon_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.multipolygon_end(idx) | ||
} | ||
fn geometrycollection_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.geometrycollection_begin(size, idx) | ||
} | ||
fn geometrycollection_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.geometrycollection_end(idx) | ||
} | ||
fn circularstring_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.circularstring_begin(size, idx) | ||
} | ||
fn circularstring_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.circularstring_end(idx) | ||
} | ||
fn compoundcurve_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.compoundcurve_begin(size, idx) | ||
} | ||
fn compoundcurve_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.compoundcurve_end(idx) | ||
} | ||
fn curvepolygon_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.curvepolygon_begin(size, idx) | ||
} | ||
fn curvepolygon_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.curvepolygon_end(idx) | ||
} | ||
fn multicurve_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.multicurve_begin(size, idx) | ||
} | ||
fn multicurve_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.multicurve_end(idx) | ||
} | ||
fn multisurface_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.multisurface_begin(size, idx) | ||
} | ||
fn multisurface_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.multisurface_end(idx) | ||
} | ||
fn triangle_begin(&mut self, tagged: bool, size: usize, idx: usize) -> Result<()> { | ||
self.inner.triangle_begin(tagged, size, idx) | ||
} | ||
fn triangle_end(&mut self, tagged: bool, idx: usize) -> Result<()> { | ||
self.inner.triangle_end(tagged, idx) | ||
} | ||
fn polyhedralsurface_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.polyhedralsurface_begin(size, idx) | ||
} | ||
fn polyhedralsurface_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.polyhedralsurface_end(idx) | ||
} | ||
fn tin_begin(&mut self, size: usize, idx: usize) -> Result<()> { | ||
self.inner.tin_begin(size, idx) | ||
} | ||
fn tin_end(&mut self, idx: usize) -> Result<()> { | ||
self.inner.tin_end(idx) | ||
} | ||
} | ||
|
||
impl<T: PropertyProcessor, F: Fn(&mut f64, &mut f64)> PropertyProcessor | ||
for WrappedXYProcessor<T, F> | ||
{ | ||
fn property(&mut self, idx: usize, name: &str, value: &ColumnValue<'_>) -> Result<bool> { | ||
self.inner.property(idx, name, value) | ||
} | ||
} | ||
|
||
impl<T: FeatureProcessor, F: Fn(&mut f64, &mut f64)> FeatureProcessor for WrappedXYProcessor<T, F> { | ||
fn dataset_begin(&mut self, name: Option<&str>) -> Result<()> { | ||
self.inner.dataset_begin(name) | ||
} | ||
fn dataset_end(&mut self) -> Result<()> { | ||
self.inner.dataset_end() | ||
} | ||
fn feature_begin(&mut self, idx: u64) -> Result<()> { | ||
self.inner.feature_begin(idx) | ||
} | ||
fn feature_end(&mut self, idx: u64) -> Result<()> { | ||
self.inner.feature_end(idx) | ||
} | ||
fn properties_begin(&mut self) -> Result<()> { | ||
self.inner.properties_begin() | ||
} | ||
fn properties_end(&mut self) -> Result<()> { | ||
self.inner.properties_end() | ||
} | ||
fn geometry_begin(&mut self) -> Result<()> { | ||
self.inner.geometry_begin() | ||
} | ||
fn geometry_end(&mut self) -> Result<()> { | ||
self.inner.geometry_end() | ||
} | ||
} | ||
|
||
#[cfg(all(feature = "with-csv", feature = "with-geojson"))] | ||
#[cfg(test)] | ||
mod test { | ||
use crate::csv::CsvWriter; | ||
use crate::geojson::GeoJsonString; | ||
use crate::{GeomProcessor, GeozeroDatasource}; | ||
use serde_json::json; | ||
|
||
fn geojson_fixture_data() -> GeoJsonString { | ||
GeoJsonString( | ||
json!({ | ||
"type": "FeatureCollection", | ||
"features": [ | ||
{ | ||
"type": "Feature", | ||
"properties": { | ||
"population": 100 | ||
}, | ||
"geometry": { | ||
"type": "Point", | ||
"coordinates": [1.0, 2.0] | ||
} | ||
}, | ||
{ | ||
"type": "Feature", | ||
"properties": { | ||
"population": 200 | ||
}, | ||
"geometry": { | ||
"type": "Point", | ||
"coordinates": [3.0, 4.0] | ||
} | ||
} | ||
] | ||
}) | ||
.to_string(), | ||
) | ||
} | ||
|
||
#[test] | ||
fn test_pre_process() { | ||
let mut geojson = geojson_fixture_data(); | ||
let mut out = Vec::new(); | ||
{ | ||
let mut transforming_csv_writer = | ||
CsvWriter::new(&mut out).pre_process_xy(|x: &mut f64, y: &mut f64| { | ||
*x += 1.0; | ||
*y += 2.0; | ||
}); | ||
geojson.process(&mut transforming_csv_writer).unwrap(); | ||
} | ||
assert_eq!( | ||
String::from_utf8(out).unwrap(), | ||
"geometry,population\nPOINT(2 4),100\nPOINT(4 6),200\n" | ||
); | ||
} | ||
|
||
#[test] | ||
fn multiple_transforms() { | ||
let mut geojson = geojson_fixture_data(); | ||
let mut out = Vec::new(); | ||
{ | ||
let mut transforming_csv_writer = CsvWriter::new(&mut out) | ||
.pre_process_xy(|x: &mut f64, y: &mut f64| { | ||
*x += 1.0; | ||
*y += 2.0; | ||
}) | ||
.pre_process_xy(|x: &mut f64, y: &mut f64| { | ||
// It might be surprising that this second transformation is applied before the first. | ||
// It makes sense if you think about each subsequent call as an "insert first", but | ||
// it's admittedly potentially confusing. | ||
*x *= 2.0; | ||
*y *= 2.0; | ||
}); | ||
geojson.process(&mut transforming_csv_writer).unwrap(); | ||
} | ||
|
||
assert_eq!( | ||
String::from_utf8(out).unwrap(), | ||
"geometry,population\nPOINT(3 6),100\nPOINT(7 10),200\n" | ||
); | ||
} | ||
} |