diff --git a/CHANGES.md b/CHANGES.md index 1ddab60..e8d4a10 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,9 +26,10 @@ geojson::ser::to_feature_collection_string(&my_structs).unwrap(); ``` * PR: -* Added `geojson::{FeatureReader, FeatureWriter}` to stream the reading/writing of your custom struct to and from GeoJSON. - * PR: +* Added `geojson::{FeatureReader, FeatureWriter}` to stream the reading/writing of your custom struct to and from GeoJSON, greatly reducing the memory required to process a FeatureCollection. + * PR: * PR: + * PR: * Added IntoIter implementation for FeatureCollection. * * Added `geojson::Result`. diff --git a/src/de.rs b/src/de.rs index 3023732..8f00f7e 100644 --- a/src/de.rs +++ b/src/de.rs @@ -150,21 +150,17 @@ pub fn deserialize_feature_collection<'de, T>( where T: Deserialize<'de>, { - let mut deserializer = serde_json::Deserializer::from_reader(feature_collection_reader); - - // PERF: rather than deserializing the entirety of the `features:` array into memory here, it'd - // be nice to stream the features. However, I ran into difficulty while trying to return any - // borrowed reference from the visitor methods (e.g. MapAccess) - let visitor = FeatureCollectionVisitor::new(); - let objects = deserializer.deserialize_map(visitor)?; - - Ok(objects.into_iter().map(|feature_value| { - let deserializer = feature_value.into_deserializer(); - let visitor = FeatureVisitor::new(); - let record: T = deserializer.deserialize_map(visitor)?; - - Ok(record) - })) + #[allow(deprecated)] + let iter = crate::FeatureIterator::new(feature_collection_reader).map( + |feature_value: Result| { + let deserializer = feature_value?.into_deserializer(); + let visitor = FeatureVisitor::new(); + let record: T = deserializer.deserialize_map(visitor)?; + + Ok(record) + }, + ); + Ok(iter) } /// Build a `Vec` of structs from a GeoJson `&str`. @@ -295,62 +291,6 @@ where Ok(deserializer.deserialize_map(visitor)?) } -struct FeatureCollectionVisitor; - -impl FeatureCollectionVisitor { - fn new() -> Self { - Self - } -} - -impl<'de> serde::de::Visitor<'de> for FeatureCollectionVisitor { - type Value = Vec; - - fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { - write!(formatter, "a valid GeoJSON Feature object") - } - - fn visit_map(self, mut map_access: A) -> std::result::Result - where - A: serde::de::MapAccess<'de>, - { - let mut has_feature_collection_type = false; - let mut features = None; - while let Some((key, value)) = map_access.next_entry::()? { - if key == "type" { - if value == JsonValue::String("FeatureCollection".to_string()) { - has_feature_collection_type = true; - } else { - return Err(Error::custom("invalid type for feature collection")); - } - } else if key == "features" { - if let JsonValue::Array(value) = value { - if features.is_some() { - return Err(Error::custom( - "Encountered more than one list of `features`", - )); - } - features = Some(value); - } else { - return Err(Error::custom("`features` had unexpected value")); - } - } else { - log::warn!("foreign members are not handled by FeatureCollection deserializer") - } - } - - if let Some(features) = features { - if has_feature_collection_type { - Ok(features) - } else { - Err(Error::custom("No `type` field was found")) - } - } else { - Err(Error::custom("No `features` field was found")) - } - } -} - struct FeatureVisitor { _marker: PhantomData, } diff --git a/src/feature_iterator.rs b/src/feature_iterator.rs index 70f48a5..9155067 100644 --- a/src/feature_iterator.rs +++ b/src/feature_iterator.rs @@ -15,6 +15,7 @@ use crate::{Feature, Result}; +use serde::Deserialize; use std::io; use std::marker::PhantomData; @@ -32,10 +33,11 @@ use std::marker::PhantomData; /// Based on example code found at . /// /// [GeoJSON Format Specification ยง 3.3](https://datatracker.ietf.org/doc/html/rfc7946#section-3.3) -pub struct FeatureIterator { +pub struct FeatureIterator<'de, R, D = Feature> { reader: R, state: State, - marker: PhantomData, + output: PhantomData, + lifetime: PhantomData<&'de ()>, } #[derive(Debug, Copy, Clone)] @@ -45,17 +47,18 @@ enum State { AfterFeatures, } -impl FeatureIterator { +impl<'de, R, D> FeatureIterator<'de, R, D> { pub fn new(reader: R) -> Self { FeatureIterator { reader, state: State::BeforeFeatures, - marker: PhantomData, + output: PhantomData, + lifetime: PhantomData, } } } -impl FeatureIterator +impl<'de, R, D> FeatureIterator<'de, R, D> where R: io::Read, { @@ -98,11 +101,12 @@ where } } -impl Iterator for FeatureIterator +impl<'de, R, D> Iterator for FeatureIterator<'de, R, D> where R: io::Read, + D: Deserialize<'de>, { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { match self.seek_to_next_feature() { @@ -124,9 +128,9 @@ where #[cfg(test)] mod tests { - use crate::FeatureIterator; - use crate::Geometry; - use crate::Value; + use super::*; + use crate::{Geometry, Value}; + use std::io::BufReader; fn fc() -> &'static str { @@ -179,7 +183,7 @@ mod tests { #[test] fn stream_read_test() { - let mut fi = FeatureIterator::new(BufReader::new(fc().as_bytes())); + let mut fi = FeatureIterator::<_, Feature>::new(BufReader::new(fc().as_bytes())); assert_eq!( Geometry { bbox: None,