From 4bf2c657ad6b350f221177eda515ab0c317229e2 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 12:20:40 +0100 Subject: [PATCH 01/13] Forward `Loggable::from_arrow2_opt` to `from_arrow_opt` --- crates/store/re_types_core/src/loggable.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/store/re_types_core/src/loggable.rs b/crates/store/re_types_core/src/loggable.rs index 1ebc3ecce1f8..0da30c5acc11 100644 --- a/crates/store/re_types_core/src/loggable.rs +++ b/crates/store/re_types_core/src/loggable.rs @@ -138,11 +138,8 @@ pub trait Loggable: 'static + Send + Sync + Clone + Sized + SizeBytes { fn from_arrow2_opt( data: &dyn arrow2::array::Array, ) -> DeserializationResult>> { - _ = data; // NOTE: do this here to avoid breaking users' autocomplete snippets - Err(crate::DeserializationError::NotImplemented { - fqname: "".to_owned(), - backtrace: _Backtrace::new_unresolved(), - }) + let boxed_arrow_array = arrow::array::ArrayRef::from(data); + Self::from_arrow_opt(boxed_arrow_array.as_ref()) } } From ed87c63f05b2019c835dc5ae6790f96d23a3b4c3 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 12:26:45 +0100 Subject: [PATCH 02/13] Re-export arrow --- crates/store/re_types_core/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/store/re_types_core/src/lib.rs b/crates/store/re_types_core/src/lib.rs index c1fd9cf2a6df..f6deee691390 100644 --- a/crates/store/re_types_core/src/lib.rs +++ b/crates/store/re_types_core/src/lib.rs @@ -194,6 +194,7 @@ pub mod macros { pub mod external { pub use anyhow; + pub use arrow; pub use arrow2; pub use re_tuid; } From 5c03ce7199b4a74817b473ef9534adcd3b0e2289 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 12:27:33 +0100 Subject: [PATCH 03/13] Add Archetype::from_arrow_components --- crates/store/re_types_core/src/archetype.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/crates/store/re_types_core/src/archetype.rs b/crates/store/re_types_core/src/archetype.rs index 9032fb1b53a5..b093a104f565 100644 --- a/crates/store/re_types_core/src/archetype.rs +++ b/crates/store/re_types_core/src/archetype.rs @@ -119,8 +119,8 @@ pub trait Archetype { /// Arrow arrays that are unknown to this [`Archetype`] will simply be ignored and a warning /// logged to stderr. #[inline] - fn from_arrow2_components( - data: impl IntoIterator)>, + fn from_arrow_components( + data: impl IntoIterator, ) -> DeserializationResult where Self: Sized, @@ -131,6 +131,23 @@ pub trait Archetype { backtrace: _Backtrace::new_unresolved(), }) } + + /// Given an iterator of Arrow arrays and their respective `ComponentNames`, deserializes them + /// into this archetype. + /// + /// Arrow arrays that are unknown to this [`Archetype`] will simply be ignored and a warning + /// logged to stderr. + #[inline] + fn from_arrow2_components( + data: impl IntoIterator)>, + ) -> DeserializationResult + where + Self: Sized, + { + Self::from_arrow_components(data.into_iter().map(|(component, arrow2_array)| { + (component, arrow::array::ArrayRef::from(arrow2_array)) + })) + } } /// Indicates that the archetype has reflection data available for it. From ca029301ccb4946899d63e3550d8581949baaf6c Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 12:28:06 +0100 Subject: [PATCH 04/13] Add Archetype::from_arrow --- crates/store/re_types_core/src/archetype.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/store/re_types_core/src/archetype.rs b/crates/store/re_types_core/src/archetype.rs index b093a104f565..601684ef68fb 100644 --- a/crates/store/re_types_core/src/archetype.rs +++ b/crates/store/re_types_core/src/archetype.rs @@ -95,6 +95,24 @@ pub trait Archetype { // --- + /// Given an iterator of Arrow arrays and their respective field metadata, deserializes them + /// into this archetype. + /// + /// Arrow arrays that are unknown to this [`Archetype`] will simply be ignored and a warning + /// logged to stderr. + #[inline] + fn from_arrow( + data: impl IntoIterator, + ) -> DeserializationResult + where + Self: Sized, + { + Self::from_arrow_components( + data.into_iter() + .map(|(field, array)| (field.name.into(), array)), + ) + } + /// Given an iterator of Arrow arrays and their respective field metadata, deserializes them /// into this archetype. /// From fb273c67db8296dec3f8fe0810cfba5c5f0bf472 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 15:23:52 +0100 Subject: [PATCH 05/13] Add arrow1 compatibility for ArrowString --- crates/store/re_types_core/src/arrow_string.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/store/re_types_core/src/arrow_string.rs b/crates/store/re_types_core/src/arrow_string.rs index 9fd8b6d4e633..8376167d8deb 100644 --- a/crates/store/re_types_core/src/arrow_string.rs +++ b/crates/store/re_types_core/src/arrow_string.rs @@ -55,12 +55,24 @@ impl ArrowString { std::str::from_utf8(self.0.as_ref()).unwrap_or("INVALID UTF-8") } + #[inline] + pub fn into_arrow_buffer(self) -> arrow::buffer::Buffer { + self.0.into() + } + #[inline] pub fn into_arrow2_buffer(self) -> arrow2::buffer::Buffer { self.0 } } +impl From for ArrowString { + #[inline] + fn from(buf: arrow::buffer::Buffer) -> Self { + Self(buf.into()) + } +} + impl From> for ArrowString { #[inline] fn from(buf: arrow2::buffer::Buffer) -> Self { From ef61e4b8c2ed029f9abf28be437466c0c9e91211 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 15:27:18 +0100 Subject: [PATCH 06/13] Fix typo --- tests/python/release_checklist/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/release_checklist/main.py b/tests/python/release_checklist/main.py index 96d4553c3f3d..a989c705b38d 100755 --- a/tests/python/release_checklist/main.py +++ b/tests/python/release_checklist/main.py @@ -32,10 +32,10 @@ def main() -> None: args = parser.parse_args() # Download test assets: - download_test_assest_path = ( + download_test_assets_path = ( Path(__file__).parent.parent.parent.joinpath("assets/download_test_assets.py").absolute() ) - subprocess.run([sys.executable, download_test_assest_path]) + subprocess.run([sys.executable, download_test_assets_path]) log_checks(args) From 00571effe00aa64ffdc36f3d2fdf89f0e323e0ac Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 15:34:27 +0100 Subject: [PATCH 07/13] Fix bug that we haven't run into yet --- .../re_types_builder/src/codegen/rust/deserializer.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs index 2b942ca13f19..eab1bbd0d87c 100644 --- a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs @@ -326,10 +326,11 @@ pub fn quote_arrow_deserializer( let quoted_field_deserializers = obj .fields .iter() - .filter(|obj_field| - // For unit fields we don't have to collect any data. - obj_field.typ != crate::Type::Unit) .enumerate() + .filter(|(_, obj_field)| { + // For unit fields we don't have to collect any data. + obj_field.typ != crate::Type::Unit + }) .map(|(i, obj_field)| { let data_dst = format_ident!("{}", obj_field.snake_case_name()); From 3e9157b40934466bec422e81d20c63a84ddccdb4 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 15:37:40 +0100 Subject: [PATCH 08/13] better naming of a variable --- .../src/codegen/rust/deserializer.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs index eab1bbd0d87c..79c02eb9838c 100644 --- a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs @@ -331,7 +331,7 @@ pub fn quote_arrow_deserializer( // For unit fields we don't have to collect any data. obj_field.typ != crate::Type::Unit }) - .map(|(i, obj_field)| { + .map(|(type_id, obj_field)| { let data_dst = format_ident!("{}", obj_field.snake_case_name()); let field_datatype = &arrow_registry.get(&obj_field.fqname); @@ -345,15 +345,15 @@ pub fn quote_arrow_deserializer( InnerRepr::NativeIterable, ); - let i = i + 1; // NOTE: +1 to account for `_null_markers` virtual arm + let type_id = type_id + 1; // NOTE: +1 to account for `_null_markers` virtual arm quote! { let #data_dst = { // NOTE: `data_src_arrays` is a runtime collection of all of the - // input's payload's union arms, while `#i` is our comptime union + // input's payload's union arms, while `#type_id` is our comptime union // arm counter… there's no guarantee it's actually there at // runtime! - if #i >= #data_src_arrays.len() { + if #type_id >= #data_src_arrays.len() { // By not returning an error but rather defaulting to an empty // vector, we introduce some kind of light forwards compatibility: // old clients that don't yet know about the new arms can still @@ -361,13 +361,13 @@ pub fn quote_arrow_deserializer( return Ok(Vec::new()); // return Err(DeserializationError::missing_union_arm( - // #quoted_datatype, #obj_field_fqname, #i, + // #quoted_datatype, #obj_field_fqname, #type_id, // )).with_context(#obj_fqname); } // NOTE: The array indexing is safe: checked above. - let #data_src = &*#data_src_arrays[#i]; - #quoted_deserializer.collect::>() + let #data_src = &*#data_src_arrays[#type_id]; + #quoted_deserializer.collect::>() } } }); From 2271107494b4a47f73a4ca1770951a5d05cee1ea Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 15:40:39 +0100 Subject: [PATCH 09/13] Remove integer suffix --- .../src/codegen/rust/deserializer.rs | 2 +- .../re_types/src/datatypes/tensor_buffer.rs | 44 +++++++++---------- .../src/testing/datatypes/affix_fuzzer3.rs | 12 ++--- .../src/testing/datatypes/affix_fuzzer4.rs | 8 ++-- .../src/datatypes/time_range_boundary.rs | 8 ++-- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs index 79c02eb9838c..509c986d7698 100644 --- a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs @@ -345,7 +345,7 @@ pub fn quote_arrow_deserializer( InnerRepr::NativeIterable, ); - let type_id = type_id + 1; // NOTE: +1 to account for `_null_markers` virtual arm + let type_id = Literal::usize_unsuffixed(type_id + 1); // NOTE: +1 to account for `_null_markers` virtual arm quote! { let #data_dst = { diff --git a/crates/store/re_types/src/datatypes/tensor_buffer.rs b/crates/store/re_types/src/datatypes/tensor_buffer.rs index 5ab47f74b755..be26f5af8f58 100644 --- a/crates/store/re_types/src/datatypes/tensor_buffer.rs +++ b/crates/store/re_types/src/datatypes/tensor_buffer.rs @@ -787,10 +787,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .with_context("rerun.datatypes.TensorBuffer"); } let u8 = { - if 1usize >= arrow_data_arrays.len() { + if 1 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[1usize]; + let arrow_data = &*arrow_data_arrays[1]; { let arrow_data = arrow_data .as_any() @@ -855,10 +855,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u16 = { - if 2usize >= arrow_data_arrays.len() { + if 2 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[2usize]; + let arrow_data = &*arrow_data_arrays[2]; { let arrow_data = arrow_data .as_any() @@ -923,10 +923,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u32 = { - if 3usize >= arrow_data_arrays.len() { + if 3 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[3usize]; + let arrow_data = &*arrow_data_arrays[3]; { let arrow_data = arrow_data .as_any() @@ -991,10 +991,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u64 = { - if 4usize >= arrow_data_arrays.len() { + if 4 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[4usize]; + let arrow_data = &*arrow_data_arrays[4]; { let arrow_data = arrow_data .as_any() @@ -1059,10 +1059,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i8 = { - if 5usize >= arrow_data_arrays.len() { + if 5 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[5usize]; + let arrow_data = &*arrow_data_arrays[5]; { let arrow_data = arrow_data .as_any() @@ -1127,10 +1127,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i16 = { - if 6usize >= arrow_data_arrays.len() { + if 6 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[6usize]; + let arrow_data = &*arrow_data_arrays[6]; { let arrow_data = arrow_data .as_any() @@ -1195,10 +1195,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i32 = { - if 7usize >= arrow_data_arrays.len() { + if 7 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[7usize]; + let arrow_data = &*arrow_data_arrays[7]; { let arrow_data = arrow_data .as_any() @@ -1263,10 +1263,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i64 = { - if 8usize >= arrow_data_arrays.len() { + if 8 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[8usize]; + let arrow_data = &*arrow_data_arrays[8]; { let arrow_data = arrow_data .as_any() @@ -1331,10 +1331,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f16 = { - if 9usize >= arrow_data_arrays.len() { + if 9 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[9usize]; + let arrow_data = &*arrow_data_arrays[9]; { let arrow_data = arrow_data .as_any() @@ -1399,10 +1399,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f32 = { - if 10usize >= arrow_data_arrays.len() { + if 10 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[10usize]; + let arrow_data = &*arrow_data_arrays[10]; { let arrow_data = arrow_data .as_any() @@ -1467,10 +1467,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f64 = { - if 11usize >= arrow_data_arrays.len() { + if 11 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[11usize]; + let arrow_data = &*arrow_data_arrays[11]; { let arrow_data = arrow_data .as_any() diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs index 53bff2f28a95..d203501e0bc8 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs @@ -288,10 +288,10 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .with_context("rerun.testing.datatypes.AffixFuzzer3"); } let degrees = { - if 1usize >= arrow_data_arrays.len() { + if 1 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[1usize]; + let arrow_data = &*arrow_data_arrays[1]; arrow_data .as_any() .downcast_ref::() @@ -306,10 +306,10 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .collect::>() }; let craziness = { - if 2usize >= arrow_data_arrays.len() { + if 2 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[2usize]; + let arrow_data = &*arrow_data_arrays[2]; { let arrow_data = arrow_data .as_any() @@ -371,10 +371,10 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .collect::>() }; let fixed_size_shenanigans = { - if 3usize >= arrow_data_arrays.len() { + if 3 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[3usize]; + let arrow_data = &*arrow_data_arrays[3]; { let arrow_data = arrow_data .as_any() diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs index 8224c5bb8998..7ad992ff3194 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs @@ -229,20 +229,20 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { .with_context("rerun.testing.datatypes.AffixFuzzer4"); } let single_required = { - if 1usize >= arrow_data_arrays.len() { + if 1 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[1usize]; + let arrow_data = &*arrow_data_arrays[1]; crate::testing::datatypes::AffixFuzzer3::from_arrow2_opt(arrow_data) .with_context("rerun.testing.datatypes.AffixFuzzer4#single_required")? .into_iter() .collect::>() }; let many_required = { - if 2usize >= arrow_data_arrays.len() { + if 2 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[2usize]; + let arrow_data = &*arrow_data_arrays[2]; { let arrow_data = arrow_data .as_any() diff --git a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs index d50c56c9086c..580c340bdad5 100644 --- a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs +++ b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs @@ -231,10 +231,10 @@ impl crate::Loggable for TimeRangeBoundary { .with_context("rerun.datatypes.TimeRangeBoundary"); } let cursor_relative = { - if 1usize >= arrow_data_arrays.len() { + if 1 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[1usize]; + let arrow_data = &*arrow_data_arrays[1]; arrow_data .as_any() .downcast_ref::() @@ -250,10 +250,10 @@ impl crate::Loggable for TimeRangeBoundary { .collect::>() }; let absolute = { - if 2usize >= arrow_data_arrays.len() { + if 2 >= arrow_data_arrays.len() { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[2usize]; + let arrow_data = &*arrow_data_arrays[2]; arrow_data .as_any() .downcast_ref::() From 24a18cbec9909c1bb9d2ba2e437cbb9516087230 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 15:41:22 +0100 Subject: [PATCH 10/13] Flip a comparison because I am a control freak --- .../src/codegen/rust/deserializer.rs | 2 +- .../re_types/src/datatypes/tensor_buffer.rs | 22 +++++++++---------- .../src/testing/datatypes/affix_fuzzer3.rs | 6 ++--- .../src/testing/datatypes/affix_fuzzer4.rs | 4 ++-- .../src/datatypes/time_range_boundary.rs | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs index 509c986d7698..3dba17918b1f 100644 --- a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs @@ -353,7 +353,7 @@ pub fn quote_arrow_deserializer( // input's payload's union arms, while `#type_id` is our comptime union // arm counter… there's no guarantee it's actually there at // runtime! - if #type_id >= #data_src_arrays.len() { + if #data_src_arrays.len() <= #type_id { // By not returning an error but rather defaulting to an empty // vector, we introduce some kind of light forwards compatibility: // old clients that don't yet know about the new arms can still diff --git a/crates/store/re_types/src/datatypes/tensor_buffer.rs b/crates/store/re_types/src/datatypes/tensor_buffer.rs index be26f5af8f58..6b55add717cf 100644 --- a/crates/store/re_types/src/datatypes/tensor_buffer.rs +++ b/crates/store/re_types/src/datatypes/tensor_buffer.rs @@ -787,7 +787,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .with_context("rerun.datatypes.TensorBuffer"); } let u8 = { - if 1 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 1 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[1]; @@ -855,7 +855,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u16 = { - if 2 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 2 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[2]; @@ -923,7 +923,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u32 = { - if 3 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 3 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[3]; @@ -991,7 +991,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u64 = { - if 4 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 4 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[4]; @@ -1059,7 +1059,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i8 = { - if 5 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 5 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[5]; @@ -1127,7 +1127,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i16 = { - if 6 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 6 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[6]; @@ -1195,7 +1195,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i32 = { - if 7 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 7 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[7]; @@ -1263,7 +1263,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i64 = { - if 8 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 8 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[8]; @@ -1331,7 +1331,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f16 = { - if 9 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 9 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[9]; @@ -1399,7 +1399,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f32 = { - if 10 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 10 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[10]; @@ -1467,7 +1467,7 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f64 = { - if 11 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 11 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[11]; diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs index d203501e0bc8..04d50d257049 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs @@ -288,7 +288,7 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .with_context("rerun.testing.datatypes.AffixFuzzer3"); } let degrees = { - if 1 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 1 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[1]; @@ -306,7 +306,7 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .collect::>() }; let craziness = { - if 2 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 2 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[2]; @@ -371,7 +371,7 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .collect::>() }; let fixed_size_shenanigans = { - if 3 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 3 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[3]; diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs index 7ad992ff3194..ecf9a988f7a8 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs @@ -229,7 +229,7 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { .with_context("rerun.testing.datatypes.AffixFuzzer4"); } let single_required = { - if 1 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 1 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[1]; @@ -239,7 +239,7 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { .collect::>() }; let many_required = { - if 2 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 2 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[2]; diff --git a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs index 580c340bdad5..92b12ba1ca91 100644 --- a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs +++ b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs @@ -231,7 +231,7 @@ impl crate::Loggable for TimeRangeBoundary { .with_context("rerun.datatypes.TimeRangeBoundary"); } let cursor_relative = { - if 1 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 1 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[1]; @@ -250,7 +250,7 @@ impl crate::Loggable for TimeRangeBoundary { .collect::>() }; let absolute = { - if 2 >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 2 { return Ok(Vec::new()); } let arrow_data = &*arrow_data_arrays[2]; From 55f630b4c575eb3efa0114289ec1c73f0b16609f Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 16:10:44 +0100 Subject: [PATCH 11/13] Do manual windowing of offset buffer --- .../src/codegen/rust/deserializer.rs | 11 ++--- .../datatypes/component_column_selector.rs | 22 +++++----- .../blueprint/datatypes/selected_columns.rs | 11 ++--- .../re_types/src/datatypes/annotation_info.rs | 11 ++--- .../src/datatypes/tensor_dimension.rs | 11 ++--- .../store/re_types/src/datatypes/utf8pair.rs | 22 +++++----- .../src/testing/components/affix_fuzzer10.rs | 11 ++--- .../src/testing/components/affix_fuzzer12.rs | 11 ++--- .../src/testing/components/affix_fuzzer13.rs | 11 ++--- .../src/testing/components/affix_fuzzer9.rs | 11 ++--- .../src/testing/datatypes/affix_fuzzer1.rs | 44 ++++++++++--------- .../src/testing/datatypes/affix_fuzzer20.rs | 11 ++--- .../src/testing/datatypes/string_component.rs | 11 ++--- .../src/blueprint/datatypes/utf8list.rs | 11 ++--- .../src/datatypes/entity_path.rs | 11 ++--- .../store/re_types_core/src/datatypes/utf8.rs | 11 ++--- .../src/datatypes/visible_time_range.rs | 11 ++--- 17 files changed, 132 insertions(+), 110 deletions(-) diff --git a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs index 3dba17918b1f..17bcc9321038 100644 --- a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs @@ -584,19 +584,20 @@ fn quote_arrow_field_deserializer( let offsets = #data_src.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), #data_src.validity(), ) - .map(|elem| elem.map(|(start, len)| { + .map(|elem| elem.map(|window| { // NOTE: Do _not_ use `Buffer::sliced`, it panics on malformed inputs. - let start = *start as usize; - let end = start + len; + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; // NOTE: It is absolutely crucial we explicitly handle the // boundchecks manually first, otherwise rustc completely chokes // when slicing the data (as in: a 100x perf drop)! - if end > #data_src_buf.len() { + if #data_src_buf.len() < end { // error context is appended below during final collection return Err(DeserializationError::offset_slice_oob( (start, end), #data_src_buf.len(), diff --git a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs index 300720dffec5..dd0694d2ab66 100644 --- a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs +++ b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs @@ -209,14 +209,15 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), @@ -270,14 +271,15 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs index 02bead6bfb67..939228de695c 100644 --- a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs +++ b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs @@ -265,15 +265,16 @@ impl ::re_types_core::Loggable for SelectedColumns { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), diff --git a/crates/store/re_types/src/datatypes/annotation_info.rs b/crates/store/re_types/src/datatypes/annotation_info.rs index 4c8e1d32667e..2c40d6bf2938 100644 --- a/crates/store/re_types/src/datatypes/annotation_info.rs +++ b/crates/store/re_types/src/datatypes/annotation_info.rs @@ -231,14 +231,15 @@ impl ::re_types_core::Loggable for AnnotationInfo { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/datatypes/tensor_dimension.rs b/crates/store/re_types/src/datatypes/tensor_dimension.rs index 4fce89954ca5..619c954eea2f 100644 --- a/crates/store/re_types/src/datatypes/tensor_dimension.rs +++ b/crates/store/re_types/src/datatypes/tensor_dimension.rs @@ -199,14 +199,15 @@ impl ::re_types_core::Loggable for TensorDimension { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/datatypes/utf8pair.rs b/crates/store/re_types/src/datatypes/utf8pair.rs index 99739ab10aa1..2eec236d7687 100644 --- a/crates/store/re_types/src/datatypes/utf8pair.rs +++ b/crates/store/re_types/src/datatypes/utf8pair.rs @@ -186,14 +186,15 @@ impl ::re_types_core::Loggable for Utf8Pair { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), @@ -241,14 +242,15 @@ impl ::re_types_core::Loggable for Utf8Pair { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs index c7a341da3438..739d7de755cc 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs @@ -104,14 +104,15 @@ impl ::re_types_core::Loggable for AffixFuzzer10 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs index fe8e4847fe44..9215792180fe 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs @@ -136,14 +136,15 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner_buf.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs index a38e831ac11e..c09c0bf4bda2 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs @@ -136,14 +136,15 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner_buf.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs index 4e9e6c414ca8..a5d0d98442cd 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs @@ -104,14 +104,15 @@ impl ::re_types_core::Loggable for AffixFuzzer9 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs index a6bdbe5c3b41..b48625e87fab 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs @@ -543,14 +543,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), @@ -600,14 +601,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), @@ -753,15 +755,16 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), @@ -874,15 +877,16 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs index 0807008bf746..36adf314259f 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs @@ -214,14 +214,15 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/testing/datatypes/string_component.rs b/crates/store/re_types/src/testing/datatypes/string_component.rs index 1b2e10502f04..735ff75abb84 100644 --- a/crates/store/re_types/src/testing/datatypes/string_component.rs +++ b/crates/store/re_types/src/testing/datatypes/string_component.rs @@ -98,14 +98,15 @@ impl ::re_types_core::Loggable for StringComponent { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs index 7dd2246d1c5c..c02e9cad9724 100644 --- a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs +++ b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs @@ -129,14 +129,15 @@ impl ::re_types_core::Loggable for Utf8List { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner_buf.len(), diff --git a/crates/store/re_types_core/src/datatypes/entity_path.rs b/crates/store/re_types_core/src/datatypes/entity_path.rs index f3b2fca0e5c8..52690454d531 100644 --- a/crates/store/re_types_core/src/datatypes/entity_path.rs +++ b/crates/store/re_types_core/src/datatypes/entity_path.rs @@ -99,14 +99,15 @@ impl crate::Loggable for EntityPath { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types_core/src/datatypes/utf8.rs b/crates/store/re_types_core/src/datatypes/utf8.rs index 052f94e220d5..79e3fe46b16e 100644 --- a/crates/store/re_types_core/src/datatypes/utf8.rs +++ b/crates/store/re_types_core/src/datatypes/utf8.rs @@ -99,14 +99,15 @@ impl crate::Loggable for Utf8 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types_core/src/datatypes/visible_time_range.rs b/crates/store/re_types_core/src/datatypes/visible_time_range.rs index fd0504caec70..91e470e85a07 100644 --- a/crates/store/re_types_core/src/datatypes/visible_time_range.rs +++ b/crates/store/re_types_core/src/datatypes/visible_time_range.rs @@ -190,14 +190,15 @@ impl crate::Loggable for VisibleTimeRange { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), From 8322542f93f9448ba563fe926e46a643f84e2616 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 16:12:18 +0100 Subject: [PATCH 12/13] Manual windowing again --- .../src/codegen/rust/deserializer.rs | 10 +- .../blueprint/datatypes/selected_columns.rs | 20 ++-- .../src/components/annotation_context.rs | 10 +- .../src/components/geo_line_string.rs | 10 +- .../re_types/src/components/line_strip2d.rs | 10 +- .../re_types/src/components/line_strip3d.rs | 10 +- crates/store/re_types/src/datatypes/blob.rs | 10 +- .../src/datatypes/class_description.rs | 20 ++-- .../re_types/src/datatypes/tensor_buffer.rs | 110 +++++++++--------- .../re_types/src/datatypes/tensor_data.rs | 10 +- .../src/testing/components/affix_fuzzer11.rs | 10 +- .../src/testing/components/affix_fuzzer12.rs | 10 +- .../src/testing/components/affix_fuzzer13.rs | 10 +- .../src/testing/components/affix_fuzzer16.rs | 10 +- .../src/testing/components/affix_fuzzer17.rs | 10 +- .../src/testing/components/affix_fuzzer18.rs | 10 +- .../src/testing/components/affix_fuzzer7.rs | 10 +- .../src/testing/datatypes/affix_fuzzer1.rs | 30 ++--- .../src/testing/datatypes/affix_fuzzer21.rs | 10 +- .../src/testing/datatypes/affix_fuzzer3.rs | 10 +- .../src/testing/datatypes/affix_fuzzer4.rs | 10 +- .../src/blueprint/datatypes/utf8list.rs | 10 +- 22 files changed, 180 insertions(+), 180 deletions(-) diff --git a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs index 17bcc9321038..25880acd6fd0 100644 --- a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs @@ -814,19 +814,19 @@ fn quote_arrow_field_deserializer( let offsets = #data_src.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), #data_src.validity(), ) - .map(|elem| elem.map(|(start, len)| { + .map(|elem| elem.map(|window| { // NOTE: Do _not_ use `Buffer::sliced`, it panics on malformed inputs. - let start = *start as usize; - let end = start + len; + let start = window[0] as usize; + let end = window[1] as usize; // NOTE: It is absolutely crucial we explicitly handle the // boundchecks manually first, otherwise rustc completely chokes // when slicing the data (as in: a 100x perf drop)! - if end > #data_src_inner.len() { + if #data_src_inner.len() < end { // error context is appended below during final collection return Err(DeserializationError::offset_slice_oob( (start, end), #data_src_inner.len(), diff --git a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs index 939228de695c..4bb0cec1fea9 100644 --- a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs +++ b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs @@ -310,15 +310,15 @@ impl ::re_types_core::Loggable for SelectedColumns { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), @@ -390,15 +390,15 @@ impl ::re_types_core::Loggable for SelectedColumns { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), diff --git a/crates/store/re_types/src/components/annotation_context.rs b/crates/store/re_types/src/components/annotation_context.rs index 09c12c378f25..c24f64f2d2fd 100644 --- a/crates/store/re_types/src/components/annotation_context.rs +++ b/crates/store/re_types/src/components/annotation_context.rs @@ -134,14 +134,14 @@ impl ::re_types_core::Loggable for AnnotationContext { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/components/geo_line_string.rs b/crates/store/re_types/src/components/geo_line_string.rs index 5188e4c88bf7..09348a64d66f 100644 --- a/crates/store/re_types/src/components/geo_line_string.rs +++ b/crates/store/re_types/src/components/geo_line_string.rs @@ -208,14 +208,14 @@ impl ::re_types_core::Loggable for GeoLineString { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/components/line_strip2d.rs b/crates/store/re_types/src/components/line_strip2d.rs index 71ecbe3149db..fcac594ffd8e 100644 --- a/crates/store/re_types/src/components/line_strip2d.rs +++ b/crates/store/re_types/src/components/line_strip2d.rs @@ -217,14 +217,14 @@ impl ::re_types_core::Loggable for LineStrip2D { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/components/line_strip3d.rs b/crates/store/re_types/src/components/line_strip3d.rs index eff94b820b1a..97e579575bc7 100644 --- a/crates/store/re_types/src/components/line_strip3d.rs +++ b/crates/store/re_types/src/components/line_strip3d.rs @@ -217,14 +217,14 @@ impl ::re_types_core::Loggable for LineStrip3D { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/blob.rs b/crates/store/re_types/src/datatypes/blob.rs index 0159c543fc9e..06e08627da94 100644 --- a/crates/store/re_types/src/datatypes/blob.rs +++ b/crates/store/re_types/src/datatypes/blob.rs @@ -127,14 +127,14 @@ impl ::re_types_core::Loggable for Blob { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/class_description.rs b/crates/store/re_types/src/datatypes/class_description.rs index dddfad4eeb7b..b6827410d9b8 100644 --- a/crates/store/re_types/src/datatypes/class_description.rs +++ b/crates/store/re_types/src/datatypes/class_description.rs @@ -323,14 +323,14 @@ impl ::re_types_core::Loggable for ClassDescription { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -393,14 +393,14 @@ impl ::re_types_core::Loggable for ClassDescription { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/tensor_buffer.rs b/crates/store/re_types/src/datatypes/tensor_buffer.rs index 6b55add717cf..cf2df863a568 100644 --- a/crates/store/re_types/src/datatypes/tensor_buffer.rs +++ b/crates/store/re_types/src/datatypes/tensor_buffer.rs @@ -823,14 +823,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -891,14 +891,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -959,14 +959,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1027,14 +1027,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1095,14 +1095,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1163,14 +1163,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1231,14 +1231,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1299,14 +1299,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1367,14 +1367,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1435,14 +1435,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1503,14 +1503,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/tensor_data.rs b/crates/store/re_types/src/datatypes/tensor_data.rs index 6e10fad1c17d..0d19cb6190a6 100644 --- a/crates/store/re_types/src/datatypes/tensor_data.rs +++ b/crates/store/re_types/src/datatypes/tensor_data.rs @@ -227,14 +227,14 @@ impl ::re_types_core::Loggable for TensorData { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer11.rs b/crates/store/re_types/src/testing/components/affix_fuzzer11.rs index 4a2e04a06eca..142b2b88218f 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer11.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer11.rs @@ -132,14 +132,14 @@ impl ::re_types_core::Loggable for AffixFuzzer11 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs index 9215792180fe..e3f13d3a9d4b 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs @@ -174,14 +174,14 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs index c09c0bf4bda2..9118de31e42a 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs @@ -174,14 +174,14 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer16.rs b/crates/store/re_types/src/testing/components/affix_fuzzer16.rs index 70fd6558ac5d..605c7831b803 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer16.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer16.rs @@ -126,14 +126,14 @@ impl ::re_types_core::Loggable for AffixFuzzer16 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer17.rs b/crates/store/re_types/src/testing/components/affix_fuzzer17.rs index f2cd5031ae0c..5c3584825e37 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer17.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer17.rs @@ -126,14 +126,14 @@ impl ::re_types_core::Loggable for AffixFuzzer17 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer18.rs b/crates/store/re_types/src/testing/components/affix_fuzzer18.rs index 2a370088b673..0ee06f26d382 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer18.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer18.rs @@ -126,14 +126,14 @@ impl ::re_types_core::Loggable for AffixFuzzer18 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer7.rs b/crates/store/re_types/src/testing/components/affix_fuzzer7.rs index 8467fb435de0..74fa0a201a01 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer7.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer7.rs @@ -124,14 +124,14 @@ impl ::re_types_core::Loggable for AffixFuzzer7 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs index b48625e87fab..4cec3891a667 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs @@ -680,14 +680,14 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -797,15 +797,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), @@ -919,15 +919,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs index e29597829b98..aa6cac91defd 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs @@ -242,14 +242,14 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs index 04d50d257049..b43ce063aba7 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs @@ -338,14 +338,14 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs index ecf9a988f7a8..bc763821d865 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs @@ -271,14 +271,14 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs index c02e9cad9724..1e1d612b53c2 100644 --- a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs +++ b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs @@ -165,14 +165,14 @@ impl ::re_types_core::Loggable for Utf8List { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), From ed5cc1cbb93a797a5d1d80f433d60423c0b9ad25 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Mon, 9 Dec 2024 16:16:58 +0100 Subject: [PATCH 13/13] Improve test fail message --- crates/store/re_types/tests/types/validity.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/store/re_types/tests/types/validity.rs b/crates/store/re_types/tests/types/validity.rs index dab289a75f92..6b3f19cbe63a 100644 --- a/crates/store/re_types/tests/types/validity.rs +++ b/crates/store/re_types/tests/types/validity.rs @@ -31,8 +31,9 @@ fn validity_checks() { let serialized = Position2D::to_arrow2_opt(bad).unwrap(); let deserialized = Position2D::from_arrow2(serialized.as_ref()); assert!(deserialized.is_err()); - assert!(matches!( - deserialized.err().unwrap(), - DeserializationError::MissingData { .. } - )); + let actual_error = deserialized.err().unwrap(); + assert!( + matches!(actual_error, DeserializationError::MissingData { .. }), + "Expected error MissingData, got {actual_error:?}", + ); }