diff --git a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs index 2b942ca13f19..25880acd6fd0 100644 --- a/crates/build/re_types_builder/src/codegen/rust/deserializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/deserializer.rs @@ -326,11 +326,12 @@ pub fn quote_arrow_deserializer( let quoted_field_deserializers = obj .fields .iter() - .filter(|obj_field| - // For unit fields we don't have to collect any data. - obj_field.typ != crate::Type::Unit) .enumerate() - .map(|(i, obj_field)| { + .filter(|(_, obj_field)| { + // For unit fields we don't have to collect any data. + obj_field.typ != crate::Type::Unit + }) + .map(|(type_id, obj_field)| { let data_dst = format_ident!("{}", obj_field.snake_case_name()); let field_datatype = &arrow_registry.get(&obj_field.fqname); @@ -344,15 +345,15 @@ pub fn quote_arrow_deserializer( InnerRepr::NativeIterable, ); - let i = i + 1; // NOTE: +1 to account for `_null_markers` virtual arm + let type_id = Literal::usize_unsuffixed(type_id + 1); // NOTE: +1 to account for `_null_markers` virtual arm quote! { let #data_dst = { // NOTE: `data_src_arrays` is a runtime collection of all of the - // input's payload's union arms, while `#i` is our comptime union + // input's payload's union arms, while `#type_id` is our comptime union // arm counter… there's no guarantee it's actually there at // runtime! - if #i >= #data_src_arrays.len() { + if #data_src_arrays.len() <= #type_id { // By not returning an error but rather defaulting to an empty // vector, we introduce some kind of light forwards compatibility: // old clients that don't yet know about the new arms can still @@ -360,13 +361,13 @@ pub fn quote_arrow_deserializer( return Ok(Vec::new()); // return Err(DeserializationError::missing_union_arm( - // #quoted_datatype, #obj_field_fqname, #i, + // #quoted_datatype, #obj_field_fqname, #type_id, // )).with_context(#obj_fqname); } // NOTE: The array indexing is safe: checked above. - let #data_src = &*#data_src_arrays[#i]; - #quoted_deserializer.collect::>() + let #data_src = &*#data_src_arrays[#type_id]; + #quoted_deserializer.collect::>() } } }); @@ -583,19 +584,20 @@ fn quote_arrow_field_deserializer( let offsets = #data_src.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), #data_src.validity(), ) - .map(|elem| elem.map(|(start, len)| { + .map(|elem| elem.map(|window| { // NOTE: Do _not_ use `Buffer::sliced`, it panics on malformed inputs. - let start = *start as usize; - let end = start + len; + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; // NOTE: It is absolutely crucial we explicitly handle the // boundchecks manually first, otherwise rustc completely chokes // when slicing the data (as in: a 100x perf drop)! - if end > #data_src_buf.len() { + if #data_src_buf.len() < end { // error context is appended below during final collection return Err(DeserializationError::offset_slice_oob( (start, end), #data_src_buf.len(), @@ -812,19 +814,19 @@ fn quote_arrow_field_deserializer( let offsets = #data_src.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), #data_src.validity(), ) - .map(|elem| elem.map(|(start, len)| { + .map(|elem| elem.map(|window| { // NOTE: Do _not_ use `Buffer::sliced`, it panics on malformed inputs. - let start = *start as usize; - let end = start + len; + let start = window[0] as usize; + let end = window[1] as usize; // NOTE: It is absolutely crucial we explicitly handle the // boundchecks manually first, otherwise rustc completely chokes // when slicing the data (as in: a 100x perf drop)! - if end > #data_src_inner.len() { + if #data_src_inner.len() < end { // error context is appended below during final collection return Err(DeserializationError::offset_slice_oob( (start, end), #data_src_inner.len(), diff --git a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs index 300720dffec5..dd0694d2ab66 100644 --- a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs +++ b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs @@ -209,14 +209,15 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), @@ -270,14 +271,15 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs index 02bead6bfb67..4bb0cec1fea9 100644 --- a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs +++ b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs @@ -265,15 +265,16 @@ impl ::re_types_core::Loggable for SelectedColumns { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), @@ -309,15 +310,15 @@ impl ::re_types_core::Loggable for SelectedColumns { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), @@ -389,15 +390,15 @@ impl ::re_types_core::Loggable for SelectedColumns { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), diff --git a/crates/store/re_types/src/components/annotation_context.rs b/crates/store/re_types/src/components/annotation_context.rs index 09c12c378f25..c24f64f2d2fd 100644 --- a/crates/store/re_types/src/components/annotation_context.rs +++ b/crates/store/re_types/src/components/annotation_context.rs @@ -134,14 +134,14 @@ impl ::re_types_core::Loggable for AnnotationContext { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/components/geo_line_string.rs b/crates/store/re_types/src/components/geo_line_string.rs index 5188e4c88bf7..09348a64d66f 100644 --- a/crates/store/re_types/src/components/geo_line_string.rs +++ b/crates/store/re_types/src/components/geo_line_string.rs @@ -208,14 +208,14 @@ impl ::re_types_core::Loggable for GeoLineString { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/components/line_strip2d.rs b/crates/store/re_types/src/components/line_strip2d.rs index 71ecbe3149db..fcac594ffd8e 100644 --- a/crates/store/re_types/src/components/line_strip2d.rs +++ b/crates/store/re_types/src/components/line_strip2d.rs @@ -217,14 +217,14 @@ impl ::re_types_core::Loggable for LineStrip2D { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/components/line_strip3d.rs b/crates/store/re_types/src/components/line_strip3d.rs index eff94b820b1a..97e579575bc7 100644 --- a/crates/store/re_types/src/components/line_strip3d.rs +++ b/crates/store/re_types/src/components/line_strip3d.rs @@ -217,14 +217,14 @@ impl ::re_types_core::Loggable for LineStrip3D { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/annotation_info.rs b/crates/store/re_types/src/datatypes/annotation_info.rs index 4c8e1d32667e..2c40d6bf2938 100644 --- a/crates/store/re_types/src/datatypes/annotation_info.rs +++ b/crates/store/re_types/src/datatypes/annotation_info.rs @@ -231,14 +231,15 @@ impl ::re_types_core::Loggable for AnnotationInfo { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/datatypes/blob.rs b/crates/store/re_types/src/datatypes/blob.rs index 0159c543fc9e..06e08627da94 100644 --- a/crates/store/re_types/src/datatypes/blob.rs +++ b/crates/store/re_types/src/datatypes/blob.rs @@ -127,14 +127,14 @@ impl ::re_types_core::Loggable for Blob { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/class_description.rs b/crates/store/re_types/src/datatypes/class_description.rs index dddfad4eeb7b..b6827410d9b8 100644 --- a/crates/store/re_types/src/datatypes/class_description.rs +++ b/crates/store/re_types/src/datatypes/class_description.rs @@ -323,14 +323,14 @@ impl ::re_types_core::Loggable for ClassDescription { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -393,14 +393,14 @@ impl ::re_types_core::Loggable for ClassDescription { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/tensor_buffer.rs b/crates/store/re_types/src/datatypes/tensor_buffer.rs index 5ab47f74b755..cf2df863a568 100644 --- a/crates/store/re_types/src/datatypes/tensor_buffer.rs +++ b/crates/store/re_types/src/datatypes/tensor_buffer.rs @@ -787,10 +787,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .with_context("rerun.datatypes.TensorBuffer"); } let u8 = { - if 1usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 1 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[1usize]; + let arrow_data = &*arrow_data_arrays[1]; { let arrow_data = arrow_data .as_any() @@ -823,14 +823,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -855,10 +855,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u16 = { - if 2usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 2 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[2usize]; + let arrow_data = &*arrow_data_arrays[2]; { let arrow_data = arrow_data .as_any() @@ -891,14 +891,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -923,10 +923,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u32 = { - if 3usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 3 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[3usize]; + let arrow_data = &*arrow_data_arrays[3]; { let arrow_data = arrow_data .as_any() @@ -959,14 +959,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -991,10 +991,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let u64 = { - if 4usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 4 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[4usize]; + let arrow_data = &*arrow_data_arrays[4]; { let arrow_data = arrow_data .as_any() @@ -1027,14 +1027,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1059,10 +1059,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i8 = { - if 5usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 5 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[5usize]; + let arrow_data = &*arrow_data_arrays[5]; { let arrow_data = arrow_data .as_any() @@ -1095,14 +1095,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1127,10 +1127,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i16 = { - if 6usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 6 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[6usize]; + let arrow_data = &*arrow_data_arrays[6]; { let arrow_data = arrow_data .as_any() @@ -1163,14 +1163,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1195,10 +1195,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i32 = { - if 7usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 7 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[7usize]; + let arrow_data = &*arrow_data_arrays[7]; { let arrow_data = arrow_data .as_any() @@ -1231,14 +1231,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1263,10 +1263,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let i64 = { - if 8usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 8 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[8usize]; + let arrow_data = &*arrow_data_arrays[8]; { let arrow_data = arrow_data .as_any() @@ -1299,14 +1299,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1331,10 +1331,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f16 = { - if 9usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 9 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[9usize]; + let arrow_data = &*arrow_data_arrays[9]; { let arrow_data = arrow_data .as_any() @@ -1367,14 +1367,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1399,10 +1399,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f32 = { - if 10usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 10 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[10usize]; + let arrow_data = &*arrow_data_arrays[10]; { let arrow_data = arrow_data .as_any() @@ -1435,14 +1435,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -1467,10 +1467,10 @@ impl ::re_types_core::Loggable for TensorBuffer { .collect::>() }; let f64 = { - if 11usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 11 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[11usize]; + let arrow_data = &*arrow_data_arrays[11]; { let arrow_data = arrow_data .as_any() @@ -1503,14 +1503,14 @@ impl ::re_types_core::Loggable for TensorBuffer { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/tensor_data.rs b/crates/store/re_types/src/datatypes/tensor_data.rs index 6e10fad1c17d..0d19cb6190a6 100644 --- a/crates/store/re_types/src/datatypes/tensor_data.rs +++ b/crates/store/re_types/src/datatypes/tensor_data.rs @@ -227,14 +227,14 @@ impl ::re_types_core::Loggable for TensorData { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/datatypes/tensor_dimension.rs b/crates/store/re_types/src/datatypes/tensor_dimension.rs index 4fce89954ca5..619c954eea2f 100644 --- a/crates/store/re_types/src/datatypes/tensor_dimension.rs +++ b/crates/store/re_types/src/datatypes/tensor_dimension.rs @@ -199,14 +199,15 @@ impl ::re_types_core::Loggable for TensorDimension { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/datatypes/utf8pair.rs b/crates/store/re_types/src/datatypes/utf8pair.rs index 99739ab10aa1..2eec236d7687 100644 --- a/crates/store/re_types/src/datatypes/utf8pair.rs +++ b/crates/store/re_types/src/datatypes/utf8pair.rs @@ -186,14 +186,15 @@ impl ::re_types_core::Loggable for Utf8Pair { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), @@ -241,14 +242,15 @@ impl ::re_types_core::Loggable for Utf8Pair { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs index c7a341da3438..739d7de755cc 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs @@ -104,14 +104,15 @@ impl ::re_types_core::Loggable for AffixFuzzer10 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer11.rs b/crates/store/re_types/src/testing/components/affix_fuzzer11.rs index 4a2e04a06eca..142b2b88218f 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer11.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer11.rs @@ -132,14 +132,14 @@ impl ::re_types_core::Loggable for AffixFuzzer11 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs index fe8e4847fe44..e3f13d3a9d4b 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs @@ -136,14 +136,15 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner_buf.len(), @@ -173,14 +174,14 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs index a38e831ac11e..9118de31e42a 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs @@ -136,14 +136,15 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner_buf.len(), @@ -173,14 +174,14 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer16.rs b/crates/store/re_types/src/testing/components/affix_fuzzer16.rs index 70fd6558ac5d..605c7831b803 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer16.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer16.rs @@ -126,14 +126,14 @@ impl ::re_types_core::Loggable for AffixFuzzer16 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer17.rs b/crates/store/re_types/src/testing/components/affix_fuzzer17.rs index f2cd5031ae0c..5c3584825e37 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer17.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer17.rs @@ -126,14 +126,14 @@ impl ::re_types_core::Loggable for AffixFuzzer17 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer18.rs b/crates/store/re_types/src/testing/components/affix_fuzzer18.rs index 2a370088b673..0ee06f26d382 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer18.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer18.rs @@ -126,14 +126,14 @@ impl ::re_types_core::Loggable for AffixFuzzer18 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer7.rs b/crates/store/re_types/src/testing/components/affix_fuzzer7.rs index 8467fb435de0..74fa0a201a01 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer7.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer7.rs @@ -124,14 +124,14 @@ impl ::re_types_core::Loggable for AffixFuzzer7 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs index 4e9e6c414ca8..a5d0d98442cd 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs @@ -104,14 +104,15 @@ impl ::re_types_core::Loggable for AffixFuzzer9 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs index a6bdbe5c3b41..4cec3891a667 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs @@ -543,14 +543,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), @@ -600,14 +601,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), @@ -678,14 +680,14 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -753,15 +755,16 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), @@ -794,15 +797,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), @@ -874,15 +877,16 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), @@ -915,15 +919,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { elem - .map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + .map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err( DeserializationError::offset_slice_oob( (start, end), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs index 0807008bf746..36adf314259f 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs @@ -214,14 +214,15 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs index e29597829b98..aa6cac91defd 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs @@ -242,14 +242,14 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs index 53bff2f28a95..b43ce063aba7 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs @@ -288,10 +288,10 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .with_context("rerun.testing.datatypes.AffixFuzzer3"); } let degrees = { - if 1usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 1 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[1usize]; + let arrow_data = &*arrow_data_arrays[1]; arrow_data .as_any() .downcast_ref::() @@ -306,10 +306,10 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .collect::>() }; let craziness = { - if 2usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 2 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[2usize]; + let arrow_data = &*arrow_data_arrays[2]; { let arrow_data = arrow_data .as_any() @@ -338,14 +338,14 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), @@ -371,10 +371,10 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { .collect::>() }; let fixed_size_shenanigans = { - if 3usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 3 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[3usize]; + let arrow_data = &*arrow_data_arrays[3]; { let arrow_data = arrow_data .as_any() diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs index 8224c5bb8998..bc763821d865 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs @@ -229,20 +229,20 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { .with_context("rerun.testing.datatypes.AffixFuzzer4"); } let single_required = { - if 1usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 1 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[1usize]; + let arrow_data = &*arrow_data_arrays[1]; crate::testing::datatypes::AffixFuzzer3::from_arrow2_opt(arrow_data) .with_context("rerun.testing.datatypes.AffixFuzzer4#single_required")? .into_iter() .collect::>() }; let many_required = { - if 2usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 2 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[2usize]; + let arrow_data = &*arrow_data_arrays[2]; { let arrow_data = arrow_data .as_any() @@ -271,14 +271,14 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types/src/testing/datatypes/string_component.rs b/crates/store/re_types/src/testing/datatypes/string_component.rs index 1b2e10502f04..735ff75abb84 100644 --- a/crates/store/re_types/src/testing/datatypes/string_component.rs +++ b/crates/store/re_types/src/testing/datatypes/string_component.rs @@ -98,14 +98,15 @@ impl ::re_types_core::Loggable for StringComponent { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types/tests/types/validity.rs b/crates/store/re_types/tests/types/validity.rs index dab289a75f92..6b3f19cbe63a 100644 --- a/crates/store/re_types/tests/types/validity.rs +++ b/crates/store/re_types/tests/types/validity.rs @@ -31,8 +31,9 @@ fn validity_checks() { let serialized = Position2D::to_arrow2_opt(bad).unwrap(); let deserialized = Position2D::from_arrow2(serialized.as_ref()); assert!(deserialized.is_err()); - assert!(matches!( - deserialized.err().unwrap(), - DeserializationError::MissingData { .. } - )); + let actual_error = deserialized.err().unwrap(); + assert!( + matches!(actual_error, DeserializationError::MissingData { .. }), + "Expected error MissingData, got {actual_error:?}", + ); } diff --git a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs index 7dd2246d1c5c..1e1d612b53c2 100644 --- a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs +++ b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs @@ -129,14 +129,15 @@ impl ::re_types_core::Loggable for Utf8List { let arrow_data_inner_buf = arrow_data_inner.values(); let offsets = arrow_data_inner.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data_inner.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_inner_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner_buf.len(), @@ -164,14 +165,14 @@ impl ::re_types_core::Loggable for Utf8List { }; let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_inner.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + if arrow_data_inner.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_inner.len(), diff --git a/crates/store/re_types_core/src/archetype.rs b/crates/store/re_types_core/src/archetype.rs index 9032fb1b53a5..601684ef68fb 100644 --- a/crates/store/re_types_core/src/archetype.rs +++ b/crates/store/re_types_core/src/archetype.rs @@ -95,6 +95,24 @@ pub trait Archetype { // --- + /// Given an iterator of Arrow arrays and their respective field metadata, deserializes them + /// into this archetype. + /// + /// Arrow arrays that are unknown to this [`Archetype`] will simply be ignored and a warning + /// logged to stderr. + #[inline] + fn from_arrow( + data: impl IntoIterator, + ) -> DeserializationResult + where + Self: Sized, + { + Self::from_arrow_components( + data.into_iter() + .map(|(field, array)| (field.name.into(), array)), + ) + } + /// Given an iterator of Arrow arrays and their respective field metadata, deserializes them /// into this archetype. /// @@ -119,8 +137,8 @@ pub trait Archetype { /// Arrow arrays that are unknown to this [`Archetype`] will simply be ignored and a warning /// logged to stderr. #[inline] - fn from_arrow2_components( - data: impl IntoIterator)>, + fn from_arrow_components( + data: impl IntoIterator, ) -> DeserializationResult where Self: Sized, @@ -131,6 +149,23 @@ pub trait Archetype { backtrace: _Backtrace::new_unresolved(), }) } + + /// Given an iterator of Arrow arrays and their respective `ComponentNames`, deserializes them + /// into this archetype. + /// + /// Arrow arrays that are unknown to this [`Archetype`] will simply be ignored and a warning + /// logged to stderr. + #[inline] + fn from_arrow2_components( + data: impl IntoIterator)>, + ) -> DeserializationResult + where + Self: Sized, + { + Self::from_arrow_components(data.into_iter().map(|(component, arrow2_array)| { + (component, arrow::array::ArrayRef::from(arrow2_array)) + })) + } } /// Indicates that the archetype has reflection data available for it. diff --git a/crates/store/re_types_core/src/arrow_string.rs b/crates/store/re_types_core/src/arrow_string.rs index 9fd8b6d4e633..8376167d8deb 100644 --- a/crates/store/re_types_core/src/arrow_string.rs +++ b/crates/store/re_types_core/src/arrow_string.rs @@ -55,12 +55,24 @@ impl ArrowString { std::str::from_utf8(self.0.as_ref()).unwrap_or("INVALID UTF-8") } + #[inline] + pub fn into_arrow_buffer(self) -> arrow::buffer::Buffer { + self.0.into() + } + #[inline] pub fn into_arrow2_buffer(self) -> arrow2::buffer::Buffer { self.0 } } +impl From for ArrowString { + #[inline] + fn from(buf: arrow::buffer::Buffer) -> Self { + Self(buf.into()) + } +} + impl From> for ArrowString { #[inline] fn from(buf: arrow2::buffer::Buffer) -> Self { diff --git a/crates/store/re_types_core/src/datatypes/entity_path.rs b/crates/store/re_types_core/src/datatypes/entity_path.rs index f3b2fca0e5c8..52690454d531 100644 --- a/crates/store/re_types_core/src/datatypes/entity_path.rs +++ b/crates/store/re_types_core/src/datatypes/entity_path.rs @@ -99,14 +99,15 @@ impl crate::Loggable for EntityPath { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs index d50c56c9086c..92b12ba1ca91 100644 --- a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs +++ b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs @@ -231,10 +231,10 @@ impl crate::Loggable for TimeRangeBoundary { .with_context("rerun.datatypes.TimeRangeBoundary"); } let cursor_relative = { - if 1usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 1 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[1usize]; + let arrow_data = &*arrow_data_arrays[1]; arrow_data .as_any() .downcast_ref::() @@ -250,10 +250,10 @@ impl crate::Loggable for TimeRangeBoundary { .collect::>() }; let absolute = { - if 2usize >= arrow_data_arrays.len() { + if arrow_data_arrays.len() <= 2 { return Ok(Vec::new()); } - let arrow_data = &*arrow_data_arrays[2usize]; + let arrow_data = &*arrow_data_arrays[2]; arrow_data .as_any() .downcast_ref::() diff --git a/crates/store/re_types_core/src/datatypes/utf8.rs b/crates/store/re_types_core/src/datatypes/utf8.rs index 052f94e220d5..79e3fe46b16e 100644 --- a/crates/store/re_types_core/src/datatypes/utf8.rs +++ b/crates/store/re_types_core/src/datatypes/utf8.rs @@ -99,14 +99,15 @@ impl crate::Loggable for Utf8 { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types_core/src/datatypes/visible_time_range.rs b/crates/store/re_types_core/src/datatypes/visible_time_range.rs index fd0504caec70..91e470e85a07 100644 --- a/crates/store/re_types_core/src/datatypes/visible_time_range.rs +++ b/crates/store/re_types_core/src/datatypes/visible_time_range.rs @@ -190,14 +190,15 @@ impl crate::Loggable for VisibleTimeRange { let arrow_data_buf = arrow_data.values(); let offsets = arrow_data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets.iter().zip(offsets.lengths()), + offsets.windows(2), arrow_data.validity(), ) .map(|elem| { - elem.map(|(start, len)| { - let start = *start as usize; - let end = start + len; - if end > arrow_data_buf.len() { + elem.map(|window| { + let start = window[0] as usize; + let end = window[1] as usize; + let len = end - start; + if arrow_data_buf.len() < end { return Err(DeserializationError::offset_slice_oob( (start, end), arrow_data_buf.len(), diff --git a/crates/store/re_types_core/src/lib.rs b/crates/store/re_types_core/src/lib.rs index c1fd9cf2a6df..f6deee691390 100644 --- a/crates/store/re_types_core/src/lib.rs +++ b/crates/store/re_types_core/src/lib.rs @@ -194,6 +194,7 @@ pub mod macros { pub mod external { pub use anyhow; + pub use arrow; pub use arrow2; pub use re_tuid; } diff --git a/crates/store/re_types_core/src/loggable.rs b/crates/store/re_types_core/src/loggable.rs index 1ebc3ecce1f8..0da30c5acc11 100644 --- a/crates/store/re_types_core/src/loggable.rs +++ b/crates/store/re_types_core/src/loggable.rs @@ -138,11 +138,8 @@ pub trait Loggable: 'static + Send + Sync + Clone + Sized + SizeBytes { fn from_arrow2_opt( data: &dyn arrow2::array::Array, ) -> DeserializationResult>> { - _ = data; // NOTE: do this here to avoid breaking users' autocomplete snippets - Err(crate::DeserializationError::NotImplemented { - fqname: "".to_owned(), - backtrace: _Backtrace::new_unresolved(), - }) + let boxed_arrow_array = arrow::array::ArrayRef::from(data); + Self::from_arrow_opt(boxed_arrow_array.as_ref()) } } diff --git a/tests/python/release_checklist/main.py b/tests/python/release_checklist/main.py index 96d4553c3f3d..a989c705b38d 100755 --- a/tests/python/release_checklist/main.py +++ b/tests/python/release_checklist/main.py @@ -32,10 +32,10 @@ def main() -> None: args = parser.parse_args() # Download test assets: - download_test_assest_path = ( + download_test_assets_path = ( Path(__file__).parent.parent.parent.joinpath("assets/download_test_assets.py").absolute() ) - subprocess.run([sys.executable, download_test_assest_path]) + subprocess.run([sys.executable, download_test_assets_path]) log_checks(args)