diff --git a/crates/re_arrow_store/src/lib.rs b/crates/re_arrow_store/src/lib.rs index 978a8b0b4a8b..4593868e9ec6 100644 --- a/crates/re_arrow_store/src/lib.rs +++ b/crates/re_arrow_store/src/lib.rs @@ -17,6 +17,7 @@ mod arrow_util; mod store; mod store_arrow; +mod store_dump; mod store_format; mod store_gc; mod store_read; diff --git a/crates/re_arrow_store/src/store.rs b/crates/re_arrow_store/src/store.rs index 0b45171cf54f..a2dba209616f 100644 --- a/crates/re_arrow_store/src/store.rs +++ b/crates/re_arrow_store/src/store.rs @@ -14,7 +14,7 @@ use re_log_types::{ // --- Data store --- -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct DataStoreConfig { /// The maximum number of rows in an indexed bucket before triggering a split. /// Does not apply to timeless data. @@ -201,6 +201,11 @@ impl DataStore { self.cluster_key } + /// See [`DataStoreConfig`] for more information about configuration. + pub fn config(&self) -> &DataStoreConfig { + &self.config + } + /// Lookup the arrow [`DataType`] of a [`re_log_types::Component`] in the internal /// `DataTypeRegistry`. pub fn lookup_datatype(&self, component: &ComponentName) -> Option<&DataType> { diff --git a/crates/re_arrow_store/src/store_dump.rs b/crates/re_arrow_store/src/store_dump.rs new file mode 100644 index 000000000000..d24f3a317454 --- /dev/null +++ b/crates/re_arrow_store/src/store_dump.rs @@ -0,0 +1,195 @@ +use ahash::HashMapExt; +use arrow2::Either; +use nohash_hasher::IntMap; +use re_log_types::{ + DataCellColumn, DataTable, ErasedTimeVec, RowIdVec, TableId, TimeRange, Timeline, +}; + +use crate::{ + store::{IndexedBucketInner, PersistentIndexedTable}, + DataStore, IndexedBucket, +}; + +// --- + +impl DataStore { + /// Serializes the entire datastore into an iterator of [`DataTable`]s. + // TODO(#1793): This shouldn't dump cluster keys that were autogenerated. + // TODO(#1794): Implement simple recompaction. + pub fn to_data_tables( + &self, + time_filter: Option<(Timeline, TimeRange)>, + ) -> impl Iterator + '_ { + let timeless = self.dump_timeless_tables(); + let temporal = if let Some(time_filter) = time_filter { + Either::Left(self.dump_temporal_tables_filtered(time_filter)) + } else { + Either::Right(self.dump_temporal_tables()) + }; + + timeless.chain(temporal) + } + + fn dump_timeless_tables(&self) -> impl Iterator + '_ { + self.timeless_tables.values().map(|table| { + crate::profile_scope!("timeless_table"); + + let PersistentIndexedTable { + ent_path, + cluster_key: _, + col_insert_id: _, + col_row_id, + col_num_instances, + columns, + } = table; + + DataTable { + table_id: TableId::random(), + col_row_id: col_row_id.clone(), + col_timelines: Default::default(), + col_entity_path: std::iter::repeat_with(|| ent_path.clone()) + .take(table.total_rows() as _) + .collect(), + col_num_instances: col_num_instances.clone(), + columns: columns.clone(), // shallow + } + }) + } + + fn dump_temporal_tables(&self) -> impl Iterator + '_ { + self.tables.values().flat_map(|table| { + crate::profile_scope!("temporal_table"); + + table.buckets.values().map(move |bucket| { + crate::profile_scope!("temporal_bucket"); + + bucket.sort_indices_if_needed(); + + let IndexedBucket { + timeline, + cluster_key: _, + inner, + } = bucket; + + let IndexedBucketInner { + is_sorted, + time_range: _, + col_time, + col_insert_id: _, + col_row_id, + col_num_instances, + columns, + size_bytes: _, + } = &*inner.read(); + debug_assert!(is_sorted); + + DataTable { + table_id: TableId::random(), + col_row_id: col_row_id.clone(), + col_timelines: [(*timeline, col_time.iter().copied().map(Some).collect())] + .into(), + col_entity_path: std::iter::repeat_with(|| table.ent_path.clone()) + .take(table.total_rows() as _) + .collect(), + col_num_instances: col_num_instances.clone(), + columns: columns.clone(), // shallow + } + }) + }) + } + + fn dump_temporal_tables_filtered( + &self, + (timeline_filter, time_filter): (Timeline, TimeRange), + ) -> impl Iterator + '_ { + self.tables + .values() + .filter_map(move |table| { + crate::profile_scope!("temporal_table_filtered"); + + if table.timeline != timeline_filter { + return None; + } + + Some(table.buckets.values().filter_map(move |bucket| { + crate::profile_scope!("temporal_bucket_filtered"); + + bucket.sort_indices_if_needed(); + + let IndexedBucket { + timeline, + cluster_key: _, + inner, + } = bucket; + + let IndexedBucketInner { + is_sorted, + time_range, + col_time, + col_insert_id: _, + col_row_id, + col_num_instances, + columns, + size_bytes: _, + } = &*inner.read(); + debug_assert!(is_sorted); + + if !time_range.intersects(time_filter) { + return None; + } + + let col_row_id: RowIdVec = + filter_column(col_time, col_row_id.iter(), time_filter).collect(); + + // NOTE: Shouldn't ever happen due to check above, but better safe than + // sorry... + debug_assert!(!col_row_id.is_empty()); + if col_row_id.is_empty() { + return None; + } + + let col_timelines = [( + *timeline, + filter_column(col_time, col_time.iter(), time_filter) + .map(Some) + .collect(), + )] + .into(); + + let col_entity_path = std::iter::repeat_with(|| table.ent_path.clone()) + .take(col_row_id.len()) + .collect(); + + let col_num_instances = + filter_column(col_time, col_num_instances.iter(), time_filter).collect(); + + let mut columns2 = IntMap::with_capacity(columns.len()); + for (component, column) in columns { + let column = filter_column(col_time, column.iter(), time_filter).collect(); + columns2.insert(*component, DataCellColumn(column)); + } + + Some(DataTable { + table_id: TableId::random(), + col_row_id, + col_timelines, + col_entity_path, + col_num_instances, + columns: columns2, + }) + })) + }) + .flatten() + } +} + +fn filter_column<'a, T: 'a + Clone>( + col_time: &'a ErasedTimeVec, + column: impl Iterator + 'a, + time_filter: TimeRange, +) -> impl Iterator + 'a { + col_time + .iter() + .zip(column) + .filter_map(move |(time, v)| time_filter.contains((*time).into()).then(|| v.clone())) +} diff --git a/crates/re_arrow_store/src/store_stats.rs b/crates/re_arrow_store/src/store_stats.rs index 58d738091269..11c91bfc6223 100644 --- a/crates/re_arrow_store/src/store_stats.rs +++ b/crates/re_arrow_store/src/store_stats.rs @@ -2,13 +2,12 @@ use nohash_hasher::IntMap; use re_log_types::{ComponentName, DataCellColumn}; use crate::{ - store::IndexedBucketInner, DataStore, DataStoreConfig, IndexedBucket, IndexedTable, - PersistentIndexedTable, + store::IndexedBucketInner, DataStore, IndexedBucket, IndexedTable, PersistentIndexedTable, }; // --- -#[derive(Default, Debug, Clone)] +#[derive(Default, Debug, Clone, PartialEq, Eq, PartialOrd)] pub struct DataStoreStats { pub total_timeless_rows: u64, pub total_timeless_size_bytes: u64, @@ -19,8 +18,6 @@ pub struct DataStoreStats { pub total_rows: u64, pub total_size_bytes: u64, - - pub config: DataStoreConfig, } impl DataStoreStats { @@ -47,8 +44,6 @@ impl DataStoreStats { total_rows, total_size_bytes, - - config: store.config.clone(), } } } diff --git a/crates/re_arrow_store/src/store_write.rs b/crates/re_arrow_store/src/store_write.rs index 7994c83a992c..2f68bedf445c 100644 --- a/crates/re_arrow_store/src/store_write.rs +++ b/crates/re_arrow_store/src/store_write.rs @@ -214,7 +214,8 @@ impl DataStore { let values = arrow2::array::UInt64Array::from_vec((0..num_instances as u64).collect_vec()) .boxed(); - let cell = DataCell::from_arrow(InstanceKey::name(), values); + let mut cell = DataCell::from_arrow(InstanceKey::name(), values); + cell.compute_size_bytes(); self.cluster_cell_cache .insert(num_instances, cell.clone() /* shallow */); diff --git a/crates/re_arrow_store/tests/data_store.rs b/crates/re_arrow_store/tests/data_store.rs index 391f4624df12..29af874ae0e3 100644 --- a/crates/re_arrow_store/tests/data_store.rs +++ b/crates/re_arrow_store/tests/data_store.rs @@ -20,7 +20,8 @@ use re_log_types::{ build_frame_nr, build_some_colors, build_some_instances, build_some_instances_from, build_some_point2d, build_some_rects, }, - Component as _, ComponentName, DataCell, DataRow, EntityPath, TimeType, Timeline, + Component as _, ComponentName, DataCell, DataRow, DataTable, EntityPath, TableId, TimeType, + Timeline, }; // TODO(#1619): introduce batching in the testing matrix @@ -41,6 +42,13 @@ fn all_components() { let assert_latest_components_at = |store: &mut DataStore, ent_path: &EntityPath, expected: Option<&[ComponentName]>| { + // Stress test save-to-disk & load-from-disk + let mut store2 = DataStore::new(store.cluster_key(), store.config().clone()); + for table in store.to_data_tables(None) { + store2.insert_table(&table).unwrap(); + } + + let mut store = store2; let timeline = Timeline::new("frame_nr", TimeType::Sequence); let components = store.all_components(&timeline, ent_path); @@ -251,6 +259,7 @@ fn latest_at() { for config in re_arrow_store::test_util::all_configs() { let mut store = DataStore::new(InstanceKey::name(), config.clone()); latest_at_impl(&mut store); + // TODO(#1619): bring back garbage collection // store.gc( // GarbageCollectionTarget::DropAtLeastPercentage(1.0), @@ -272,32 +281,43 @@ fn latest_at_impl(store: &mut DataStore) { let frame3: TimeInt = 3.into(); let frame4: TimeInt = 4.into(); - // helper to insert a row both as a temporal and timeless payload - let insert = |store: &mut DataStore, row| { + // helper to insert a table both as a temporal and timeless payload + let insert_table = |store: &mut DataStore, table: &DataTable| { // insert temporal - store.insert_row(row).unwrap(); + store.insert_table(table).unwrap(); // insert timeless - let mut row_timeless = (*row).clone(); - row_timeless.timepoint = Default::default(); - store.insert_row(&row_timeless).unwrap(); + let mut table_timeless = table.clone(); + table_timeless.col_timelines = Default::default(); + store.insert_table(&table_timeless).unwrap(); }; let (instances1, colors1) = (build_some_instances(3), build_some_colors(3)); let row1 = test_row!(ent_path @ [build_frame_nr(frame1)] => 3; [instances1.clone(), colors1]); - insert(store, &row1); let points2 = build_some_point2d(3); let row2 = test_row!(ent_path @ [build_frame_nr(frame2)] => 3; [instances1, points2]); - insert(store, &row2); let points3 = build_some_point2d(10); let row3 = test_row!(ent_path @ [build_frame_nr(frame3)] => 10; [points3]); - insert(store, &row3); let colors4 = build_some_colors(5); let row4 = test_row!(ent_path @ [build_frame_nr(frame4)] => 5; [colors4]); - insert(store, &row4); + + insert_table( + store, + &DataTable::from_rows( + TableId::random(), + [row1.clone(), row2.clone(), row3.clone(), row4.clone()], + ), + ); + + // Stress test save-to-disk & load-from-disk + let mut store2 = DataStore::new(store.cluster_key(), store.config().clone()); + for table in store.to_data_tables(None) { + store2.insert_table(&table).unwrap(); + } + let mut store = store2; if let err @ Err(_) = store.sanity_check() { store.sort_indices_if_needed(); @@ -310,7 +330,7 @@ fn latest_at_impl(store: &mut DataStore) { let components_all = &[ColorRGBA::name(), Point2D::name()]; let df = polars_util::latest_components( - store, + &store, &LatestAtQuery::new(timeline_frame_nr, frame_nr), &ent_path, components_all, @@ -433,10 +453,17 @@ fn range_impl(store: &mut DataStore) { // A single timepoint might have several of those! That's one of the behaviors specific to // range queries. #[allow(clippy::type_complexity)] - let mut assert_range_components = + let assert_range_components = |time_range: TimeRange, components: [ComponentName; 2], rows_at_times: &[(Option, &[(ComponentName, &DataRow)])]| { + // Stress test save-to-disk & load-from-disk + let mut store2 = DataStore::new(store.cluster_key(), store.config().clone()); + for table in store.to_data_tables(None) { + store2.insert_table(&table).unwrap(); + } + let mut store = store2; + let mut expected_timeless = Vec::::new(); let mut expected_at_times: IntMap> = Default::default(); @@ -456,7 +483,7 @@ fn range_impl(store: &mut DataStore) { let components = [InstanceKey::name(), components[0], components[1]]; let query = RangeQuery::new(timeline_frame_nr, time_range); let dfs = polars_util::range_components( - store, + &store, &query, &ent_path, components[1], diff --git a/crates/re_arrow_store/tests/dump.rs b/crates/re_arrow_store/tests/dump.rs new file mode 100644 index 000000000000..5293870bf882 --- /dev/null +++ b/crates/re_arrow_store/tests/dump.rs @@ -0,0 +1,236 @@ +//! Dumping a datastore to log messages and back. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use itertools::Itertools; +use re_arrow_store::{test_row, DataStore, DataStoreStats, TimeInt, TimeRange, Timeline}; +use re_log_types::{ + component_types::InstanceKey, + datagen::{ + build_frame_nr, build_log_time, build_some_colors, build_some_instances, build_some_point2d, + }, + Component as _, DataTable, EntityPath, TableId, +}; + +// --- Dump --- + +#[test] +fn data_store_dump() { + init_logs(); + + for mut config in re_arrow_store::test_util::all_configs() { + // NOTE: insert IDs aren't serialized and can be different across runs. + config.store_insert_ids = false; + + let mut store1 = DataStore::new(InstanceKey::name(), config.clone()); + let mut store2 = DataStore::new(InstanceKey::name(), config.clone()); + let mut store3 = DataStore::new(InstanceKey::name(), config.clone()); + + data_store_dump_impl(&mut store1, &mut store2, &mut store3); + } +} + +fn data_store_dump_impl(store1: &mut DataStore, store2: &mut DataStore, store3: &mut DataStore) { + // helper to insert a table both as a temporal and timeless payload + let insert_table = |store: &mut DataStore, table: &DataTable| { + // insert temporal + store.insert_table(table).unwrap(); + + // insert timeless + let mut table_timeless = table.clone(); + table_timeless.col_timelines = Default::default(); + store.insert_table(&table_timeless).unwrap(); + }; + + let ent_paths = ["this/that", "other", "yet/another/one"]; + let tables = ent_paths + .iter() + .map(|ent_path| create_insert_table(*ent_path)) + .collect_vec(); + + // Fill the first store. + for table in &tables { + insert_table(store1, table); + } + if let err @ Err(_) = store1.sanity_check() { + store1.sort_indices_if_needed(); + eprintln!("{store1}"); + err.unwrap(); + } + + // Dump the first store into the second one. + for table in store1.to_data_tables(None) { + store2.insert_table(&table).unwrap(); + } + if let err @ Err(_) = store2.sanity_check() { + store2.sort_indices_if_needed(); + eprintln!("{store2}"); + err.unwrap(); + } + + // Dump the second store into the third one. + for table in store2.to_data_tables(None) { + store3.insert_table(&table).unwrap(); + } + if let err @ Err(_) = store3.sanity_check() { + store3.sort_indices_if_needed(); + eprintln!("{store3}"); + err.unwrap(); + } + + let store1_df = store1.to_dataframe(); + let store2_df = store2.to_dataframe(); + let store3_df = store3.to_dataframe(); + assert!( + store1_df == store2_df, + "First & second stores differ:\n{store1_df}\n{store2_df}" + ); + assert!( + store1_df == store3_df, + "First & third stores differ:\n{store1_df}\n{store3_df}" + ); + + let store1_stats = DataStoreStats::from_store(store1); + let store2_stats = DataStoreStats::from_store(store2); + let store3_stats = DataStoreStats::from_store(store3); + assert!( + store1_stats <= store2_stats, + "First store should have <= amount of data of second store:\n\ + {store1_stats:#?}\n{store2_stats:#?}" + ); + assert!( + store2_stats <= store3_stats, + "Second store should have <= amount of data of third store:\n\ + {store2_stats:#?}\n{store3_stats:#?}" + ); +} + +// --- Time-based filtering --- + +#[test] +fn data_store_dump_filtered() { + init_logs(); + + for mut config in re_arrow_store::test_util::all_configs() { + // NOTE: insert IDs aren't serialized and can be different across runs. + config.store_insert_ids = false; + + let mut store1 = DataStore::new(InstanceKey::name(), config.clone()); + let mut store2 = DataStore::new(InstanceKey::name(), config.clone()); + + data_store_dump_filtered_impl(&mut store1, &mut store2); + } +} + +fn data_store_dump_filtered_impl(store1: &mut DataStore, store2: &mut DataStore) { + let timeline_frame_nr = Timeline::new_sequence("frame_nr"); + let timeline_log_time = Timeline::new_temporal("log_time"); + let frame1: TimeInt = 1.into(); + let frame2: TimeInt = 2.into(); + let frame3: TimeInt = 3.into(); + let frame4: TimeInt = 4.into(); + + let ent_paths = ["this/that", "other", "yet/another/one"]; + let tables = ent_paths + .iter() + .map(|ent_path| create_insert_table(*ent_path)) + .collect_vec(); + + // Fill the first store. + for table in &tables { + store1.insert_table(table).unwrap(); + } + if let err @ Err(_) = store1.sanity_check() { + store1.sort_indices_if_needed(); + eprintln!("{store1}"); + err.unwrap(); + } + + // Dump frame1 from the first store into the second one. + for table in store1.to_data_tables((timeline_frame_nr, TimeRange::new(frame1, frame1)).into()) { + store2.insert_table(&table).unwrap(); + } + // Dump frame2 from the first store into the second one. + for table in store1.to_data_tables((timeline_frame_nr, TimeRange::new(frame2, frame2)).into()) { + store2.insert_table(&table).unwrap(); + } + // Dump frame3 from the first store into the second one. + for table in store1.to_data_tables((timeline_frame_nr, TimeRange::new(frame3, frame3)).into()) { + store2.insert_table(&table).unwrap(); + } + // Dump the other frame3 from the first store into the second one. + for table in store1.to_data_tables((timeline_log_time, TimeRange::new(frame3, frame3)).into()) { + store2.insert_table(&table).unwrap(); + } + // Dump frame4 from the first store into the second one. + for table in store1.to_data_tables((timeline_frame_nr, TimeRange::new(frame4, frame4)).into()) { + store2.insert_table(&table).unwrap(); + } + if let err @ Err(_) = store2.sanity_check() { + store2.sort_indices_if_needed(); + eprintln!("{store2}"); + err.unwrap(); + } + + let store1_df = store1.to_dataframe(); + let store2_df = store2.to_dataframe(); + assert!( + store1_df == store2_df, + "First & second stores differ:\n{store1_df}\n{store2_df}" + ); + + let store1_stats = DataStoreStats::from_store(store1); + let store2_stats = DataStoreStats::from_store(store2); + assert!( + store1_stats <= store2_stats, + "First store should have <= amount of data of second store:\n\ + {store1_stats:#?}\n{store2_stats:#?}" + ); +} + +// --- + +pub fn init_logs() { + static INIT: AtomicBool = AtomicBool::new(false); + + if INIT + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + re_log::setup_native_logging(); + } +} + +fn create_insert_table(ent_path: impl Into) -> DataTable { + let ent_path = ent_path.into(); + + let frame1: TimeInt = 1.into(); + let frame2: TimeInt = 2.into(); + let frame3: TimeInt = 3.into(); + let frame4: TimeInt = 4.into(); + + let (instances1, colors1) = (build_some_instances(3), build_some_colors(3)); + let row1 = test_row!(ent_path @ [ + build_frame_nr(frame1), + ] => 3; [instances1.clone(), colors1]); + + let points2 = build_some_point2d(3); + let row2 = test_row!(ent_path @ [ + build_frame_nr(frame2), + ] => 3; [instances1, points2]); + + let points3 = build_some_point2d(10); + let row3 = test_row!(ent_path @ [ + build_log_time(frame3.into()) /* ! */, build_frame_nr(frame3), + ] => 10; [points3]); + + let colors4 = build_some_colors(5); + let row4 = test_row!(ent_path @ [ + build_frame_nr(frame4), + ] => 5; [colors4]); + + let mut table = DataTable::from_rows(TableId::random(), [row1, row2, row3, row4]); + table.compute_all_size_bytes(); + + table +} diff --git a/crates/re_log_types/src/data_row.rs b/crates/re_log_types/src/data_row.rs index 62dac0388ec2..460a645bb20a 100644 --- a/crates/re_log_types/src/data_row.rs +++ b/crates/re_log_types/src/data_row.rs @@ -115,7 +115,6 @@ impl RowId { pub const ZERO: Self = Self(re_tuid::Tuid::ZERO); #[inline] - #[cfg(not(target_arch = "wasm32"))] pub fn random() -> Self { Self(re_tuid::Tuid::random()) } diff --git a/crates/re_log_types/src/data_table.rs b/crates/re_log_types/src/data_table.rs index 22267de70c3d..3af2aba910fb 100644 --- a/crates/re_log_types/src/data_table.rs +++ b/crates/re_log_types/src/data_table.rs @@ -150,7 +150,6 @@ impl TableId { pub const ZERO: Self = Self(re_tuid::Tuid::ZERO); #[inline] - #[cfg(not(target_arch = "wasm32"))] pub fn random() -> Self { Self(re_tuid::Tuid::random()) } @@ -405,6 +404,13 @@ impl DataTable { } } } + + // handle potential sparseness + for (timeline, col_time) in &mut col_timelines { + if timepoint.get(timeline).is_none() { + col_time.push(None); + } + } } // Pre-allocate all columns (one per component). diff --git a/crates/re_log_types/src/time_range.rs b/crates/re_log_types/src/time_range.rs index 645e8aff889a..68b7e4a7f159 100644 --- a/crates/re_log_types/src/time_range.rs +++ b/crates/re_log_types/src/time_range.rs @@ -38,6 +38,7 @@ impl TimeRange { self.min.as_i64().abs_diff(self.max.as_i64()) } + #[inline] pub fn center(&self) -> TimeInt { self.min + TimeInt::from((self.abs_length() / 2) as i64) } @@ -47,6 +48,11 @@ impl TimeRange { self.min <= time && time <= self.max } + #[inline] + pub fn intersects(&self, other: Self) -> bool { + self.min <= other.max && self.max >= other.min + } + #[inline] pub fn union(&self, other: Self) -> Self { Self { diff --git a/crates/re_viewer/src/app.rs b/crates/re_viewer/src/app.rs index 40bec9455028..32062b79534d 100644 --- a/crates/re_viewer/src/app.rs +++ b/crates/re_viewer/src/app.rs @@ -7,7 +7,7 @@ use itertools::Itertools as _; use nohash_hasher::IntMap; use poll_promise::Promise; -use re_arrow_store::DataStoreStats; +use re_arrow_store::{DataStoreConfig, DataStoreStats}; use re_data_store::log_db::LogDb; use re_format::format_number; use re_log_types::{ApplicationId, LogMsg, RecordingId}; @@ -389,6 +389,7 @@ impl App { &mut self, ui: &mut egui::Ui, gpu_resource_stats: &WgpuResourcePoolStatistics, + store_config: &DataStoreConfig, store_stats: &DataStoreStats, ) { let frame = egui::Frame { @@ -405,6 +406,7 @@ impl App { ui, &self.startup_options.memory_limit, gpu_resource_stats, + store_config, store_stats, ); }); @@ -471,9 +473,11 @@ impl eframe::App for App { render_ctx.gpu_resources.statistics() }; + let store_config = self.log_db().entity_db.data_store.config().clone(); let store_stats = DataStoreStats::from_store(&self.log_db().entity_db.data_store); - self.memory_panel.update(&gpu_resource_stats, &store_stats); // do first, before doing too many allocations + // do first, before doing too many allocations + self.memory_panel.update(&gpu_resource_stats, &store_stats); self.check_keyboard_shortcuts(egui_ctx); @@ -502,7 +506,7 @@ impl eframe::App for App { top_panel(ui, frame, self, &gpu_resource_stats); - self.memory_panel_ui(ui, &gpu_resource_stats, &store_stats); + self.memory_panel_ui(ui, &gpu_resource_stats, &store_config, &store_stats); let log_db = self.log_dbs.entry(self.state.selected_rec_id).or_default(); let selected_app_id = log_db diff --git a/crates/re_viewer/src/ui/memory_panel.rs b/crates/re_viewer/src/ui/memory_panel.rs index 1a390ed267a8..c2e9f9450059 100644 --- a/crates/re_viewer/src/ui/memory_panel.rs +++ b/crates/re_viewer/src/ui/memory_panel.rs @@ -1,4 +1,4 @@ -use re_arrow_store::DataStoreStats; +use re_arrow_store::{DataStoreConfig, DataStoreStats}; use re_format::{format_bytes, format_number}; use re_memory::{util::sec_since_start, MemoryHistory, MemoryLimit, MemoryUse}; use re_renderer::WgpuResourcePoolStatistics; @@ -40,6 +40,7 @@ impl MemoryPanel { ui: &mut egui::Ui, limit: &MemoryLimit, gpu_resource_stats: &WgpuResourcePoolStatistics, + store_config: &DataStoreConfig, store_stats: &DataStoreStats, ) { crate::profile_function!(); @@ -52,7 +53,7 @@ impl MemoryPanel { .min_width(250.0) .default_width(300.0) .show_inside(ui, |ui| { - Self::left_side(ui, limit, gpu_resource_stats, store_stats); + Self::left_side(ui, limit, gpu_resource_stats, store_config, store_stats); }); egui::CentralPanel::default().show_inside(ui, |ui| { @@ -65,6 +66,7 @@ impl MemoryPanel { ui: &mut egui::Ui, limit: &MemoryLimit, gpu_resource_stats: &WgpuResourcePoolStatistics, + store_config: &DataStoreConfig, store_stats: &DataStoreStats, ) { ui.strong("Rerun Viewer resource usage"); @@ -81,7 +83,7 @@ impl MemoryPanel { ui.separator(); ui.collapsing("Datastore Resources", |ui| { - Self::store_stats(ui, store_stats); + Self::store_stats(ui, store_config, store_stats); }); } @@ -178,12 +180,14 @@ impl MemoryPanel { }); } - fn store_stats(ui: &mut egui::Ui, store_stats: &DataStoreStats) { + fn store_stats( + ui: &mut egui::Ui, + store_config: &DataStoreConfig, + store_stats: &DataStoreStats, + ) { egui::Grid::new("store config grid") .num_columns(3) .show(ui, |ui| { - let DataStoreStats { config, .. } = store_stats; - ui.label(egui::RichText::new("Limits").italics()); ui.label("Row limit"); ui.end_row(); @@ -201,7 +205,7 @@ impl MemoryPanel { ui.end_row(); ui.label("Temporal:"); - label_rows(ui, config.indexed_bucket_num_rows); + label_rows(ui, store_config.indexed_bucket_num_rows); ui.end_row(); }); @@ -218,7 +222,6 @@ impl MemoryPanel { total_temporal_buckets, total_rows, total_size_bytes, - config: _, } = *store_stats; ui.label(egui::RichText::new("Stats").italics());