diff --git a/crates/re_data_store/src/log_db.rs b/crates/re_data_store/src/log_db.rs index fc721474d7594..37a485ae4c401 100644 --- a/crates/re_data_store/src/log_db.rs +++ b/crates/re_data_store/src/log_db.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use nohash_hasher::IntMap; use re_arrow_store::{DataStoreConfig, TimeInt}; @@ -159,33 +161,31 @@ impl EntityDb { /// A in-memory database built from a stream of [`LogMsg`]es. #[derive(Default)] pub struct LogDb { - /// Messages in the order they arrived - chronological_row_ids: Vec, - log_messages: ahash::HashMap, - - /// Data that was logged with [`TimePoint::timeless`]. - /// We need to re-insert those in any new timelines - /// that are created after they were logged. - timeless_row_ids: Vec, + /// All [`EntityPathOpMsg`]s ever received. + entity_op_msgs: BTreeMap, /// Set by whomever created this [`LogDb`]. pub data_source: Option, /// Comes in a special message, [`LogMsg::BeginRecordingMsg`]. - recording_info: Option, + recording_msg: Option, /// Where we store the entities. pub entity_db: EntityDb, } impl LogDb { + pub fn recording_msg(&self) -> Option<&BeginRecordingMsg> { + self.recording_msg.as_ref() + } + pub fn recording_info(&self) -> Option<&RecordingInfo> { - self.recording_info.as_ref() + self.recording_msg().map(|msg| &msg.info) } pub fn recording_id(&self) -> RecordingId { - if let Some(info) = &self.recording_info { - info.recording_id + if let Some(msg) = &self.recording_msg { + msg.info.recording_id } else { RecordingId::ZERO } @@ -203,11 +203,16 @@ impl LogDb { self.entity_db.tree.num_timeless_messages() } + pub fn len(&self) -> usize { + self.entity_db.data_store.total_timeless_rows() as usize + + self.entity_db.data_store.total_temporal_rows() as usize + } + pub fn is_empty(&self) -> bool { - self.log_messages.is_empty() + self.len() == 0 } - pub fn add(&mut self, msg: LogMsg) -> Result<(), Error> { + pub fn add(&mut self, msg: &LogMsg) -> Result<(), Error> { crate::profile_function!(); match &msg { @@ -218,38 +223,27 @@ impl LogDb { time_point, path_op, } = msg; + self.entity_op_msgs.insert(*row_id, msg.clone()); self.entity_db.add_path_op(*row_id, time_point, path_op); } LogMsg::ArrowMsg(inner) => self.entity_db.try_add_arrow_msg(inner)?, LogMsg::Goodbye(_) => {} } - // TODO(#1619): the following only makes sense because, while we support sending and - // receiving batches, we don't actually do so yet. - // We need to stop storing raw `LogMsg`s before we can benefit from our batching. - self.chronological_row_ids.push(msg.id()); - self.log_messages.insert(msg.id(), msg); - Ok(()) } fn add_begin_recording_msg(&mut self, msg: &BeginRecordingMsg) { - self.recording_info = Some(msg.info.clone()); + self.recording_msg = Some(msg.clone()); } - pub fn len(&self) -> usize { - self.log_messages.len() - } - - /// In the order they arrived - pub fn chronological_log_messages(&self) -> impl Iterator { - self.chronological_row_ids - .iter() - .filter_map(|id| self.get_log_msg(id)) + /// Returns an iterator over all [`EntityPathOpMsg`]s that have been written to this `LogDb`. + pub fn iter_entity_op_msgs(&self) -> impl Iterator { + self.entity_op_msgs.values() } - pub fn get_log_msg(&self, row_id: &RowId) -> Option<&LogMsg> { - self.log_messages.get(row_id) + pub fn get_entity_op_msg(&self, row_id: &RowId) -> Option<&EntityPathOpMsg> { + self.entity_op_msgs.get(row_id) } /// Free up some RAM by forgetting the older parts of all timelines. @@ -263,26 +257,15 @@ impl LogDb { let cutoff_times = self.entity_db.data_store.oldest_time_per_timeline(); let Self { - chronological_row_ids, - log_messages, - timeless_row_ids, + entity_op_msgs, data_source: _, - recording_info: _, + recording_msg: _, entity_db, } = self; { - crate::profile_scope!("chronological_row_ids"); - chronological_row_ids.retain(|row_id| !drop_row_ids.contains(row_id)); - } - - { - crate::profile_scope!("log_messages"); - log_messages.retain(|row_id, _| !drop_row_ids.contains(row_id)); - } - { - crate::profile_scope!("timeless_row_ids"); - timeless_row_ids.retain(|row_id| !drop_row_ids.contains(row_id)); + crate::profile_scope!("entity_op_msgs"); + entity_op_msgs.retain(|row_id, _| !drop_row_ids.contains(row_id)); } entity_db.purge(&cutoff_times, &drop_row_ids); diff --git a/crates/re_log_types/src/encoding.rs b/crates/re_log_types/src/encoding.rs index ed564b1abad1d..5ddb1d54d08da 100644 --- a/crates/re_log_types/src/encoding.rs +++ b/crates/re_log_types/src/encoding.rs @@ -107,6 +107,17 @@ mod encoder { } encoder.finish() } + + pub fn encode_owned( + messages: impl Iterator, + write: impl std::io::Write, + ) -> Result<(), EncodeError> { + let mut encoder = Encoder::new(write)?; + for message in messages { + encoder.append(&message)?; + } + encoder.finish() + } } #[cfg(feature = "save")] diff --git a/crates/re_log_types/src/lib.rs b/crates/re_log_types/src/lib.rs index 70c88d0c597b2..ed7a556307616 100644 --- a/crates/re_log_types/src/lib.rs +++ b/crates/re_log_types/src/lib.rs @@ -187,20 +187,6 @@ pub enum LogMsg { Goodbye(RowId), } -impl LogMsg { - pub fn id(&self) -> RowId { - match self { - Self::BeginRecordingMsg(msg) => msg.row_id, - Self::EntityPathOpMsg(msg) => msg.row_id, - Self::Goodbye(row_id) => *row_id, - // TODO(#1619): the following only makes sense because, while we support sending and - // receiving batches, we don't actually do so yet. - // We need to stop storing raw `LogMsg`s before we can benefit from our batching. - Self::ArrowMsg(msg) => msg.table_id.into_row_id(), - } - } -} - impl_into_enum!(BeginRecordingMsg, LogMsg, BeginRecordingMsg); impl_into_enum!(EntityPathOpMsg, LogMsg, EntityPathOpMsg); impl_into_enum!(ArrowMsg, LogMsg, ArrowMsg); diff --git a/crates/re_viewer/src/app.rs b/crates/re_viewer/src/app.rs index e806d5db0414e..57612b6bd52c8 100644 --- a/crates/re_viewer/src/app.rs +++ b/crates/re_viewer/src/app.rs @@ -695,7 +695,7 @@ impl App { log_db.data_source = Some(self.rx.source().clone()); } - if let Err(err) = log_db.add(msg) { + if let Err(err) = log_db.add(&msg) { re_log::error!("Failed to add incoming msg: {err}"); }; @@ -906,8 +906,6 @@ fn preview_files_being_dropped(egui_ctx: &egui::Context) { enum PanelSelection { #[default] Viewport, - - EventLog, } #[derive(Default, serde::Deserialize, serde::Serialize)] @@ -930,8 +928,6 @@ struct AppState { /// Which view panel is currently being shown panel_selection: PanelSelection, - event_log_view: crate::event_log_view::EventLogView, - selection_panel: crate::selection_panel::SelectionPanel, time_panel: crate::time_panel::TimePanel, @@ -959,7 +955,6 @@ impl AppState { selected_rec_id, recording_configs, panel_selection, - event_log_view, blueprints, selection_panel, time_panel, @@ -1004,7 +999,6 @@ impl AppState { .entry(selected_app_id) .or_insert_with(|| Blueprint::new(ui.ctx())) .blueprint_panel_and_viewport(&mut ctx, ui), - PanelSelection::EventLog => event_log_view.ui(&mut ctx, ui), }); // move time last, so we get to see the first data first! @@ -1536,16 +1530,6 @@ fn main_view_selector_ui(ui: &mut egui::Ui, app: &mut App) { { ui.close_menu(); } - if ui - .selectable_value( - &mut app.state.panel_selection, - PanelSelection::EventLog, - "Event Log", - ) - .clicked() - { - ui.close_menu(); - } }); } } @@ -1751,44 +1735,36 @@ fn save_database_to_file( path: std::path::PathBuf, time_selection: Option<(re_data_store::Timeline, TimeRangeF)>, ) -> impl FnOnce() -> anyhow::Result { - use re_log_types::{EntityPathOpMsg, TimeInt}; - - let msgs = match time_selection { - // Fast path: no query, just dump everything. - None => log_db - .chronological_log_messages() - .cloned() - .collect::>(), - - // Query path: time to filter! - Some((timeline, range)) => { - use std::ops::RangeInclusive; - let range: RangeInclusive = range.min.floor()..=range.max.ceil(); - log_db - .chronological_log_messages() - .filter(|msg| { - match msg { - LogMsg::BeginRecordingMsg(_) | LogMsg::Goodbye(_) => { - true // timeless - } - LogMsg::EntityPathOpMsg(EntityPathOpMsg { time_point, .. }) => { - time_point.is_timeless() || { - let is_within_range = time_point - .get(&timeline) - .map_or(false, |t| range.contains(t)); - is_within_range - } - } - LogMsg::ArrowMsg(_) => { - // TODO(john) - false - } - } - }) - .cloned() - .collect::>() - } - }; + use re_arrow_store::TimeRange; + + crate::profile_scope!("dump_messages"); + + let begin_rec_msg = log_db + .recording_msg() + .map(|msg| LogMsg::BeginRecordingMsg(msg.clone())); + + let ent_op_msgs = log_db + .iter_entity_op_msgs() + .map(|msg| LogMsg::EntityPathOpMsg(msg.clone())) + .collect_vec(); + + let time_filter = time_selection.map(|(timeline, range)| { + ( + timeline, + TimeRange::new(range.min.floor(), range.max.ceil()), + ) + }); + let data_msgs = log_db + .entity_db + .data_store + .to_data_tables(time_filter) + .map(|table| LogMsg::ArrowMsg(table.to_arrow_msg().unwrap())) + .collect_vec(); + + let msgs = std::iter::once(begin_rec_msg) + .flatten() + .chain(ent_op_msgs) + .chain(data_msgs); move || { crate::profile_scope!("save_to_file"); @@ -1797,7 +1773,7 @@ fn save_database_to_file( let file = std::fs::File::create(path.as_path()) .with_context(|| format!("Failed to create file at {path:?}"))?; - re_log_types::encoding::encode(msgs.iter(), file) + re_log_types::encoding::encode_owned(msgs, file) .map(|_| path) .context("Message encode") } @@ -1811,7 +1787,7 @@ fn load_rrd_to_log_db(mut read: impl std::io::Read) -> anyhow::Result { let mut log_db = LogDb::default(); for msg in decoder { - log_db.add(msg?)?; + log_db.add(&msg?)?; } Ok(log_db) } diff --git a/crates/re_viewer/src/lib.rs b/crates/re_viewer/src/lib.rs index a248d82f521e6..a04d9782746fd 100644 --- a/crates/re_viewer/src/lib.rs +++ b/crates/re_viewer/src/lib.rs @@ -14,7 +14,7 @@ mod viewer_analytics; pub(crate) use misc::{mesh_loader, Item, TimeControl, TimeView, ViewerContext}; use re_log_types::PythonVersion; -pub(crate) use ui::{event_log_view, memory_panel, selection_panel, time_panel, UiVerbosity}; +pub(crate) use ui::{memory_panel, selection_panel, time_panel, UiVerbosity}; pub use app::{App, StartupOptions}; pub use remote_viewer_app::RemoteViewerApp; diff --git a/crates/re_viewer/src/misc/item.rs b/crates/re_viewer/src/misc/item.rs index 096dc815599c7..6c6a67ff73ad1 100644 --- a/crates/re_viewer/src/misc/item.rs +++ b/crates/re_viewer/src/misc/item.rs @@ -38,7 +38,7 @@ impl Item { Item::InstancePath(space_view_id, _) => space_view_id .map(|space_view_id| blueprint.viewport.space_view(&space_view_id).is_some()) .unwrap_or(true), - Item::RowId(row_id) => log_db.get_log_msg(row_id).is_some(), + Item::RowId(row_id) => log_db.get_entity_op_msg(row_id).is_some(), Item::SpaceView(space_view_id) => { blueprint.viewport.space_view(space_view_id).is_some() } diff --git a/crates/re_viewer/src/ui/data_ui/mod.rs b/crates/re_viewer/src/ui/data_ui/mod.rs index 09580773c4490..330acdd853a0b 100644 --- a/crates/re_viewer/src/ui/data_ui/mod.rs +++ b/crates/re_viewer/src/ui/data_ui/mod.rs @@ -25,6 +25,7 @@ pub enum UiVerbosity { Small, /// At most this height + #[allow(dead_code)] MaxHeight(f32), /// Display a reduced set, used for hovering. diff --git a/crates/re_viewer/src/ui/data_ui/row_id.rs b/crates/re_viewer/src/ui/data_ui/row_id.rs index 603bd924a0d4f..cbca4a1cd472b 100644 --- a/crates/re_viewer/src/ui/data_ui/row_id.rs +++ b/crates/re_viewer/src/ui/data_ui/row_id.rs @@ -17,7 +17,7 @@ impl DataUi for RowId { ctx.row_id_button(ui, *self); } UiVerbosity::All | UiVerbosity::Reduced => { - if let Some(msg) = ctx.log_db.get_log_msg(self) { + if let Some(msg) = ctx.log_db.get_entity_op_msg(self) { msg.data_ui(ctx, ui, verbosity, query); } else { ctx.row_id_button(ui, *self); diff --git a/crates/re_viewer/src/ui/event_log_view.rs b/crates/re_viewer/src/ui/event_log_view.rs deleted file mode 100644 index 209830b2d4140..0000000000000 --- a/crates/re_viewer/src/ui/event_log_view.rs +++ /dev/null @@ -1,234 +0,0 @@ -use itertools::Itertools as _; - -use re_arrow_store::{LatestAtQuery, TimeInt}; -use re_format::format_number; -use re_log_types::{BeginRecordingMsg, DataTable, EntityPathOpMsg, LogMsg, RecordingInfo}; - -use crate::{UiVerbosity, ViewerContext}; - -use super::data_ui::DataUi; - -/// An event log, a table of all log messages. -#[derive(Default, serde::Deserialize, serde::Serialize)] -#[serde(default)] -pub(crate) struct EventLogView {} - -impl EventLogView { - #[allow(clippy::unused_self)] - pub fn ui(&mut self, ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui) { - crate::profile_function!(); - - let messages = { - crate::profile_scope!("Collecting messages"); - ctx.log_db.chronological_log_messages().collect_vec() - }; - - egui::Frame { - inner_margin: re_ui::ReUi::view_padding().into(), - ..egui::Frame::default() - } - .show(ui, |ui| { - ui.label(format!("{} log lines", format_number(ctx.log_db.len()))); - ui.separator(); - - egui::ScrollArea::horizontal() - .auto_shrink([false; 2]) - .show(ui, |ui| { - message_table(ctx, ui, &messages); - }); - }); - } -} - -pub(crate) fn message_table(ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui, messages: &[&LogMsg]) { - crate::profile_function!(); - - use egui_extras::{Column, TableBuilder}; - - TableBuilder::new(ui) - .max_scroll_height(f32::INFINITY) // Fill up whole height - .cell_layout(egui::Layout::left_to_right(egui::Align::Center)) - .resizable(true) - .column(Column::initial(100.0).at_least(50.0).clip(true)) // row_id - .column(Column::initial(130.0).at_least(50.0).clip(true)) // message type - .columns( - // timeline(s): - Column::auto().clip(true).at_least(50.0), - ctx.log_db.timelines().count(), - ) - .column(Column::auto().clip(true).at_least(50.0)) // path - .column(Column::remainder()) // payload - .header(re_ui::ReUi::table_header_height(), |mut header| { - re_ui::ReUi::setup_table_header(&mut header); - header.col(|ui| { - ui.strong("MsgID"); - }); - header.col(|ui| { - ui.strong("Message Type"); - }); - for timeline in ctx.log_db.timelines() { - header.col(|ui| { - ctx.timeline_button(ui, timeline); - }); - } - header.col(|ui| { - ui.strong("Path"); - }); - header.col(|ui| { - ui.strong("Payload"); - }); - }) - .body(|mut body| { - re_ui::ReUi::setup_table_body(&mut body); - - // for MANY messages, `heterogeneous_rows` is too slow. TODO(emilk): how many? - if messages.len() < 10_000_000 { - body.heterogeneous_rows( - messages.iter().copied().map(row_height), - |index, mut row| { - let msg = messages[index]; - table_row(ctx, &mut row, msg, row_height(msg)); - }, - ); - } else { - let row_height = re_ui::ReUi::table_line_height(); - body.rows(row_height, messages.len(), |index, mut row| { - table_row(ctx, &mut row, messages[index], row_height); - }); - } - }); -} - -fn row_height(_msg: &LogMsg) -> f32 { - // TODO(emilk): make rows with images (tensors) higher! - re_ui::ReUi::table_line_height() -} - -fn table_row( - ctx: &mut ViewerContext<'_>, - row: &mut egui_extras::TableRow<'_, '_>, - msg: &LogMsg, - row_height: f32, -) { - match msg { - LogMsg::BeginRecordingMsg(msg) => { - let BeginRecordingMsg { row_id, info } = msg; - let RecordingInfo { - application_id, - recording_id, - started, - recording_source, - is_official_example, - } = info; - - row.col(|ui| { - ctx.row_id_button(ui, *row_id); - }); - row.col(|ui| { - ui.monospace("BeginRecordingMsg"); - ui.label(format!("Source: {recording_source}")); - ui.label(format!("Official example: {is_official_example}")); - }); - for _ in ctx.log_db.timelines() { - row.col(|ui| { - ui.label("-"); - }); - } - row.col(|ui| { - ui.label(started.format()); - }); - row.col(|ui| { - ui.monospace(format!("{application_id} - {recording_id:?}")); - }); - } - LogMsg::EntityPathOpMsg(msg) => { - let EntityPathOpMsg { - row_id, - time_point, - path_op, - } = msg; - - row.col(|ui| { - ctx.row_id_button(ui, *row_id); - }); - row.col(|ui| { - ui.monospace("EntityPathOpMsg"); - }); - for timeline in ctx.log_db.timelines() { - row.col(|ui| { - if let Some(value) = time_point.get(timeline) { - ctx.time_button(ui, timeline, *value); - } - }); - } - row.col(|ui| { - ctx.entity_path_button(ui, None, path_op.entity_path()); - }); - row.col(|ui| { - let timeline = *ctx.rec_cfg.time_ctrl.timeline(); - let query = LatestAtQuery::new( - timeline, - time_point.get(&timeline).copied().unwrap_or(TimeInt::MAX), - ); - path_op.data_ui(ctx, ui, UiVerbosity::All, &query); - }); - } - // NOTE: This really only makes sense because we don't yet have batches with more than a - // single row at the moment... and by the time we do, the event log view will have - // disappeared entirely. - LogMsg::ArrowMsg(msg) => match DataTable::from_arrow_msg(msg) { - Ok(table) => { - for datarow in table.to_rows() { - row.col(|ui| { - ctx.row_id_button(ui, datarow.row_id()); - }); - row.col(|ui| { - ui.monospace("ArrowMsg"); - }); - for timeline in ctx.log_db.timelines() { - row.col(|ui| { - if let Some(value) = datarow.timepoint().get(timeline) { - ctx.time_button(ui, timeline, *value); - } - }); - } - row.col(|ui| { - ctx.entity_path_button(ui, None, datarow.entity_path()); - }); - - row.col(|ui| { - let timeline = *ctx.rec_cfg.time_ctrl.timeline(); - let query = LatestAtQuery::new( - timeline, - datarow - .timepoint() - .get(&timeline) - .copied() - .unwrap_or(TimeInt::MAX), - ); - datarow.cells().data_ui( - ctx, - ui, - UiVerbosity::MaxHeight(row_height), - &query, - ); - }); - } - } - Err(err) => { - re_log::error_once!("Bad arrow payload: {err}",); - row.col(|ui| { - ui.label("Bad Arrow Payload".to_owned()); - }); - } - }, - LogMsg::Goodbye(row_id) => { - row.col(|ui| { - ctx.row_id_button(ui, *row_id); - }); - row.col(|ui| { - ui.monospace("Goodbye"); - }); - } - } -} diff --git a/crates/re_viewer/src/ui/mod.rs b/crates/re_viewer/src/ui/mod.rs index 3c89fdb5bdbc1..da730d21b5e71 100644 --- a/crates/re_viewer/src/ui/mod.rs +++ b/crates/re_viewer/src/ui/mod.rs @@ -16,7 +16,6 @@ mod view_time_series; mod viewport; pub(crate) mod data_ui; -pub(crate) mod event_log_view; pub(crate) mod memory_panel; pub(crate) mod selection_panel; pub(crate) mod time_panel;