Drive blueprints off of a DataStore (#2010)

 - Uses the new serde-helper from: #2004 to stick space-views in the data-store
 - Refactors the frame loop to follow this basic logic:
   - Materialize the blueprint from the store
   - Save a snapshot of the materialized blueprint
   - Run most of the legacy code as is
   - If the blueprint has been modified then save the modifications back to the store
 - In the internals of the Python, this introduces a new `global_blueprint_stream()`
 - Adds a few python APIs that send data to the stream.
 - RecordingId is now a string, and we use a special condition of RecordingId == AppId to determine that a blueprint is the "default blueprint" for an app.
 - The default behavior of rr.init() now uses this special recording-id, which means in the common case your blueprint API calls do almost exactly what you want, but an expert can still opt out using `add_to_app_default_blueprint=False` (this might need a better name), in which case they get complete control over a new blueprint.
 - SpaceViewIds generated by the app use a hash of the spaceview name to avoid repeated appends.
 - The "selected blueprint" is determined based on app-id. There is a "currently selected" blueprint for each app, which defaults to the special global blueprint.

Author: jleibs

Cargo.lock

@@ -1,5 +1,5 @@
-use re_arrow_store::LatestAtQuery;
-use re_log_types::{DeserializableComponent, EntityPath};
+#[cfg(feature = "serde")]
+use re_log_types::EntityPath;
 
 #[cfg(feature = "serde")]
 use crate::EditableAutoValue;
@@ -8,7 +8,7 @@ use crate::EditableAutoValue;
 
 /// Properties for a collection of entities.
 #[cfg(feature = "serde")]
-#[derive(Clone, Default)]
+#[derive(Clone, Default, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 pub struct EntityPropertyMap {
     props: nohash_hasher::IntMap<EntityPath, EntityProperties>,
@@ -186,39 +186,3 @@ impl Default for ColorMapper {
         Self::Colormap(Colormap::default())
     }
 }
-
-// ----------------------------------------------------------------------------
-
-/// Get the latest value for a given [`re_log_types::Component`].
-///
-/// This assumes that the row we get from the store only contains a single instance for this
-/// component; it will log a warning otherwise.
-///
-/// This should only be used for "mono-components" such as `Transform` and `Tensor`.
-pub fn query_latest_single<C: DeserializableComponent>(
-    data_store: &re_arrow_store::DataStore,
-    entity_path: &EntityPath,
-    query: &LatestAtQuery,
-) -> Option<C>
-where
-    for<'b> &'b C::ArrayType: IntoIterator,
-{
-    crate::profile_function!();
-
-    // Although it would be nice to use the `re_query` helpers for this, we would need to move
-    // this out of re_data_store to avoid a circular dep. Since we don't need to do a join for
-    // single components this is easy enough.
-
-    let (_, cells) = data_store.latest_at(query, entity_path, C::name(), &[C::name()])?;
-    let cell = cells.get(0)?.as_ref()?;
-
-    let mut iter = cell.try_to_native::<C>().ok()?;
-
-    let component = iter.next();
-
-    if iter.next().is_some() {
-        re_log::warn_once!("Unexpected batch for {} at: {}", C::name(), entity_path);
-    }
-
-    component
-}

crates/re_data_store/src/entity_properties.rs

@@ -10,11 +10,13 @@ pub mod entity_properties;
 pub mod entity_tree;
 mod instance_path;
 pub mod log_db;
+mod util;
 
 pub use entity_properties::*;
 pub use entity_tree::*;
 pub use instance_path::*;
 pub use log_db::LogDb;
+pub use util::*;
 
 #[cfg(feature = "serde")]
 pub use editable_auto_value::EditableAutoValue;

crates/re_data_store/src/lib.rs

@@ -6,7 +6,7 @@ use re_arrow_store::{DataStoreConfig, TimeInt};
 use re_log_types::{
     component_types::InstanceKey, ArrowMsg, Component as _, ComponentPath, DataCell, DataRow,
     DataTable, EntityPath, EntityPathHash, EntityPathOpMsg, LogMsg, PathOp, RecordingId,
-    RecordingInfo, RowId, SetRecordingInfo, TimePoint, Timeline,
+    RecordingInfo, RecordingType, RowId, SetRecordingInfo, TimePoint, Timeline,
 };
 
 use crate::{Error, TimesPerTimeline};
@@ -69,7 +69,8 @@ impl EntityDb {
         Ok(())
     }
 
-    fn try_add_data_row(&mut self, row: &DataRow) -> Result<(), Error> {
+    // TODO(jleibs): If this shouldn't be public, chain together other setters
+    pub fn try_add_data_row(&mut self, row: &DataRow) -> Result<(), Error> {
         for (&timeline, &time_int) in row.timepoint().iter() {
             self.times_per_timeline.insert(timeline, time_int);
         }
@@ -195,6 +196,10 @@ impl LogDb {
         self.recording_msg().map(|msg| &msg.info)
     }
 
+    pub fn recording_type(&self) -> RecordingType {
+        self.recording_id.variant
+    }
+
     pub fn recording_id(&self) -> &RecordingId {
         &self.recording_id
     }
@@ -241,7 +246,7 @@ impl LogDb {
         Ok(())
     }
 
-    fn add_begin_recording_msg(&mut self, msg: &SetRecordingInfo) {
+    pub fn add_begin_recording_msg(&mut self, msg: &SetRecordingInfo) {
         self.recording_msg = Some(msg.clone());
     }
 

crates/re_data_store/src/log_db.rs

@@ -0,0 +1,88 @@
+use re_arrow_store::LatestAtQuery;
+use re_log_types::{
+    DataRow, DeserializableComponent, EntityPath, RowId, SerializableComponent, TimeInt, TimePoint,
+    Timeline,
+};
+
+use crate::LogDb;
+
+// ----------------------------------------------------------------------------
+
+/// Get the latest value for a given [`re_log_types::Component`].
+///
+/// This assumes that the row we get from the store only contains a single instance for this
+/// component; it will log a warning otherwise.
+///
+/// This should only be used for "mono-components" such as `Transform` and `Tensor`.
+pub fn query_latest_single<C: DeserializableComponent>(
+    data_store: &re_arrow_store::DataStore,
+    entity_path: &EntityPath,
+    query: &LatestAtQuery,
+) -> Option<C>
+where
+    for<'b> &'b C::ArrayType: IntoIterator,
+{
+    crate::profile_function!();
+
+    // Although it would be nice to use the `re_query` helpers for this, we would need to move
+    // this out of re_data_store to avoid a circular dep. Since we don't need to do a join for
+    // single components this is easy enough.
+
+    let (_, cells) = data_store.latest_at(query, entity_path, C::name(), &[C::name()])?;
+    let cell = cells.get(0)?.as_ref()?;
+
+    let mut iter = cell.try_to_native::<C>().ok()?;
+
+    let component = iter.next();
+
+    if iter.next().is_some() {
+        re_log::warn_once!("Unexpected batch for {} at: {}", C::name(), entity_path);
+    }
+
+    component
+}
+
+/// Get the latest value for a given [`re_log_types::Component`] assuming it is timeless.
+///
+/// This assumes that the row we get from the store only contains a single instance for this
+/// component; it will log a warning otherwise.
+pub fn query_timeless_single<C: DeserializableComponent>(
+    data_store: &re_arrow_store::DataStore,
+    entity_path: &EntityPath,
+) -> Option<C>
+where
+    for<'b> &'b C::ArrayType: IntoIterator,
+{
+    let query = re_arrow_store::LatestAtQuery::new(Timeline::default(), TimeInt::MAX);
+    query_latest_single(data_store, entity_path, &query)
+}
+
+// ----------------------------------------------------------------------------
+
+/// Store a single value for a given [`re_log_types::Component`].
+pub fn store_one_component<C: SerializableComponent>(
+    log_db: &mut LogDb,
+    entity_path: &EntityPath,
+    timepoint: &TimePoint,
+    component: C,
+) {
+    let mut row = DataRow::from_cells1(
+        RowId::random(),
+        entity_path.clone(),
+        timepoint.clone(),
+        1,
+        [component].as_slice(),
+    );
+    row.compute_all_size_bytes();
+
+    match log_db.entity_db.try_add_data_row(&row) {
+        Ok(()) => {}
+        Err(err) => {
+            re_log::warn_once!(
+                "Failed to store component {}.{}: {err}",
+                entity_path,
+                C::name(),
+            );
+        }
+    }
+}

crates/re_data_store/src/util.rs

@@ -39,6 +39,7 @@ impl DataUi for SetRecordingInfo {
             started,
             recording_source,
             is_official_example,
+            recording_type,
         } = info;
 
         egui::Grid::new("fields").num_columns(2).show(ui, |ui| {
@@ -61,6 +62,10 @@ impl DataUi for SetRecordingInfo {
             ui.monospace("is_official_example:");
             ui.label(format!("{is_official_example}"));
             ui.end_row();
+
+            ui.monospace("recording_type:");
+            ui.label(format!("{recording_type}"));
+            ui.end_row();
         });
     }
 }

crates/re_data_ui/src/log_msg.rs

@@ -198,6 +198,7 @@ fn test_encode_decode() {
                 rustc_version: String::new(),
                 llvm_version: String::new(),
             },
+            recording_type: re_log_types::RecordingType::Data,
         },
     })];
 

crates/re_log_encoding/src/decoder.rs

@@ -120,8 +120,8 @@ impl std::fmt::Display for RecordingType {
 #[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 pub struct RecordingId {
-    variant: RecordingType,
-    id: Arc<String>,
+    pub variant: RecordingType,
+    pub id: Arc<String>,
 }
 
 impl RecordingId {
@@ -148,6 +148,11 @@ impl RecordingId {
             id: Arc::new(str),
         }
     }
+
+    #[inline]
+    pub fn as_str(&self) -> &str {
+        self.id.as_str()
+    }
 }
 
 impl std::fmt::Display for RecordingId {
@@ -187,6 +192,10 @@ impl ApplicationId {
     pub fn unknown() -> Self {
         Self("unknown_app_id".to_owned())
     }
+
+    pub fn as_str(&self) -> &str {
+        self.0.as_str()
+    }
 }
 
 impl std::fmt::Display for ApplicationId {
@@ -261,6 +270,16 @@ pub struct RecordingInfo {
     pub started: Time,
 
     pub recording_source: RecordingSource,
+
+    pub recording_type: RecordingType,
+}
+
+impl RecordingInfo {
+    /// Whether this `RecordingInfo` is the default used when a user is not explicitly
+    /// creating their own blueprint.
+    pub fn is_app_default_blueprint(&self) -> bool {
+        self.application_id.as_str() == self.recording_id.as_str()
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]

crates/re_log_types/src/lib.rs

@@ -24,6 +24,9 @@ pub struct MemoryHistory {
 
     /// Bytes used by the datastore according to its own accounting.
     pub counted_store: History<i64>,
+
+    /// Bytes used by the blueprint store according to its own accounting.
+    pub counted_blueprint: History<i64>,
 }
 
 impl Default for MemoryHistory {
@@ -35,6 +38,7 @@ impl Default for MemoryHistory {
             counted: History::new(0..max_elems, max_seconds),
             counted_gpu: History::new(0..max_elems, max_seconds),
             counted_store: History::new(0..max_elems, max_seconds),
+            counted_blueprint: History::new(0..max_elems, max_seconds),
         }
     }
 }
@@ -46,15 +50,22 @@ impl MemoryHistory {
             counted,
             counted_gpu,
             counted_store,
+            counted_blueprint,
         } = self;
         resident.is_empty()
             && counted.is_empty()
             && counted_gpu.is_empty()
             && counted_store.is_empty()
+            && counted_blueprint.is_empty()
     }
 
     /// Add data to history
-    pub fn capture(&mut self, counted_gpu: Option<i64>, counted_store: Option<i64>) {
+    pub fn capture(
+        &mut self,
+        counted_gpu: Option<i64>,
+        counted_store: Option<i64>,
+        counted_blueprint: Option<i64>,
+    ) {
         let mem_use = crate::MemoryUse::capture();
         let now = crate::util::sec_since_start();
 
@@ -70,5 +81,8 @@ impl MemoryHistory {
         if let Some(counted_store) = counted_store {
             self.counted_store.add(now, counted_store);
         }
+        if let Some(counted_blueprint) = counted_blueprint {
+            self.counted_blueprint.add(now, counted_blueprint);
+        }
     }
 }

crates/re_memory/src/memory_history.rs

@@ -133,7 +133,7 @@ pub enum Projection {
 }
 
 /// How [`Size::AUTO`] is interpreted.
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 pub struct AutoSizeConfig {
     /// Determines the point radius when [`crate::Size::AUTO`].

crates/re_renderer/src/view_builder.rs

@@ -167,6 +167,7 @@ pub fn new_recording_info(
             rustc_version: env!("RE_BUILD_RUSTC_VERSION").into(),
             llvm_version: env!("RE_BUILD_LLVM_VERSION").into(),
         },
+        recording_type: re_log_types::RecordingType::Data,
     }
 }