Merge dbsp/live-source into dbsp/pushdown

# Conflicts:
#	context/code/source.md
#	context/test-evidence/core.md
#	context/test-evidence/improvise-incremental.md
#	crates/improvise-core/src/model/source.rs

crates/improvise-core/src/model/over_tests.rs

@@ -219,6 +219,48 @@ fn a_model_over_a_source_refuses_a_write() {
     over.set_cell(key("East", "A", "Revenue"), CellValue::Number(1.0));
 }
 
+#[test]
+#[should_panic(expected = "read-only")]
+fn a_model_over_a_source_refuses_a_clear() {
+    let mut over = lent(&fixture());
+    over.clear_cell(&key("East", "A", "Revenue"));
+}
+
+#[test]
+#[should_panic(expected = "read-only")]
+fn a_model_over_a_source_refuses_to_remove_a_category() {
+    let mut over = lent(&fixture());
+    over.remove_category("Product");
+}
+
+#[test]
+#[should_panic(expected = "read-only")]
+fn a_model_over_a_source_refuses_to_remove_an_item() {
+    let mut over = lent(&fixture());
+    over.remove_item("Product", "A");
+}
+
+#[test]
+fn putting_a_populated_model_over_a_source_drops_its_own_cells() {
+    let owned = fixture();
+    assert!(owned.data.iter_cells().count() > 0);
+    let rows = owned
+        .data
+        .iter_cells()
+        .map(|(k, v)| (k, v.clone()))
+        .collect();
+    let over = owned.clone().over(Arc::new(TableSource { rows }));
+    assert_eq!(over.data.iter_cells().count(), 0, "no stale second copy");
+    let cell = key("East", "A", "Revenue");
+    assert_eq!(
+        over.get_cell(&cell),
+        None,
+        "the owned store is empty, the source is not"
+    );
+    let read = over.evaluate_aggregated(&cell, Aggregation::Exact);
+    assert_eq!(read, owned.evaluate_aggregated(&cell, Aggregation::Exact));
+}
+
 #[test]
 fn the_records_a_drill_freezes_are_read_from_the_source() {
     use crate::view::layout::drill_snapshot;

crates/improvise-core/src/model/source.rs

@@ -61,6 +61,23 @@ impl Numbers {
         numbers
     }
 
+    /// Numbers from what an engine computed: how many, their exact sum and the least and
+    /// greatest that are not NaN. Extremes that contradict the count or are NaN are dropped.
+    pub fn from_parts(
+        count: u64,
+        sum: ExactSum,
+        least: Option<f64>,
+        greatest: Option<f64>,
+    ) -> Self {
+        let held = |x: Option<f64>| x.filter(|x| count > 0 && !x.is_nan());
+        Numbers {
+            count,
+            sum,
+            least: held(least),
+            greatest: held(greatest),
+        }
+    }
+
     pub fn add(&mut self, x: f64) {
         self.count += 1;
         self.sum.add(x, 1);
@@ -114,9 +131,9 @@ fn merged(a: Option<f64>, b: Option<f64>, pick: fn(f64, f64) -> f64) -> Option<f
     }
 }
 
-/// The values that are not numbers, as far as unanimity needs them.
+/// What the values that are not numbers agree on: nothing held, one value, or disagreement.
 #[derive(Debug, Clone, Default, PartialEq)]
-enum Others {
+pub enum Agreement {
     #[default]
     None,
     One(CellValue),
@@ -128,7 +145,7 @@ enum Others {
 #[derive(Debug, Clone, Default)]
 pub struct Summary {
     numbers: Numbers,
-    others: Others,
+    others: Agreement,
 }
 
 impl Summary {
@@ -138,6 +155,11 @@ impl Summary {
         summary
     }
 
+    /// A summary from what an engine computed: the numbers, and what the rest agree on.
+    pub fn from_parts(numbers: Numbers, others: Agreement) -> Self {
+        Summary { numbers, others }
+    }
+
     pub fn add(&mut self, value: &CellValue) {
         match value.as_f64() {
             Some(x) => self.numbers.add(x),
@@ -147,18 +169,18 @@ impl Summary {
 
     fn add_other(&mut self, value: &CellValue) {
         self.others = match std::mem::take(&mut self.others) {
-            Others::None => Others::One(value.clone()),
-            Others::One(held) if held == *value => Others::One(held),
-            _ => Others::Mixed,
+            Agreement::None => Agreement::One(value.clone()),
+            Agreement::One(held) if held == *value => Agreement::One(held),
+            _ => Agreement::Mixed,
         };
     }
 
     pub fn merge(&mut self, other: &Summary) {
         self.numbers.merge(&other.numbers);
         match &other.others {
-            Others::None => {}
-            Others::One(v) => self.add_other(v),
-            Others::Mixed => self.others = Others::Mixed,
+            Agreement::None => {}
+            Agreement::One(v) => self.add_other(v),
+            Agreement::Mixed => self.others = Agreement::Mixed,
         }
     }
 
@@ -169,8 +191,8 @@ impl Summary {
 
     pub fn reduce(&self, func: &AggFunc) -> Option<CellValue> {
         match (&self.others, self.numbers.count()) {
-            (Others::None, _) => self.numbers.reduce(func).map(CellValue::Number),
-            (Others::One(only), 0) => Some(only.clone()),
+            (Agreement::None, _) => self.numbers.reduce(func).map(CellValue::Number),
+            (Agreement::One(only), 0) => Some(only.clone()),
             _ => Some(CellValue::Error("mixed".to_string())),
         }
     }
@@ -180,8 +202,8 @@ impl Summary {
 /// its readers (a database) does so through interior mutability of its own.
 pub type SharedSource = std::sync::Arc<dyn CellSource + Send + Sync>;
 
-/// A source that reports its changes. Its cells are the state after the changes it has drained,
-/// so a snapshot read now plus the changes drained later counts no change twice or not at all.
+/// A source that reports its changes; its cells are the state as of its last drain.
+/// A drain is destructive: a feed has one drainer (`context/code/source.md`).
 pub trait ChangeSource: CellSource {
     /// The changes since the last drain, oldest first; reads see them from now on.
     fn drain_changes(&self) -> Vec<CellChange>;
@@ -244,7 +266,10 @@ impl CellSource for DataStore {
         &'a self,
         partial: &'a [(String, String)],
     ) -> Box<dyn Iterator<Item = CellKey> + 'a> {
-        Box::new(self.cells(partial).map(|(key, _)| key))
+        match partial.is_empty() {
+            true => Box::new(self.iter_cells().map(|(key, _)| key)),
+            false => Box::new(self.matching_cells(partial).into_iter().map(|(key, _)| key)),
+        }
     }
 
     fn cells<'a>(
@@ -708,4 +733,61 @@ mod tests {
         }
         assert!(asked > 40, "only {asked} stems were compared");
     }
+
+    /// The parts an engine would compute for `values`, found without `Summary::of`.
+    fn parts_of(values: &[CellValue]) -> Summary {
+        let numbers: Vec<f64> = values.iter().filter_map(|v| v.as_f64()).collect();
+        let mut sum = ExactSum::new();
+        numbers.iter().for_each(|x| sum.add(*x, 1));
+        let finite = || numbers.iter().copied().filter(|x| !x.is_nan());
+        let least = finite().min_by(f64::total_cmp);
+        let greatest = finite().max_by(f64::total_cmp);
+        let rest: Vec<&CellValue> = values.iter().filter(|v| v.as_f64().is_none()).collect();
+        let agreement = match rest.split_first() {
+            None => Agreement::None,
+            Some((first, others)) if others.iter().all(|v| v == first) => {
+                Agreement::One((*first).clone())
+            }
+            Some(_) => Agreement::Mixed,
+        };
+        let parts = Numbers::from_parts(numbers.len() as u64, sum, least, greatest);
+        Summary::from_parts(parts, agreement)
+    }
+
+    #[test]
+    fn a_summary_built_from_an_engines_parts_reduces_as_one_built_from_the_values() {
+        for values in sequences(3) {
+            let refs: Vec<&CellValue> = values.iter().collect();
+            for func in &FUNCS {
+                let want = reference(&refs, func);
+                let got = parts_of(&values).reduce(func);
+                assert!(
+                    same(&want, &got),
+                    "{func:?} of {values:?}: {want:?} != {got:?}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn parts_that_contradict_themselves_are_made_consistent() {
+        let none = Numbers::from_parts(0, ExactSum::new(), Some(1.0), Some(f64::NAN));
+        assert_eq!(none.reduce(&AggFunc::Min), None, "no numbers, no extremes");
+        let nan = Numbers::from_parts(2, ExactSum::new(), Some(f64::NAN), None);
+        let min = nan.reduce(&AggFunc::Min).unwrap();
+        assert!(min.is_nan(), "a NaN is never an extreme");
+    }
+
+    #[test]
+    fn keys_of_a_store_are_its_cells_keys_without_the_values() {
+        let rows = rows();
+        let store = store_of(&rows);
+        let all: Vec<CellKey> = store.keys(&[]).collect();
+        assert_eq!(all, rows.iter().map(|(k, _)| k.clone()).collect::<Vec<_>>());
+        let east = vec![pair("R", "E")];
+        let under: Vec<CellKey> = store.keys(&east).collect();
+        let want: Vec<CellKey> = store.cells(&east).map(|c| c.0).collect();
+        assert_eq!(under, want);
+        assert!(!under.is_empty());
+    }
 }

crates/improvise-core/src/model/types.rs

@@ -430,13 +430,19 @@ impl Model {
         }
     }
 
-    /// This model, reading its raw cells from `source` and not `data`: read-only.
-    /// Why: `context/code/source.md`.
+    /// This model, reading its raw cells from `source` and not `data`: read-only. Its own
+    /// cells are dropped, so no stale second copy is left to read. Why: `context/code/source.md`.
     pub fn over(mut self, source: SharedSource) -> Self {
+        self.data = DataStore::new();
         self.external = Some(source);
         self
     }
 
+    /// Refuses, in every build, a write the source would never see.
+    fn writable(&self) {
+        assert!(self.external.is_none(), "{READ_ONLY}");
+    }
+
     /// Add a pivot category, enforcing `MAX_CATEGORIES` over regular ones.
     /// The caller registers it on every view — normally `Workbook`.
     pub fn add_category(&mut self, name: impl Into<String>) -> Result<CategoryId> {
@@ -480,6 +486,7 @@ impl Model {
     /// Remove a category, its cells and the formulas targeting it. The caller
     /// removes it from the views — normally `Workbook`.
     pub fn remove_category(&mut self, name: &str) {
+        self.writable();
         if !self.categories.contains_key(name) || CategoryKind::of_name(name).is_virtual() {
             return;
         }
@@ -502,6 +509,7 @@ impl Model {
     /// An item, the formulas targeting it and the cells naming it. The sweep
     /// is `remove_category`'s one level down — `context/code/core.md` §9.
     pub fn remove_item(&mut self, cat_name: &str, item_name: &str) {
+        self.writable();
         if let Some(cat) = self.categories.get_mut(cat_name) {
             cat.remove_item(item_name);
         }
@@ -552,12 +560,12 @@ impl Model {
     }
 
     pub fn set_cell(&mut self, key: CellKey, value: CellValue) {
-        debug_assert!(self.external.is_none(), "{READ_ONLY}");
+        self.writable();
         self.data.set(key, value);
     }
 
     pub fn clear_cell(&mut self, key: &CellKey) {
-        debug_assert!(self.external.is_none(), "{READ_ONLY}");
+        self.writable();
         self.data.remove(key);
     }
 

crates/improvise-core/src/view/layout.rs

@@ -1348,6 +1348,32 @@ mod tests {
         assert_eq!(layout.col_label(0), "X");
     }
 
+    /// The records a records-mode grid draws come from the source the model is over.
+    #[test]
+    fn records_mode_reads_the_records_from_a_source() {
+        use crate::model::source::TableSource;
+        let mut wb = records_workbook();
+        let v = wb.active_view_mut();
+        v.set_axis("_Index", Axis::Row);
+        v.set_axis("_Dim", Axis::Column);
+        let rows = wb.model.data.iter_cells().map(|(k, v)| (k, v.clone()));
+        let over = wb.model.clone().over(std::sync::Arc::new(TableSource {
+            rows: rows.collect(),
+        }));
+        let (owned, lent) = (
+            GridLayout::new(&wb.model, wb.active_view()),
+            GridLayout::new(&over, wb.active_view()),
+        );
+        assert!(lent.is_records_mode());
+        assert_eq!(lent.row_count(), 1, "the source holds the records");
+        for col in 0..owned.col_count() {
+            assert_eq!(lent.records_display(0, col), owned.records_display(0, col));
+        }
+        let revenue = (0..lent.col_count()).find(|&c| lent.col_label(c) == "Revenue");
+        let revenue = revenue.unwrap();
+        assert_eq!(lent.records_display(0, revenue).as_deref(), Some("100"));
+    }
+
     #[test]
     fn records_mode_activated_when_index_and_dim_on_axes() {
         let mut wb = records_workbook();