From 56fc9922b67fa97b580ce7169913e3cfb2b8db1a Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 11:50:20 -0700 Subject: [PATCH 01/12] feat(protocol): the lossless wire form, stamps, notices and the client end of an edge (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- Cargo.lock | 3 + crates/improvise-protocol/Cargo.toml | 4 + .../improvise-protocol/src/delivery/client.rs | 349 ++++++++++++++++++ .../src/delivery/client_tests.rs | 290 +++++++++++++++ .../src/delivery/exchange.rs | 94 +++++ crates/improvise-protocol/src/delivery/mod.rs | 18 + .../improvise-protocol/src/delivery/notice.rs | 300 +++++++++++++++ .../improvise-protocol/src/delivery/wire.rs | 325 ++++++++++++++++ crates/improvise-protocol/src/lib.rs | 1 + 9 files changed, 1384 insertions(+) create mode 100644 crates/improvise-protocol/src/delivery/client.rs create mode 100644 crates/improvise-protocol/src/delivery/client_tests.rs create mode 100644 crates/improvise-protocol/src/delivery/exchange.rs create mode 100644 crates/improvise-protocol/src/delivery/mod.rs create mode 100644 crates/improvise-protocol/src/delivery/notice.rs create mode 100644 crates/improvise-protocol/src/delivery/wire.rs diff --git a/Cargo.lock b/Cargo.lock index d09a5591..1936579d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -981,9 +981,12 @@ name = "improvise-protocol" version = "0.1.0-rc2" dependencies = [ "improvise-core", + "numtower", + "proptest", "serde", "serde_json", "unicode-width", + "zset", ] [[package]] diff --git a/crates/improvise-protocol/Cargo.toml b/crates/improvise-protocol/Cargo.toml index 62c4a498..530fe1f1 100644 --- a/crates/improvise-protocol/Cargo.toml +++ b/crates/improvise-protocol/Cargo.toml @@ -8,6 +8,9 @@ repository = "https://github.com/fiddlerwoaroof/improvise" [dependencies] improvise-core = { path = "../improvise-core" } +# ADR 009 clause 9: the wire form carries a correction as Z-set arithmetic, a value as ADR 005 canonical text. +zset = { path = "../zset" } +numtower = { path = "../numtower" } serde = { version = "1", features = ["derive"] } # For the one width rule both front ends measure a header label with # (`AxisFrame::header_widths`). A terminal column is a display cell, not a @@ -24,3 +27,4 @@ test-support = ["dep:serde_json"] [dev-dependencies] serde_json = "1" +proptest = "1" diff --git a/crates/improvise-protocol/src/delivery/client.rs b/crates/improvise-protocol/src/delivery/client.rs new file mode 100644 index 00000000..25dcd395 --- /dev/null +++ b/crates/improvise-protocol/src/delivery/client.rs @@ -0,0 +1,349 @@ +//! The client end of an edge: watermarks, the held window, and when to ask (ADR 009 clause 9). +//! Why each choice: `context/code/delivery.md` §4. + +use std::collections::{BTreeMap, BTreeSet}; + +use zset::ZSet; + +use super::exchange::{Ask, Correction, CorrectionBody, Edge, EdgeError, Reply, Request}; +use super::exchange::{SessionToken, Window}; +use super::notice::{AxisChange, Flags, Moved, Notice, Relabel, ShapeKey, Stamp}; +use super::wire::{Cell, WireKey, decode}; + +/// What the shape's consumer must do with the structure around the cells. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Event { + Axis(AxisChange), + Relabel(Relabel), + Flags(Flags), + /// Positions and labels are unknown: re-derive them from what is held. + Resync, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Want { + Correction { + watermark: Option, + }, + Fetch { + watermark: Option, + keys: BTreeSet, + through: Stamp, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Outcome { + /// Applied; `delta` is what changed in the held cells. + Applied { delta: ZSet }, + /// Already covered by the watermark: nothing was done. + Duplicate, + /// Starts after the watermark, or in another epoch: ask again from the watermark. + Gap, + /// Names a shape this client does not read. + Untracked, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Synced { + Settled, + Failed, + /// The rounds ran out with work left: a defect in the edge, not a state to wait out. + Exhausted, +} + +#[derive(Debug, Clone)] +struct Tracked { + window: Window, + held: ZSet, + watermark: Option, + seen: Option, + /// Notices received and not yet fetched: `(publication, what moved)`. + unapplied: Vec<(Stamp, Moved)>, + stale: bool, +} + +#[derive(Debug, Clone, Default)] +pub struct EdgeClient { + session: Option, + shapes: BTreeMap, + events_through: Option, +} + +impl EdgeClient { + pub fn new(session: Option) -> Self { + Self { + session, + ..Self::default() + } + } + + /// Start reading `shape`; the first thing it wants is a full refetch. + pub fn track(&mut self, shape: ShapeKey, window: Window) { + self.shapes.insert( + shape, + Tracked { + window, + held: ZSet::new(), + watermark: None, + seen: None, + unapplied: Vec::new(), + stale: true, + }, + ); + } + + pub fn held(&self, shape: &ShapeKey) -> Option<&ZSet> { + self.shapes.get(shape).map(|t| &t.held) + } + + pub fn watermark(&self, shape: &ShapeKey) -> Option { + self.shapes.get(shape).and_then(|t| t.watermark) + } + + pub fn wants(&self, shape: &ShapeKey) -> Option { + let t = self.shapes.get(shape)?; + let needs_all = t.stale || t.unapplied.iter().any(|(_, m)| *m == Moved::Correct); + match (needs_all, t.seen) { + (true, _) => Some(Want::Correction { + watermark: t.watermark, + }), + (false, Some(through)) if !t.unapplied.is_empty() => Some(Want::Fetch { + watermark: t.watermark, + keys: t + .unapplied + .iter() + .flat_map(|(_, m)| match m { + Moved::Keys(keys) => keys.iter().cloned(), + Moved::Correct => unreachable!("handled above"), + }) + .collect(), + through, + }), + _ => None, + } + } + + pub fn request(&self, shape: &ShapeKey, want: Want) -> Option { + let t = self.shapes.get(shape)?; + Some(Request { + session: self.session.clone(), + shape: shape.clone(), + window: t.window, + ask: match want { + Want::Correction { watermark } => Ask::Correction { watermark }, + Want::Fetch { + watermark, + keys, + through, + } => Ask::Fetch { + watermark, + keys, + through, + }, + }, + }) + } + + /// A pushed notice. Duplicates are ignored, a gap or a new epoch makes every shape ask. + pub fn on_notice(&mut self, notice: &Notice) -> Vec { + let end = notice.stamp(); + for (shape, t) in &mut self.shapes { + let contiguous = match t.seen { + Some(seen) if seen.same_epoch(&end) => match notice.range.to <= seen.publication { + true => continue, + false => notice.range.from <= seen.publication, + }, + _ => false, + }; + t.seen = Some(end); + if !contiguous || notice.evicted.contains(shape) { + t.unapplied.push((end, Moved::Correct)); + continue; + } + match notice.moved.get(shape) { + Some(Moved::Keys(keys)) if keys.is_empty() => {} + Some(moved) => t.unapplied.push((end, moved.clone())), + None => {} + } + let current = t.watermark.is_some_and(|w| w.same_epoch(&end)); + if t.unapplied.is_empty() && !t.stale && current { + t.watermark = Some(end); + } + } + self.events_of(notice) + } + + /// The connection dropped: every shape asks for a correction from its watermark. + pub fn connection_lost(&mut self) { + self.shapes.values_mut().for_each(|t| t.stale = true); + } + + pub fn request_failed(&mut self, shape: &ShapeKey) { + if let Some(t) = self.shapes.get_mut(shape) { + t.stale = true; + } + } + + /// A notice that could not be read: what it said is lost, so everything asks. + pub fn unreadable(&mut self) { + self.connection_lost(); + self.events_through = None; + } + + fn events_of(&mut self, n: &Notice) -> Vec { + let end = n.stamp(); + let resync = vec![Event::Resync]; + let usable = match self.events_through { + Some(t) if t.same_epoch(&end) => { + if n.range.to <= t.publication { + return Vec::new(); + } + n.range.from == t.publication + } + _ => false, + }; + self.events_through = Some(end); + match usable { + false => resync, + true => n + .axis + .iter() + .cloned() + .map(Event::Axis) + .chain(n.relabels.iter().cloned().map(Event::Relabel)) + .chain(n.flags.any().then_some(Event::Flags(n.flags))) + .collect(), + } + } + + /// A correction. It applies only at the watermark it starts from: `held + (target - held)`. + pub fn apply(&mut self, c: &Correction) -> (Outcome, Vec) { + let Some(t) = self.shapes.get_mut(&c.shape) else { + return (Outcome::Untracked, Vec::new()); + }; + let target = decode(&c.target); + let covered = t + .watermark + .is_some_and(|w| w.same_epoch(&c.through) && w.publication >= c.through.publication); + let starts_here = match (&c.from, t.watermark) { + (Some(from), Some(w)) => w.same_epoch(from) && w.publication >= from.publication, + _ => false, + }; + let delta = match (&c.body, covered) { + (_, true) => { + t.stale &= t.watermark != Some(c.through); + return (Outcome::Duplicate, Vec::new()); + } + (CorrectionBody::Window, false) => &target - &t.held, + (CorrectionBody::Moved { .. }, false) if !starts_here => { + return (Outcome::Gap, Vec::new()); + } + (CorrectionBody::Moved { keys }, false) => { + let held_there: ZSet = t + .held + .iter() + .filter(|((k, _), _)| keys.contains(k)) + .map(|(cell, w)| (cell.clone(), w)) + .collect(); + &target - &held_there + } + }; + t.held += δ + t.watermark = Some(c.through); + t.stale = match &c.body { + CorrectionBody::Window => false, + CorrectionBody::Moved { .. } => t.stale, + }; + let seen_epoch = t.seen.map(|s| (s.host, s.generation)); + t.unapplied.retain(|(at, _)| { + Some((at.host, at.generation)) == seen_epoch + && !(at.same_epoch(&c.through) && at.publication <= c.through.publication) + }); + let seen_ahead = t + .seen + .is_some_and(|s| s.same_epoch(&c.through) && s.publication >= c.through.publication); + if !seen_ahead { + t.seen = Some(c.through); + } + let mut events = match &c.body { + CorrectionBody::Window => { + self.events_through = Some(c.through); + vec![Event::Resync] + } + CorrectionBody::Moved { .. } => Vec::new(), + }; + if let Some(carried) = &c.carried { + events.extend(self.events_of(carried)); + } + (Outcome::Applied { delta }, events) + } + + /// Ask until `shape` wants nothing, at most `rounds` times. + pub fn sync( + &mut self, + edge: &mut impl Edge, + shape: &ShapeKey, + rounds: usize, + ) -> (Synced, Vec) { + let mut events = Vec::new(); + for _ in 0..rounds { + let Some(want) = self.wants(shape) else { + return (Synced::Settled, events); + }; + let Some(request) = self.request(shape, want) else { + return (Synced::Settled, events); + }; + match edge.call(request) { + Ok(Reply::Correction(c)) => events.extend(self.apply(&c).1), + Ok(Reply::Notices(list)) => { + list.iter().for_each(|n| events.extend(self.on_notice(n))); + } + Err(EdgeError::Failed) => { + self.request_failed(shape); + return (Synced::Failed, events); + } + Err(EdgeError::Unreadable) => { + self.unreadable(); + return (Synced::Failed, events); + } + } + } + match self.wants(shape) { + None => (Synced::Settled, events), + Some(_) => (Synced::Exhausted, events), + } + } + + /// Pull instead of listening: ask for the notices after the horizon seen. + pub fn poll(&mut self, edge: &mut impl Edge, shape: &ShapeKey) -> (Synced, Vec) { + let (Some(t), Some(after)) = ( + self.shapes.get(shape), + self.shapes.get(shape).and_then(|t| t.seen), + ) else { + return self.sync(edge, shape, 4); + }; + let request = Request { + session: self.session.clone(), + shape: shape.clone(), + window: t.window, + ask: Ask::Notices { after }, + }; + let mut events = Vec::new(); + match edge.call(request) { + Ok(Reply::Notices(list)) => list.iter().for_each(|n| events.extend(self.on_notice(n))), + Ok(Reply::Correction(c)) => events.extend(self.apply(&c).1), + Err(EdgeError::Failed) => { + self.request_failed(shape); + return (Synced::Failed, events); + } + Err(EdgeError::Unreadable) => { + self.unreadable(); + return (Synced::Failed, events); + } + } + let (synced, more) = self.sync(edge, shape, 4); + events.extend(more); + (synced, events) + } +} diff --git a/crates/improvise-protocol/src/delivery/client_tests.rs b/crates/improvise-protocol/src/delivery/client_tests.rs new file mode 100644 index 00000000..0121d5e9 --- /dev/null +++ b/crates/improvise-protocol/src/delivery/client_tests.rs @@ -0,0 +1,290 @@ +//! The client end of an edge against hand-built notices and corrections. + +use super::client::*; +use super::exchange::*; +use super::notice::*; +use super::wire::{Cell, WireElem, WireKey, WireValue, decode, encode}; + +fn at(host: u64, publication: u64) -> Stamp { + Stamp { + host, + generation: 1, + publication, + } +} + +fn shape() -> ShapeKey { + ShapeKey("s".into()) +} + +fn window() -> Window { + Window { + top: 0, + left: 0, + rows: 10, + cols: 10, + } +} + +fn key(n: u8) -> WireKey { + WireKey(vec![("C".into(), n.to_string())]) +} + +fn cell(k: u8, v: &str) -> Cell { + (key(k), WireValue::Text(v.into())) +} + +fn target(cells: &[Cell]) -> Vec { + encode(&cells.iter().cloned().map(|c| (c, 1)).collect()) +} + +fn window_correction(through: Stamp, cells: &[Cell]) -> Correction { + Correction { + shape: shape(), + at: through, + from: None, + through, + body: CorrectionBody::Window, + target: target(cells), + carried: None, + } +} + +fn notice(host: u64, publication: u64, moved: &[u8]) -> Notice { + let mut n = Notice::empty(PubRange::one(at(host, publication))); + if !moved.is_empty() { + n.moved.insert( + shape(), + Moved::Keys(moved.iter().map(|k| key(*k)).collect()), + ); + } + n +} + +fn synced_at(publication: u64, cells: &[Cell]) -> EdgeClient { + let mut c = EdgeClient::default(); + c.track(shape(), window()); + c.apply(&window_correction(at(1, publication), cells)); + assert_eq!(c.wants(&shape()), None, "the fixture must start settled"); + c +} + +fn moved_correction(from: u64, through: u64, keys: &[u8], cells: &[Cell]) -> Correction { + Correction { + shape: shape(), + at: at(1, through), + from: Some(at(1, from)), + through: at(1, through), + body: CorrectionBody::Moved { + keys: keys.iter().map(|k| key(*k)).collect(), + }, + target: target(cells), + carried: None, + } +} + +#[test] +fn a_tracked_shape_first_wants_a_full_refetch() { + let mut c = EdgeClient::default(); + c.track(shape(), window()); + assert_eq!( + c.wants(&shape()), + Some(Want::Correction { watermark: None }) + ); +} + +#[test] +fn consecutive_notices_ask_for_no_correction_and_advance_the_watermark() { + let mut c = synced_at(4, &[cell(1, "a")]); + c.on_notice(¬ice(1, 5, &[])); + c.on_notice(¬ice(1, 6, &[])); + assert_eq!(c.wants(&shape()), None); + assert_eq!(c.watermark(&shape()), Some(at(1, 6))); +} + +#[test] +fn a_moved_key_wants_a_fetch_and_not_a_correction() { + let mut c = synced_at(4, &[cell(1, "a")]); + c.on_notice(¬ice(1, 5, &[1, 2])); + assert_eq!( + c.wants(&shape()), + Some(Want::Fetch { + watermark: Some(at(1, 4)), + keys: [key(1), key(2)].into(), + through: at(1, 5) + }) + ); +} + +#[test] +fn a_dropped_notice_is_detected() { + let mut c = synced_at(4, &[cell(1, "a")]); + c.on_notice(¬ice(1, 6, &[])); + assert_eq!( + c.wants(&shape()), + Some(Want::Correction { + watermark: Some(at(1, 4)) + }) + ); +} + +#[test] +fn a_notice_from_another_host_or_generation_asks_for_a_correction() { + for foreign in [ + notice(2, 5, &[]), + Notice::empty(PubRange::one(Stamp { + generation: 2, + ..at(1, 5) + })), + ] { + let mut c = synced_at(4, &[cell(1, "a")]); + c.on_notice(&foreign); + assert!( + matches!(c.wants(&shape()), Some(Want::Correction { .. })), + "{foreign:?}" + ); + } +} + +#[test] +fn an_eviction_and_an_oversized_notice_each_ask_for_a_correction() { + let mut evicted = notice(1, 5, &[]); + evicted.evicted.insert(shape()); + let mut oversized = notice(1, 5, &[]); + oversized.moved.insert(shape(), Moved::Correct); + for n in [evicted, oversized] { + let mut c = synced_at(4, &[cell(1, "a")]); + c.on_notice(&n); + assert!(matches!(c.wants(&shape()), Some(Want::Correction { .. }))); + } +} + +#[test] +fn a_duplicate_notice_changes_nothing() { + let mut c = synced_at(4, &[]); + c.on_notice(¬ice(1, 5, &[3])); + let before = c.wants(&shape()); + assert_eq!(c.on_notice(¬ice(1, 5, &[3])), Vec::new()); + assert_eq!(c.wants(&shape()), before); +} + +#[test] +fn a_correction_equals_the_target_and_a_second_one_is_the_zero_zset() { + let mut c = EdgeClient::default(); + c.track(shape(), window()); + let fix = window_correction(at(1, 4), &[cell(1, "a"), cell(2, "b")]); + let (first, _) = c.apply(&fix); + let Outcome::Applied { delta } = first else { + panic!("{first:?}") + }; + assert_eq!(delta.len(), 2); + assert_eq!(c.held(&shape()), Some(&decode(&fix.target))); + + let later = window_correction(at(1, 5), &[cell(1, "a"), cell(2, "b")]); + let (second, _) = c.apply(&later); + let Outcome::Applied { delta } = second else { + panic!("{second:?}") + }; + assert!(delta.is_zero(), "correct state corrected: {delta:?}"); +} + +#[test] +fn a_correction_replaces_what_is_held_not_adds_to_it() { + let mut c = synced_at(4, &[cell(1, "a"), cell(2, "b")]); + c.apply(&window_correction(at(1, 5), &[cell(1, "z")])); + assert_eq!(c.held(&shape()), Some(&decode(&target(&[cell(1, "z")])))); +} + +#[test] +fn a_correction_already_covered_is_idempotent_and_never_regresses() { + let mut c = synced_at(4, &[cell(1, "a")]); + c.apply(&window_correction(at(1, 7), &[cell(1, "new")])); + let late = window_correction(at(1, 5), &[cell(1, "old")]); + assert_eq!(c.apply(&late).0, Outcome::Duplicate); + assert_eq!(c.held(&shape()), Some(&decode(&target(&[cell(1, "new")])))); + assert_eq!(c.watermark(&shape()), Some(at(1, 7))); +} + +#[test] +fn a_moved_correction_applies_only_at_the_watermark_it_starts_from() { + let mut c = synced_at(4, &[cell(1, "a"), cell(2, "b")]); + let ahead = moved_correction(6, 8, &[1], &[cell(1, "x")]); + assert_eq!(c.apply(&ahead).0, Outcome::Gap); + assert_eq!(c.watermark(&shape()), Some(at(1, 4))); + + let fits = moved_correction(4, 6, &[1], &[cell(1, "x")]); + assert!(matches!(c.apply(&fits).0, Outcome::Applied { .. })); + assert_eq!(c.apply(&fits).0, Outcome::Duplicate); + let expect = decode(&target(&[cell(1, "x"), cell(2, "b")])); + assert_eq!(c.held(&shape()), Some(&expect)); + assert_eq!(c.watermark(&shape()), Some(at(1, 6))); +} + +#[test] +fn a_moved_key_the_target_no_longer_holds_is_removed() { + let mut c = synced_at(4, &[cell(1, "a"), cell(2, "b")]); + c.apply(&moved_correction(4, 5, &[2], &[])); + assert_eq!(c.held(&shape()), Some(&decode(&target(&[cell(1, "a")])))); +} + +#[test] +fn a_dropped_connection_or_failed_request_asks_from_the_watermark() { + let mut c = synced_at(4, &[cell(1, "a")]); + c.connection_lost(); + assert_eq!( + c.wants(&shape()), + Some(Want::Correction { + watermark: Some(at(1, 4)) + }) + ); + c.apply(&moved_correction(4, 4, &[], &[])); + assert_eq!(c.wants(&shape()), None, "an empty answer settles it"); + + c.request_failed(&shape()); + assert!(matches!(c.wants(&shape()), Some(Want::Correction { .. }))); +} + +#[test] +fn a_fetch_does_not_advance_past_the_notices_it_did_not_name() { + let mut c = synced_at(4, &[cell(1, "a")]); + c.on_notice(¬ice(1, 5, &[1])); + c.on_notice(¬ice(1, 6, &[2])); + c.apply(&moved_correction(4, 5, &[1], &[cell(1, "x")])); + assert_eq!(c.watermark(&shape()), Some(at(1, 5))); + assert_eq!( + c.wants(&shape()), + Some(Want::Fetch { + watermark: Some(at(1, 5)), + keys: [key(2)].into(), + through: at(1, 6) + }) + ); +} + +#[test] +fn carried_axis_changes_are_applied_once_and_in_order() { + let mut c = synced_at(4, &[]); + let mut n = notice(1, 5, &[]); + n.axis = vec![ + AxisChange::CategoryAdded { + category: "X".into(), + }, + AxisChange::ItemInserted { + category: "X".into(), + item: "i".into(), + at: 0, + }, + ]; + let events = c.on_notice(&n); + assert_eq!(events.len(), 2, "{events:?}"); + let mut carried = moved_correction(4, 5, &[], &[]); + carried.carried = Some(n); + let (_, again) = c.apply(&carried); + assert_eq!(again, Vec::new(), "the same changes must not apply twice"); +} + +#[test] +fn a_gap_in_structure_asks_the_consumer_to_resync() { + let mut c = synced_at(4, &[]); + assert_eq!(c.on_notice(¬ice(1, 7, &[])), vec![Event::Resync]); +} diff --git a/crates/improvise-protocol/src/delivery/exchange.rs b/crates/improvise-protocol/src/delivery/exchange.rs new file mode 100644 index 00000000..adccc3d4 --- /dev/null +++ b/crates/improvise-protocol/src/delivery/exchange.rs @@ -0,0 +1,94 @@ +//! Requests, corrections and the edge between a client and the host (ADR 009 clause 9). +//! Why each choice: `context/code/delivery.md` §3. + +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; + +use super::notice::{Notice, ShapeKey, Stamp}; +use super::wire::{WireElem, WireKey, wide}; + +/// The cross-product rectangle a request costs O(visible) for. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Window { + #[serde(with = "wide")] + pub top: u128, + #[serde(with = "wide")] + pub left: u128, + #[serde(with = "wide")] + pub rows: u128, + #[serde(with = "wide")] + pub cols: u128, +} + +impl Window { + pub fn contains(&self, row: u128, col: u128) -> bool { + (self.top..self.top.saturating_add(self.rows)).contains(&row) + && (self.left..self.left.saturating_add(self.cols)).contains(&col) + } +} + +/// Opaque transport, never read by the host (clause 9, "an edge is a session"). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SessionToken(pub String); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Ask { + /// Everything since `watermark`; `None` is a full refetch, `target - empty`. + Correction { watermark: Option }, + /// The current values of `keys`, covering the publications up to `through`. + Fetch { + watermark: Option, + keys: BTreeSet, + through: Stamp, + }, + /// The notices after `after`, if the host's log still has them. + Notices { after: Stamp }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Request { + pub session: Option, + pub shape: ShapeKey, + pub window: Window, + pub ask: Ask, +} + +/// What a correction replaces on the client: the keys it names, or the whole window. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum CorrectionBody { + Moved { keys: BTreeSet }, + Window, +} + +/// `target`, restricted as `body` says, read at `at` and covering `(from, through]`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Correction { + pub shape: ShapeKey, + pub at: Stamp, + pub from: Option, + pub through: Stamp, + pub body: CorrectionBody, + pub target: Vec, + /// The axis changes, relabels and flags of `(from, through]`, in order, as a merged notice. + pub carried: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Reply { + Correction(Correction), + Notices(Vec), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeError { + /// The request did not complete: dropped, timed out or refused by the transport. + Failed, + /// What arrived could not be read. + Unreadable, +} + +/// One end of an edge: a request goes out and a reply, or a failure, comes back. +pub trait Edge { + fn call(&mut self, request: Request) -> Result; +} diff --git a/crates/improvise-protocol/src/delivery/mod.rs b/crates/improvise-protocol/src/delivery/mod.rs new file mode 100644 index 00000000..38ef9804 --- /dev/null +++ b/crates/improvise-protocol/src/delivery/mod.rs @@ -0,0 +1,18 @@ +//! Delivery (ADR 009 clause 9): the wire form, the stamp and notice, the correction +//! and the client end of an edge. The host end is in `improvise-command`. +//! Why: `context/code/delivery.md`. + +pub mod client; +pub mod exchange; +pub mod notice; +pub mod wire; + +pub use client::{EdgeClient, Event, Outcome, Synced, Want}; +pub use exchange::{ + Ask, Correction, CorrectionBody, Edge, EdgeError, Reply, Request, SessionToken, Window, +}; +pub use notice::{AxisChange, Flags, Moved, Notice, PubRange, Relabel, ShapeKey, Stamp}; +pub use wire::{Cell, WireElem, WireKey, WireNum, WireValue}; + +#[cfg(test)] +mod client_tests; diff --git a/crates/improvise-protocol/src/delivery/notice.rs b/crates/improvise-protocol/src/delivery/notice.rs new file mode 100644 index 00000000..b4491dcf --- /dev/null +++ b/crates/improvise-protocol/src/delivery/notice.rs @@ -0,0 +1,300 @@ +//! The stamp, the change notice and the merge of notices (ADR 009 clause 9). +//! Why each choice: `context/code/delivery.md` §2. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use super::wire::{WireKey, wide}; + +/// A logical position in a host's stream: never a wall clock. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct Stamp { + #[serde(with = "wide")] + pub host: u64, + #[serde(with = "wide")] + pub generation: u64, + #[serde(with = "wide")] + pub publication: u64, +} + +impl Stamp { + pub fn same_epoch(&self, other: &Stamp) -> bool { + self.host == other.host && self.generation == other.generation + } + + /// A stamp from another host instance or generation is behind every current one. + pub fn is_behind(&self, current: &Stamp) -> bool { + !self.same_epoch(current) || self.publication < current.publication + } +} + +/// The publications `(from, to]` one notice covers, in one epoch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct PubRange { + #[serde(with = "wide")] + pub host: u64, + #[serde(with = "wide")] + pub generation: u64, + #[serde(with = "wide")] + pub from: u64, + #[serde(with = "wide")] + pub to: u64, +} + +impl PubRange { + pub fn one(at: Stamp) -> Self { + Self { + host: at.host, + generation: at.generation, + from: at.publication - 1, + to: at.publication, + } + } + + pub fn end(&self) -> Stamp { + Stamp { + host: self.host, + generation: self.generation, + publication: self.to, + } + } + + pub fn start(&self) -> Stamp { + Stamp { + host: self.host, + generation: self.generation, + publication: self.from, + } + } +} + +/// A named query: the key operator state is selected by (clause 7). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct ShapeKey(pub String); + +/// A change to the items an axis is made of. The client re-derives positions from these. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum AxisChange { + CategoryAdded { + category: String, + }, + CategoryRemoved { + category: String, + }, + ItemInserted { + category: String, + item: String, + at: usize, + }, + ItemRemoved { + category: String, + item: String, + }, + ItemsReordered { + category: String, + items: Vec, + }, +} + +/// A display label that moved: the id and what to show. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Relabel { + pub id: String, + pub display: String, +} + +/// What moved that no cell key names. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Flags { + pub formulas: bool, + pub measure_agg: bool, + pub views: bool, + pub default_view: bool, +} + +impl Flags { + pub fn any(&self) -> bool { + self.formulas || self.measure_agg || self.views || self.default_view + } + + pub fn union(self, other: Flags) -> Flags { + Flags { + formulas: self.formulas || other.formulas, + measure_agg: self.measure_agg || other.measure_agg, + views: self.views || other.views, + default_view: self.default_view || other.default_view, + } + } +} + +/// The output keys of one shape that moved, or "correct" when the list was too long to send. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Moved { + Keys(BTreeSet), + Correct, +} + +impl Moved { + fn union(self, other: Moved) -> Moved { + match (self, other) { + (Moved::Keys(mut a), Moved::Keys(b)) => { + a.extend(b); + Moved::Keys(a) + } + _ => Moved::Correct, + } + } +} + +/// One committed command, or a run of them merged. Addressed to no client. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Notice { + pub range: PubRange, + pub axis: Vec, + pub relabels: Vec, + pub flags: Flags, + pub evicted: BTreeSet, + pub moved: BTreeMap, +} + +impl Notice { + pub fn empty(range: PubRange) -> Self { + Self { + range, + axis: Vec::new(), + relabels: Vec::new(), + flags: Flags::default(), + evicted: BTreeSet::new(), + moved: BTreeMap::new(), + } + } + + pub fn stamp(&self) -> Stamp { + self.range.end() + } + + /// `self` followed by `next`, if `next` starts where `self` ends. Anything else + /// has a gap or another epoch in it, and merging would hide a loss. + pub fn merge(mut self, next: Notice) -> Result { + let adjacent = self.range.end() == next.range.start(); + match adjacent { + false => Err((self, next)), + true => { + self.range.to = next.range.to; + self.axis.extend(next.axis); + self.relabels.extend(next.relabels); + self.flags = self.flags.union(next.flags); + self.evicted.extend(next.evicted); + for (shape, moved) in next.moved { + let merged = match self.moved.remove(&shape) { + Some(before) => before.union(moved), + None => moved, + }; + self.moved.insert(shape, merged); + } + Ok(self) + } + } + } + + /// This notice as one shape reads it: other shapes' key lists are not its business. + pub fn for_shape(&self, shape: &ShapeKey) -> Notice { + Notice { + moved: self + .moved + .get(shape) + .map(|m| BTreeMap::from([(shape.clone(), m.clone())])) + .unwrap_or_default(), + ..self.clone() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + pub(crate) fn stamp(publication: u64) -> Stamp { + Stamp { + host: 7, + generation: 1, + publication, + } + } + + pub(crate) fn key(n: u8) -> WireKey { + WireKey(vec![("C".into(), n.to_string())]) + } + + pub(crate) fn notice(publication: u64, moved: &[(&str, &[u8])]) -> Notice { + let mut n = Notice::empty(PubRange::one(stamp(publication))); + for (shape, keys) in moved { + n.moved.insert( + ShapeKey((*shape).into()), + Moved::Keys(keys.iter().map(|k| key(*k)).collect()), + ); + } + n + } + + #[test] + fn a_stamp_from_another_epoch_is_behind_every_current_one() { + let now = stamp(5); + assert!(stamp(4).is_behind(&now)); + assert!(!stamp(5).is_behind(&now)); + let other_host = Stamp { host: 8, ..stamp(99) }; + let other_generation = Stamp { generation: 2, ..stamp(99) }; + assert!(other_host.is_behind(&now), "a restart is behind"); + assert!(other_generation.is_behind(&now), "a swap is behind"); + } + + #[test] + fn adjacent_notices_merge_into_the_range_they_cover() { + let merged = notice(3, &[("a", &[1])]) + .merge(notice(4, &[("a", &[2]), ("b", &[9])])) + .unwrap(); + assert_eq!((merged.range.from, merged.range.to), (2, 4)); + assert_eq!( + merged.moved[&ShapeKey("a".into())], + Moved::Keys([key(1), key(2)].into()) + ); + assert!(merged.moved.contains_key(&ShapeKey("b".into()))); + } + + #[test] + fn a_gap_does_not_merge() { + let (a, b) = notice(3, &[]).merge(notice(5, &[])).unwrap_err(); + assert_eq!((a.range.to, b.range.to), (3, 5)); + let foreign = Notice::empty(PubRange::one(Stamp { host: 9, ..stamp(4) })); + assert!(notice(3, &[]).merge(foreign).is_err()); + } + + #[test] + fn merging_keeps_axis_changes_in_order_and_correct_absorbs_keys() { + let mut a = notice(1, &[("s", &[1])]); + a.axis.push(AxisChange::CategoryAdded { category: "X".into() }); + let mut b = notice(2, &[]); + b.moved.insert(ShapeKey("s".into()), Moved::Correct); + b.axis.push(AxisChange::ItemRemoved { + category: "X".into(), + item: "i".into(), + }); + b.flags.formulas = true; + let m = a.merge(b).unwrap(); + assert_eq!(m.axis.len(), 2); + assert!(matches!(m.axis[0], AxisChange::CategoryAdded { .. })); + assert!(matches!(m.axis[1], AxisChange::ItemRemoved { .. })); + assert_eq!(m.moved[&ShapeKey("s".into())], Moved::Correct); + assert!(m.flags.formulas && !m.flags.views); + } + + #[test] + fn a_notice_round_trips_and_a_wide_stamp_crosses_as_a_string() { + let wide_host = Stamp { host: u64::MAX, ..stamp(1 << 53) }; + let n = Notice::empty(PubRange::one(wide_host)); + let json = serde_json::to_string(&n).unwrap(); + assert!(json.contains(&format!("\"{}\"", u64::MAX)), "{json}"); + assert_eq!(serde_json::from_str::(&json).unwrap(), n); + } +} diff --git a/crates/improvise-protocol/src/delivery/wire.rs b/crates/improvise-protocol/src/delivery/wire.rs new file mode 100644 index 00000000..7686f805 --- /dev/null +++ b/crates/improvise-protocol/src/delivery/wire.rs @@ -0,0 +1,325 @@ +//! The lossless wire form of what flows on a delivery edge (ADR 009 clause 9). +//! Why each choice: `context/code/delivery.md` §1. + +use improvise_core::model::cell::{CellKey, CellValue}; +use numtower::Num; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use zset::ZSet; + +const EXACT_JSON: u128 = 1 << 53; + +/// An unsigned integer: a JSON number below 2^53, a decimal string above it. +pub mod wide { + use super::*; + + #[derive(Serialize, Deserialize)] + #[serde(untagged)] + enum Repr { + Small(u64), + Wide(String), + } + + pub fn serialize, S: Serializer>(n: &T, s: S) -> Result { + let n: u128 = (*n).into(); + match n < EXACT_JSON { + true => Repr::Small(n as u64).serialize(s), + false => Repr::Wide(n.to_string()).serialize(s), + } + } + + pub fn deserialize<'de, T: TryFrom, D: Deserializer<'de>>(d: D) -> Result { + let n: u128 = match Repr::deserialize(d)? { + Repr::Small(n) => n.into(), + Repr::Wide(text) => text.parse().map_err(serde::de::Error::custom)?, + }; + T::try_from(n).map_err(|_| serde::de::Error::custom("integer out of range")) + } +} + +/// A signed weight, with the same rule as [`wide`]. +pub mod wide_weight { + use super::*; + + #[derive(Serialize, Deserialize)] + #[serde(untagged)] + enum Repr { + Small(i64), + Wide(String), + } + + pub fn serialize(n: &i64, s: S) -> Result { + match u128::from(n.unsigned_abs()) < EXACT_JSON { + true => Repr::Small(*n).serialize(s), + false => Repr::Wide(n.to_string()).serialize(s), + } + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + match Repr::deserialize(d)? { + Repr::Small(n) => Ok(n), + Repr::Wide(text) => text.parse().map_err(serde::de::Error::custom), + } + } +} + +/// A number in ADR 005's canonical text. Equality is textual, so the sign of a zero survives. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct WireNum(String); + +impl WireNum { + pub fn of(n: &Num) -> Self { + Self(n.to_string()) + } + + pub fn of_f64(x: f64) -> Self { + Self::of(&Num::inexact(x)) + } + + /// `None` for text the tower does not write, so one value has one spelling. + pub fn parse(text: &str) -> Option { + let n = Num::from_canonical(text)?; + (n.to_string() == text).then(|| Self(text.to_string())) + } + + pub fn text(&self) -> &str { + &self.0 + } + + pub fn to_num(&self) -> Num { + Num::from_canonical(&self.0).expect("a WireNum holds canonical text by construction") + } +} + +impl Serialize for WireNum { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for WireNum { + fn deserialize>(d: D) -> Result { + let text = String::deserialize(d)?; + Self::parse(&text) + .ok_or_else(|| serde::de::Error::custom(format!("{text:?} is not a canonical number"))) + } +} + +/// A cell's value on the wire. Text is byte-exact; a number never crosses as a double. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum WireValue { + Num(WireNum), + Text(String), + Error(String), +} + +impl WireValue { + pub fn of(value: &CellValue) -> Self { + match value { + CellValue::Number(x) => Self::Num(WireNum::of_f64(*x)), + CellValue::Text(t) => Self::Text(t.clone()), + CellValue::Error(e) => Self::Error(e.clone()), + } + } + + /// Exact numbers narrow to `f64` here, because `CellValue` is still an `f64` (ADR 005). + pub fn to_cell_value(&self) -> CellValue { + match self { + Self::Num(n) => CellValue::Number(n.to_num().to_f64()), + Self::Text(t) => CellValue::Text(t.clone()), + Self::Error(e) => CellValue::Error(e.clone()), + } + } +} + +/// A cell named by its coordinate tuple (ids once ADR 007 lands). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct WireKey(pub Vec<(String, String)>); + +impl WireKey { + pub fn of(key: &CellKey) -> Self { + Self(key.0.clone()) + } +} + +pub type Cell = (WireKey, WireValue); + +/// One element of a Z-set on the wire. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WireElem { + pub key: WireKey, + pub value: WireValue, + #[serde(with = "wide_weight")] + pub weight: i64, +} + +pub fn encode(z: &ZSet) -> Vec { + z.iter() + .map(|((key, value), weight)| WireElem { + key: key.clone(), + value: value.clone(), + weight, + }) + .collect() +} + +pub fn decode(elems: &[WireElem]) -> ZSet { + elems + .iter() + .map(|e| ((e.key.clone(), e.value.clone()), e.weight)) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + fn json_round Deserialize<'de>>(v: &T) -> T { + serde_json::from_str(&serde_json::to_string(v).expect("serializes")).expect("reads back") + } + + #[test] + fn a_float_crosses_in_canonical_text_with_the_sign_of_zero() { + for (x, text) in [ + (0.1, "#i0.1"), + (-0.0, "#i-0.0"), + (0.0, "#i0.0"), + (f64::INFINITY, "inf"), + (f64::NEG_INFINITY, "-inf"), + (f64::NAN, "nan"), + ] { + let sent = WireValue::of(&CellValue::Number(x)); + let json = serde_json::to_string(&sent).unwrap(); + assert_eq!(json, format!(r#"{{"Num":"{text}"}}"#), "{x:?}"); + let CellValue::Number(y) = json_round(&sent).to_cell_value() else { + panic!("a number came back as something else") + }; + match x.is_nan() { + true => assert!(y.is_nan()), + false => assert_eq!(x.to_bits(), y.to_bits(), "{x:?} came back {y:?}"), + } + } + } + + #[test] + fn every_nan_is_the_one_token() { + let payload = f64::from_bits(0x7ff8_dead_beef_0001); + assert!(payload.is_nan()); + assert_eq!(WireNum::of_f64(payload), WireNum::of_f64(f64::NAN)); + } + + #[test] + fn exact_numbers_round_trip_beyond_a_double() { + let wide_decimal = Num::from_canonical("1234567890123456789.0123456789").unwrap(); + let fraction = Num::ratio(i128::MAX / 3, 7).unwrap(); + for n in [wide_decimal, fraction] { + let back = json_round(&WireNum::of(&n)).to_num(); + assert_eq!(back.to_string(), n.to_string()); + assert_eq!(back.is_exact(), n.is_exact()); + assert!(back.is_exact(), "{n} lost its exactness: {back}"); + } + } + + #[test] + fn text_that_the_tower_does_not_write_is_refused() { + for bad in ["1e3", " 1", "+1", "#i1", "1.50x", "0x10", ""] { + assert_eq!(WireNum::parse(bad), None, "{bad:?}"); + let json = serde_json::to_string(bad).unwrap(); + assert!(serde_json::from_str::(&json).is_err(), "{bad:?}"); + } + } + + #[test] + fn wide_integers_cross_as_strings_and_narrow_ones_as_numbers() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct P( + #[serde(with = "wide")] u128, + #[serde(with = "wide_weight")] i64, + ); + let below = (1u128 << 53) - 1; + assert_eq!( + serde_json::to_string(&P(below, -3)).unwrap(), + format!("[{below},-3]") + ); + let p = P(u128::MAX, i64::MIN); + let json = serde_json::to_string(&p).unwrap(); + assert_eq!(json, format!(r#"["{}","{}"]"#, u128::MAX, i64::MIN)); + assert_eq!(json_round(&p), p); + assert_eq!(json_round(&P(1 << 53, 1 << 53)), P(1 << 53, 1 << 53)); + let past = r#"["340282366920938463463374607431768211456",1]"#; + assert!(serde_json::from_str::

(past).is_err()); + } + + #[test] + fn text_is_byte_exact() { + for t in ["", " é ", "e\u{301}", "\u{0}\n\r\"\\", "\u{1F600}"] { + let v = WireValue::Text(t.to_string()); + assert_eq!(json_round(&v), v, "{t:?}"); + } + } + + fn any_f64() -> impl Strategy { + prop_oneof![ + any::(), + Just(-0.0), + Just(f64::INFINITY), + Just(f64::NEG_INFINITY), + Just(f64::NAN), + any::().prop_map(f64::from_bits), + ] + } + + fn a_decimal(mantissa: u64, scale: usize) -> Option { + let digits = format!("{mantissa:0>w$}", w = scale + 1); + let (whole, frac) = digits.split_at(digits.len() - scale); + let text = match scale { + 0 => whole.to_string(), + _ => format!("{whole}.{frac}"), + }; + Num::from_canonical(&text) + } + + fn any_value() -> impl Strategy { + prop_oneof![ + any_f64().prop_map(|x| WireValue::of(&CellValue::Number(x))), + (any::(), 1..i128::MAX) + .prop_filter_map("fits", |(n, d)| Num::ratio(n, d)) + .prop_map(|n| WireValue::Num(WireNum::of(&n))), + (any::(), 0usize..=28) + .prop_filter_map("decimal", |(m, s)| a_decimal(m, s)) + .prop_map(|n| WireValue::Num(WireNum::of(&n))), + any::().prop_map(WireValue::Text), + any::().prop_map(WireValue::Error), + ] + } + + proptest! { + #[test] + fn a_value_is_the_identity_through_the_wire(v in any_value()) { + prop_assert_eq!(json_round(&v), v); + } + + #[test] + fn a_float_keeps_every_bit_but_the_nan_payload(x in any_f64()) { + let sent = WireValue::of(&CellValue::Number(x)); + let CellValue::Number(y) = json_round(&sent).to_cell_value() else { + panic!("a number came back as something else") + }; + match x.is_nan() { + true => prop_assert!(y.is_nan()), + false => prop_assert_eq!(x.to_bits(), y.to_bits()), + } + } + + #[test] + fn a_zset_is_the_identity_through_the_wire( + elems in proptest::collection::vec((0u8..4, any_value(), -3i64..=3), 0..12) + ) { + let z: ZSet = elems + .into_iter() + .map(|(k, v, w)| ((WireKey(vec![("c".into(), k.to_string())]), v), w)) + .collect(); + prop_assert_eq!(decode(&json_round(&encode(&z))), z); + } + } +} diff --git a/crates/improvise-protocol/src/lib.rs b/crates/improvise-protocol/src/lib.rs index 6ac68f17..130e1391 100644 --- a/crates/improvise-protocol/src/lib.rs +++ b/crates/improvise-protocol/src/lib.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; pub mod basis; pub mod client; +pub mod delivery; pub mod face; pub mod frame; pub mod help; From a4c309642f964a2b9ef4d5f8d6dc37c56a5aced4 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 11:54:51 -0700 Subject: [PATCH 02/12] fix(numtower): a zero ExactSum has the sign IEEE gives its addends (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/test-evidence/numtower.md | 19 +++++ crates/numtower/src/sum.rs | 25 +++++- .../tests/zero_sign.proptest-regressions | 10 +++ crates/numtower/tests/zero_sign.rs | 80 +++++++++++++++++++ 4 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 crates/numtower/tests/zero_sign.proptest-regressions create mode 100644 crates/numtower/tests/zero_sign.rs diff --git a/context/test-evidence/numtower.md b/context/test-evidence/numtower.md index c57949a3..3e04045f 100644 --- a/context/test-evidence/numtower.md +++ b/context/test-evidence/numtower.md @@ -466,3 +466,22 @@ factors (3, 77, 2^20, 3^40), so the lcm and the product differ. The rival combines over the product of the denominators. Its value is right, so only the lcm assertion can see it. That is why the property asserts the denominator before the value. + +### a_sum_of_negative_zeros_is_negative_zero + +`ExactSum` kept only limbs, so every zero sum read `+0.0` and a lone `-0.0` addend lost its sign +(ADR 009 clause 8 asks for IEEE's). Injected fault: `negative_zero()` always false. This test, the +property below and the retraction property all fail, each by name. + +### a_zero_sum_has_the_sign_ieee_gives_the_same_addends + +The oracle is an IEEE fold from `-0.0` over the same addends. The generator builds zero sums on +purpose (zeros of either sign plus cancelling pairs, shuffled), because filtering random addends +for a zero sum rejected nearly every case. Rival: the sign rule without the `others` count +(`neg_zeros > 0` alone) fails `one_positive_zero_makes_the_zero_positive_and_so_does_a_cancellation` +and both properties, since one `+0.0` or one cancelling pair must make the zero positive. + +### a_retracted_addend_leaves_the_sign_of_the_rest + +Adds `keep` and `gone`, then retracts `gone`; the read and the sum itself must equal a fresh sum +of `keep`. It is why both counters are net weights and why `PartialEq` compares the sign. diff --git a/crates/numtower/src/sum.rs b/crates/numtower/src/sum.rs index 366edbb5..63f7a97a 100644 --- a/crates/numtower/src/sum.rs +++ b/crates/numtower/src/sum.rs @@ -27,6 +27,10 @@ pub struct ExactSum { nan: i128, pos_inf: i128, neg_inf: i128, + /// Net weight of the `-0.0` addends, and of every other finite one: a zero sum is + /// `-0.0` only when the first is positive and the second is nil (IEEE). + neg_zeros: i128, + others: i128, } /// Equal sums, not equal representations: carries propagated, `pending` ignored. @@ -36,7 +40,9 @@ impl PartialEq for ExactSum { propagate(&mut mine); propagate(&mut theirs); let counts = |s: &ExactSum| (s.nan, s.pos_inf, s.neg_inf); - mine == theirs && counts(self) == counts(o) + mine == theirs + && counts(self) == counts(o) + && self.negative_zero() == o.negative_zero() } } @@ -50,6 +56,8 @@ impl Default for ExactSum { nan: 0, pos_inf: 0, neg_inf: 0, + neg_zeros: 0, + others: 0, } } } @@ -66,6 +74,10 @@ impl ExactSum { (false, true, true) => self.pos_inf += weight as i128, (false, true, false) => self.neg_inf += weight as i128, (false, false, _) => { + match x == 0.0 && x.is_sign_negative() { + true => self.neg_zeros += weight as i128, + false => self.others += weight as i128, + } let magnitude = weight.unsigned_abs(); for j in 0..3 { let digit = (magnitude >> (30 * j)) & (CHUNK - 1); @@ -122,6 +134,12 @@ impl ExactSum { self.nan += o.nan; self.pos_inf += o.pos_inf; self.neg_inf += o.neg_inf; + self.neg_zeros += o.neg_zeros; + self.others += o.others; + } + + fn negative_zero(&self) -> bool { + self.neg_zeros > 0 && self.others == 0 } /// The exact sum rounded to the nearest f64, ties to even. @@ -130,7 +148,10 @@ impl ExactSum { (true, _, _) | (false, true, true) => f64::NAN, (false, true, false) => f64::INFINITY, (false, false, true) => f64::NEG_INFINITY, - (false, false, false) => ratio_to_f64(&self.total(), &(BigInt::one() << 1074)), + (false, false, false) => match self.negative_zero() { + true => -0.0, + false => ratio_to_f64(&self.total(), &(BigInt::one() << 1074)), + }, } } diff --git a/crates/numtower/tests/zero_sign.proptest-regressions b/crates/numtower/tests/zero_sign.proptest-regressions new file mode 100644 index 00000000..671a5deb --- /dev/null +++ b/crates/numtower/tests/zero_sign.proptest-regressions @@ -0,0 +1,10 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 7c90e519b344ca86376f5a0b214d39873c790abd54bb140d369dac66bf4157dd # shrinks to mut addends = [], pairs = [] +cc 05e479a0606a42e8b9d31945d9ee2c548bff3b487f85189dff10c750ca1a2708 # shrinks to keep = [-3.0, 1e-300, 3.0], gone = [] +cc ba72a5a0e795ac7bb6f0833b21286f9da2cd9ec82c20224550e0fe030791e906 # shrinks to addends = [-0.0] +cc 1b8f452c472f86228458b4a4ce0e904c63335d40d3aaab9c0e7606a98b54d771 # shrinks to keep = [-0.0], gone = [] diff --git a/crates/numtower/tests/zero_sign.rs b/crates/numtower/tests/zero_sign.rs new file mode 100644 index 00000000..18c7e06f --- /dev/null +++ b/crates/numtower/tests/zero_sign.rs @@ -0,0 +1,80 @@ +//! ADR 009 step 8's last test: the sign of a zero sum is IEEE's. Evidence: `context/test-evidence/numtower.md`. + +use numtower::ExactSum; +use proptest::prelude::*; + +fn addend() -> impl Strategy { + prop_oneof![ + Just(-0.0), + Just(0.0), + Just(0.5), + Just(-0.5), + Just(3.0), + Just(-3.0), + ] +} + +/// IEEE addition from `-0.0`, the identity that keeps a `-0.0` addend's sign. +fn ieee(addends: &[f64]) -> f64 { + addends.iter().fold(-0.0, |sum, x| sum + x) +} + +fn exact(addends: &[f64]) -> f64 { + let mut sum = ExactSum::new(); + addends.iter().for_each(|x| sum.add(*x, 1)); + sum.value() +} + +#[test] +fn a_sum_of_negative_zeros_is_negative_zero() { + assert_eq!(exact(&[-0.0, -0.0]).to_bits(), (-0.0f64).to_bits()); + assert_eq!(exact(&[-0.0]).to_bits(), (-0.0f64).to_bits()); +} + +#[test] +fn one_positive_zero_makes_the_zero_positive_and_so_does_a_cancellation() { + assert_eq!(exact(&[-0.0, 0.0]).to_bits(), 0.0f64.to_bits()); + assert_eq!(exact(&[0.5, -0.5]).to_bits(), 0.0f64.to_bits()); + assert_eq!(exact(&[-0.0, 0.5, -0.5]).to_bits(), 0.0f64.to_bits()); +} + +#[test] +fn an_empty_sum_is_positive_zero() { + assert_eq!(ExactSum::new().value().to_bits(), 0.0f64.to_bits()); +} + +/// Addends whose exact sum is zero: some zeros of either sign, and cancelling pairs. +fn zero_sum() -> impl Strategy> { + let zeros = proptest::collection::vec(prop_oneof![Just(-0.0), Just(0.0)], 0..5); + let pairs = proptest::collection::vec(prop_oneof![Just(0.5), Just(-3.0)], 0..3); + (zeros, pairs) + .prop_map(|(mut all, pairs)| { + all.extend(pairs.iter().flat_map(|p| [*p, -*p])); + all + }) + .prop_filter("non-empty", |all| !all.is_empty()) + .prop_shuffle() +} + +proptest! { + #[test] + fn a_zero_sum_has_the_sign_ieee_gives_the_same_addends(addends in zero_sum()) { + let want = ieee(&addends); + prop_assert_eq!(want, 0.0, "the generator must build zero sums"); + prop_assert_eq!(exact(&addends).to_bits(), want.to_bits()); + } + + #[test] + fn a_retracted_addend_leaves_the_sign_of_the_rest( + keep in zero_sum(), + gone in proptest::collection::vec(addend(), 0..6), + ) { + let mut sum = ExactSum::new(); + keep.iter().chain(&gone).for_each(|x| sum.add(*x, 1)); + gone.iter().for_each(|x| sum.add(*x, -1)); + prop_assert_eq!(sum.value().to_bits(), ieee(&keep).to_bits()); + let mut fresh = ExactSum::new(); + keep.iter().for_each(|x| fresh.add(*x, 1)); + prop_assert_eq!(sum, fresh); + } +} From 19cea8ededa047ad37ec4d5a2888331611506f70 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 11:59:30 -0700 Subject: [PATCH 03/12] feat(command): the host end of delivery, one notice per committed command, with the log and corrections (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- Cargo.lock | 2 + crates/improvise-command/Cargo.toml | 2 + crates/improvise-command/src/ui/delivery.rs | 505 ++++++++++ .../src/ui/delivery_tests.rs | 866 ++++++++++++++++++ crates/improvise-command/src/ui/host.rs | 64 +- crates/improvise-command/src/ui/mod.rs | 4 + .../improvise-protocol/src/delivery/client.rs | 55 +- .../src/delivery/client_tests.rs | 6 +- .../src/delivery/exchange.rs | 4 +- 9 files changed, 1480 insertions(+), 28 deletions(-) create mode 100644 crates/improvise-command/src/ui/delivery.rs create mode 100644 crates/improvise-command/src/ui/delivery_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 1936579d..be376f6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -909,11 +909,13 @@ dependencies = [ "improvise-gp", "improvise-io", "improvise-protocol", + "proptest", "serde", "serde_json", "tempfile", "unicode-width", "web-time", + "zset", ] [[package]] diff --git a/crates/improvise-command/Cargo.toml b/crates/improvise-command/Cargo.toml index bc1ef6e9..7d8e6a4c 100644 --- a/crates/improvise-command/Cargo.toml +++ b/crates/improvise-command/Cargo.toml @@ -12,6 +12,7 @@ improvise-protocol = { path = "../improvise-protocol" } improvise-formula = { path = "../improvise-formula" } improvise-io = { path = "../improvise-io" } improvise-gp = { path = "../improvise-gp" } +zset = { path = "../zset" } anyhow = "1" serde = { version = "1", features = ["derive"] } # Parses JSON a user's numbers cross, so it must read back the f64 written: per crate, @@ -26,3 +27,4 @@ web-time = "1" improvise-core = { path = "../improvise-core", features = ["test-support"] } improvise-protocol = { path = "../improvise-protocol", features = ["test-support"] } tempfile = "3" +proptest = "1" diff --git a/crates/improvise-command/src/ui/delivery.rs b/crates/improvise-command/src/ui/delivery.rs new file mode 100644 index 00000000..96c6b4cf --- /dev/null +++ b/crates/improvise-command/src/ui/delivery.rs @@ -0,0 +1,505 @@ +//! The host end of delivery: one notice per committed command, the shared bounded +//! log, and corrections (ADR 009 clause 9). Why: `context/code/delivery.md` §5-§7. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::sync::Arc; + +use improvise_core::view::layout::GridLayout; +use improvise_core::workbook::Workbook; +use improvise_protocol::delivery::wire::{self, Cell, WireKey, WireValue}; +use improvise_protocol::delivery::{ + Ask, AxisChange, Correction, CorrectionBody, Edge, EdgeError, Flags, Moved, Notice, PubRange, + Reply, Request, ShapeKey, Stamp, Window, +}; +use zset::ZSet; + +use crate::ui::host::Host; + +/// The most notices the log keeps, and the most keys one shape lists in one notice. +pub const LOG_CAPACITY: usize = 1024; +pub const MAX_KEYS: usize = 4096; + +const EAGER: &str = "eager:view/"; + +/// The Eager stand-in for a shape key: the view it draws (`context/code/delivery.md` §5). +pub fn shape_of_view(view: &str) -> ShapeKey { + ShapeKey(format!("{EAGER}{view}")) +} + +fn view_of(shape: &ShapeKey) -> Option<&str> { + shape.0.strip_prefix(EAGER) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Placed { + row: u128, + col: u128, + value: WireValue, +} + +/// A shape's whole output, keyed by tuple identity with the position attached. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Output { + cells: BTreeMap, +} + +impl Output { + pub fn of(wb: &Workbook, view: &str) -> Option { + let model = &wb.model; + let layout = GridLayout::with_frozen_records(model, wb.views.get(view)?, None); + let eval = layout.evaluator(model); + let mut cells = BTreeMap::new(); + for row in 0..layout.row_count() { + for col in 0..layout.col_count() { + let Some(key) = layout.cell_key(row, col) else { + continue; + }; + let value = match layout.is_records_mode() { + true => layout.records_value(row, col), + false => eval.cell(&key), + }; + if let Some(value) = value { + cells.insert( + WireKey::of(&key), + Placed { + row: row as u128, + col: col as u128, + value: WireValue::of(&value), + }, + ); + } + } + } + Some(Output { cells }) + } + + /// The cells in `window` as a Z-set of weight one: the target a correction is read from. + pub fn target(&self, window: &Window, only: Option<&BTreeSet>) -> ZSet { + self.cells + .iter() + .filter(|(k, p)| window.contains(p.row, p.col) && only.is_none_or(|s| s.contains(k))) + .map(|(k, p)| ((k.clone(), p.value.clone()), 1)) + .collect() + } + + /// The keys whose value, or presence, differs. A moved position is an axis change, not a key. + fn moved_since(&self, before: &Output) -> BTreeSet { + let changed = self + .cells + .iter() + .filter(|(k, p)| before.cells.get(*k).is_none_or(|b| b.value != p.value)); + let gone = before.cells.keys().filter(|k| !self.cells.contains_key(*k)); + changed.map(|(k, _)| k).chain(gone).cloned().collect() + } +} + +/// What a notice reports besides cells: the items on each axis and the document-wide settings. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +struct Structure { + categories: Vec<(String, Vec)>, + formulas: Vec, + measure_agg: String, + views: String, + default_view: String, +} + +impl Structure { + fn of(wb: &Workbook) -> Self { + let model = &wb.model; + let aggs: BTreeMap<_, _> = model.measure_agg.iter().collect(); + Self { + categories: model + .categories + .iter() + .map(|(name, cat)| (name.clone(), cat.items.keys().cloned().collect())) + .collect(), + formulas: model.formulas().iter().map(|f| f.raw.clone()).collect(), + measure_agg: format!("{aggs:?}"), + views: format!("{:?}", wb.views), + default_view: wb.active_view.clone(), + } + } + + fn since(&self, before: &Structure) -> (Vec, Flags) { + let flags = Flags { + formulas: self.formulas != before.formulas, + measure_agg: self.measure_agg != before.measure_agg, + views: self.views != before.views, + default_view: self.default_view != before.default_view, + }; + (axis_changes(&before.categories, &self.categories), flags) + } +} + +fn axis_changes(before: &[(String, Vec)], after: &[(String, Vec)]) -> Vec { + let held: BTreeMap<_, _> = before.iter().map(|(c, i)| (c.as_str(), i)).collect(); + let now: BTreeSet<_> = after.iter().map(|(c, _)| c.as_str()).collect(); + let gone = before + .iter() + .filter(|(c, _)| !now.contains(c.as_str())) + .map(|(category, _)| AxisChange::CategoryRemoved { + category: category.clone(), + }); + let rest = after.iter().flat_map(|(category, items)| { + let Some(was) = held.get(category.as_str()) else { + let added = AxisChange::CategoryAdded { + category: category.clone(), + }; + let inserted = items.iter().enumerate().map(|(at, item)| AxisChange::ItemInserted { + category: category.clone(), + item: item.clone(), + at, + }); + return std::iter::once(added).chain(inserted).collect::>(); + }; + item_changes(category, was, items) + }); + gone.chain(rest).collect() +} + +fn item_changes(category: &str, was: &[String], now: &[String]) -> Vec { + let removed = was + .iter() + .filter(|i| !now.contains(i)) + .map(|item| AxisChange::ItemRemoved { + category: category.into(), + item: item.clone(), + }); + let inserted = now + .iter() + .enumerate() + .filter(|(_, i)| !was.contains(i)) + .map(|(at, item)| AxisChange::ItemInserted { + category: category.into(), + item: item.clone(), + at, + }); + let kept_before: Vec<_> = was.iter().filter(|i| now.contains(i)).collect(); + let kept_after: Vec<_> = now.iter().filter(|i| was.contains(i)).collect(); + let reordered = (kept_before != kept_after).then(|| AxisChange::ItemsReordered { + category: category.into(), + items: now.to_vec(), + }); + removed.chain(inserted).chain(reordered).collect() +} + +#[derive(Debug, Clone)] +struct Live { + view: String, + since: u64, + last_touch: u64, + output: Arc, +} + +/// The publisher, the shared log and the live shapes of one document. Cheap to clone, +/// so a frame that fails can put it back (`context/code/delivery.md` §6). +#[derive(Debug, Clone)] +pub struct Delivery { + host: u64, + generation: u64, + publication: u64, + log: VecDeque>, + capacity: usize, + max_keys: usize, + live: BTreeMap, + structure: Arc, + evicted: BTreeSet, + outbox: Vec>, +} + +impl Delivery { + /// `host` is a random instance id, never a time (clause 9). + pub fn new(host: u64, wb: &Workbook, generation: u64) -> Self { + Self { + host, + generation, + publication: 0, + log: VecDeque::new(), + capacity: LOG_CAPACITY, + max_keys: MAX_KEYS, + live: BTreeMap::new(), + structure: Arc::new(Structure::of(wb)), + evicted: BTreeSet::new(), + outbox: Vec::new(), + } + } + + pub fn set_limits(&mut self, capacity: usize, max_keys: usize) { + self.capacity = capacity; + self.max_keys = max_keys; + } + + pub fn stamp(&self) -> Stamp { + Stamp { + host: self.host, + generation: self.generation, + publication: self.publication, + } + } + + pub fn log_len(&self) -> usize { + self.log.len() + } + + pub fn is_live(&self, shape: &ShapeKey) -> bool { + self.live.contains_key(shape) + } + + /// Notices published since the last call, for the transport to push. + pub fn take_outbox(&mut self) -> Vec { + std::mem::take(&mut self.outbox) + .into_iter() + .map(|n| (*n).clone()) + .collect() + } + + /// One command committed: publish its notice. The document is read once, here, and every + /// read after it sees this state until the next commit. + pub fn commit(&mut self, wb: &Workbook, generation: u64) -> Stamp { + let new_epoch = generation != self.generation; + if new_epoch { + self.generation = generation; + self.log.clear(); + } + self.publication += 1; + let range = PubRange::one(self.stamp()); + let structure = Arc::new(Structure::of(wb)); + let (axis, flags) = match new_epoch { + true => (Vec::new(), all_flags()), + false => structure.since(&self.structure), + }; + self.structure = structure; + + let mut notice = Notice::empty(range); + notice.axis = axis; + notice.flags = flags; + notice.evicted = std::mem::take(&mut self.evicted); + let max_keys = self.max_keys; + self.live.retain(|shape, live| { + let Some(output) = Output::of(wb, &live.view) else { + notice.evicted.insert(shape.clone()); + return false; + }; + let keys = output.moved_since(&live.output); + let moved = match (new_epoch, keys.len() > max_keys) { + (false, false) => Moved::Keys(keys), + _ => Moved::Correct, + }; + if moved != Moved::Keys(BTreeSet::new()) { + notice.moved.insert(shape.clone(), moved); + } + live.output = Arc::new(output); + true + }); + let notice = Arc::new(notice); + self.log.push_back(Arc::clone(¬ice)); + while self.log.len() > self.capacity { + self.log.pop_front(); + } + self.outbox.push(notice); + self.stamp() + } + + /// A request names `shape`: it goes live if it was not, and is not idle. + pub fn touch(&mut self, shape: &ShapeKey, wb: &Workbook, now: u64) { + match self.live.get_mut(shape) { + Some(live) => live.last_touch = now, + None => { + let Some(view) = view_of(shape) else { return }; + let Some(output) = Output::of(wb, view) else { + return; + }; + self.live.insert( + shape.clone(), + Live { + view: view.to_string(), + since: self.publication, + last_touch: now, + output: Arc::new(output), + }, + ); + } + } + } + + /// Drop shapes no request has named for `idle`, a local duration in the caller's unit. + pub fn evict_idle(&mut self, now: u64, idle: u64) -> Vec { + let gone: Vec<_> = self + .live + .iter() + .filter(|(_, l)| now.saturating_sub(l.last_touch) >= idle) + .map(|(s, _)| s.clone()) + .collect(); + for shape in &gone { + self.live.remove(shape); + self.evicted.insert(shape.clone()); + } + gone + } + + /// Answer a request from the last committed state. + pub fn answer(&mut self, wb: &Workbook, request: &Request, now: u64) -> Reply { + self.touch(&request.shape, wb, now); + match &request.ask { + Ask::Notices { after } => match self.covered(&request.shape, Some(after)) { + Some(after) => Reply::Notices(self.notices_after(&request.shape, after, None)), + None => Reply::Correction(self.window(request, None, self.stamp(), None)), + }, + Ask::Correction { watermark } => match self.covered(&request.shape, watermark.as_ref()) + { + Some(from) => self.moved(request, from, self.stamp(), None, false), + None => Reply::Correction(self.window(request, None, self.stamp(), None)), + }, + Ask::Fetch { + watermark, + keys, + through, + window, + } => { + let sane = through.same_epoch(&self.stamp()) + && through.publication <= self.publication; + match (self.covered(&request.shape, watermark.as_ref()), sane) { + (Some(from), true) if from.publication <= through.publication => { + self.moved(request, from, *through, Some(keys), *window) + } + _ => Reply::Correction(self.window(request, None, self.stamp(), None)), + } + } + } + } + + /// The watermark, if the log and the shape's liveness reach back to it. + fn covered(&self, shape: &ShapeKey, watermark: Option<&Stamp>) -> Option { + let w = *watermark?; + let live = self.live.get(shape)?; + let log_start = self.log.front().map_or(self.publication, |n| n.range.from); + let reaches = w.publication <= self.publication + && (w.publication == self.publication + || (w.publication >= log_start && w.publication >= live.since)); + (w.same_epoch(&self.stamp()) && reaches).then_some(w) + } + + fn notices_after(&self, shape: &ShapeKey, after: Stamp, through: Option) -> Vec { + self.log + .iter() + .filter(|n| n.range.to > after.publication) + .filter(|n| through.is_none_or(|t| n.range.to <= t.publication)) + .map(|n| n.for_shape(shape)) + .collect() + } + + fn window( + &self, + request: &Request, + from: Option, + through: Stamp, + carried: Option, + ) -> Correction { + let target = self + .live + .get(&request.shape) + .map(|l| l.output.target(&request.window, None)) + .unwrap_or_default(); + Correction { + shape: request.shape.clone(), + at: self.stamp(), + from, + through, + body: CorrectionBody::Window, + target: wire::encode(&target), + carried, + } + } + + fn moved( + &self, + request: &Request, + from: Stamp, + through: Stamp, + only: Option<&BTreeSet>, + window: bool, + ) -> Reply { + let covering = self.notices_after(&request.shape, from, Some(through)); + let oversized = covering + .iter() + .any(|n| n.moved.get(&request.shape) == Some(&Moved::Correct)); + let carried = covering + .iter() + .cloned() + .reduce(|a, b| a.merge(b).unwrap_or_else(|(_, b)| b)); + let restructured = covering + .iter() + .any(|n| !n.axis.is_empty() || !n.relabels.is_empty()); + if oversized { + return Reply::Correction(self.window(request, None, self.stamp(), None)); + } + if window || restructured { + return Reply::Correction(self.window(request, Some(from), through, carried)); + } + let keys: BTreeSet = match only { + Some(keys) => keys.clone(), + None => covering + .iter() + .filter_map(|n| n.moved.get(&request.shape)) + .flat_map(|m| match m { + Moved::Keys(k) => k.iter().cloned().collect::>(), + Moved::Correct => Vec::new(), + }) + .collect(), + }; + let target = self + .live + .get(&request.shape) + .map(|l| l.output.target(&request.window, Some(&keys))) + .unwrap_or_default(); + Reply::Correction(Correction { + shape: request.shape.clone(), + at: self.stamp(), + from: Some(from), + through, + body: CorrectionBody::Moved { keys }, + target: wire::encode(&target), + carried, + }) + } +} + +fn all_flags() -> Flags { + Flags { + formulas: true, + measure_agg: true, + views: true, + default_view: true, + } +} + +/// An edge that is a function call: the host in this process. +pub struct InProcess<'a> { + pub host: &'a mut Host, + pub now: u64, +} + +impl Edge for InProcess<'_> { + fn call(&mut self, request: Request) -> Result { + Ok(self.host.deliver(&request, self.now)) + } +} + +/// An edge whose request and reply cross as JSON text: the adapter at a wire boundary. +pub struct WireEdge(pub E); + +impl Edge for WireEdge { + fn call(&mut self, request: Request) -> Result { + let sent = serde_json::to_string(&request).map_err(|_| EdgeError::Unreadable)?; + let arrived = serde_json::from_str(&sent).map_err(|_| EdgeError::Unreadable)?; + let reply = self.0.call(arrived)?; + let sent = serde_json::to_string(&reply).map_err(|_| EdgeError::Unreadable)?; + serde_json::from_str(&sent).map_err(|_| EdgeError::Unreadable) + } +} + +/// A notice as it crosses a wire. +pub fn notice_over_the_wire(notice: &Notice) -> Result { + let sent = serde_json::to_string(notice).map_err(|_| EdgeError::Unreadable)?; + serde_json::from_str(&sent).map_err(|_| EdgeError::Unreadable) +} diff --git a/crates/improvise-command/src/ui/delivery_tests.rs b/crates/improvise-command/src/ui/delivery_tests.rs new file mode 100644 index 00000000..d5fa8f3c --- /dev/null +++ b/crates/improvise-command/src/ui/delivery_tests.rs @@ -0,0 +1,866 @@ +//! Delivery against the Eager path: the host publishes, a client folds, and the client's +//! held cells must end at what the grid draws. Evidence: `context/test-evidence/delivery.md`. + +use std::collections::VecDeque; + +use improvise_core::model::cell::{CellKey, CellValue}; +use improvise_core::model::record::RecordHash; +use improvise_core::view::layout::GridLayout; +use improvise_core::view::Axis; +use improvise_core::workbook::Workbook; +use improvise_protocol::delivery::wire::{self, Cell, WireKey, WireValue}; +use improvise_protocol::delivery::{ + Ask, AxisChange, CorrectionBody, Edge, EdgeClient, EdgeError, Moved, Notice, Outcome, Reply, + Request, SessionToken, ShapeKey, Stamp, Synced, Window, +}; +use improvise_protocol::{ClientFrame, ModelCommand, Rejection, RecordCell, ServerFrame}; +use zset::ZSet; + +use super::app::App; +use super::delivery::{InProcess, WireEdge, notice_over_the_wire, shape_of_view}; +use super::host::Host; +use super::project::SessionId; + +fn workbook() -> Workbook { + let mut wb = Workbook::new("Test"); + wb.add_category("Region").unwrap(); + for item in ["North", "South"] { + wb.add_item("Region", item).unwrap(); + } + wb.add_item("_Measure", "Revenue").unwrap(); + for (region, value) in [("North", 10.0), ("South", 20.0)] { + let key = CellKey::new(vec![ + ("Region".to_string(), region.to_string()), + ("_Measure".to_string(), "Revenue".to_string()), + ]); + wb.model.set_cell(key, CellValue::Number(value)); + } + wb.active_view_mut().set_axis("Region", Axis::Row); + wb.active_view_mut().set_axis("_Measure", Axis::Column); + wb +} + +fn host_instance(instance: u64) -> Host { + let mut app = App::new(workbook(), None); + app.rebuild_layout(); + Host::with_instance(app, instance) +} + +fn host() -> Host { + host_instance(1) +} + +fn shape() -> ShapeKey { + shape_of_view("Default") +} + +fn everything() -> Window { + Window { + top: 0, + left: 0, + rows: 1000, + cols: 1000, + } +} + +fn record(region: &str) -> RecordCell { + let key = CellKey::new(vec![ + ("Region".to_string(), region.to_string()), + ("_Measure".to_string(), "Revenue".to_string()), + ]); + RecordCell { + record: RecordHash::of(&key), + measure: Some("Revenue".to_string()), + } +} + +fn set(region: &str, value: f64) -> ModelCommand { + ModelCommand::SetCell { + view: "Default".into(), + cell: record(region), + value: CellValue::Number(value), + } +} + +fn add_item(item: &str) -> ModelCommand { + ModelCommand::AddItem { + category: "Region".into(), + item: item.into(), + } +} + +/// A frame of caught-up commands: the first one's basis states every record the frame names. +fn frame(host: &Host, cmds: Vec) -> ClientFrame { + let mut wire: Vec = cmds + .into_iter() + .map(|c| crate::ui::reduce::caught_up(host.app(), c)) + .collect(); + let records: Vec<_> = wire + .iter() + .filter_map(|c| match c { + improvise_protocol::Command::Model { basis, .. } => Some(basis.records.clone()), + _ => None, + }) + .flatten() + .collect(); + if let Some(improvise_protocol::Command::Model { basis, .. }) = wire.first_mut() { + basis.records = records; + } + ClientFrame::Commands(wire) +} + +fn send(host: &mut Host, id: SessionId, cmds: Vec) -> Vec { + let f = frame(host, cmds); + host.handle(id, f).into_iter().map(|a| a.msg).collect() +} + +fn refused(answer: &[ServerFrame]) -> bool { + answer + .iter() + .any(|m| matches!(m, ServerFrame::Rejected(_))) +} + +/// What the grid draws, derived without the delivery module's own reader. +fn drawn(host: &Host, window: &Window) -> ZSet { + let wb = &host.app().model_state.workbook; + let layout = GridLayout::with_frozen_records(&wb.model, &wb.views["Default"], None); + let mut cells = ZSet::new(); + for row in 0..layout.row_count() { + for col in 0..layout.col_count() { + let Some(key) = layout.cell_key(row, col) else { + continue; + }; + if let Some(value) = layout.evaluate(&wb.model, &key) + && window.contains(row as u128, col as u128) + { + cells.insert((WireKey::of(&key), WireValue::of(&value)), 1); + } + } + } + cells +} + +#[derive(Clone, Copy, Debug, PartialEq)] +enum Fault { + DropRequest, + DropReply, + Garble, +} + +/// A transport that loses things on a script and remembers every request that reached it. +struct Flaky<'a> { + host: &'a mut Host, + script: VecDeque>, + reached: Vec, +} + +impl<'a> Flaky<'a> { + fn new(host: &'a mut Host) -> Self { + Self { + host, + script: VecDeque::new(), + reached: Vec::new(), + } + } + + fn scripted(host: &'a mut Host, script: Vec>) -> Self { + Self { + script: script.into(), + ..Self::new(host) + } + } +} + +impl Edge for Flaky<'_> { + fn call(&mut self, request: Request) -> Result { + let fault = self.script.pop_front().flatten(); + if fault == Some(Fault::DropRequest) { + return Err(EdgeError::Failed); + } + self.reached.push(request.clone()); + let reply = self.host.deliver(&request, 0); + match fault { + Some(Fault::DropReply) => Err(EdgeError::Failed), + Some(Fault::Garble) => Err(EdgeError::Unreadable), + _ => Ok(reply), + } + } +} + +fn session() -> Option { + Some(SessionToken("s-1".into())) +} + +/// A client that has fetched everything the host holds. +fn settled(host: &mut Host) -> EdgeClient { + let mut client = EdgeClient::new(session()); + client.track(shape(), everything()); + let (state, _) = client.sync(&mut WireEdge(InProcess { host, now: 0 }), &shape(), 6); + assert_eq!(state, Synced::Settled); + client +} + +fn connect(host: &mut Host) -> SessionId { + host.connect(10, 10).0 +} + +fn asks(reached: &[Request]) -> Vec<&Ask> { + reached.iter().map(|r| &r.ask).collect() +} + +fn deliver_all(client: &mut EdgeClient, notices: &[Notice]) { + for n in notices { + client.on_notice(¬ice_over_the_wire(n).expect("a notice crosses the wire")); + } +} + +fn held_equals_drawn(client: &EdgeClient, host: &Host) { + let want = drawn(host, &everything()); + assert_eq!(client.held(&shape()), Some(&want)); +} + +// ── one notice per committed command ─────────────────────────────────────── + +#[test] +fn a_frame_of_two_commands_publishes_one_notice_each_and_no_client_asks_for_a_correction() { + let mut host = host(); + let id = connect(&mut host); + let mut client = settled(&mut host); + let before = host.delivery().stamp(); + host.take_notices(); + + let answer = send(&mut host, id, vec![add_item("East"), set("North", 5.0)]); + assert!(!refused(&answer), "{answer:?}"); + let notices = host.take_notices(); + + let stamps: Vec = notices.iter().map(|n| n.stamp().publication).collect(); + assert_eq!( + stamps, + vec![before.publication + 1, before.publication + 2], + "one notice per committed command, numbered consecutively" + ); + deliver_all(&mut client, ¬ices); + let mut edge = Flaky::new(&mut host); + let (state, _) = client.sync(&mut edge, &shape(), 6); + assert_eq!(state, Synced::Settled); + assert!( + asks(&edge.reached) + .iter() + .all(|a| !matches!(a, Ask::Correction { .. })), + "no correction was asked for: {:?}", + edge.reached + ); + held_equals_drawn(&client, &host); +} + +#[test] +fn a_command_that_moves_nothing_still_publishes_and_a_view_command_does_not() { + let mut host = host(); + let id = connect(&mut host); + host.take_notices(); + send(&mut host, id, vec![set("North", 1.0)]); + send(&mut host, id, vec![set("North", 1.0)]); + let same = host.take_notices(); + assert_eq!(same.len(), 2); + assert_eq!(same[1].moved, Default::default(), "an equal write moves nothing"); + + host.handle( + id, + ClientFrame::command(improvise_protocol::Command::SetSelected { row: 1, col: 0 }), + ); + assert!(host.take_notices().is_empty(), "the cursor is the view store's"); +} + +#[test] +fn a_notice_names_the_moved_cells_including_a_formula_cell_it_did_not_write() { + let mut host = host(); + let id = connect(&mut host); + send( + &mut host, + id, + vec![ModelCommand::AddFormula { + raw: "Double = Revenue * 2".into(), + target_category: "_Measure".into(), + }], + ); + let _ = settled(&mut host); + host.take_notices(); + send(&mut host, id, vec![set("North", 4.0)]); + let notice = host.take_notices().remove(0); + let Some(Moved::Keys(keys)) = notice.moved.get(&shape()) else { + panic!("the live shape must be listed: {notice:?}") + }; + let measures: Vec<&str> = keys + .iter() + .flat_map(|k| k.0.iter()) + .filter(|(c, _)| c == "_Measure") + .map(|(_, i)| i.as_str()) + .collect(); + assert!(measures.contains(&"Revenue") && measures.contains(&"Double"), "{keys:?}"); +} + +#[test] +fn a_structural_command_reports_the_axis_change_and_a_formula_the_flag() { + let mut host = host(); + let id = connect(&mut host); + host.take_notices(); + send(&mut host, id, vec![add_item("East")]); + send(&mut host, id, vec![ModelCommand::RemoveItem { + category: "Region".into(), + item: "North".into(), + }]); + send( + &mut host, + id, + vec![ModelCommand::AddFormula { + raw: "Double = Revenue * 2".into(), + target_category: "_Measure".into(), + }], + ); + let notices = host.take_notices(); + assert_eq!( + notices[0].axis, + vec![AxisChange::ItemInserted { + category: "Region".into(), + item: "East".into(), + at: 2 + }] + ); + assert_eq!( + notices[1].axis, + vec![AxisChange::ItemRemoved { + category: "Region".into(), + item: "North".into() + }] + ); + assert!(notices[2].flags.formulas && !notices[1].flags.formulas); +} + +// ── loss, restart, refusal ───────────────────────────────────────────────── + +#[test] +fn a_dropped_notice_is_detected_and_a_correction_repairs_it() { + let mut host = host(); + let id = connect(&mut host); + let mut client = settled(&mut host); + host.take_notices(); + send(&mut host, id, vec![set("North", 1.0)]); + send(&mut host, id, vec![set("South", 2.0)]); + let notices = host.take_notices(); + deliver_all(&mut client, ¬ices[1..]); + + let mut edge = Flaky::new(&mut host); + let (state, _) = client.sync(&mut edge, &shape(), 6); + assert_eq!(state, Synced::Settled); + assert!( + matches!(asks(&edge.reached)[0], Ask::Correction { watermark: Some(_) }), + "{:?}", + edge.reached + ); + held_equals_drawn(&client, &host); +} + +#[test] +fn a_reconnect_after_a_host_restart_asks_for_a_correction_from_the_old_watermark() { + let mut first = host_instance(1); + let id = connect(&mut first); + send(&mut first, id, vec![set("North", 9.0)]); + let mut client = settled(&mut first); + let old = client.watermark(&shape()).expect("settled"); + + let mut second = host_instance(2); + client.connection_lost(); + let mut edge = Flaky::new(&mut second); + let (state, _) = client.sync(&mut edge, &shape(), 6); + assert_eq!(state, Synced::Settled); + assert_eq!( + asks(&edge.reached)[0], + &Ask::Correction { watermark: Some(old) } + ); + assert_ne!(client.watermark(&shape()).unwrap().host, old.host); + held_equals_drawn(&client, &second); +} + +#[test] +fn a_notice_from_a_restarted_host_asks_for_a_correction_too() { + let mut first = host_instance(1); + let mut client = settled(&mut first); + let mut second = host_instance(2); + let id = connect(&mut second); + send(&mut second, id, vec![set("North", 3.0)]); + deliver_all(&mut client, &second.take_notices()); + let mut edge = Flaky::new(&mut second); + client.sync(&mut edge, &shape(), 6); + assert!(matches!(asks(&edge.reached)[0], Ask::Correction { .. })); + held_equals_drawn(&client, &second); +} + +#[test] +fn a_refused_command_consumes_no_publication_number() { + let mut host = host(); + let id = connect(&mut host); + for n in 0..crate::model::MAX_CATEGORIES - 1 { + host.app_mut() + .model_state + .workbook + .add_category(format!("Filler{n}")) + .unwrap(); + } + host.app_mut().rebuild_layout(); + let before = host.delivery().stamp(); + host.take_notices(); + + let answer = send( + &mut host, + id, + vec![add_item("West"), ModelCommand::AddCategory("Overflow".into())], + ); + assert!( + answer + .iter() + .any(|m| matches!(m, ServerFrame::Rejected(Rejection::MutationRefused))), + "the fixture must reach the refusal: {answer:?}" + ); + assert_eq!(host.delivery().stamp(), before, "nothing was committed"); + assert!(host.take_notices().is_empty()); + + send(&mut host, id, vec![set("North", 1.0)]); + assert_eq!( + host.take_notices()[0].stamp().publication, + before.publication + 1, + "the next commit takes the next number" + ); +} + +#[test] +fn a_refused_frame_is_never_read_back() { + let mut host = host(); + let id = connect(&mut host); + for n in 0..crate::model::MAX_CATEGORIES - 1 { + host.app_mut() + .model_state + .workbook + .add_category(format!("Filler{n}")) + .unwrap(); + } + host.app_mut().rebuild_layout(); + let mut client = settled(&mut host); + let committed = drawn(&host, &everything()); + send( + &mut host, + id, + vec![set("North", 77.0), ModelCommand::AddCategory("Overflow".into())], + ); + let mut edge = Flaky::new(&mut host); + client.connection_lost(); + client.sync(&mut edge, &shape(), 6); + assert_eq!(client.held(&shape()), Some(&committed)); + let leaked = client + .held(&shape()) + .unwrap() + .iter() + .any(|((_, v), _)| *v == WireValue::of(&CellValue::Number(77.0))); + assert!(!leaked, "a value the failed frame wrote was read"); +} + +#[test] +fn commands_before_a_refused_one_are_rolled_back_with_their_notices_under_adr_001() { + let mut host = host(); + let id = connect(&mut host); + for n in 0..crate::model::MAX_CATEGORIES - 1 { + host.app_mut() + .model_state + .workbook + .add_category(format!("Filler{n}")) + .unwrap(); + } + host.app_mut().rebuild_layout(); + host.take_notices(); + send( + &mut host, + id, + vec![add_item("West"), ModelCommand::AddCategory("Overflow".into())], + ); + assert!( + host.take_notices().is_empty(), + "ADR 001 clause 4 undoes the whole frame; ADR 009 (via ADR 006) would keep `West`'s notice" + ); +} + +#[test] +fn a_dropped_connection_or_a_failed_request_triggers_a_correction_and_the_session_resumes() { + let mut host = host(); + let id = connect(&mut host); + let mut client = settled(&mut host); + let at = client.watermark(&shape()).unwrap(); + send(&mut host, id, vec![set("North", 8.0)]); + deliver_all(&mut client, &host.take_notices()); + + for fault in [Fault::DropRequest, Fault::DropReply, Fault::Garble] { + let mut edge = Flaky::scripted(&mut host, vec![Some(fault)]); + let (state, _) = client.sync(&mut edge, &shape(), 6); + assert_eq!(state, Synced::Failed, "{fault:?}"); + assert!( + matches!(client.wants(&shape()), Some(improvise_protocol::delivery::Want::Correction { .. })), + "{fault:?}: a failed request asks for a correction" + ); + } + let mut edge = Flaky::new(&mut host); + let (state, _) = client.sync(&mut edge, &shape(), 6); + assert_eq!(state, Synced::Settled); + assert_eq!( + asks(&edge.reached)[0], + &Ask::Correction { watermark: Some(at) }, + "it resumes from the watermark it held" + ); + assert!(edge.reached.iter().all(|r| r.session == session())); + held_equals_drawn(&client, &host); + + client.connection_lost(); + let (state, _) = client.sync(&mut Flaky::new(&mut host), &shape(), 6); + assert_eq!(state, Synced::Settled, "and a dropped connection settles too"); +} + +// ── the log, windows, eviction ───────────────────────────────────────────── + +fn reply_for(host: &mut Host, ask: Ask, window: Window) -> improvise_protocol::delivery::Correction { + let request = Request { + session: session(), + shape: shape(), + window, + ask, + }; + match host.deliver(&request, 0) { + Reply::Correction(c) => c, + other => panic!("expected a correction: {other:?}"), + } +} + +#[test] +fn a_watermark_the_log_covers_gets_the_moved_keys_and_one_it_does_not_gets_the_window() { + let mut host = host(); + host.limit_delivery(3, 4096); + let id = connect(&mut host); + let client = settled(&mut host); + let w = client.watermark(&shape()).unwrap(); + for n in 0..2 { + send(&mut host, id, vec![set("North", n as f64)]); + } + let near = reply_for(&mut host, Ask::Correction { watermark: Some(w) }, everything()); + assert!(matches!(near.body, CorrectionBody::Moved { .. }), "{near:?}"); + assert_eq!(near.from, Some(w)); + + for n in 2..6 { + send(&mut host, id, vec![set("North", n as f64)]); + } + assert_eq!(host.delivery().log_len(), 3, "the log is bounded"); + let far = reply_for(&mut host, Ask::Correction { watermark: Some(w) }, everything()); + assert!(matches!(far.body, CorrectionBody::Window), "{far:?}"); + assert_eq!(far.from, None); +} + +#[test] +fn an_oversized_notice_says_correct_and_costs_every_reader_a_correction() { + let mut host = host(); + host.limit_delivery(64, 1); + let id = connect(&mut host); + let mut client = settled(&mut host); + host.take_notices(); + send(&mut host, id, vec![set("North", 1.0)]); + let small = host.take_notices(); + assert!(matches!(small[0].moved.get(&shape()), Some(Moved::Keys(k)) if k.len() == 1)); + + let formula = ModelCommand::AddFormula { + raw: "Double = Revenue * 2".into(), + target_category: "_Measure".into(), + }; + send(&mut host, id, vec![formula]); + let big = host.take_notices(); + assert_eq!(big[0].moved.get(&shape()), Some(&Moved::Correct)); + deliver_all(&mut client, &small); + deliver_all(&mut client, &big); + let mut edge = Flaky::new(&mut host); + client.sync(&mut edge, &shape(), 6); + assert!(matches!(asks(&edge.reached)[0], Ask::Correction { .. })); + held_equals_drawn(&client, &host); +} + +#[test] +fn a_correction_carries_only_the_clients_window() { + let mut host = host(); + let id = connect(&mut host); + let top_left = Window { top: 0, left: 0, rows: 1, cols: 1 }; + let all = reply_for(&mut host, Ask::Correction { watermark: None }, everything()); + let one = reply_for(&mut host, Ask::Correction { watermark: None }, top_left); + assert_eq!(all.target.len(), 2); + assert_eq!(one.target.len(), 1, "{one:?}"); + send(&mut host, id, vec![set("South", 5.0)]); + let w = host.delivery().stamp(); + let _ = w; + let moved = reply_for( + &mut host, + Ask::Correction { watermark: Some(Stamp { publication: w.publication - 1, ..w }) }, + top_left, + ); + assert!(moved.target.is_empty(), "South is outside the window: {moved:?}"); +} + +#[test] +fn an_evicted_shape_is_announced_and_its_reader_corrects() { + let mut host = host(); + let id = connect(&mut host); + let mut client = settled(&mut host); + assert!(host.delivery().is_live(&shape())); + let gone = host.evict_idle(100, 50); + assert_eq!(gone, vec![shape()]); + assert!(!host.delivery().is_live(&shape())); + host.take_notices(); + send(&mut host, id, vec![set("North", 2.0)]); + let notices = host.take_notices(); + assert!(notices[0].evicted.contains(&shape())); + deliver_all(&mut client, ¬ices); + assert!(matches!( + client.wants(&shape()), + Some(improvise_protocol::delivery::Want::Correction { .. }) + )); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + held_equals_drawn(&client, &host); + assert!(host.delivery().is_live(&shape()), "naming it again revives it"); +} + +#[test] +fn a_shape_naming_no_view_is_answered_with_nothing_and_never_goes_live() { + let mut host = host(); + let ghost = shape_of_view("Nope"); + let c = { + let request = Request { + session: None, + shape: ghost.clone(), + window: everything(), + ask: Ask::Correction { watermark: None }, + }; + match host.deliver(&request, 0) { + Reply::Correction(c) => c, + other => panic!("{other:?}"), + } + }; + assert!(c.target.is_empty()); + assert!(!host.delivery().is_live(&ghost)); +} + +#[test] +fn a_document_swap_is_a_new_epoch_behind_every_old_watermark() { + let storage = crate::storage::MemoryFs::new(); + let mut other = Workbook::new("Other"); + other.add_category("Product").unwrap(); + other.add_item("Product", "Widget").unwrap(); + other.add_item("_Measure", "Units").unwrap(); + other.active_view_mut().set_axis("Product", Axis::Row); + other.active_view_mut().set_axis("_Measure", Axis::Column); + crate::persistence::save_to(&storage, &other, std::path::Path::new("other.improv")).unwrap(); + let mut app = App::new(workbook(), None); + app.storage = Box::new(storage); + app.rebuild_layout(); + let mut host = Host::with_instance(app, 1); + let id = connect(&mut host); + let mut client = settled(&mut host); + let old = client.watermark(&shape()).unwrap(); + host.take_notices(); + + let load = ModelCommand::LoadModel("other.improv".into()); + let answer = send(&mut host, id, vec![load]); + assert!(!refused(&answer), "{answer:?}"); + let notices = host.take_notices(); + let now = notices.last().expect("the swap is a committed command").stamp(); + assert!(old.is_behind(&now)); + assert_ne!(old.generation, now.generation); + assert!(notices[0].flags.formulas && notices[0].flags.views); + + deliver_all(&mut client, ¬ices); + let mut edge = Flaky::new(&mut host); + client.sync(&mut edge, &shape(), 6); + assert!(matches!(asks(&edge.reached)[0], Ask::Correction { .. })); + held_equals_drawn(&client, &host); +} + +#[test] +fn a_pull_only_client_follows_the_log_without_a_correction() { + let mut host = host(); + let id = connect(&mut host); + let mut client = settled(&mut host); + send(&mut host, id, vec![set("North", 6.0)]); + send(&mut host, id, vec![set("South", 7.0)]); + let mut edge = Flaky::new(&mut host); + // The client has seen no pushes, so its horizon is its watermark. + let (state, _) = client.poll(&mut edge, &shape()); + assert_eq!(state, Synced::Settled); + assert!(matches!(asks(&edge.reached)[0], Ask::Notices { .. })); + assert!( + asks(&edge.reached) + .iter() + .all(|a| !matches!(a, Ask::Correction { .. })) + ); + held_equals_drawn(&client, &host); +} + +#[test] +fn a_request_can_be_retried_and_answers_the_same() { + let mut host = host(); + let id = connect(&mut host); + let client = settled(&mut host); + send(&mut host, id, vec![set("North", 6.0)]); + let w = client.watermark(&shape()).unwrap(); + let a = reply_for(&mut host, Ask::Correction { watermark: Some(w) }, everything()); + let b = reply_for(&mut host, Ask::Correction { watermark: Some(w) }, everything()); + assert_eq!(a, b); +} + +// ── lossless through the wire ────────────────────────────────────────────── + +#[test] +fn values_keep_every_bit_from_the_cell_to_the_client() { + let mut host = host(); + let id = connect(&mut host); + let mut client = settled(&mut host); + let values = [0.1 + 0.2, -0.0, f64::INFINITY, f64::NEG_INFINITY, f64::NAN, 1e308, 5e-324]; + for (n, v) in values.iter().enumerate() { + let region = ["North", "South"][n % 2]; + send(&mut host, id, vec![set(region, *v)]); + deliver_all(&mut client, &host.take_notices()); + let mut edge = WireEdge(Flaky::new(&mut host)); + assert_eq!(client.sync(&mut edge, &shape(), 6).0, Synced::Settled); + let held = client.held(&shape()).unwrap(); + let got = held.iter().find_map(|((k, v), _)| { + (k.0.contains(&("Region".to_string(), region.to_string()))).then(|| v.to_cell_value()) + }); + let Some(CellValue::Number(got)) = got else { + panic!("{region} holds no number after {v:?}") + }; + match v.is_nan() { + true => assert!(got.is_nan()), + false => assert_eq!(got.to_bits(), v.to_bits(), "{v:?} became {got:?}"), + } + } +} + +// ── the whole thing, at random ───────────────────────────────────────────── + +mod random { + use proptest::prelude::*; + + use super::*; + + #[derive(Debug, Clone)] + enum Op { + Set(bool, usize), + Add(usize), + Remove(usize), + Formula, + } + + fn op() -> impl Strategy { + prop_oneof![ + 4 => (any::(), 0usize..8).prop_map(|(r, v)| Op::Set(r, v)), + 2 => (0usize..3).prop_map(Op::Add), + 1 => (0usize..3).prop_map(Op::Remove), + 1 => Just(Op::Formula), + ] + } + + const VALUES: [f64; 8] = [1.0, 2.5, -0.0, 0.0, f64::INFINITY, f64::NEG_INFINITY, f64::NAN, 0.30000000000000004]; + const EXTRA: [&str; 3] = ["East", "West", "North"]; + + fn command(op: &Op) -> ModelCommand { + match op { + Op::Set(north, v) => set(["South", "North"][*north as usize], VALUES[*v]), + Op::Add(i) => add_item(EXTRA[*i]), + Op::Remove(i) => ModelCommand::RemoveItem { + category: "Region".into(), + item: EXTRA[*i].into(), + }, + Op::Formula => ModelCommand::AddFormula { + raw: "Double = Revenue * 2".into(), + target_category: "_Measure".into(), + }, + } + } + + #[derive(Debug, Clone)] + struct Weather { + deliver_notice: bool, + lose_connection: bool, + sync: Vec>, + } + + fn weather() -> impl Strategy { + ( + any::(), + 0u8..8, + proptest::collection::vec(proptest::option::of(0u8..3), 0..3), + ) + .prop_map(|(deliver_notice, lose, sync)| Weather { + deliver_notice, + lose_connection: lose == 0, + sync, + }) + } + + fn fault(code: u8) -> Fault { + [Fault::DropRequest, Fault::DropReply, Fault::Garble][code as usize] + } + + fn windows() -> impl Strategy { + (0u128..2, 0u128..2, 1u128..6, 1u128..4).prop_map(|(top, left, rows, cols)| Window { + top, + left, + rows, + cols, + }) + } + + proptest! { + #[test] + fn a_corrected_state_is_the_target_and_correcting_it_again_is_the_zero_zset( + steps in proptest::collection::vec((op(), weather()), 1..12), + window in windows(), + ) { + let mut host = host(); + let id = connect(&mut host); + let mut client = EdgeClient::new(session()); + client.track(shape(), window); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + host.take_notices(); + + for (op, weather) in &steps { + send(&mut host, id, vec![command(op)]); + let notices = host.take_notices(); + if weather.deliver_notice { + deliver_all(&mut client, ¬ices); + } + if weather.lose_connection { + client.connection_lost(); + } + let script = weather.sync.iter().map(|f| f.map(fault)).collect(); + client.sync(&mut Flaky::scripted(&mut host, script), &shape(), 6); + } + + let (state, _) = client.poll(&mut WireEdge(Flaky::new(&mut host)), &shape()); + prop_assert_eq!(state, Synced::Settled); + let want = drawn(&host, &window); + prop_assert_eq!(client.held(&shape()), Some(&want)); + + let full = reply_for(&mut host, Ask::Correction { watermark: None }, window); + let correction = &wire::decode(&full.target) - client.held(&shape()).unwrap(); + prop_assert!(correction.is_zero(), "correcting a correct state: {correction:?}"); + + let before = client.held(&shape()).cloned(); + let (outcome, _) = client.apply(&full); + match outcome { + Outcome::Applied { delta } => prop_assert!(delta.is_zero()), + Outcome::Duplicate => {} + other => prop_assert!(false, "{other:?}"), + } + prop_assert_eq!(client.held(&shape()).cloned(), before); + } + } +} diff --git a/crates/improvise-command/src/ui/host.rs b/crates/improvise-command/src/ui/host.rs index b9434734..58553143 100644 --- a/crates/improvise-command/src/ui/host.rs +++ b/crates/improvise-command/src/ui/host.rs @@ -2,9 +2,11 @@ //! *document*, many clients per `App` — see `context/code/command.md`. use crate::ui::app::App; +use crate::ui::delivery::Delivery; use crate::ui::effect; use crate::ui::project::{Applied, Broadcast, Dirty, SessionId, Sessions}; use crate::ui::reduce::{self, Checked, CheckedBatch}; +use improvise_protocol::delivery::{Notice, Reply, Request, ShapeKey}; use improvise_protocol::{AppMode, Command, Rejection}; use improvise_protocol::{ClientFrame, ServerFrame}; @@ -22,16 +24,58 @@ pub struct Addressed { pub struct Host { app: App, sessions: Sessions, + delivery: Delivery, } impl Host { pub fn new(app: App) -> Self { + use std::hash::{BuildHasher, Hasher}; + let instance = std::collections::hash_map::RandomState::new() + .build_hasher() + .finish(); + Self::with_instance(app, instance) + } + + /// A host with a chosen instance id, which must differ from every earlier run's + /// (`context/code/delivery.md` §6). + pub fn with_instance(app: App, instance: u64) -> Self { + let delivery = Delivery::new( + instance, + &app.model_state.workbook, + app.document_generation(), + ); Self { app, sessions: Sessions::default(), + delivery, } } + pub fn delivery(&self) -> &Delivery { + &self.delivery + } + + /// Answer a delivery request from the last committed state. `now` is a local monotonic count. + pub fn deliver(&mut self, request: &Request, now: u64) -> Reply { + self.delivery + .answer(&self.app.model_state.workbook, request, now) + } + + /// Forget shapes no request has named for `idle` (a local duration in `now`'s unit). + pub fn evict_idle(&mut self, now: u64, idle: u64) -> Vec { + self.delivery.evict_idle(now, idle) + } + + /// How many notices the shared log keeps, and how many keys one shape lists in a notice. + pub fn limit_delivery(&mut self, capacity: usize, max_keys: usize) { + self.delivery.set_limits(capacity, max_keys); + } + + /// Notices published since the last call, for the transport to push. + pub fn take_notices(&mut self) -> Vec { + self.delivery.take_outbox() + } + pub fn app(&self) -> &App { &self.app } @@ -224,11 +268,12 @@ impl Host { fn mutating_frame(&mut self, id: SessionId, batch: CheckedBatch<'_>) -> Vec { let undo = self.app.snapshot(); let mirrors = self.sessions.clone(); + let published = self.delivery.clone(); let mut out = Vec::new(); let mut asker_reset = false; for cmd in batch { if let Some(reason) = reduce::broken_chain(&self.app, cmd.command()) { - return self.undo_frame(id, undo, mirrors, reason); + return self.undo_frame(id, (undo, mirrors, published), reason); } // Only a mutation runs `apply_effects`, which resets both flags — so // after anything else they are STALE. A refusal need not abort (cz8d). @@ -252,7 +297,7 @@ impl Host { out.extend(answered); let failed = self.app.last_batch_refused() || self.app.last_batch_aborted(); if mutation && failed { - return self.undo_frame(id, undo, mirrors, Rejection::MutationRefused); + return self.undo_frame(id, (undo, mirrors, published), Rejection::MutationRefused); } } // Last, after everything the frame owed: a client folds the answer to @@ -269,12 +314,12 @@ impl Host { fn undo_frame( &mut self, id: SessionId, - undo: crate::ui::app::AppSnapshot, - mirrors: Sessions, + (undo, mirrors, published): (crate::ui::app::AppSnapshot, Sessions, Delivery), reason: Rejection, ) -> Vec { self.app.restore(undo); self.sessions = mirrors; + self.delivery = published; vec![Addressed { to: id, msg: ServerFrame::Rejected(reason), @@ -284,7 +329,16 @@ impl Host { /// The one door (gotcha 36); the match that follows is the mechanism (gotcha 39): /// asking for the `Dirty` means naming the case where a `Snapshot` is owed. fn command(&mut self, id: SessionId, cmd: Checked<'_>) -> Vec { - match self.sessions.apply_checked(&mut self.app, id, cmd) { + let mutation = matches!(cmd.command(), Command::Model { .. }); + let applied = self.sessions.apply_checked(&mut self.app, id, cmd); + let landed = !matches!(applied, Applied::Rejected { .. }) + && !(self.app.last_batch_refused() || self.app.last_batch_aborted()); + if mutation && landed { + let generation = self.app.document_generation(); + self.delivery + .commit(&self.app.model_state.workbook, generation); + } + match applied { // The document was replaced: nothing to diff and nothing to broadcast, // each snapshot carrying a whole view slice and its seed. Applied::Rebased { snapshots } => snapshots diff --git a/crates/improvise-command/src/ui/mod.rs b/crates/improvise-command/src/ui/mod.rs index 9026994d..12d9d679 100644 --- a/crates/improvise-command/src/ui/mod.rs +++ b/crates/improvise-command/src/ui/mod.rs @@ -1,7 +1,11 @@ pub mod app; pub mod cat_tree; +pub mod delivery; pub mod effect; pub mod host; pub mod project; pub mod reduce; pub mod viewport; + +#[cfg(test)] +mod delivery_tests; diff --git a/crates/improvise-protocol/src/delivery/client.rs b/crates/improvise-protocol/src/delivery/client.rs index 25dcd395..88259a36 100644 --- a/crates/improvise-protocol/src/delivery/client.rs +++ b/crates/improvise-protocol/src/delivery/client.rs @@ -29,6 +29,7 @@ pub enum Want { watermark: Option, keys: BTreeSet, through: Stamp, + window: bool, }, } @@ -60,6 +61,8 @@ struct Tracked { seen: Option, /// Notices received and not yet fetched: `(publication, what moved)`. unapplied: Vec<(Stamp, Moved)>, + /// The last notice that moved an axis: until a window is read past it, positions are unknown. + restructured: Option, stale: bool, } @@ -88,6 +91,7 @@ impl EdgeClient { watermark: None, seen: None, unapplied: Vec::new(), + restructured: None, stale: true, }, ); @@ -104,21 +108,24 @@ impl EdgeClient { pub fn wants(&self, shape: &ShapeKey) -> Option { let t = self.shapes.get(shape)?; let needs_all = t.stale || t.unapplied.iter().any(|(_, m)| *m == Moved::Correct); + let keys: BTreeSet = t + .unapplied + .iter() + .flat_map(|(_, m)| match m { + Moved::Keys(keys) => keys.iter().cloned().collect::>(), + Moved::Correct => Vec::new(), + }) + .collect(); + let owed = !t.unapplied.is_empty() || t.restructured.is_some(); match (needs_all, t.seen) { (true, _) => Some(Want::Correction { watermark: t.watermark, }), - (false, Some(through)) if !t.unapplied.is_empty() => Some(Want::Fetch { + (false, Some(through)) if owed => Some(Want::Fetch { watermark: t.watermark, - keys: t - .unapplied - .iter() - .flat_map(|(_, m)| match m { - Moved::Keys(keys) => keys.iter().cloned(), - Moved::Correct => unreachable!("handled above"), - }) - .collect(), + keys, through, + window: t.restructured.is_some(), }), _ => None, } @@ -136,10 +143,12 @@ impl EdgeClient { watermark, keys, through, + window, } => Ask::Fetch { watermark, keys, through, + window, }, }, }) @@ -161,13 +170,16 @@ impl EdgeClient { t.unapplied.push((end, Moved::Correct)); continue; } + if !notice.axis.is_empty() || !notice.relabels.is_empty() { + t.restructured = Some(end); + } match notice.moved.get(shape) { Some(Moved::Keys(keys)) if keys.is_empty() => {} Some(moved) => t.unapplied.push((end, moved.clone())), None => {} } let current = t.watermark.is_some_and(|w| w.same_epoch(&end)); - if t.unapplied.is_empty() && !t.stale && current { + if t.unapplied.is_empty() && t.restructured.is_none() && !t.stale && current { t.watermark = Some(end); } } @@ -251,10 +263,15 @@ impl EdgeClient { }; t.held += δ t.watermark = Some(c.through); - t.stale = match &c.body { - CorrectionBody::Window => false, - CorrectionBody::Moved { .. } => t.stale, - }; + let window = matches!(c.body, CorrectionBody::Window); + t.stale &= !window; + if window + && t + .restructured + .is_some_and(|r| !r.same_epoch(&c.through) || r.publication <= c.through.publication) + { + t.restructured = None; + } let seen_epoch = t.seen.map(|s| (s.host, s.generation)); t.unapplied.retain(|(at, _)| { Some((at.host, at.generation)) == seen_epoch @@ -266,16 +283,14 @@ impl EdgeClient { if !seen_ahead { t.seen = Some(c.through); } - let mut events = match &c.body { - CorrectionBody::Window => { + let events = match (&c.carried, window) { + (Some(carried), _) => self.events_of(carried), + (None, true) => { self.events_through = Some(c.through); vec![Event::Resync] } - CorrectionBody::Moved { .. } => Vec::new(), + (None, false) => Vec::new(), }; - if let Some(carried) = &c.carried { - events.extend(self.events_of(carried)); - } (Outcome::Applied { delta }, events) } diff --git a/crates/improvise-protocol/src/delivery/client_tests.rs b/crates/improvise-protocol/src/delivery/client_tests.rs index 0121d5e9..aa770870 100644 --- a/crates/improvise-protocol/src/delivery/client_tests.rs +++ b/crates/improvise-protocol/src/delivery/client_tests.rs @@ -111,7 +111,8 @@ fn a_moved_key_wants_a_fetch_and_not_a_correction() { Some(Want::Fetch { watermark: Some(at(1, 4)), keys: [key(1), key(2)].into(), - through: at(1, 5) + through: at(1, 5), + window: false }) ); } @@ -256,7 +257,8 @@ fn a_fetch_does_not_advance_past_the_notices_it_did_not_name() { Some(Want::Fetch { watermark: Some(at(1, 5)), keys: [key(2)].into(), - through: at(1, 6) + through: at(1, 6), + window: false }) ); } diff --git a/crates/improvise-protocol/src/delivery/exchange.rs b/crates/improvise-protocol/src/delivery/exchange.rs index adccc3d4..eaef636a 100644 --- a/crates/improvise-protocol/src/delivery/exchange.rs +++ b/crates/improvise-protocol/src/delivery/exchange.rs @@ -36,11 +36,13 @@ pub struct SessionToken(pub String); pub enum Ask { /// Everything since `watermark`; `None` is a full refetch, `target - empty`. Correction { watermark: Option }, - /// The current values of `keys`, covering the publications up to `through`. + /// The current values of `keys`, covering the publications up to `through`. `window` + /// asks for the whole window too: an axis change moved positions, not values. Fetch { watermark: Option, keys: BTreeSet, through: Stamp, + window: bool, }, /// The notices after `after`, if the host's log still has them. Notices { after: Stamp }, From e3f5cee29a9ad9a76d3ec6a5a50c07a92ef29ca2 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:11:47 -0700 Subject: [PATCH 04/12] fix(numtower): a nonzero ExactSum is never negative zero (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/test-evidence/numtower.md | 8 ++++++++ crates/numtower/src/sum.rs | 10 ++++------ crates/numtower/tests/zero_sign.rs | 9 +++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/context/test-evidence/numtower.md b/context/test-evidence/numtower.md index 3e04045f..f2c475cb 100644 --- a/context/test-evidence/numtower.md +++ b/context/test-evidence/numtower.md @@ -485,3 +485,11 @@ and both properties, since one `+0.0` or one cancelling pair must make the zero Adds `keep` and `gone`, then retracts `gone`; the read and the sum itself must equal a fresh sum of `keep`. It is why both counters are net weights and why `PartialEq` compares the sign. + +### a_sum_that_is_not_zero_never_reads_as_negative_zero + +The first version of the sign rule returned `-0.0` whenever the `-0.0` count was positive and the +other count was nil, without checking that the finite total was zero. The weights here (+5.0 and -7.0 +net to one "other" addend, and the `-0.0` is held) make the counts say "negative zero" over a total of +-2. Found by the existing `the_accumulator_reads_the_exact_sum_or_its_stated_roundings`, which read +`#i-0.0` for a nonzero sum; the rival is that version. diff --git a/crates/numtower/src/sum.rs b/crates/numtower/src/sum.rs index 63f7a97a..ce882009 100644 --- a/crates/numtower/src/sum.rs +++ b/crates/numtower/src/sum.rs @@ -40,9 +40,7 @@ impl PartialEq for ExactSum { propagate(&mut mine); propagate(&mut theirs); let counts = |s: &ExactSum| (s.nan, s.pos_inf, s.neg_inf); - mine == theirs - && counts(self) == counts(o) - && self.negative_zero() == o.negative_zero() + mine == theirs && counts(self) == counts(o) && self.negative_zero() == o.negative_zero() } } @@ -148,9 +146,9 @@ impl ExactSum { (true, _, _) | (false, true, true) => f64::NAN, (false, true, false) => f64::INFINITY, (false, false, true) => f64::NEG_INFINITY, - (false, false, false) => match self.negative_zero() { - true => -0.0, - false => ratio_to_f64(&self.total(), &(BigInt::one() << 1074)), + (false, false, false) => match (self.negative_zero(), self.total()) { + (true, total) if total.is_zero() => -0.0, + (_, total) => ratio_to_f64(&total, &(BigInt::one() << 1074)), }, } } diff --git a/crates/numtower/tests/zero_sign.rs b/crates/numtower/tests/zero_sign.rs index 18c7e06f..ae7cabbf 100644 --- a/crates/numtower/tests/zero_sign.rs +++ b/crates/numtower/tests/zero_sign.rs @@ -78,3 +78,12 @@ proptest! { prop_assert_eq!(sum, fresh); } } + +#[test] +fn a_sum_that_is_not_zero_never_reads_as_negative_zero() { + let mut sum = ExactSum::new(); + sum.add(-0.0, 1); + sum.add(5.0, 1); + sum.add(7.0, -1); + assert_eq!(sum.value(), -2.0); +} From 0e6ad9e96a9b7ab543f225b85a63e3b2f02e89ea Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:11:48 -0700 Subject: [PATCH 05/12] refactor(protocol): box the large delivery variants, fmt and the axis-change fetch (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- crates/improvise-command/src/ui/delivery.rs | 40 +-- .../src/ui/delivery_tests.rs | 230 ++++++++++++++---- .../improvise-protocol/src/delivery/client.rs | 6 +- .../src/delivery/exchange.rs | 2 +- crates/improvise-protocol/src/delivery/mod.rs | 5 +- .../improvise-protocol/src/delivery/notice.rs | 30 ++- 6 files changed, 238 insertions(+), 75 deletions(-) diff --git a/crates/improvise-command/src/ui/delivery.rs b/crates/improvise-command/src/ui/delivery.rs index 96c6b4cf..4f52ed0a 100644 --- a/crates/improvise-command/src/ui/delivery.rs +++ b/crates/improvise-command/src/ui/delivery.rs @@ -131,7 +131,10 @@ impl Structure { } } -fn axis_changes(before: &[(String, Vec)], after: &[(String, Vec)]) -> Vec { +fn axis_changes( + before: &[(String, Vec)], + after: &[(String, Vec)], +) -> Vec { let held: BTreeMap<_, _> = before.iter().map(|(c, i)| (c.as_str(), i)).collect(); let now: BTreeSet<_> = after.iter().map(|(c, _)| c.as_str()).collect(); let gone = before @@ -145,11 +148,14 @@ fn axis_changes(before: &[(String, Vec)], after: &[(String, Vec) let added = AxisChange::CategoryAdded { category: category.clone(), }; - let inserted = items.iter().enumerate().map(|(at, item)| AxisChange::ItemInserted { - category: category.clone(), - item: item.clone(), - at, - }); + let inserted = items + .iter() + .enumerate() + .map(|(at, item)| AxisChange::ItemInserted { + category: category.clone(), + item: item.clone(), + at, + }); return std::iter::once(added).chain(inserted).collect::>(); }; item_changes(category, was, items) @@ -343,12 +349,12 @@ impl Delivery { match &request.ask { Ask::Notices { after } => match self.covered(&request.shape, Some(after)) { Some(after) => Reply::Notices(self.notices_after(&request.shape, after, None)), - None => Reply::Correction(self.window(request, None, self.stamp(), None)), + None => Reply::Correction(Box::new(self.window(request, None, self.stamp(), None))), }, Ask::Correction { watermark } => match self.covered(&request.shape, watermark.as_ref()) { Some(from) => self.moved(request, from, self.stamp(), None, false), - None => Reply::Correction(self.window(request, None, self.stamp(), None)), + None => Reply::Correction(Box::new(self.window(request, None, self.stamp(), None))), }, Ask::Fetch { watermark, @@ -356,13 +362,15 @@ impl Delivery { through, window, } => { - let sane = through.same_epoch(&self.stamp()) - && through.publication <= self.publication; + let sane = + through.same_epoch(&self.stamp()) && through.publication <= self.publication; match (self.covered(&request.shape, watermark.as_ref()), sane) { (Some(from), true) if from.publication <= through.publication => { self.moved(request, from, *through, Some(keys), *window) } - _ => Reply::Correction(self.window(request, None, self.stamp(), None)), + _ => { + Reply::Correction(Box::new(self.window(request, None, self.stamp(), None))) + } } } } @@ -426,15 +434,15 @@ impl Delivery { let carried = covering .iter() .cloned() - .reduce(|a, b| a.merge(b).unwrap_or_else(|(_, b)| b)); + .reduce(|a, b| a.merge(b).unwrap_or_else(|e| e.1)); let restructured = covering .iter() .any(|n| !n.axis.is_empty() || !n.relabels.is_empty()); if oversized { - return Reply::Correction(self.window(request, None, self.stamp(), None)); + return Reply::Correction(Box::new(self.window(request, None, self.stamp(), None))); } if window || restructured { - return Reply::Correction(self.window(request, Some(from), through, carried)); + return Reply::Correction(Box::new(self.window(request, Some(from), through, carried))); } let keys: BTreeSet = match only { Some(keys) => keys.clone(), @@ -452,7 +460,7 @@ impl Delivery { .get(&request.shape) .map(|l| l.output.target(&request.window, Some(&keys))) .unwrap_or_default(); - Reply::Correction(Correction { + Reply::Correction(Box::new(Correction { shape: request.shape.clone(), at: self.stamp(), from: Some(from), @@ -460,7 +468,7 @@ impl Delivery { body: CorrectionBody::Moved { keys }, target: wire::encode(&target), carried, - }) + })) } } diff --git a/crates/improvise-command/src/ui/delivery_tests.rs b/crates/improvise-command/src/ui/delivery_tests.rs index d5fa8f3c..0841dc86 100644 --- a/crates/improvise-command/src/ui/delivery_tests.rs +++ b/crates/improvise-command/src/ui/delivery_tests.rs @@ -5,15 +5,15 @@ use std::collections::VecDeque; use improvise_core::model::cell::{CellKey, CellValue}; use improvise_core::model::record::RecordHash; -use improvise_core::view::layout::GridLayout; use improvise_core::view::Axis; +use improvise_core::view::layout::GridLayout; use improvise_core::workbook::Workbook; use improvise_protocol::delivery::wire::{self, Cell, WireKey, WireValue}; use improvise_protocol::delivery::{ Ask, AxisChange, CorrectionBody, Edge, EdgeClient, EdgeError, Moved, Notice, Outcome, Reply, Request, SessionToken, ShapeKey, Stamp, Synced, Window, }; -use improvise_protocol::{ClientFrame, ModelCommand, Rejection, RecordCell, ServerFrame}; +use improvise_protocol::{ClientFrame, ModelCommand, RecordCell, Rejection, ServerFrame}; use zset::ZSet; use super::app::App; @@ -115,9 +115,7 @@ fn send(host: &mut Host, id: SessionId, cmds: Vec) -> Vec bool { - answer - .iter() - .any(|m| matches!(m, ServerFrame::Rejected(_))) + answer.iter().any(|m| matches!(m, ServerFrame::Rejected(_))) } /// What the grid draws, derived without the delivery module's own reader. @@ -262,13 +260,20 @@ fn a_command_that_moves_nothing_still_publishes_and_a_view_command_does_not() { send(&mut host, id, vec![set("North", 1.0)]); let same = host.take_notices(); assert_eq!(same.len(), 2); - assert_eq!(same[1].moved, Default::default(), "an equal write moves nothing"); + assert_eq!( + same[1].moved, + Default::default(), + "an equal write moves nothing" + ); host.handle( id, ClientFrame::command(improvise_protocol::Command::SetSelected { row: 1, col: 0 }), ); - assert!(host.take_notices().is_empty(), "the cursor is the view store's"); + assert!( + host.take_notices().is_empty(), + "the cursor is the view store's" + ); } #[test] @@ -296,7 +301,10 @@ fn a_notice_names_the_moved_cells_including_a_formula_cell_it_did_not_write() { .filter(|(c, _)| c == "_Measure") .map(|(_, i)| i.as_str()) .collect(); - assert!(measures.contains(&"Revenue") && measures.contains(&"Double"), "{keys:?}"); + assert!( + measures.contains(&"Revenue") && measures.contains(&"Double"), + "{keys:?}" + ); } #[test] @@ -305,10 +313,14 @@ fn a_structural_command_reports_the_axis_change_and_a_formula_the_flag() { let id = connect(&mut host); host.take_notices(); send(&mut host, id, vec![add_item("East")]); - send(&mut host, id, vec![ModelCommand::RemoveItem { - category: "Region".into(), - item: "North".into(), - }]); + send( + &mut host, + id, + vec![ModelCommand::RemoveItem { + category: "Region".into(), + item: "North".into(), + }], + ); send( &mut host, id, @@ -353,7 +365,10 @@ fn a_dropped_notice_is_detected_and_a_correction_repairs_it() { let (state, _) = client.sync(&mut edge, &shape(), 6); assert_eq!(state, Synced::Settled); assert!( - matches!(asks(&edge.reached)[0], Ask::Correction { watermark: Some(_) }), + matches!( + asks(&edge.reached)[0], + Ask::Correction { watermark: Some(_) } + ), "{:?}", edge.reached ); @@ -375,7 +390,9 @@ fn a_reconnect_after_a_host_restart_asks_for_a_correction_from_the_old_watermark assert_eq!(state, Synced::Settled); assert_eq!( asks(&edge.reached)[0], - &Ask::Correction { watermark: Some(old) } + &Ask::Correction { + watermark: Some(old) + } ); assert_ne!(client.watermark(&shape()).unwrap().host, old.host); held_equals_drawn(&client, &second); @@ -413,7 +430,10 @@ fn a_refused_command_consumes_no_publication_number() { let answer = send( &mut host, id, - vec![add_item("West"), ModelCommand::AddCategory("Overflow".into())], + vec![ + add_item("West"), + ModelCommand::AddCategory("Overflow".into()), + ], ); assert!( answer @@ -436,20 +456,21 @@ fn a_refused_command_consumes_no_publication_number() { fn a_refused_frame_is_never_read_back() { let mut host = host(); let id = connect(&mut host); - for n in 0..crate::model::MAX_CATEGORIES - 1 { - host.app_mut() - .model_state - .workbook - .add_category(format!("Filler{n}")) - .unwrap(); - } - host.app_mut().rebuild_layout(); let mut client = settled(&mut host); let committed = drawn(&host, &everything()); - send( + let answer = send( &mut host, id, - vec![set("North", 77.0), ModelCommand::AddCategory("Overflow".into())], + vec![ + set("North", 77.0), + ModelCommand::LoadModel("missing.improv".into()), + ], + ); + assert!( + answer + .iter() + .any(|m| matches!(m, ServerFrame::Rejected(Rejection::MutationRefused))), + "the write must have run before the refusal: {answer:?}" ); let mut edge = Flaky::new(&mut host); client.connection_lost(); @@ -479,7 +500,10 @@ fn commands_before_a_refused_one_are_rolled_back_with_their_notices_under_adr_00 send( &mut host, id, - vec![add_item("West"), ModelCommand::AddCategory("Overflow".into())], + vec![ + add_item("West"), + ModelCommand::AddCategory("Overflow".into()), + ], ); assert!( host.take_notices().is_empty(), @@ -501,7 +525,10 @@ fn a_dropped_connection_or_a_failed_request_triggers_a_correction_and_the_sessio let (state, _) = client.sync(&mut edge, &shape(), 6); assert_eq!(state, Synced::Failed, "{fault:?}"); assert!( - matches!(client.wants(&shape()), Some(improvise_protocol::delivery::Want::Correction { .. })), + matches!( + client.wants(&shape()), + Some(improvise_protocol::delivery::Want::Correction { .. }) + ), "{fault:?}: a failed request asks for a correction" ); } @@ -510,7 +537,9 @@ fn a_dropped_connection_or_a_failed_request_triggers_a_correction_and_the_sessio assert_eq!(state, Synced::Settled); assert_eq!( asks(&edge.reached)[0], - &Ask::Correction { watermark: Some(at) }, + &Ask::Correction { + watermark: Some(at) + }, "it resumes from the watermark it held" ); assert!(edge.reached.iter().all(|r| r.session == session())); @@ -518,12 +547,20 @@ fn a_dropped_connection_or_a_failed_request_triggers_a_correction_and_the_sessio client.connection_lost(); let (state, _) = client.sync(&mut Flaky::new(&mut host), &shape(), 6); - assert_eq!(state, Synced::Settled, "and a dropped connection settles too"); + assert_eq!( + state, + Synced::Settled, + "and a dropped connection settles too" + ); } // ── the log, windows, eviction ───────────────────────────────────────────── -fn reply_for(host: &mut Host, ask: Ask, window: Window) -> improvise_protocol::delivery::Correction { +fn reply_for( + host: &mut Host, + ask: Ask, + window: Window, +) -> improvise_protocol::delivery::Correction { let request = Request { session: session(), shape: shape(), @@ -531,7 +568,7 @@ fn reply_for(host: &mut Host, ask: Ask, window: Window) -> improvise_protocol::d ask, }; match host.deliver(&request, 0) { - Reply::Correction(c) => c, + Reply::Correction(c) => *c, other => panic!("expected a correction: {other:?}"), } } @@ -546,15 +583,26 @@ fn a_watermark_the_log_covers_gets_the_moved_keys_and_one_it_does_not_gets_the_w for n in 0..2 { send(&mut host, id, vec![set("North", n as f64)]); } - let near = reply_for(&mut host, Ask::Correction { watermark: Some(w) }, everything()); - assert!(matches!(near.body, CorrectionBody::Moved { .. }), "{near:?}"); + let near = reply_for( + &mut host, + Ask::Correction { watermark: Some(w) }, + everything(), + ); + assert!( + matches!(near.body, CorrectionBody::Moved { .. }), + "{near:?}" + ); assert_eq!(near.from, Some(w)); for n in 2..6 { send(&mut host, id, vec![set("North", n as f64)]); } assert_eq!(host.delivery().log_len(), 3, "the log is bounded"); - let far = reply_for(&mut host, Ask::Correction { watermark: Some(w) }, everything()); + let far = reply_for( + &mut host, + Ask::Correction { watermark: Some(w) }, + everything(), + ); assert!(matches!(far.body, CorrectionBody::Window), "{far:?}"); assert_eq!(far.from, None); } @@ -589,7 +637,12 @@ fn an_oversized_notice_says_correct_and_costs_every_reader_a_correction() { fn a_correction_carries_only_the_clients_window() { let mut host = host(); let id = connect(&mut host); - let top_left = Window { top: 0, left: 0, rows: 1, cols: 1 }; + let top_left = Window { + top: 0, + left: 0, + rows: 1, + cols: 1, + }; let all = reply_for(&mut host, Ask::Correction { watermark: None }, everything()); let one = reply_for(&mut host, Ask::Correction { watermark: None }, top_left); assert_eq!(all.target.len(), 2); @@ -599,10 +652,68 @@ fn a_correction_carries_only_the_clients_window() { let _ = w; let moved = reply_for( &mut host, - Ask::Correction { watermark: Some(Stamp { publication: w.publication - 1, ..w }) }, + Ask::Correction { + watermark: Some(Stamp { + publication: w.publication - 1, + ..w + }), + }, top_left, ); - assert!(moved.target.is_empty(), "South is outside the window: {moved:?}"); + assert!( + moved.target.is_empty(), + "South is outside the window: {moved:?}" + ); +} + +#[test] +fn an_axis_change_that_shifts_an_unmoved_cell_into_the_window_refetches_the_window() { + let mut host = host(); + let id = connect(&mut host); + let one = Window { + top: 0, + left: 0, + rows: 1, + cols: 1, + }; + let mut client = EdgeClient::new(session()); + client.track(shape(), one); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + host.take_notices(); + + let gone = ModelCommand::RemoveItem { + category: "Region".into(), + item: "North".into(), + }; + send(&mut host, id, vec![gone]); + let notices = host.take_notices(); + assert_eq!( + notices[0] + .moved + .get(&shape()) + .map(|m| matches!(m, Moved::Keys(k) if k.len() == 1)), + Some(true) + ); + deliver_all(&mut client, ¬ices); + let mut edge = Flaky::new(&mut host); + client.sync(&mut edge, &shape(), 6); + assert!( + matches!(asks(&edge.reached)[0], Ask::Fetch { window: true, .. }), + "{:?}", + edge.reached + ); + assert_eq!(client.held(&shape()), Some(&drawn(&host, &one))); + let south = WireKey(vec![ + ("Region".into(), "South".into()), + ("_Measure".into(), "Revenue".into()), + ]); + assert!( + client + .held(&shape()) + .unwrap() + .iter() + .any(|((k, _), _)| *k == south) + ); } #[test] @@ -625,7 +736,10 @@ fn an_evicted_shape_is_announced_and_its_reader_corrects() { )); client.sync(&mut Flaky::new(&mut host), &shape(), 6); held_equals_drawn(&client, &host); - assert!(host.delivery().is_live(&shape()), "naming it again revives it"); + assert!( + host.delivery().is_live(&shape()), + "naming it again revives it" + ); } #[test] @@ -640,7 +754,7 @@ fn a_shape_naming_no_view_is_answered_with_nothing_and_never_goes_live() { ask: Ask::Correction { watermark: None }, }; match host.deliver(&request, 0) { - Reply::Correction(c) => c, + Reply::Correction(c) => *c, other => panic!("{other:?}"), } }; @@ -671,7 +785,10 @@ fn a_document_swap_is_a_new_epoch_behind_every_old_watermark() { let answer = send(&mut host, id, vec![load]); assert!(!refused(&answer), "{answer:?}"); let notices = host.take_notices(); - let now = notices.last().expect("the swap is a committed command").stamp(); + let now = notices + .last() + .expect("the swap is a committed command") + .stamp(); assert!(old.is_behind(&now)); assert_ne!(old.generation, now.generation); assert!(notices[0].flags.formulas && notices[0].flags.views); @@ -710,8 +827,16 @@ fn a_request_can_be_retried_and_answers_the_same() { let client = settled(&mut host); send(&mut host, id, vec![set("North", 6.0)]); let w = client.watermark(&shape()).unwrap(); - let a = reply_for(&mut host, Ask::Correction { watermark: Some(w) }, everything()); - let b = reply_for(&mut host, Ask::Correction { watermark: Some(w) }, everything()); + let a = reply_for( + &mut host, + Ask::Correction { watermark: Some(w) }, + everything(), + ); + let b = reply_for( + &mut host, + Ask::Correction { watermark: Some(w) }, + everything(), + ); assert_eq!(a, b); } @@ -722,7 +847,15 @@ fn values_keep_every_bit_from_the_cell_to_the_client() { let mut host = host(); let id = connect(&mut host); let mut client = settled(&mut host); - let values = [0.1 + 0.2, -0.0, f64::INFINITY, f64::NEG_INFINITY, f64::NAN, 1e308, 5e-324]; + let values = [ + 0.1 + 0.2, + -0.0, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + 1e308, + 5e-324, + ]; for (n, v) in values.iter().enumerate() { let region = ["North", "South"][n % 2]; send(&mut host, id, vec![set(region, *v)]); @@ -767,7 +900,16 @@ mod random { ] } - const VALUES: [f64; 8] = [1.0, 2.5, -0.0, 0.0, f64::INFINITY, f64::NEG_INFINITY, f64::NAN, 0.30000000000000004]; + const VALUES: [f64; 8] = [ + 1.0, + 2.5, + -0.0, + 0.0, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + 0.30000000000000004, + ]; const EXTRA: [&str; 3] = ["East", "West", "North"]; fn command(op: &Op) -> ModelCommand { diff --git a/crates/improvise-protocol/src/delivery/client.rs b/crates/improvise-protocol/src/delivery/client.rs index 88259a36..67ce2800 100644 --- a/crates/improvise-protocol/src/delivery/client.rs +++ b/crates/improvise-protocol/src/delivery/client.rs @@ -266,9 +266,9 @@ impl EdgeClient { let window = matches!(c.body, CorrectionBody::Window); t.stale &= !window; if window - && t - .restructured - .is_some_and(|r| !r.same_epoch(&c.through) || r.publication <= c.through.publication) + && t.restructured.is_some_and(|r| { + !r.same_epoch(&c.through) || r.publication <= c.through.publication + }) { t.restructured = None; } diff --git a/crates/improvise-protocol/src/delivery/exchange.rs b/crates/improvise-protocol/src/delivery/exchange.rs index eaef636a..b8162604 100644 --- a/crates/improvise-protocol/src/delivery/exchange.rs +++ b/crates/improvise-protocol/src/delivery/exchange.rs @@ -78,7 +78,7 @@ pub struct Correction { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum Reply { - Correction(Correction), + Correction(Box), Notices(Vec), } diff --git a/crates/improvise-protocol/src/delivery/mod.rs b/crates/improvise-protocol/src/delivery/mod.rs index 38ef9804..6d85bee0 100644 --- a/crates/improvise-protocol/src/delivery/mod.rs +++ b/crates/improvise-protocol/src/delivery/mod.rs @@ -1,6 +1,5 @@ -//! Delivery (ADR 009 clause 9): the wire form, the stamp and notice, the correction -//! and the client end of an edge. The host end is in `improvise-command`. -//! Why: `context/code/delivery.md`. +//! Delivery (ADR 009 clause 9): the wire form, notices, corrections and the client end. +//! The host end is in `improvise-command`; why: `context/code/delivery.md`. pub mod client; pub mod exchange; diff --git a/crates/improvise-protocol/src/delivery/notice.rs b/crates/improvise-protocol/src/delivery/notice.rs index b4491dcf..616bf04d 100644 --- a/crates/improvise-protocol/src/delivery/notice.rs +++ b/crates/improvise-protocol/src/delivery/notice.rs @@ -176,10 +176,10 @@ impl Notice { /// `self` followed by `next`, if `next` starts where `self` ends. Anything else /// has a gap or another epoch in it, and merging would hide a loss. - pub fn merge(mut self, next: Notice) -> Result { + pub fn merge(mut self, next: Notice) -> Result> { let adjacent = self.range.end() == next.range.start(); match adjacent { - false => Err((self, next)), + false => Err(Box::new((self, next))), true => { self.range.to = next.range.to; self.axis.extend(next.axis); @@ -243,8 +243,14 @@ mod tests { let now = stamp(5); assert!(stamp(4).is_behind(&now)); assert!(!stamp(5).is_behind(&now)); - let other_host = Stamp { host: 8, ..stamp(99) }; - let other_generation = Stamp { generation: 2, ..stamp(99) }; + let other_host = Stamp { + host: 8, + ..stamp(99) + }; + let other_generation = Stamp { + generation: 2, + ..stamp(99) + }; assert!(other_host.is_behind(&now), "a restart is behind"); assert!(other_generation.is_behind(&now), "a swap is behind"); } @@ -264,16 +270,21 @@ mod tests { #[test] fn a_gap_does_not_merge() { - let (a, b) = notice(3, &[]).merge(notice(5, &[])).unwrap_err(); + let (a, b) = *notice(3, &[]).merge(notice(5, &[])).unwrap_err(); assert_eq!((a.range.to, b.range.to), (3, 5)); - let foreign = Notice::empty(PubRange::one(Stamp { host: 9, ..stamp(4) })); + let foreign = Notice::empty(PubRange::one(Stamp { + host: 9, + ..stamp(4) + })); assert!(notice(3, &[]).merge(foreign).is_err()); } #[test] fn merging_keeps_axis_changes_in_order_and_correct_absorbs_keys() { let mut a = notice(1, &[("s", &[1])]); - a.axis.push(AxisChange::CategoryAdded { category: "X".into() }); + a.axis.push(AxisChange::CategoryAdded { + category: "X".into(), + }); let mut b = notice(2, &[]); b.moved.insert(ShapeKey("s".into()), Moved::Correct); b.axis.push(AxisChange::ItemRemoved { @@ -291,7 +302,10 @@ mod tests { #[test] fn a_notice_round_trips_and_a_wide_stamp_crosses_as_a_string() { - let wide_host = Stamp { host: u64::MAX, ..stamp(1 << 53) }; + let wide_host = Stamp { + host: u64::MAX, + ..stamp(1 << 53) + }; let n = Notice::empty(PubRange::one(wide_host)); let json = serde_json::to_string(&n).unwrap(); assert!(json.contains(&format!("\"{}\"", u64::MAX)), "{json}"); From eb15df6c368523251fa86729cfd21b8e95b186cb Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:11:48 -0700 Subject: [PATCH 06/12] docs(context): the reasoning and the evidence for delivery (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/code/delivery.md | 198 ++++++++++++++++++++++++++++++ context/test-evidence/delivery.md | 95 ++++++++++++++ scripts/build-doc-site.mjs | 2 + 3 files changed, 295 insertions(+) create mode 100644 context/code/delivery.md create mode 100644 context/test-evidence/delivery.md diff --git a/context/code/delivery.md b/context/code/delivery.md new file mode 100644 index 00000000..2ee9781c --- /dev/null +++ b/context/code/delivery.md @@ -0,0 +1,198 @@ +# Delivery: the reasoning + +ADR 009 step 8 (improvise-z2vr): notices per committed command, watermarks, corrections, the shared +bounded log, the lossless wire and the edge adapters. The decisions are the ADR's clause 9 +(`context/adr/009-incremental-evaluation.md`). This file records how the code reads them, the places +the ADR left a choice open, and the places it could not be satisfied yet. The evidence for each test is +in `context/test-evidence/delivery.md`. + +**What is built, and against what.** Neither the incremental backend nor ADR 008's render model +exists. So the source of every target is the existing Eager path: the cells a `GridLayout` draws for a +view. Everything below is correct against that source and does not depend on what replaces it. The +circuit will supply "moved output keys" where `Delivery::commit` diffs two `Output`s today, and +nothing else changes. + +| piece | where | +|---|---| +| wire values, keys, Z-set elements | `improvise-protocol/src/delivery/wire.rs` | +| stamp, range, notice, merge | `.../notice.rs` | +| request, correction, `Edge` | `.../exchange.rs` | +| the client end: watermark, held cells, when to ask | `.../client.rs` | +| the publisher, the log, live shapes, answering | `improvise-command/src/ui/delivery.rs` | +| `Host` integration, in-process and JSON edges | `ui/host.rs`, `ui/delivery.rs` | + +## 1. The wire form + +A number crosses as ADR 005's canonical text through `numtower::Num`: `#i0.1`, `#i-0.0`, `inf`, +`-inf`, `nan`. `WireNum` accepts only text the tower itself writes (`parse` re-prints and compares), +because a second spelling of one value would make textual equality wrong. Equality of wire values is +textual, so `-0.0` and `+0.0` are different wire values. That is the opposite of the cell delta in +clause 3, which makes them equal; the two do not meet, because the delta is a store-side signal and a +correction is a value the client displays. + +An integer is a JSON number below 2^53 and a decimal string at or above it (`wide`, `wide_weight`). +That covers stamps, positions, window sizes and weights. Text is a JSON string, byte-exact. + +`CellValue` is still an `f64`, so `WireValue::of` writes it through `Num::inexact` and +`to_cell_value` narrows an exact number to `f64`. The round-trip properties therefore run over +28-digit decimals and `i128`-wide fractions at the `WireNum` level, where they are lossless, and over +`f64` at the `CellValue` level, where only the NaN payload is dropped. + +`CellKey` is not `Serialize`, so a cell is named by `WireKey`, its `(category, item)` tuple. Under +ADR 007 those become ids; nothing here reads the strings. + +## 2. Stamps and notices + +`Stamp` is `(host instance, document_generation, publication)`. A stamp from another host or +generation is behind every current one (`Stamp::is_behind`). The publication number increments once +per committed command, and it keeps counting across a document swap rather than restarting, so a +number is never reused within one host instance. + +**What "a committed command" is.** The host publishes one notice for each `Command::Model` that +lands. A view command (cursor, mode, a view switch) publishes nothing: it is view-store state and +the model store has no business announcing it. A write that changes nothing still publishes, with +empty `moved`, because the contract is one notice per commit and a client counts them. + +**A notice names moved keys per live shape**, plus axis changes, relabels, flags, evicted shapes. Axis +changes are computed from the categories' item lists before and after (`axis_changes`): insert and +remove by item, and a reorder when the surviving items changed order. `Relabel` exists and is never +emitted: with names as identity a rename is a remove and an insert, and display writes arrive with +ADR 007. Flags come from comparing the formula texts, `measure_agg`, the views and the default view. + +`Notice::merge` joins two notices only when the second starts where the first ends (same epoch). +Anything else returns both unchanged, so a merge cannot hide a loss. `Moved::Correct` absorbs key +lists in a merge. + +## 3. Requests and corrections + +A request is a shape key, a window and one of three asks. `Ask::Correction` is "bring me up from my +watermark"; `Ask::Fetch` is "the current values of these keys, covering the publications up to +`through`"; `Ask::Notices` is "the notices after this stamp". + +**A correction's elements are `target`, not `target - held`.** The host does not know what the client +holds, so it sends the target restricted as the body says, and the client computes `target - held` +with ordinary Z-set subtraction. A `Moved` body restricts both sides to the named keys, so a key the +target no longer holds is removed. A `Window` body replaces everything held. + +**`through` is not `at`.** `at` is the state the values were read at, always the latest committed one. +`through` is how far the correction advances the client's watermark. They differ for a `Fetch`: +the client named the keys of the notices it has seen, and a later commit may have moved other keys +whose notice it has not seen yet. Advancing to `at` would make that notice look already applied, so +the watermark advances only to `through` and the later notice refetches its keys. The values are +at least as new as the watermark claims, never older. + +**An axis change makes the correction a window refetch.** A moved key list says which values changed, +not which keys sit in the client's window, and an insert or remove shifts cells into and out of it +without moving them. The ADR has the client re-derive positions from the axis changes and fetch what +it lacks. That needs ADR 008's render model, which this layer does not have. So the client records a +notice that moved an axis (`restructured`), asks `Fetch { window: true }`, and the host answers with a +`Window` body whose `carried` notice holds the axis changes, in order. This is a fetch, not a +correction, so the required test "a frame `[AddItem, SetCell]` ... and no client asks for a correction" +holds: it asserts no `Ask::Correction` was sent. When the render model exists the client can narrow +the fetch to the cells that entered its window. + +## 4. The client + +`EdgeClient` holds, per shape: the window, the held cells (a `ZSet`), the **watermark** (applied +through), `seen` (the last notice received), the notices received and not yet fetched, `restructured` +and `stale`. + +`on_notice` ignores a duplicate (`to <= seen`), treats a gap, another host or generation, a shape in the +evicted list, or `Moved::Correct` as "ask for a correction", and otherwise queues the moved keys. The +watermark advances over a notice only when nothing is owed, so it never claims a key it has not fetched. + +`apply` takes a correction. It is a duplicate if the watermark already covers `through`; a `Moved` +body is a gap unless it starts at or below the watermark. Otherwise `held += target - held`, the +watermark becomes `through`, and the queued notices it covers are dropped. A late reply older than +the watermark therefore cannot regress the state, which is what makes a retry safe. + +`connection_lost`, `request_failed` and `unreadable` set `stale`, which `wants` turns into +`Ask::Correction` from the watermark the client held. A duplicate whose `through` equals the watermark +also clears `stale`: the host has said there is nothing newer. + +**A lost final notice is invisible to a listener.** The next notice shows the gap, but if none comes the +client never learns. `poll` is the answer: it asks `Ask::Notices { after: seen }`, which the host +answers from the log. The property test ends every history with a `poll` for this reason. + +`Event`s (`Axis`, `Relabel`, `Flags`, `Resync`) are what the shape's consumer applies to its +structure. They are deduplicated by `events_through`, because an axis change is not idempotent. +`Resync` means positions are unknown: re-derive them from what is held. + +## 5. The Eager stand-in for a shape key + +The real key is the circuit's: axes, None categories, filters, pages, `measure_agg`, in canonical id +form. Here a shape key is `eager:view/` (`shape_of_view`) and the output is that view's layout. +That is finer than the real key, which is always safe: two views that would share state do not, and +nothing else is lost. A view that no longer exists is announced as evicted. + +`Output` is the shape's whole output, keyed by `WireKey` with the position attached. `target` restricts +it to a window. `moved_since` compares two outputs by value and presence; a changed position is an axis +change, not a key. + +**This costs O(grid) per live shape per commit.** It stands in for the circuit's delta, and it is the +reason the module exists beside, not instead of, the incremental work: nothing else in the host reads +the whole grid per command. + +## 6. The host + +`Delivery` is one value per document, held by `Host`. `Host::handle` calls `Delivery::commit` after +each `Command::Model` that lands, and `Host::deliver` answers a `Request`. + +**Reads see only the last committed state.** `answer` reads `Live::output`, which `commit` refreshed, +and never the workbook. A value a failed command wrote is not in it. + +**Rollback.** `Host::mutating_frame` clones the `Delivery` beside the app snapshot and the session +mirrors, and `undo_frame` puts it back. The clone is cheap (`Arc` for outputs, notices and structure, +bounded by the log's capacity). So a refused frame consumes no publication number and publishes nothing. + +**The ADR and ADR 001 disagree here, and ADR 001 wins until ADR 006 exists.** Clause 9 says a frame +whose later command fails keeps its earlier commands and their notices, "each command commits on its own +(ADR 006)". Today's host undoes the whole frame (ADR 001 clause 4, `Host::mutating_frame`), and ADR 006 is +not in the tree. The required test "a refused command consumes no publication number, and the commands +before it in its frame stay published" is split in two. The first half is tested. The second is +tested as its opposite, `commands_before_a_refused_one_are_rolled_back_with_their_notices_under_adr_001`, +which is the test to flip when per-command commit lands: the publisher already supports it, since it +publishes whatever the host tells it committed. + +**The host instance** is a random id, never a time (`Host::new` takes one from `RandomState`). +`RandomState` has no entropy on `wasm32-unknown-unknown`, so a worker must pass a real one through +`Host::with_instance`. A restarted host with the same id would accept a stale watermark as current. + +**Eviction** (`evict_idle`) takes a local count and an idle length in the same unit, never a wall clock +compared across machines. An evicted key is announced in the next notice and forgotten; naming it again +revives it with `since` set to the current publication, so a watermark older than that is answered with +a window. + +## 7. Answering + +`covered(shape, watermark)` is true when the watermark is in the current epoch, is not ahead of the +host, and either equals the current publication or is no older than both the log's start and the time +the shape went live. Only then are the moved keys of `(watermark, now]` complete. + +- covered, only moved keys: a `Moved` body with those keys' current values in the window; +- covered, with axis changes or relabels in range: a `Window` body with `carried`; +- a `Moved::Correct` in range, or not covered: a `Window` body from `None`, which is `target - empty`. + +A notice lists no keys for a shape whose list passed `MAX_KEYS`; it says `Moved::Correct` instead, and +costs every reader a correction. `LOG_CAPACITY` bounds the single shared log. Both are set per host by +`limit_delivery` so a test can reach them. + +The token in a request is carried and echoed by the client and never read by the host. + +## 8. Edge adapters + +`Edge` is one end of a stream: `call(Request) -> Result`. `InProcess` is the +function call; `WireEdge` passes request and reply through JSON text, the adapter at a wire boundary, +so the lossless properties are exercised by every test that uses it. Neither adds a `ClientFrame` or +`ServerFrame` variant: the page does not read notices yet, and adding the frames means changing the +browser and the worker together (ADR 008 replaces `Projection`). A transport pushes +`Host::take_notices()`. + +## Not done, and why + +- **The notices are not sent by the socket or worker hosts.** They need frame variants and the page + (above). +- **A twin at the serving end** (a server-side watermark or filter) is not built; the ADR allows it and + does not require it. +- **A pushed filter** is the client's `window`, applied on every request. +- **Hidden-item overlays, pruning queries and column widths** are ADR 008 and clause 10. diff --git a/context/test-evidence/delivery.md b/context/test-evidence/delivery.md new file mode 100644 index 00000000..c617c311 --- /dev/null +++ b/context/test-evidence/delivery.md @@ -0,0 +1,95 @@ +# Test evidence: delivery + +What each delivery test guards and which injected fault it was seen to fail on. Keyed by test name. +The reasoning is in `context/code/delivery.md`. Each fault was put into the source one at a time, +the suite run, the fault reverted with `git checkout -- `, and the suite run again. + +The host-side tests are in `crates/improvise-command/src/ui/delivery_tests.rs`, the client's in +`crates/improvise-protocol/src/delivery/client_tests.rs`, the wire's in `delivery/wire.rs`. + +## The injected faults, by what failed + +- **The host reports no moved keys** (`let keys = BTreeSet::new()` in `Delivery::commit`): seven + host-side tests fail: `a_dropped_connection_or_a_failed_request_triggers_a_correction_and_the_session_resumes`, + `a_pull_only_client_follows_the_log_without_a_correction`, + `a_notice_names_the_moved_cells_including_a_formula_cell_it_did_not_write`, + `values_keep_every_bit_from_the_cell_to_the_client`, + `an_oversized_notice_says_correct_and_costs_every_reader_a_correction`, + `a_dropped_notice_is_detected_and_a_correction_repairs_it` and the random property. + `a_frame_of_two_commands_publishes_one_notice_each_and_no_client_asks_for_a_correction` still + passes, on purpose: its `AddItem` makes the client refetch the window, which also fetches the write. +- **The client treats every notice as contiguous** (`false => true` in `on_notice`): + `a_dropped_notice_is_detected`, `a_dropped_notice_is_detected_and_a_correction_repairs_it` and the + random property. +- **A failed frame does not put the `Delivery` back** (`undo_frame` drops `published`): + `a_refused_command_consumes_no_publication_number`, + `commands_before_a_refused_one_are_rolled_back_with_their_notices_under_adr_001` and + `a_refused_frame_is_never_read_back`. +- **A moved correction replaces instead of subtracting held** (`target` for `target - held_there`): + `a_moved_correction_applies_only_at_the_watermark_it_starts_from` and + `a_moved_key_the_target_no_longer_holds_is_removed`. The same for a window correction (`target` for + `target - held`): `a_correction_equals_the_target_and_a_second_one_is_the_zero_zset`, + `a_correction_replaces_what_is_held_not_adds_to_it` and + `a_correction_already_covered_is_idempotent_and_never_regresses`. +- **A wire number loses the sign of zero** (`Num::inexact(x + 0.0)` in `WireNum::of_f64`): + `a_float_crosses_in_canonical_text_with_the_sign_of_zero` and + `a_float_keeps_every_bit_but_the_nan_payload`. +- **`covered` ignores the log's start and the shape's liveness**: + `a_watermark_the_log_covers_gets_the_moved_keys_and_one_it_does_not_gets_the_window` and + `an_evicted_shape_is_announced_and_its_reader_corrects`. +- **The client does not remember an axis change** (`restructured` never set): + `an_axis_change_that_shifts_an_unmoved_cell_into_the_window_refetches_the_window`. The host's own + `restructured` test alone fails only the random property, since the client's `window` flag also forces + a window. +- **`Stamp::same_epoch` ignores the host instance**: + `a_notice_from_another_host_or_generation_asks_for_a_correction`, + `a_stamp_from_another_epoch_is_behind_every_current_one`, + `a_reconnect_after_a_host_restart_asks_for_a_correction_from_the_old_watermark` and + `a_notice_from_a_restarted_host_asks_for_a_correction_too`. + +## Entries + +### a_refused_frame_is_never_read_back + +The first fixture filled the document to the category limit and then refused the frame at its basis +check, so the write never ran and the test passed with the `Delivery` not restored. A fixture that fails +in a wider shape than the fault: the frame never reached `mutating_frame`. It now writes North, then +loads a missing document, and asserts `Rejection::MutationRefused` so the write is known to have run. The +rival above fails it with `77.0` held. + +### a_corrected_state_is_the_target_and_correcting_it_again_is_the_zero_zset + +The random property: histories of up to twelve single-command frames (writes of `0.0`, `-0.0`, both +infinities, NaN and `0.1 + 0.2`, item adds and removes, a formula), a random window, notices delivered +or dropped, connections lost, requests dropped, replies dropped and garbled. It ends with a `poll`, then +asserts the client holds exactly what the grid draws in its window, and that `target - held` is the zero +Z-set. The oracle (`drawn`) reads `GridLayout` itself and shares nothing with `Output`. + +Without the closing `poll` a lost last notice is not detectable by a listener, and the property +reported it as a stale client (minimal case: one `Remove`, notice dropped, no later notice). + +The first version of the property also failed on a window of one cell after a remove, and that was a +real gap in the correction, not in the test: the remove moved South into the window without moving its +value. It is the reason for the `restructured` rule (`delivery.md` §3). + +### a_frame_of_two_commands_publishes_one_notice_each_and_no_client_asks_for_a_correction + +Step 8's first required test. The frame's first command carries one basis stating both records +(`Host`'s frame rule: the first mutation's basis stands for the frame), which the test helper builds. + +### commands_before_a_refused_one_are_rolled_back_with_their_notices_under_adr_001 + +Step 8 requires that the commands before a refused one stay published. Today's host undoes the whole +frame (ADR 001 clause 4), so this test asserts the opposite and names the reason. Flip it when ADR 006 +makes each command commit on its own. + +### a_dropped_connection_or_a_failed_request_triggers_a_correction_and_the_session_resumes + +The three transport faults (request dropped, reply dropped, reply unreadable) each leave the client +asking for `Ask::Correction` from the watermark it held, and the next request carries the same session +token and that watermark. + +### an_axis_change_that_shifts_an_unmoved_cell_into_the_window_refetches_the_window + +A one-cell window, a remove that moves South into it. The notice lists only North, so a client that +fetches moved keys alone ends with an empty window. diff --git a/scripts/build-doc-site.mjs b/scripts/build-doc-site.mjs index 0ccf5cc6..9414b1c3 100644 --- a/scripts/build-doc-site.mjs +++ b/scripts/build-doc-site.mjs @@ -69,6 +69,7 @@ const SECTIONS = [ ["context/code/gp.md", "code/gp", "gp-tree and improvise-gp"], ["context/code/numtower.md", "code/numtower", "numtower, the numeric tower"], ["context/code/zset.md", "code/zset", "zset, the Z-set group"], + ["context/code/delivery.md", "code/delivery", "delivery, notices and corrections"], ["context/code/kb-tool.md", "code/kb-tool", "The kb tool"], ["context/code/tooling.md", "code/tooling", "The repository's own checkers"], ], @@ -89,6 +90,7 @@ const SECTIONS = [ ["context/test-evidence/gp.md", "test-evidence/gp", "gp-tree and improvise-gp"], ["context/test-evidence/numtower.md", "test-evidence/numtower", "numtower"], ["context/test-evidence/zset.md", "test-evidence/zset", "zset"], + ["context/test-evidence/delivery.md", "test-evidence/delivery", "delivery"], ["context/test-evidence/tooling.md", "test-evidence/tooling", "The repository's own gates"], ["context/test-evidence/layout-cost.md", "test-evidence/layout-cost", "Layout and keypress cost"], ["context/test-evidence/scale-bisect.md", "test-evidence/scale-bisect", "Scale bisect"], From 02bb43576237798024db69545bd72bcb869b1b01 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:12:01 -0700 Subject: [PATCH 07/12] chore(numtower): drop proptest seeds recorded against the first, wrong oracle (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- crates/numtower/tests/zero_sign.proptest-regressions | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 crates/numtower/tests/zero_sign.proptest-regressions diff --git a/crates/numtower/tests/zero_sign.proptest-regressions b/crates/numtower/tests/zero_sign.proptest-regressions deleted file mode 100644 index 671a5deb..00000000 --- a/crates/numtower/tests/zero_sign.proptest-regressions +++ /dev/null @@ -1,10 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 7c90e519b344ca86376f5a0b214d39873c790abd54bb140d369dac66bf4157dd # shrinks to mut addends = [], pairs = [] -cc 05e479a0606a42e8b9d31945d9ee2c548bff3b487f85189dff10c750ca1a2708 # shrinks to keep = [-3.0, 1e-300, 3.0], gone = [] -cc ba72a5a0e795ac7bb6f0833b21286f9da2cd9ec82c20224550e0fe030791e906 # shrinks to addends = [-0.0] -cc 1b8f452c472f86228458b4a4ce0e904c63335d40d3aaab9c0e7606a98b54d771 # shrinks to keep = [-0.0], gone = [] From 826ea70cd6fe4cca95c9d55aebe32b08b91ec3a3 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:23:59 -0700 Subject: [PATCH 08/12] chore(nix): regenerate Cargo.nix for the zset and numtower edges (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- Cargo.nix | 658 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 652 insertions(+), 6 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index b78b6f31..e06305f8 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -149,6 +149,26 @@ rec { packageId = "improvise-ws-server"; }; + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; + "numtower" = rec { + packageId = "numtower"; + build = internal.buildRustCrateWithFeatures { + packageId = "numtower"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; + "zset" = rec { + packageId = "zset"; + build = internal.buildRustCrateWithFeatures { + packageId = "zset"; + }; + # Debug support which might change between releases. # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; @@ -403,6 +423,22 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "arrayvec" = rec { + crateName = "arrayvec"; + version = "0.7.8"; + edition = "2018"; + sha256 = "0mmd8lrijbvg1qp4c5zis5dq41a3mjv2rb6bxkyj9kwaw2k6gyyk"; + authors = [ + "bluss" + ]; + features = { + "borsh" = [ "dep:borsh" ]; + "default" = [ "std" ]; + "serde" = [ "dep:serde" ]; + "zeroize" = [ "dep:zeroize" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "atomic" = rec { crateName = "atomic"; version = "0.6.1"; @@ -603,6 +639,93 @@ rec { "zeroize" = [ "dep:zeroize" ]; }; }; + "borsh" = rec { + crateName = "borsh"; + version = "1.8.1"; + edition = "2018"; + crateBin = []; + sha256 = "02p2lhz49944wc6j5dzr42y1mkrv0znbicg3cl61b9bbda25sg2m"; + authors = [ + "Near Inc " + ]; + dependencies = [ + { + name = "borsh-derive"; + packageId = "borsh-derive"; + optional = true; + } + { + name = "bytes"; + packageId = "bytes"; + optional = true; + usesDefaultFeatures = false; + } + ]; + buildDependencies = [ + { + name = "cfg_aliases"; + packageId = "cfg_aliases"; + } + ]; + features = { + "ascii" = [ "dep:ascii" ]; + "borsh-derive" = [ "dep:borsh-derive" ]; + "bson" = [ "dep:bson" ]; + "bytes" = [ "dep:bytes" ]; + "default" = [ "std" ]; + "derive" = [ "borsh-derive" ]; + "hashbrown" = [ "dep:hashbrown" ]; + "indexmap" = [ "dep:indexmap" ]; + "std" = [ "bytes?/std" ]; + "unstable__schema" = [ "derive" "borsh-derive/schema" ]; + "uuid" = [ "dep:uuid" ]; + }; + resolvedDefaultFeatures = [ "borsh-derive" "derive" "std" "unstable__schema" ]; + }; + "borsh-derive" = rec { + crateName = "borsh-derive"; + version = "1.8.1"; + edition = "2018"; + sha256 = "0mlzrbwzcr7qaky1iap09m0nrzp6crjd79sinj4iz808cxjzxk8j"; + procMacro = true; + libName = "borsh_derive"; + authors = [ + "Near Inc " + ]; + dependencies = [ + { + name = "once_cell"; + packageId = "once_cell"; + } + { + name = "proc-macro-crate"; + packageId = "proc-macro-crate"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 3.0.6"; + features = [ "full" "fold" ]; + } + ]; + devDependencies = [ + { + name = "syn"; + packageId = "syn 3.0.6"; + features = [ "full" "fold" "parsing" ]; + } + ]; + features = { + }; + resolvedDefaultFeatures = [ "default" "schema" ]; + }; "bumpalo" = rec { crateName = "bumpalo"; version = "3.20.3"; @@ -2147,6 +2270,42 @@ rec { }; resolvedDefaultFeatures = [ "more_lengths" ]; }; + "getrandom 0.2.17" = rec { + crateName = "getrandom"; + version = "0.2.17"; + edition = "2018"; + sha256 = "1l2ac6jfj9xhpjjgmcx6s1x89bbnw9x6j9258yy6xjkzpq0bqapz"; + authors = [ + "The Rand Project Developers" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (target."unix" or false); + } + { + name = "wasi"; + packageId = "wasi"; + usesDefaultFeatures = false; + target = { target, features }: ("wasi" == target."os" or null); + } + ]; + features = { + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "js" = [ "wasm-bindgen" "js-sys" ]; + "js-sys" = [ "dep:js-sys" ]; + "rustc-dep-of-std" = [ "compiler_builtins" "core" "libc/rustc-dep-of-std" "wasi/rustc-dep-of-std" ]; + "wasm-bindgen" = [ "dep:wasm-bindgen" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "getrandom 0.3.4" = rec { crateName = "getrandom"; version = "0.3.4"; @@ -2750,6 +2909,10 @@ rec { name = "web-time"; packageId = "web-time"; } + { + name = "zset"; + packageId = "zset"; + } ]; devDependencies = [ { @@ -2762,6 +2925,10 @@ rec { packageId = "improvise-protocol"; features = [ "test-support" ]; } + { + name = "proptest"; + packageId = "proptest"; + } { name = "tempfile"; packageId = "tempfile"; @@ -2982,6 +3149,10 @@ rec { name = "improvise-core"; packageId = "improvise-core"; } + { + name = "numtower"; + packageId = "numtower"; + } { name = "serde"; packageId = "serde"; @@ -2997,8 +3168,16 @@ rec { name = "unicode-width"; packageId = "unicode-width"; } + { + name = "zset"; + packageId = "zset"; + } ]; devDependencies = [ + { + name = "proptest"; + packageId = "proptest"; + } { name = "serde_json"; packageId = "serde_json"; @@ -3785,6 +3964,39 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "std" ]; }; + "num-bigint" = rec { + crateName = "num-bigint"; + version = "0.4.8"; + edition = "2021"; + sha256 = "0ry3xjal8f5xhdinani268ci13h14mf7j4w0y1gflfzhw3knk7n8"; + libName = "num_bigint"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "default" = [ "std" ]; + "quickcheck" = [ "dep:quickcheck" ]; + "rand" = [ "dep:rand" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "num-integer/std" "num-traits/std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "num-conv" = rec { crateName = "num-conv"; version = "0.2.2"; @@ -3822,6 +4034,67 @@ rec { ]; }; + "num-integer" = rec { + crateName = "num-integer"; + version = "0.1.47"; + edition = "2018"; + sha256 = "02z1p3azy6p10n99skrab4a6hhfd4amf2i9gm8sxqd1p9dfxkqkw"; + libName = "num_integer"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "num-traits/std" ]; + }; + resolvedDefaultFeatures = [ "i128" "std" ]; + }; + "num-rational" = rec { + crateName = "num-rational"; + version = "0.4.2"; + edition = "2021"; + sha256 = "093qndy02817vpgcqjnj139im3jl7vkq4h68kykdqqh577d18ggq"; + libName = "num_rational"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "num-bigint"; + packageId = "num-bigint"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + features = { + "default" = [ "num-bigint" "std" ]; + "num-bigint" = [ "dep:num-bigint" ]; + "num-bigint-std" = [ "num-bigint/std" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "num-bigint?/std" "num-integer/std" "num-traits/std" ]; + }; + resolvedDefaultFeatures = [ "default" "num-bigint" "num-bigint-std" "std" ]; + }; "num-traits" = rec { crateName = "num-traits"; version = "0.2.19"; @@ -3841,7 +4114,7 @@ rec { "default" = [ "std" ]; "libm" = [ "dep:libm" ]; }; - resolvedDefaultFeatures = [ "default" "std" ]; + resolvedDefaultFeatures = [ "default" "i128" "std" ]; }; "num_threads" = rec { crateName = "num_threads"; @@ -3859,6 +4132,54 @@ rec { } ]; + }; + "numtower" = rec { + crateName = "numtower"; + version = "0.1.0"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./crates/numtower; }; + dependencies = [ + { + name = "num-bigint"; + packageId = "num-bigint"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + } + { + name = "num-rational"; + packageId = "num-rational"; + usesDefaultFeatures = false; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "rust_decimal"; + packageId = "rust_decimal"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + ]; + devDependencies = [ + { + name = "num-rational"; + packageId = "num-rational"; + features = [ "num-bigint-std" ]; + } + { + name = "proptest"; + packageId = "proptest"; + } + ]; + }; "once_cell" = rec { crateName = "once_cell"; @@ -4406,6 +4727,25 @@ rec { }; resolvedDefaultFeatures = [ "simd" "std" ]; }; + "proc-macro-crate" = rec { + crateName = "proc-macro-crate"; + version = "3.5.0"; + edition = "2021"; + sha256 = "0kv1g1d1zjwxlgcaba2qlshzyy32j03xic8rskqlcr5mnblsfyz6"; + libName = "proc_macro_crate"; + authors = [ + "Bastian Köcher " + ]; + dependencies = [ + { + name = "toml_edit"; + packageId = "toml_edit"; + usesDefaultFeatures = false; + features = [ "parse" ]; + } + ]; + + }; "proc-macro2" = rec { crateName = "proc-macro2"; version = "1.0.107"; @@ -4463,7 +4803,7 @@ rec { } { name = "rand_chacha"; - packageId = "rand_chacha"; + packageId = "rand_chacha 0.9.0"; usesDefaultFeatures = false; } { @@ -4616,6 +4956,19 @@ rec { "The Rust Project Developers" ]; dependencies = [ + { + name = "libc"; + packageId = "libc"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (target."unix" or false); + } + { + name = "rand_chacha"; + packageId = "rand_chacha 0.3.1"; + optional = true; + usesDefaultFeatures = false; + } { name = "rand_core"; packageId = "rand_core 0.6.4"; @@ -4632,7 +4985,7 @@ rec { "std" = [ "rand_core/std" "rand_chacha/std" "alloc" "getrandom" "libc" ]; "std_rng" = [ "rand_chacha" ]; }; - resolvedDefaultFeatures = [ "small_rng" ]; + resolvedDefaultFeatures = [ "alloc" "getrandom" "libc" "rand_chacha" "small_rng" "std" ]; }; "rand 0.9.5" = rec { crateName = "rand"; @@ -4646,7 +4999,7 @@ rec { dependencies = [ { name = "rand_chacha"; - packageId = "rand_chacha"; + packageId = "rand_chacha 0.9.0"; optional = true; usesDefaultFeatures = false; } @@ -4666,7 +5019,37 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "os_rng" "std" ]; }; - "rand_chacha" = rec { + "rand_chacha 0.3.1" = rec { + crateName = "rand_chacha"; + version = "0.3.1"; + edition = "2018"; + sha256 = "123x2adin558xbhvqb8w4f6syjsdkmqff8cxwhmjacpsl1ihmhg6"; + authors = [ + "The Rand Project Developers" + "The Rust Project Developers" + "The CryptoCorrosion Contributors" + ]; + dependencies = [ + { + name = "ppv-lite86"; + packageId = "ppv-lite86"; + usesDefaultFeatures = false; + features = [ "simd" ]; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + } + ]; + features = { + "default" = [ "std" ]; + "serde" = [ "dep:serde" ]; + "serde1" = [ "serde" ]; + "std" = [ "ppv-lite86/std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; + "rand_chacha 0.9.0" = rec { crateName = "rand_chacha"; version = "0.9.0"; edition = "2021"; @@ -4722,12 +5105,20 @@ rec { "The Rand Project Developers" "The Rust Project Developers" ]; + dependencies = [ + { + name = "getrandom"; + packageId = "getrandom 0.2.17"; + optional = true; + } + ]; features = { "getrandom" = [ "dep:getrandom" ]; "serde" = [ "dep:serde" ]; "serde1" = [ "serde" ]; "std" = [ "alloc" "getrandom" "getrandom/std" ]; }; + resolvedDefaultFeatures = [ "alloc" "getrandom" "std" ]; }; "rand_core 0.9.5" = rec { crateName = "rand_core"; @@ -5311,6 +5702,147 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; }; + "rust_decimal" = rec { + crateName = "rust_decimal"; + version = "1.43.0"; + edition = "2021"; + sha256 = "0fnd56zidby90qy98ffkmx7rjcvng7ssdglz37f43b6wflp2flvn"; + authors = [ + "Paul Mason " + ]; + dependencies = [ + { + name = "arrayvec"; + packageId = "arrayvec"; + usesDefaultFeatures = false; + } + { + name = "borsh"; + packageId = "borsh"; + optional = true; + usesDefaultFeatures = false; + features = [ "derive" "unstable__schema" ]; + } + { + name = "bytes"; + packageId = "bytes"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "rand"; + packageId = "rand 0.8.8"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "rand"; + packageId = "rand 0.9.5"; + rename = "rand-0_9"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde_json"; + packageId = "serde_json"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: ("wasm32" == target."arch" or null); + } + ]; + devDependencies = [ + { + name = "bytes"; + packageId = "bytes"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch" or null)); + } + { + name = "rand"; + packageId = "rand 0.8.8"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch" or null)); + features = [ "getrandom" ]; + } + { + name = "rand"; + packageId = "rand 0.9.5"; + rename = "rand-0_9"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch" or null)); + features = [ "thread_rng" ]; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + usesDefaultFeatures = false; + target = { target, features }: ("wasm32" == target."arch" or null); + } + ]; + features = { + "borsh" = [ "dep:borsh" "std" ]; + "bytemuck" = [ "c-repr" "dep:bytemuck" "dep:bytemuck_derive" ]; + "db-diesel-mysql" = [ "diesel/mysql_backend" "std" ]; + "db-diesel-postgres" = [ "diesel/postgres_backend" "std" ]; + "db-diesel2-mysql" = [ "db-diesel-mysql" ]; + "db-diesel2-postgres" = [ "db-diesel-postgres" ]; + "db-postgres" = [ "dep:bytes" "dep:postgres-types" "std" ]; + "db-tokio-postgres" = [ "dep:bytes" "dep:postgres-types" "std" ]; + "default" = [ "serde" "std" ]; + "diesel" = [ "dep:diesel" ]; + "macros" = [ "dep:rust_decimal_macros" ]; + "maths-nopanic" = [ "maths" ]; + "ndarray" = [ "dep:ndarray" ]; + "proptest" = [ "dep:proptest" ]; + "rand" = [ "dep:rand" ]; + "rand-0_9" = [ "dep:rand-0_9" ]; + "rocket-traits" = [ "dep:rocket" "std" ]; + "rust-fuzz" = [ "dep:arbitrary" ]; + "serde" = [ "dep:serde" "wasm-bindgen?/serde" ]; + "serde-arbitrary-precision" = [ "serde-with-arbitrary-precision" ]; + "serde-bincode" = [ "serde-str" ]; + "serde-float" = [ "serde-with-float" ]; + "serde-str" = [ "serde-with-str" ]; + "serde-with-arbitrary-precision" = [ "serde" "serde_json/arbitrary_precision" "serde_json/std" "zmij" ]; + "serde-with-float" = [ "serde" ]; + "serde-with-str" = [ "serde" ]; + "serde_json" = [ "dep:serde_json" ]; + "std" = [ "arrayvec/std" "wasm-bindgen?/std" "borsh?/std" "bytes?/std" "rand?/std" "rand-0_9?/std" "serde?/std" "serde_json?/std" ]; + "tokio-pg" = [ "db-tokio-postgres" ]; + "tokio-postgres" = [ "dep:tokio-postgres" ]; + "wasm" = [ "dep:wasm-bindgen" ]; + "zmij" = [ "dep:zmij" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "rustc_version" = rec { crateName = "rustc_version"; version = "0.4.1"; @@ -6110,7 +6642,7 @@ rec { "proc-macro" = [ "proc-macro2/proc-macro" "quote?/proc-macro" ]; "test" = [ "syn-test-suite/all-features" ]; }; - resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; + resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "fold" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; "tempfile" = rec { crateName = "tempfile"; @@ -6785,6 +7317,82 @@ rec { }; resolvedDefaultFeatures = [ "connect" "default" "handshake" "stream" ]; }; + "toml_datetime" = rec { + crateName = "toml_datetime"; + version = "1.1.2+spec-1.1.0"; + edition = "2024"; + sha256 = "0lrhcmqvhjr259w4f2kijya7fgi0kpmhg9fb3m144v3cj1kxg1ib"; + dependencies = [ + { + name = "serde_core"; + packageId = "serde_core"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "serde_core?/alloc" ]; + "default" = [ "std" ]; + "serde" = [ "dep:serde_core" ]; + "std" = [ "alloc" "serde_core?/std" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; + "toml_edit" = rec { + crateName = "toml_edit"; + version = "0.25.17+spec-1.1.0"; + edition = "2024"; + sha256 = "174w1b2fjg1jb0wi7384v1m59g7ga79458i022gafjakpddisr73"; + dependencies = [ + { + name = "indexmap"; + packageId = "indexmap"; + features = [ "std" ]; + } + { + name = "toml_datetime"; + packageId = "toml_datetime"; + } + { + name = "toml_parser"; + packageId = "toml_parser"; + optional = true; + } + { + name = "winnow"; + packageId = "winnow"; + optional = true; + } + ]; + features = { + "debug" = [ "toml_parser?/debug" "dep:anstream" "dep:anstyle" "display" ]; + "default" = [ "parse" "display" ]; + "display" = [ "dep:toml_writer" ]; + "parse" = [ "dep:toml_parser" "dep:winnow" ]; + "serde" = [ "dep:serde_core" "toml_datetime/serde" "dep:serde_spanned" ]; + }; + resolvedDefaultFeatures = [ "parse" ]; + }; + "toml_parser" = rec { + crateName = "toml_parser"; + version = "1.1.5+spec-1.1.0"; + edition = "2024"; + sha256 = "0k3lljyi4zxchdklaqghkwbl7wkd2ab1w16hlyniqzid0fl979ms"; + dependencies = [ + { + name = "winnow"; + packageId = "winnow"; + usesDefaultFeatures = false; + } + ]; + features = { + "debug" = [ "std" "dep:anstream" "dep:anstyle" ]; + "default" = [ "std" ]; + "simd" = [ "winnow/simd" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; "tungstenite" = rec { crateName = "tungstenite"; version = "0.30.0"; @@ -8447,6 +9055,31 @@ rec { }; resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; + "winnow" = rec { + crateName = "winnow"; + version = "1.0.4"; + edition = "2021"; + sha256 = "10fzxipa7lx16172p3aca9j60hzbqgjki2f95kqksd5qywcp7f93"; + dependencies = [ + { + name = "memchr"; + packageId = "memchr"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "ascii" = [ "parser" ]; + "binary" = [ "parser" ]; + "debug" = [ "std" "dep:anstream" "dep:anstyle" "dep:is_terminal_polyfill" "dep:terminal_size" ]; + "default" = [ "std" "ascii" "binary" ]; + "simd" = [ "dep:memchr" ]; + "std" = [ "alloc" "memchr?/std" ]; + "unstable-doc" = [ "alloc" "std" "ascii" "binary" "simd" "unstable-recover" ]; + "unstable-recover" = [ "parser" ]; + }; + resolvedDefaultFeatures = [ "alloc" "ascii" "binary" "default" "parser" "std" ]; + }; "wit-bindgen" = rec { crateName = "wit-bindgen"; version = "0.57.1"; @@ -8559,6 +9192,19 @@ rec { "no-panic" = [ "dep:no-panic" ]; }; }; + "zset" = rec { + crateName = "zset"; + version = "0.1.0"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./crates/zset; }; + devDependencies = [ + { + name = "proptest"; + packageId = "proptest"; + } + ]; + + }; }; # From f1485f80d4f899729dc40072f7886a30ad010cb2 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:33:07 -0700 Subject: [PATCH 09/12] fix(protocol): older epochs and dead hosts cannot regress a client, pulls are per shape, wire integers are canonical (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- .../improvise-protocol/src/delivery/client.rs | 32 ++- crates/improvise-protocol/src/delivery/mod.rs | 3 + .../improvise-protocol/src/delivery/notice.rs | 3 +- .../src/delivery/review_tests.rs | 183 ++++++++++++++++++ .../improvise-protocol/src/delivery/wire.rs | 25 ++- crates/numtower/src/sum.rs | 4 +- crates/numtower/tests/zero_sign.rs | 13 ++ 7 files changed, 254 insertions(+), 9 deletions(-) create mode 100644 crates/improvise-protocol/src/delivery/review_tests.rs diff --git a/crates/improvise-protocol/src/delivery/client.rs b/crates/improvise-protocol/src/delivery/client.rs index 67ce2800..d9d47f42 100644 --- a/crates/improvise-protocol/src/delivery/client.rs +++ b/crates/improvise-protocol/src/delivery/client.rs @@ -156,8 +156,16 @@ impl EdgeClient { /// A pushed notice. Duplicates are ignored, a gap or a new epoch makes every shape ask. pub fn on_notice(&mut self, notice: &Notice) -> Vec { + self.accept(notice, None) + } + + /// `only` limits the notice to one shape: a pulled notice carries no other shape's keys. + fn accept(&mut self, notice: &Notice, only: Option<&ShapeKey>) -> Vec { let end = notice.stamp(); for (shape, t) in &mut self.shapes { + if only.is_some_and(|o| o != shape) { + continue; + } let contiguous = match t.seen { Some(seen) if seen.same_epoch(&end) => match notice.range.to <= seen.publication { true => continue, @@ -170,7 +178,7 @@ impl EdgeClient { t.unapplied.push((end, Moved::Correct)); continue; } - if !notice.axis.is_empty() || !notice.relabels.is_empty() { + if !notice.axis.is_empty() || !notice.relabels.is_empty() || notice.flags.views { t.restructured = Some(end); } match notice.moved.get(shape) { @@ -235,6 +243,12 @@ impl EdgeClient { return (Outcome::Untracked, Vec::new()); }; let target = decode(&c.target); + let older = |s: Option| { + s.is_some_and(|s| s.host == c.through.host && c.through.generation < s.generation) + }; + if older(t.watermark) || older(t.seen) { + return (Outcome::Duplicate, Vec::new()); + } let covered = t .watermark .is_some_and(|w| w.same_epoch(&c.through) && w.publication >= c.through.publication); @@ -245,6 +259,15 @@ impl EdgeClient { let delta = match (&c.body, covered) { (_, true) => { t.stale &= t.watermark != Some(c.through); + let live = c.through; + t.unapplied + .retain(|(at, _)| at.same_epoch(&live) && at.publication > live.publication); + if t.restructured.is_some_and(|r| !r.same_epoch(&live)) { + t.restructured = None; + } + if t.seen.is_some_and(|s| !s.same_epoch(&live)) { + t.seen = Some(live); + } return (Outcome::Duplicate, Vec::new()); } (CorrectionBody::Window, false) => &target - &t.held, @@ -312,7 +335,8 @@ impl EdgeClient { match edge.call(request) { Ok(Reply::Correction(c)) => events.extend(self.apply(&c).1), Ok(Reply::Notices(list)) => { - list.iter().for_each(|n| events.extend(self.on_notice(n))); + list.iter() + .for_each(|n| events.extend(self.accept(n, Some(shape)))); } Err(EdgeError::Failed) => { self.request_failed(shape); @@ -346,7 +370,9 @@ impl EdgeClient { }; let mut events = Vec::new(); match edge.call(request) { - Ok(Reply::Notices(list)) => list.iter().for_each(|n| events.extend(self.on_notice(n))), + Ok(Reply::Notices(list)) => list + .iter() + .for_each(|n| events.extend(self.accept(n, Some(shape)))), Ok(Reply::Correction(c)) => events.extend(self.apply(&c).1), Err(EdgeError::Failed) => { self.request_failed(shape); diff --git a/crates/improvise-protocol/src/delivery/mod.rs b/crates/improvise-protocol/src/delivery/mod.rs index 6d85bee0..bb30c512 100644 --- a/crates/improvise-protocol/src/delivery/mod.rs +++ b/crates/improvise-protocol/src/delivery/mod.rs @@ -15,3 +15,6 @@ pub use wire::{Cell, WireElem, WireKey, WireNum, WireValue}; #[cfg(test)] mod client_tests; + +#[cfg(test)] +mod review_tests; diff --git a/crates/improvise-protocol/src/delivery/notice.rs b/crates/improvise-protocol/src/delivery/notice.rs index 616bf04d..476c27fb 100644 --- a/crates/improvise-protocol/src/delivery/notice.rs +++ b/crates/improvise-protocol/src/delivery/notice.rs @@ -85,7 +85,8 @@ pub enum AxisChange { ItemInserted { category: String, item: String, - at: usize, + #[serde(with = "wide")] + at: u128, }, ItemRemoved { category: String, diff --git a/crates/improvise-protocol/src/delivery/review_tests.rs b/crates/improvise-protocol/src/delivery/review_tests.rs new file mode 100644 index 00000000..3074d0d2 --- /dev/null +++ b/crates/improvise-protocol/src/delivery/review_tests.rs @@ -0,0 +1,183 @@ +//! Findings from the review of #348: each is a hand-built history that failed before its fix. + +use serde::{Deserialize, Serialize}; + +use super::client::*; +use super::exchange::*; +use super::notice::*; +use super::wire::{self, Cell, WireElem, WireKey, WireValue, encode}; + +fn at(host: u64, generation: u64, publication: u64) -> Stamp { + Stamp { + host, + generation, + publication, + } +} + +fn shape(name: &str) -> ShapeKey { + ShapeKey(name.into()) +} + +fn window() -> Window { + Window { + top: 0, + left: 0, + rows: 10, + cols: 10, + } +} + +fn key(n: u8) -> WireKey { + WireKey(vec![("C".into(), n.to_string())]) +} + +fn cell(k: u8, v: &str) -> Cell { + (key(k), WireValue::Text(v.into())) +} + +fn target(cells: &[Cell]) -> Vec { + encode(&cells.iter().cloned().map(|c| (c, 1)).collect()) +} + +fn window_correction(name: &str, through: Stamp, cells: &[Cell]) -> Correction { + Correction { + shape: shape(name), + at: through, + from: None, + through, + body: CorrectionBody::Window, + target: target(cells), + carried: None, + } +} + +fn synced(name: &str, through: Stamp, cells: &[Cell]) -> EdgeClient { + let mut c = EdgeClient::default(); + c.track(shape(name), window()); + c.apply(&window_correction(name, through, cells)); + c +} + +#[test] +fn a_late_window_reply_from_an_older_generation_does_not_regress() { + let mut c = synced("s", at(1, 2, 5), &[cell(1, "new")]); + let late = window_correction("s", at(1, 1, 9), &[cell(1, "old")]); + assert_eq!(c.apply(&late).0, Outcome::Duplicate); + assert_eq!(c.watermark(&shape("s")), Some(at(1, 2, 5))); + let kept = wire::decode(&target(&[cell(1, "new")])); + assert_eq!(c.held(&shape("s")), Some(&kept)); +} + +#[test] +fn a_late_notice_from_a_dead_host_settles_against_the_live_one() { + let mut c = synced("s", at(2, 1, 4), &[cell(1, "a")]); + c.on_notice(&Notice::empty(PubRange::one(at(1, 1, 9)))); + assert!(matches!( + c.wants(&shape("s")), + Some(Want::Correction { .. }) + )); + let live = window_correction("s", at(2, 1, 4), &[cell(1, "a")]); + assert_eq!(c.apply(&live).0, Outcome::Duplicate); + assert_eq!( + c.wants(&shape("s")), + None, + "the dead host's notice is dropped" + ); + c.on_notice(&Notice::empty(PubRange::one(at(2, 1, 5)))); + assert_eq!( + c.wants(&shape("s")), + None, + "and the live host's next one is contiguous" + ); +} + +struct Replies(Vec); + +impl Edge for Replies { + fn call(&mut self, _: Request) -> Result { + match self.0.is_empty() { + true => Err(EdgeError::Failed), + false => Ok(self.0.remove(0)), + } + } +} + +#[test] +fn a_poll_for_one_shape_leaves_the_other_shapes_alone() { + let (a, b) = (shape("a"), shape("b")); + let mut c = EdgeClient::default(); + for s in [&a, &b] { + c.track(s.clone(), window()); + c.apply(&window_correction(&s.0, at(1, 1, 4), &[])); + } + let mut pulled = Notice::empty(PubRange::one(at(1, 1, 5))); + pulled.moved.insert(a.clone(), Moved::Keys([key(1)].into())); + let mut edge = Replies(vec![Reply::Notices(vec![pulled])]); + c.poll(&mut edge, &a); + assert_eq!(c.watermark(&b), Some(at(1, 1, 4)), "b was not asked about"); + let mut next = Notice::empty(PubRange::one(at(1, 1, 6))); + next.moved.insert(b.clone(), Moved::Keys([key(2)].into())); + c.on_notice(&next); + assert!( + matches!(c.wants(&b), Some(Want::Correction { .. })), + "b missed publication 5, and must see the gap: {:?}", + c.wants(&b) + ); +} + +#[test] +fn a_views_flag_makes_the_client_fetch_its_window() { + let mut c = synced("s", at(1, 1, 4), &[cell(1, "a")]); + let mut n = Notice::empty(PubRange::one(at(1, 1, 5))); + n.flags.views = true; + c.on_notice(&n); + assert!(matches!( + c.wants(&shape("s")), + Some(Want::Fetch { window: true, .. }) + )); +} + +#[test] +fn wide_integers_are_canonical_on_input() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct P( + #[serde(with = "wire::wide")] u64, + #[serde(with = "wire::wide_weight")] i64, + ); + let bad = [ + r#"[9007199254740992,1]"#, + r#"["5",1]"#, + r#"["+9007199254740992",1]"#, + r#"["09007199254740992",1]"#, + r#"[1,9007199254740992]"#, + r#"[1,"5"]"#, + r#"[1,"-5"]"#, + r#"[1,"+9007199254740992"]"#, + r#"[1,"-9007199254740992 "]"#, + ]; + for text in bad { + assert!(serde_json::from_str::

(text).is_err(), "{text}"); + } + let good = r#"["9007199254740992","-9007199254740992"]"#; + assert_eq!( + serde_json::from_str::

(good).unwrap(), + P(1 << 53, -(1 << 53)) + ); + assert_eq!( + serde_json::from_str::

("[9007199254740991,-9007199254740991]").unwrap(), + P((1 << 53) - 1, -((1 << 53) - 1)) + ); +} + +#[test] +fn an_item_insert_position_crosses_as_a_wide_integer() { + let change = AxisChange::ItemInserted { + category: "C".into(), + item: "i".into(), + at: 1 << 60, + }; + let json = serde_json::to_string(&change).unwrap(); + assert!(json.contains(r#""at":"1152921504606846976""#), "{json}"); + assert_eq!(serde_json::from_str::(&json).unwrap(), change); +} diff --git a/crates/improvise-protocol/src/delivery/wire.rs b/crates/improvise-protocol/src/delivery/wire.rs index 7686f805..a66fbf33 100644 --- a/crates/improvise-protocol/src/delivery/wire.rs +++ b/crates/improvise-protocol/src/delivery/wire.rs @@ -29,8 +29,17 @@ pub mod wide { pub fn deserialize<'de, T: TryFrom, D: Deserializer<'de>>(d: D) -> Result { let n: u128 = match Repr::deserialize(d)? { - Repr::Small(n) => n.into(), - Repr::Wide(text) => text.parse().map_err(serde::de::Error::custom)?, + Repr::Small(n) if u128::from(n) < EXACT_JSON => n.into(), + Repr::Small(_) => { + return Err(serde::de::Error::custom("a wide integer must be a string")); + } + Repr::Wide(text) => { + let n: u128 = text.parse().map_err(serde::de::Error::custom)?; + match n >= EXACT_JSON && n.to_string() == text { + true => n, + false => return Err(serde::de::Error::custom("not a canonical wide integer")), + } + } }; T::try_from(n).map_err(|_| serde::de::Error::custom("integer out of range")) } @@ -56,8 +65,16 @@ pub mod wide_weight { pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { match Repr::deserialize(d)? { - Repr::Small(n) => Ok(n), - Repr::Wide(text) => text.parse().map_err(serde::de::Error::custom), + Repr::Small(n) if u128::from(n.unsigned_abs()) < EXACT_JSON => Ok(n), + Repr::Small(_) => Err(serde::de::Error::custom("a wide weight must be a string")), + Repr::Wide(text) => { + let n: i64 = text.parse().map_err(serde::de::Error::custom)?; + let wide = u128::from(n.unsigned_abs()) >= EXACT_JSON; + match wide && n.to_string() == text { + true => Ok(n), + false => Err(serde::de::Error::custom("not a canonical wide weight")), + } + } } } } diff --git a/crates/numtower/src/sum.rs b/crates/numtower/src/sum.rs index ce882009..c7d2c307 100644 --- a/crates/numtower/src/sum.rs +++ b/crates/numtower/src/sum.rs @@ -40,7 +40,9 @@ impl PartialEq for ExactSum { propagate(&mut mine); propagate(&mut theirs); let counts = |s: &ExactSum| (s.nan, s.pos_inf, s.neg_inf); - mine == theirs && counts(self) == counts(o) && self.negative_zero() == o.negative_zero() + let zero = mine.iter().all(|l| *l == 0); + let same_zero = !zero || self.negative_zero() == o.negative_zero(); + mine == theirs && counts(self) == counts(o) && same_zero } } diff --git a/crates/numtower/tests/zero_sign.rs b/crates/numtower/tests/zero_sign.rs index ae7cabbf..215821ba 100644 --- a/crates/numtower/tests/zero_sign.rs +++ b/crates/numtower/tests/zero_sign.rs @@ -87,3 +87,16 @@ fn a_sum_that_is_not_zero_never_reads_as_negative_zero() { sum.add(7.0, -1); assert_eq!(sum.value(), -2.0); } + +#[test] +fn equal_values_compare_equal_whatever_the_zero_counts_say() { + let mut with = ExactSum::new(); + with.add(-0.0, 1); + with.add(5.0, 1); + with.add(7.0, -1); + let mut without = ExactSum::new(); + without.add(5.0, 1); + without.add(7.0, -1); + assert_eq!(with.value(), without.value()); + assert_eq!(with, without); +} From edf474d2a352f6fbd8326cccbb3c40a936daab17 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:36:09 -0700 Subject: [PATCH 10/12] fix(command): linear item diffs, no delivery work while nobody listens, a bounded outbox, view changes refetch the window (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- crates/improvise-command/src/ui/delivery.rs | 104 +++++++++--- .../src/ui/delivery_tests.rs | 160 +++++++++++++++++- crates/improvise-command/src/ui/host.rs | 12 +- 3 files changed, 248 insertions(+), 28 deletions(-) diff --git a/crates/improvise-command/src/ui/delivery.rs b/crates/improvise-command/src/ui/delivery.rs index 4f52ed0a..d1bd0005 100644 --- a/crates/improvise-command/src/ui/delivery.rs +++ b/crates/improvise-command/src/ui/delivery.rs @@ -1,7 +1,7 @@ //! The host end of delivery: one notice per committed command, the shared bounded //! log, and corrections (ADR 009 clause 9). Why: `context/code/delivery.md` §5-§7. -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque}; use std::sync::Arc; use improvise_core::view::layout::GridLayout; @@ -103,8 +103,15 @@ struct Structure { default_view: String, } +#[cfg(test)] +thread_local! { + pub(crate) static STRUCTURE_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + impl Structure { fn of(wb: &Workbook) -> Self { + #[cfg(test)] + STRUCTURE_READS.with(|n| n.set(n.get() + 1)); let model = &wb.model; let aggs: BTreeMap<_, _> = model.measure_agg.iter().collect(); Self { @@ -154,7 +161,7 @@ fn axis_changes( .map(|(at, item)| AxisChange::ItemInserted { category: category.clone(), item: item.clone(), - at, + at: at as u128, }); return std::iter::once(added).chain(inserted).collect::>(); }; @@ -163,26 +170,55 @@ fn axis_changes( gone.chain(rest).collect() } -fn item_changes(category: &str, was: &[String], now: &[String]) -> Vec { - let removed = was +/// What changed between two item lists, in one pass over each: set lookups, never `contains`. +pub(crate) struct ItemDiff<'a, T> { + pub removed: Vec<&'a T>, + pub inserted: Vec<(usize, &'a T)>, + pub reordered: bool, +} + +pub(crate) fn diff_items<'a, T: Eq + std::hash::Hash>( + was: &'a [T], + now: &'a [T], +) -> ItemDiff<'a, T> { + let had: HashSet<&T> = was.iter().collect(); + let has: HashSet<&T> = now.iter().collect(); + let removed = was.iter().filter(|i| !has.contains(i)).collect(); + let inserted = now .iter() - .filter(|i| !now.contains(i)) + .enumerate() + .filter(|(_, i)| !had.contains(i)) + .collect(); + let kept_before = was.iter().filter(|i| has.contains(i)); + let kept_after = now.iter().filter(|i| had.contains(i)); + ItemDiff { + removed, + inserted, + reordered: !kept_before.eq(kept_after), + } +} + +fn item_changes(category: &str, was: &[String], now: &[String]) -> Vec { + if was == now { + return Vec::new(); + } + let diff = diff_items(was, now); + let removed = diff + .removed + .into_iter() .map(|item| AxisChange::ItemRemoved { category: category.into(), item: item.clone(), }); - let inserted = now - .iter() - .enumerate() - .filter(|(_, i)| !was.contains(i)) + let inserted = diff + .inserted + .into_iter() .map(|(at, item)| AxisChange::ItemInserted { category: category.into(), item: item.clone(), - at, + at: at as u128, }); - let kept_before: Vec<_> = was.iter().filter(|i| now.contains(i)).collect(); - let kept_after: Vec<_> = now.iter().filter(|i| was.contains(i)).collect(); - let reordered = (kept_before != kept_after).then(|| AxisChange::ItemsReordered { + let reordered = diff.reordered.then(|| AxisChange::ItemsReordered { category: category.into(), items: now.to_vec(), }); @@ -208,14 +244,15 @@ pub struct Delivery { capacity: usize, max_keys: usize, live: BTreeMap, - structure: Arc, + structure: Option>, + attached: bool, evicted: BTreeSet, outbox: Vec>, } impl Delivery { /// `host` is a random instance id, never a time (clause 9). - pub fn new(host: u64, wb: &Workbook, generation: u64) -> Self { + pub fn new(host: u64, generation: u64) -> Self { Self { host, generation, @@ -224,12 +261,22 @@ impl Delivery { capacity: LOG_CAPACITY, max_keys: MAX_KEYS, live: BTreeMap::new(), - structure: Arc::new(Structure::of(wb)), + structure: None, + attached: false, evicted: BTreeSet::new(), outbox: Vec::new(), } } + /// A transport is (or is not) draining notices. Without one, and with no live shape, a + /// commit only counts: nothing is read, diffed or kept. + pub fn set_attached(&mut self, on: bool, wb: &Workbook) { + self.attached = on; + if on && self.structure.is_none() { + self.structure = Some(Arc::new(Structure::of(wb))); + } + } + pub fn set_limits(&mut self, capacity: usize, max_keys: usize) { self.capacity = capacity; self.max_keys = max_keys; @@ -268,13 +315,19 @@ impl Delivery { self.log.clear(); } self.publication += 1; + if !self.attached && self.live.is_empty() { + self.log.clear(); + self.structure = None; + self.evicted.clear(); + return self.stamp(); + } let range = PubRange::one(self.stamp()); let structure = Arc::new(Structure::of(wb)); - let (axis, flags) = match new_epoch { - true => (Vec::new(), all_flags()), - false => structure.since(&self.structure), + let (axis, flags) = match (new_epoch, self.structure.as_deref()) { + (false, Some(before)) => structure.since(before), + _ => (Vec::new(), all_flags()), }; - self.structure = structure; + self.structure = Some(structure); let mut notice = Notice::empty(range); notice.axis = axis; @@ -302,7 +355,11 @@ impl Delivery { while self.log.len() > self.capacity { self.log.pop_front(); } - self.outbox.push(notice); + if self.attached { + self.outbox.push(notice); + let excess = self.outbox.len().saturating_sub(self.capacity); + self.outbox.drain(..excess); + } self.stamp() } @@ -312,6 +369,9 @@ impl Delivery { Some(live) => live.last_touch = now, None => { let Some(view) = view_of(shape) else { return }; + if self.structure.is_none() { + self.structure = Some(Arc::new(Structure::of(wb))); + } let Some(output) = Output::of(wb, view) else { return; }; @@ -437,7 +497,7 @@ impl Delivery { .reduce(|a, b| a.merge(b).unwrap_or_else(|e| e.1)); let restructured = covering .iter() - .any(|n| !n.axis.is_empty() || !n.relabels.is_empty()); + .any(|n| !n.axis.is_empty() || !n.relabels.is_empty() || n.flags.views); if oversized { return Reply::Correction(Box::new(self.window(request, None, self.stamp(), None))); } diff --git a/crates/improvise-command/src/ui/delivery_tests.rs b/crates/improvise-command/src/ui/delivery_tests.rs index 0841dc86..e1343647 100644 --- a/crates/improvise-command/src/ui/delivery_tests.rs +++ b/crates/improvise-command/src/ui/delivery_tests.rs @@ -43,7 +43,9 @@ fn workbook() -> Workbook { fn host_instance(instance: u64) -> Host { let mut app = App::new(workbook(), None); app.rebuild_layout(); - Host::with_instance(app, instance) + let mut host = Host::with_instance(app, instance); + host.attach_notices(); + host } fn host() -> Host { @@ -776,6 +778,7 @@ fn a_document_swap_is_a_new_epoch_behind_every_old_watermark() { app.storage = Box::new(storage); app.rebuild_layout(); let mut host = Host::with_instance(app, 1); + host.attach_notices(); let id = connect(&mut host); let mut client = settled(&mut host); let old = client.watermark(&shape()).unwrap(); @@ -876,6 +879,161 @@ fn values_keep_every_bit_from_the_cell_to_the_client() { } } +// ── cost: what a commit does when nobody is listening ───────────────────── + +/// An item whose every comparison is counted, so the cost is a number and not a clock. +#[derive(Clone, Debug)] +struct Counted(usize); + +thread_local! { + static COMPARISONS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +impl PartialEq for Counted { + fn eq(&self, o: &Self) -> bool { + COMPARISONS.with(|c| c.set(c.get() + 1)); + self.0 == o.0 + } +} + +impl Eq for Counted {} + +impl std::hash::Hash for Counted { + fn hash(&self, h: &mut H) { + self.0.hash(h); + } +} + +#[test] +fn diffing_a_category_compares_each_item_a_bounded_number_of_times() { + use super::delivery::diff_items; + for n in [1000usize, 2000, 4000] { + let was: Vec = (0..n).map(Counted).collect(); + let mut now = was.clone(); + now.remove(n / 2); + now.push(Counted(n + 1)); + COMPARISONS.with(|c| c.set(0)); + let diff = diff_items(&was, &now); + let comparisons = COMPARISONS.with(|c| c.get()); + assert_eq!(diff.removed.len() + diff.inserted.len(), 2); + assert!( + comparisons <= 8 * n, + "{n} items cost {comparisons} comparisons" + ); + } +} + +#[test] +fn a_commit_with_nothing_live_and_no_transport_reads_nothing() { + use super::delivery::STRUCTURE_READS; + let mut app = App::new(workbook(), None); + app.rebuild_layout(); + let mut host = Host::with_instance(app, 1); + let id = connect(&mut host); + STRUCTURE_READS.with(|c| c.set(0)); + for n in 0..3 { + send(&mut host, id, vec![set("North", n as f64)]); + } + assert_eq!(STRUCTURE_READS.with(|c| c.get()), 0); + assert_eq!( + host.delivery().stamp().publication, + 3, + "the count still moves" + ); + assert_eq!(host.delivery().log_len(), 0); + assert!(host.take_notices().is_empty()); +} + +#[test] +fn a_shape_that_goes_live_after_idle_commits_still_hears_the_axis_changes() { + let mut app = App::new(workbook(), None); + app.rebuild_layout(); + let mut host = Host::with_instance(app, 1); + let id = connect(&mut host); + send(&mut host, id, vec![set("North", 1.0)]); + let mut client = settled(&mut host); + host.attach_notices(); + send(&mut host, id, vec![add_item("East")]); + let notices = host.take_notices(); + assert_eq!(notices[0].axis.len(), 1, "{:?}", notices[0]); + deliver_all(&mut client, ¬ices); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + held_equals_drawn(&client, &host); +} + +#[test] +fn the_outbox_and_the_clone_a_frame_takes_are_bounded() { + let mut host = host(); + host.limit_delivery(4, 4096); + let id = connect(&mut host); + let mut client = settled(&mut host); + for n in 0..20 { + send(&mut host, id, vec![set("North", n as f64)]); + } + let queued = host.take_notices(); + assert_eq!( + queued.len(), + 4, + "a transport that never drains keeps the newest four" + ); + deliver_all(&mut client, &queued); + let mut edge = Flaky::new(&mut host); + client.sync(&mut edge, &shape(), 6); + assert!( + matches!(asks(&edge.reached)[0], Ask::Correction { .. }), + "the gap is seen" + ); + held_equals_drawn(&client, &host); +} + +// ── view geometry moves positions without moving a key ──────────────────── + +fn transpose() -> ModelCommand { + ModelCommand::TransposeAxes { + view: "Default".into(), + } +} + +fn south_window() -> Window { + Window { + top: 1, + left: 0, + rows: 1, + cols: 1, + } +} + +#[test] +fn a_transpose_empties_a_partial_window_of_the_cell_that_left_it() { + let mut host = host(); + let id = connect(&mut host); + let mut client = EdgeClient::new(session()); + client.track(shape(), south_window()); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + assert_eq!(client.held(&shape()).map(|h| h.len()), Some(1)); + host.take_notices(); + send(&mut host, id, vec![transpose()]); + let notices = host.take_notices(); + assert!(notices[0].flags.views); + deliver_all(&mut client, ¬ices); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + assert_eq!(client.held(&shape()), Some(&drawn(&host, &south_window()))); + assert_eq!(client.held(&shape()).map(|h| h.len()), Some(0)); +} + +#[test] +fn a_dropped_views_notice_is_answered_with_the_window_not_an_empty_key_list() { + let mut host = host(); + let id = connect(&mut host); + let mut client = EdgeClient::new(session()); + client.track(shape(), south_window()); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + send(&mut host, id, vec![transpose()]); + client.connection_lost(); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + assert_eq!(client.held(&shape()), Some(&drawn(&host, &south_window()))); +} + // ── the whole thing, at random ───────────────────────────────────────────── mod random { diff --git a/crates/improvise-command/src/ui/host.rs b/crates/improvise-command/src/ui/host.rs index 58553143..78b3a95f 100644 --- a/crates/improvise-command/src/ui/host.rs +++ b/crates/improvise-command/src/ui/host.rs @@ -39,11 +39,7 @@ impl Host { /// A host with a chosen instance id, which must differ from every earlier run's /// (`context/code/delivery.md` §6). pub fn with_instance(app: App, instance: u64) -> Self { - let delivery = Delivery::new( - instance, - &app.model_state.workbook, - app.document_generation(), - ); + let delivery = Delivery::new(instance, app.document_generation()); Self { app, sessions: Sessions::default(), @@ -71,6 +67,12 @@ impl Host { self.delivery.set_limits(capacity, max_keys); } + /// A transport will drain `take_notices`; until then nothing is queued for it. + pub fn attach_notices(&mut self) { + self.delivery + .set_attached(true, &self.app.model_state.workbook); + } + /// Notices published since the last call, for the transport to push. pub fn take_notices(&mut self) -> Vec { self.delivery.take_outbox() From bfaccae5d46e08223b16ddd0f368a69e985db1a6 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:42:29 -0700 Subject: [PATCH 11/12] test(command): the delivery property delays, splits, repeats and replays, and covers swaps, restarts, limits and eviction (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/code/delivery.md | 39 +++- context/test-evidence/delivery.md | 55 ++++++ .../src/ui/delivery_tests.rs | 184 ++++++++++++++++-- 3 files changed, 253 insertions(+), 25 deletions(-) diff --git a/context/code/delivery.md b/context/code/delivery.md index 2ee9781c..bebe2aeb 100644 --- a/context/code/delivery.md +++ b/context/code/delivery.md @@ -85,11 +85,15 @@ at least as new as the watermark claims, never older. not which keys sit in the client's window, and an insert or remove shifts cells into and out of it without moving them. The ADR has the client re-derive positions from the axis changes and fetch what it lacks. That needs ADR 008's render model, which this layer does not have. So the client records a -notice that moved an axis (`restructured`), asks `Fetch { window: true }`, and the host answers with a +notice that moved an axis or set `flags.views` (`restructured`), asks `Fetch { window: true }`, and the host answers with a `Window` body whose `carried` notice holds the axis changes, in order. This is a fetch, not a correction, so the required test "a frame `[AddItem, SetCell]` ... and no client asks for a correction" holds: it asserts no `Ask::Correction` was sent. When the render model exists the client can narrow -the fetch to the cells that entered its window. +the fetch to the cells that entered its window (improvise-bw4p.9). + +A view command (`SetAxis`, `TransposeAxes`, a page selection, a hide) moves positions and no key, and +reaches the client only as `flags.views`. So that flag counts as an axis change on both sides: the client +fetches its window, and the host answers a correction over a range holding the flag with a window body. ## 4. The client @@ -106,6 +110,14 @@ body is a gap unless it starts at or below the watermark. Otherwise `held += tar watermark becomes `through`, and the queued notices it covers are dropped. A late reply older than the watermark therefore cannot regress the state, which is what makes a retry safe. +**A correction from an older generation of the same host is a duplicate, whatever the watermark.** Epochs +of different hosts are not ordered, so a late reply from a dead host is told apart only by being +covered: a duplicate answer prunes every queued entry from another epoch and moves `seen` onto the live +epoch, so a late notice from a dead host costs one extra round and cannot livelock `sync`. + +`poll` hands a pulled notice only to the shape it asked about. A notice carries only that shape's keys, +and applying it to another tracked shape would advance that shape past keys it never saw. + `connection_lost`, `request_failed` and `unreadable` set `stale`, which `wants` turns into `Ask::Correction` from the watermark the client held. A duplicate whose `through` equals the watermark also clears `stale`: the host has said there is nothing newer. @@ -142,8 +154,8 @@ each `Command::Model` that lands, and `Host::deliver` answers a `Request`. and never the workbook. A value a failed command wrote is not in it. **Rollback.** `Host::mutating_frame` clones the `Delivery` beside the app snapshot and the session -mirrors, and `undo_frame` puts it back. The clone is cheap (`Arc` for outputs, notices and structure, -bounded by the log's capacity). So a refused frame consumes no publication number and publishes nothing. +mirrors, and `undo_frame` puts it back. The clone is cheap: `Arc` for outputs, notices and structure, +and both the log and the outbox hold at most `capacity` notices. So a refused frame consumes no publication number and publishes nothing. **The ADR and ADR 001 disagree here, and ADR 001 wins until ADR 006 exists.** Clause 9 says a frame whose later command fails keeps its earlier commands and their notices, "each command commits on its own @@ -151,12 +163,22 @@ whose later command fails keeps its earlier commands and their notices, "each co not in the tree. The required test "a refused command consumes no publication number, and the commands before it in its frame stay published" is split in two. The first half is tested. The second is tested as its opposite, `commands_before_a_refused_one_are_rolled_back_with_their_notices_under_adr_001`, -which is the test to flip when per-command commit lands: the publisher already supports it, since it +which is the test to flip when per-command commit lands (improvise-bw4p.8): the publisher already supports it, since it publishes whatever the host tells it committed. +**Nothing is published while nobody can hear it.** With no live shape and no attached transport +(`Host::attach_notices`), a commit only advances the publication number: no structure is read, nothing is +diffed, the log is cleared and the queued structure is forgotten. A shape that goes live, or a transport +that attaches, re-reads the structure, so its first diff is against a known state. Without this every +host pays for delivery on every command while nothing consumes it. The outbox exists only while a +transport is attached, and keeps the newest `capacity` notices; a transport that falls behind sees a gap +and asks for a correction. The axis diff is linear in the items (`diff_items`, set lookups), and its test +counts comparisons on a counting item type rather than reading a clock. + **The host instance** is a random id, never a time (`Host::new` takes one from `RandomState`). `RandomState` has no entropy on `wasm32-unknown-unknown`, so a worker must pass a real one through `Host::with_instance`. A restarted host with the same id would accept a stale watermark as current. +`Worker` still calls `Host::new` (improvise-bw4p.7). **Eviction** (`evict_idle`) takes a local count and an idle length in the same unit, never a wall clock compared across machines. An evicted key is announced in the next notice and forgotten; naming it again @@ -185,13 +207,16 @@ The token in a request is carried and echoed by the client and never read by the function call; `WireEdge` passes request and reply through JSON text, the adapter at a wire boundary, so the lossless properties are exercised by every test that uses it. Neither adds a `ClientFrame` or `ServerFrame` variant: the page does not read notices yet, and adding the frames means changing the -browser and the worker together (ADR 008 replaces `Projection`). A transport pushes -`Host::take_notices()`. +browser and the worker together (ADR 008 replaces `Projection`; improvise-bw4p.10). A transport +pushes `Host::take_notices()`. ## Not done, and why - **The notices are not sent by the socket or worker hosts.** They need frame variants and the page (above). +- **Positions and column totals are not on the wire.** A correction carries `(key, value)` and no row or + column, and `Output` holds body cells only. The ADR attaches the position when output is sent and lists + totals among the moved keys. Both need the render model and the totals' own keys (ADR 008, clause 10). - **A twin at the serving end** (a server-side watermark or filter) is not built; the ADR allows it and does not require it. - **A pushed filter** is the client's `window`, applied on every request. diff --git a/context/test-evidence/delivery.md b/context/test-evidence/delivery.md index c617c311..c4561144 100644 --- a/context/test-evidence/delivery.md +++ b/context/test-evidence/delivery.md @@ -93,3 +93,58 @@ token and that watermark. A one-cell window, a remove that moves South into it. The notice lists only North, so a client that fetches moved keys alone ends with an empty window. + +## Review of #348, round one + +Each test below was written against the behaviour the review reported and failed before its fix, or, for +the cost tests, was run against a quadratic rival. + +### diffing_a_category_compares_each_item_a_bounded_number_of_times + +Items are a type whose `eq` is counted. Rival: `contains` on the slices in `diff_items`, which makes the +count quadratic and fails every size. A wall-clock test would have passed on a fast machine. + +### a_commit_with_nothing_live_and_no_transport_reads_nothing + +Counts `Structure::of` calls. Rival: the idle skip removed. + +### the_outbox_and_the_clone_a_frame_takes_are_bounded + +Rival: the outbox truncation removed, so twenty commits leave twenty queued. + +### a_late_window_reply_from_an_older_generation_does_not_regress + +Rival: the older-generation guard disabled (fails this test in the protocol crate). + +### a_late_notice_from_a_dead_host_settles_against_the_live_one + +Rival: the duplicate arm no longer prunes other epochs' entries; the random property also fails. + +### a_poll_for_one_shape_leaves_the_other_shapes_alone + +Rival: pulled notices applied to every shape. + +### a_transpose_empties_a_partial_window_of_the_cell_that_left_it + +A transpose moves no key. Rival: `flags.views` not counted as a structural change in the client and the +host. `a_dropped_views_notice_is_answered_with_the_window_not_an_empty_key_list` fails with the host +alone changed. + +### wide_integers_are_canonical_on_input + +Both spellings are refused outside their range: a JSON number at or above 2^53, a string below it, a +sign, leading zeros and trailing space. + +### an_item_insert_position_crosses_as_a_wide_integer + +`ItemInserted.at` is a `u128` through `wide`. + +### a_fetch_covers_only_the_notices_it_named_when_a_later_commit_is_already_in_the_state + +Rival: a fetch answered with `through: at`. The first version of the random property did not catch it (it +delivered each notice before the next commit); the property now splits, delays, duplicates and replays +notices and old replies, and fails the rival on every run, which this test also does by name. + +### equal_values_compare_equal_whatever_the_zero_counts_say + +`ExactSum`'s equality compared the sign-of-zero counters even when the total was nonzero. diff --git a/crates/improvise-command/src/ui/delivery_tests.rs b/crates/improvise-command/src/ui/delivery_tests.rs index e1343647..6d943328 100644 --- a/crates/improvise-command/src/ui/delivery_tests.rs +++ b/crates/improvise-command/src/ui/delivery_tests.rs @@ -152,6 +152,7 @@ struct Flaky<'a> { host: &'a mut Host, script: VecDeque>, reached: Vec, + replies: Vec, } impl<'a> Flaky<'a> { @@ -160,6 +161,7 @@ impl<'a> Flaky<'a> { host, script: VecDeque::new(), reached: Vec::new(), + replies: Vec::new(), } } @@ -179,6 +181,7 @@ impl Edge for Flaky<'_> { } self.reached.push(request.clone()); let reply = self.host.deliver(&request, 0); + self.replies.push(reply.clone()); match fault { Some(Fault::DropReply) => Err(EdgeError::Failed), Some(Fault::Garble) => Err(EdgeError::Unreadable), @@ -1034,6 +1037,28 @@ fn a_dropped_views_notice_is_answered_with_the_window_not_an_empty_key_list() { assert_eq!(client.held(&shape()), Some(&drawn(&host, &south_window()))); } +#[test] +fn a_fetch_covers_only_the_notices_it_named_when_a_later_commit_is_already_in_the_state() { + let mut host = host(); + let id = connect(&mut host); + let mut client = settled(&mut host); + host.take_notices(); + send(&mut host, id, vec![set("North", 1.0)]); + send(&mut host, id, vec![set("South", 2.0)]); + let notices = host.take_notices(); + deliver_all(&mut client, ¬ices[..1]); + let mut edge = Flaky::new(&mut host); + client.sync(&mut edge, &shape(), 6); + assert_eq!( + client.watermark(&shape()), + Some(notices[0].stamp()), + "through, not at" + ); + deliver_all(&mut client, ¬ices[1..]); + client.sync(&mut Flaky::new(&mut host), &shape(), 6); + held_equals_drawn(&client, &host); +} + // ── the whole thing, at random ───────────────────────────────────────────── mod random { @@ -1044,17 +1069,31 @@ mod random { #[derive(Debug, Clone)] enum Op { Set(bool, usize), + Both(usize, usize), + Clear(bool), Add(usize), Remove(usize), Formula, + Transpose, + TightLimits, + Evict, + Swap, + Restart, } fn op() -> impl Strategy { prop_oneof![ - 4 => (any::(), 0usize..8).prop_map(|(r, v)| Op::Set(r, v)), - 2 => (0usize..3).prop_map(Op::Add), - 1 => (0usize..3).prop_map(Op::Remove), + 6 => (any::(), 0usize..8).prop_map(|(r, v)| Op::Set(r, v)), + 4 => (0usize..8, 0usize..8).prop_map(|(a, b)| Op::Both(a, b)), + 2 => any::().prop_map(Op::Clear), + 3 => (0usize..3).prop_map(Op::Add), + 2 => (0usize..3).prop_map(Op::Remove), 1 => Just(Op::Formula), + 1 => Just(Op::Transpose), + 1 => Just(Op::TightLimits), + 1 => Just(Op::Evict), + 1 => Just(Op::Swap), + 1 => Just(Op::Restart), ] } @@ -1070,9 +1109,25 @@ mod random { ]; const EXTRA: [&str; 3] = ["East", "West", "North"]; - fn command(op: &Op) -> ModelCommand { + fn region(north: bool) -> &'static str { + ["South", "North"][north as usize] + } + + fn commands(op: &Op) -> Vec { match op { - Op::Set(north, v) => set(["South", "North"][*north as usize], VALUES[*v]), + Op::Both(a, b) => vec![set("North", VALUES[*a]), set("South", VALUES[*b])], + other => command(other).into_iter().collect(), + } + } + + fn command(op: &Op) -> Option { + Some(match op { + Op::Both(..) => return None, + Op::Set(north, v) => set(region(*north), VALUES[*v]), + Op::Clear(north) => ModelCommand::ClearCell { + view: "Default".into(), + cell: record(region(*north)), + }, Op::Add(i) => add_item(EXTRA[*i]), Op::Remove(i) => ModelCommand::RemoveItem { category: "Region".into(), @@ -1082,26 +1137,49 @@ mod random { raw: "Double = Revenue * 2".into(), target_category: "_Measure".into(), }, - } + Op::Transpose => transpose(), + Op::Swap => ModelCommand::LoadModel("other.improv".into()), + Op::TightLimits | Op::Evict | Op::Restart => return None, + }) + } + + /// How a step's notices reach the client. + #[derive(Debug, Clone, Copy)] + enum Arrival { + Dropped, + Now, + Twice, + Split, + Late(usize), } #[derive(Debug, Clone)] struct Weather { - deliver_notice: bool, + arrival: Arrival, lose_connection: bool, sync: Vec>, + replay: Option, } fn weather() -> impl Strategy { + let arrival = prop_oneof![ + 2 => Just(Arrival::Dropped), + 4 => Just(Arrival::Now), + 1 => Just(Arrival::Twice), + 4 => Just(Arrival::Split), + 3 => (1usize..4).prop_map(Arrival::Late), + ]; ( - any::(), + arrival, 0u8..8, proptest::collection::vec(proptest::option::of(0u8..3), 0..3), + proptest::option::of(0usize..16), ) - .prop_map(|(deliver_notice, lose, sync)| Weather { - deliver_notice, + .prop_map(|(arrival, lose, sync, replay)| Weather { + arrival, lose_connection: lose == 0, sync, + replay, }) } @@ -1118,32 +1196,102 @@ mod random { }) } + /// A host over `wb` that can load a second document, as a restarted host would be. + fn host_over(wb: Workbook, instance: u64) -> Host { + let storage = crate::storage::MemoryFs::new(); + let mut other = Workbook::new("Other"); + other.add_category("Product").unwrap(); + other.add_item("Product", "Widget").unwrap(); + other.add_item("_Measure", "Units").unwrap(); + other.active_view_mut().set_axis("Product", Axis::Row); + other.active_view_mut().set_axis("_Measure", Axis::Column); + crate::persistence::save_to(&storage, &other, std::path::Path::new("other.improv")) + .unwrap(); + let mut app = App::new(wb, None); + app.storage = Box::new(storage); + app.rebuild_layout(); + let mut host = Host::with_instance(app, instance); + host.attach_notices(); + host + } + + fn arrive( + client: &mut EdgeClient, + held_back: &mut Vec<(usize, Notice)>, + step: usize, + w: &Weather, + fresh: Vec, + ) { + for (i, n) in fresh.into_iter().enumerate() { + match w.arrival { + Arrival::Split if i == 0 => deliver_all(client, &[n]), + Arrival::Split => held_back.push((step + 1, n)), + Arrival::Dropped => {} + Arrival::Now => deliver_all(client, &[n]), + Arrival::Twice => deliver_all(client, &[n.clone(), n]), + Arrival::Late(k) => held_back.push((step + k, n)), + } + } + let (due, later): (Vec<_>, Vec<_>) = held_back.drain(..).partition(|(at, _)| *at <= step); + *held_back = later; + let notices: Vec = due.into_iter().map(|(_, n)| n).collect(); + deliver_all(client, ¬ices); + } + proptest! { #[test] fn a_corrected_state_is_the_target_and_correcting_it_again_is_the_zero_zset( steps in proptest::collection::vec((op(), weather()), 1..12), window in windows(), ) { - let mut host = host(); - let id = connect(&mut host); + let mut instance = 1; + let mut host = host_over(workbook(), instance); + let mut id = connect(&mut host); let mut client = EdgeClient::new(session()); client.track(shape(), window); client.sync(&mut Flaky::new(&mut host), &shape(), 6); host.take_notices(); + let mut held_back: Vec<(usize, Notice)> = Vec::new(); + let mut old: Vec = Vec::new(); - for (op, weather) in &steps { - send(&mut host, id, vec![command(op)]); - let notices = host.take_notices(); - if weather.deliver_notice { - deliver_all(&mut client, ¬ices); + for (step, (op, weather)) in steps.iter().enumerate() { + match (commands(op), op) { + (cmds, _) if !cmds.is_empty() => { + send(&mut host, id, cmds); + } + (_, Op::TightLimits) => host.limit_delivery(2, 1), + (_, Op::Evict) => { + host.evict_idle(10, 0); + } + _ => { + instance += 1; + let wb = host.app().model_state.workbook.clone(); + host = host_over(wb, instance); + id = connect(&mut host); + held_back.clear(); + } } + let fresh = host.take_notices(); + arrive(&mut client, &mut held_back, step, weather, fresh); if weather.lose_connection { client.connection_lost(); } + if let Some(i) = weather.replay.filter(|_| !old.is_empty()) { + match &old[i % old.len()] { + Reply::Correction(c) => { + client.apply(c); + } + Reply::Notices(list) => deliver_all(&mut client, list), + } + } let script = weather.sync.iter().map(|f| f.map(fault)).collect(); - client.sync(&mut Flaky::scripted(&mut host, script), &shape(), 6); + let mut edge = Flaky::scripted(&mut host, script); + client.sync(&mut edge, &shape(), 6); + old.append(&mut edge.replies); } + let rest: Vec = held_back.drain(..).map(|(_, n)| n).collect(); + deliver_all(&mut client, &rest); let (state, _) = client.poll(&mut WireEdge(Flaky::new(&mut host)), &shape()); prop_assert_eq!(state, Synced::Settled); let want = drawn(&host, &window); From c2f47538616ff9f436a3c65fc20972b63f0cd433 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-delivery)" Date: Sat, 10 Oct 2026 12:59:41 -0700 Subject: [PATCH 12/12] fix(protocol): a reply from an epoch the client has left changes nothing (improvise-z2vr) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/code/delivery.md | 12 ++++-- context/test-evidence/delivery.md | 26 ++++++++++- .../src/ui/delivery_tests.rs | 43 +++++++++++++++++++ .../improvise-protocol/src/delivery/client.rs | 16 ++++++- .../src/delivery/review_tests.rs | 20 +++++++++ 5 files changed, 111 insertions(+), 6 deletions(-) diff --git a/context/code/delivery.md b/context/code/delivery.md index bebe2aeb..30eb8605 100644 --- a/context/code/delivery.md +++ b/context/code/delivery.md @@ -110,10 +110,14 @@ body is a gap unless it starts at or below the watermark. Otherwise `held += tar watermark becomes `through`, and the queued notices it covers are dropped. A late reply older than the watermark therefore cannot regress the state, which is what makes a retry safe. -**A correction from an older generation of the same host is a duplicate, whatever the watermark.** Epochs -of different hosts are not ordered, so a late reply from a dead host is told apart only by being -covered: a duplicate answer prunes every queued entry from another epoch and moves `seen` onto the live -epoch, so a late notice from a dead host costs one extra round and cannot livelock `sync`. +**A correction from a generation or host the client has left is a duplicate, whatever the +watermark.** A shape records the epochs its watermark has moved out of (`left`, the newest 64), and a +correction from one of them, or from an older generation of the host it is on, changes nothing. Epochs of +different hosts are not ordered, so a client that has never applied anything from the newer host cannot +tell a late reply from a dead one from a legitimate first answer, and applies it; once it has moved on, the +dead host's replies are refused. A duplicate answer also prunes every queued entry from another epoch +and moves `seen` onto the live epoch, so a late notice from a dead host costs one extra round and cannot +livelock `sync`. `poll` hands a pulled notice only to the shape it asked about. A notice carries only that shape's keys, and applying it to another tracked shape would advance that shape past keys it never saw. diff --git a/context/test-evidence/delivery.md b/context/test-evidence/delivery.md index c4561144..6ef10e55 100644 --- a/context/test-evidence/delivery.md +++ b/context/test-evidence/delivery.md @@ -143,8 +143,32 @@ sign, leading zeros and trailing space. Rival: a fetch answered with `through: at`. The first version of the random property did not catch it (it delivered each notice before the next commit); the property now splits, delays, duplicates and replays -notices and old replies, and fails the rival on every run, which this test also does by name. +notices and old replies, and fails the rival in most runs (17 of 20 at 256 cases, measured on the +author's machine); this test fails it on every run, by name. ### equal_values_compare_equal_whatever_the_zero_counts_say `ExactSum`'s equality compared the sign-of-zero counters even when the total was nonzero. + +## Review of #348, round two + +### a_late_window_reply_from_a_dead_host_does_not_regress + +A client that has applied a correction from host 2 receives a late `Window` reply from host 1. Epochs of +different hosts are unordered, so the first fix (older generation of the same host) did not cover it. +Rival: the `left` check removed; this test and the two below fail. + +### a_reply_from_a_generation_the_client_left_does_not_return + +Generations 1, 2, 3 of one host; a late generation-2 reply. Rival: both guards removed. + +### a_reply_held_across_a_host_restart_cannot_pull_the_client_back + +The host-side form: the reply is computed and lost, the host restarts, the client settles against the new +host, and the old reply then arrives. Rival: the `left` check removed. + +**The random property, round two.** The property now checks after every step that the watermark's epoch never goes back to one it had left +(so a mid-history regression cannot be repaired by the closing `poll`), and replays an earlier reply after +every restart and swap. Rival: the `left` check removed fails it. The older-generation guard on its own is +not separable from `left` in the histories the property reaches, so it is held by +`a_late_window_reply_from_an_older_generation_does_not_regress` alone. diff --git a/crates/improvise-command/src/ui/delivery_tests.rs b/crates/improvise-command/src/ui/delivery_tests.rs index 6d943328..10e05a0b 100644 --- a/crates/improvise-command/src/ui/delivery_tests.rs +++ b/crates/improvise-command/src/ui/delivery_tests.rs @@ -1059,6 +1059,33 @@ fn a_fetch_covers_only_the_notices_it_named_when_a_later_commit_is_already_in_th held_equals_drawn(&client, &host); } +#[test] +fn a_reply_held_across_a_host_restart_cannot_pull_the_client_back() { + let mut first = host_instance(1); + let id = connect(&mut first); + let mut client = settled(&mut first); + send(&mut first, id, vec![set("North", 99.0)]); + let mut lost = Flaky::scripted(&mut first, vec![Some(Fault::DropReply)]); + client.connection_lost(); + client.sync(&mut lost, &shape(), 6); + let in_flight = lost.replies.remove(0); + + let mut second = host_instance(2); + client.connection_lost(); + client.sync(&mut Flaky::new(&mut second), &shape(), 6); + held_equals_drawn(&client, &second); + let at = client.watermark(&shape()).unwrap(); + + let Reply::Correction(late) = in_flight else { + panic!("the lost reply was a correction") + }; + assert_eq!(client.apply(&late).0, Outcome::Duplicate); + assert_eq!(client.watermark(&shape()), Some(at)); + held_equals_drawn(&client, &second); + let (state, _) = client.sync(&mut Flaky::new(&mut second), &shape(), 6); + assert_eq!(state, Synced::Settled); +} + // ── the whole thing, at random ───────────────────────────────────────────── mod random { @@ -1253,6 +1280,7 @@ mod random { host.take_notices(); let mut held_back: Vec<(usize, Notice)> = Vec::new(); let mut old: Vec = Vec::new(); + let mut epochs: Vec<(u64, u64)> = Vec::new(); for (step, (op, weather)) in steps.iter().enumerate() { match (commands(op), op) { @@ -1288,6 +1316,21 @@ mod random { let mut edge = Flaky::scripted(&mut host, script); client.sync(&mut edge, &shape(), 6); old.append(&mut edge.replies); + let moved_host = matches!(op, Op::Restart | Op::Swap); + if let (true, Some(Reply::Correction(c))) = (moved_host, old.get(step % old.len().max(1))) { + client.apply(c); + } + if let Some(w) = client.watermark(&shape()) { + let now = (w.host, w.generation); + let seen_at = epochs.iter().position(|e| *e == now); + prop_assert!( + seen_at.is_none_or(|i| i + 1 == epochs.len()), + "the client went back to a superseded epoch {now:?} after {epochs:?}" + ); + if seen_at.is_none() { + epochs.push(now); + } + } } let rest: Vec = held_back.drain(..).map(|(_, n)| n).collect(); diff --git a/crates/improvise-protocol/src/delivery/client.rs b/crates/improvise-protocol/src/delivery/client.rs index d9d47f42..49d899c6 100644 --- a/crates/improvise-protocol/src/delivery/client.rs +++ b/crates/improvise-protocol/src/delivery/client.rs @@ -53,6 +53,9 @@ pub enum Synced { Exhausted, } +/// How many abandoned epochs a shape remembers. +const LEFT_EPOCHS: usize = 64; + #[derive(Debug, Clone)] struct Tracked { window: Window, @@ -63,6 +66,8 @@ struct Tracked { unapplied: Vec<(Stamp, Moved)>, /// The last notice that moved an axis: until a window is read past it, positions are unknown. restructured: Option, + /// Epochs the watermark has moved out of: nothing from them applies again. + left: Vec<(u64, u64)>, stale: bool, } @@ -92,6 +97,7 @@ impl EdgeClient { seen: None, unapplied: Vec::new(), restructured: None, + left: Vec::new(), stale: true, }, ); @@ -246,7 +252,10 @@ impl EdgeClient { let older = |s: Option| { s.is_some_and(|s| s.host == c.through.host && c.through.generation < s.generation) }; - if older(t.watermark) || older(t.seen) { + if older(t.watermark) + || older(t.seen) + || t.left.contains(&(c.through.host, c.through.generation)) + { return (Outcome::Duplicate, Vec::new()); } let covered = t @@ -285,6 +294,11 @@ impl EdgeClient { } }; t.held += δ + if let Some(w) = t.watermark.filter(|w| !w.same_epoch(&c.through)) { + t.left.push((w.host, w.generation)); + let excess = t.left.len().saturating_sub(LEFT_EPOCHS); + t.left.drain(..excess); + } t.watermark = Some(c.through); let window = matches!(c.body, CorrectionBody::Window); t.stale &= !window; diff --git a/crates/improvise-protocol/src/delivery/review_tests.rs b/crates/improvise-protocol/src/delivery/review_tests.rs index 3074d0d2..04ac8a88 100644 --- a/crates/improvise-protocol/src/delivery/review_tests.rs +++ b/crates/improvise-protocol/src/delivery/review_tests.rs @@ -181,3 +181,23 @@ fn an_item_insert_position_crosses_as_a_wide_integer() { assert!(json.contains(r#""at":"1152921504606846976""#), "{json}"); assert_eq!(serde_json::from_str::(&json).unwrap(), change); } + +#[test] +fn a_late_window_reply_from_a_dead_host_does_not_regress() { + let mut c = synced("s", at(1, 1, 9), &[cell(1, "old")]); + c.apply(&window_correction("s", at(2, 1, 3), &[cell(1, "new")])); + let late = window_correction("s", at(1, 1, 12), &[cell(1, "dead")]); + assert_eq!(c.apply(&late).0, Outcome::Duplicate); + assert_eq!(c.watermark(&shape("s")), Some(at(2, 1, 3))); + let kept = wire::decode(&target(&[cell(1, "new")])); + assert_eq!(c.held(&shape("s")), Some(&kept)); +} + +#[test] +fn a_reply_from_a_generation_the_client_left_does_not_return() { + let mut c = synced("s", at(1, 1, 9), &[cell(1, "old")]); + c.apply(&window_correction("s", at(1, 2, 1), &[cell(1, "new")])); + c.apply(&window_correction("s", at(1, 3, 1), &[cell(1, "newer")])); + let late = window_correction("s", at(1, 2, 7), &[cell(1, "gen2")]); + assert_eq!(c.apply(&late).0, Outcome::Duplicate); +}