fiddlerwoaroof/improvise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #359 from fiddlerwoaroof/fix/delivery-left-epoch-livelock

Browse Source 
fix(delivery): the live host's answer is never refused as a left epoch (improvise-z2vr)
This commit is contained in:
Ed Langley
2026-10-10 18:42:49 -07:00
committed by GitHub
parent c83d81c510 42c2fcb041
commit ec5112cfe8
4 changed files with 131 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/improvise-command/src/ui/delivery_tests.rs
+75 -15
View File
@@ -1086,6 +1086,45 @@ fn a_reply_held_across_a_host_restart_cannot_pull_the_client_back() {
assert_eq!(state, Synced::Settled);
}
#[test]
fn a_late_reply_from_a_host_the_client_never_joined_cannot_strand_it() {
let window = Window {
top: 0,
left: 0,
rows: 1,
cols: 1,
};
let mut first = host_instance(1);
let mut client = EdgeClient::new(session());
client.track(shape(), window);
client.sync(&mut Flaky::new(&mut first), &shape(), 6);
let mut second = host_instance(2);
let id = connect(&mut second);
send(&mut second, id, vec![set("South", 1.0)]);
let mut lost = Flaky::scripted(&mut second, vec![Some(Fault::DropReply)]);
client.connection_lost();
client.sync(&mut lost, &shape(), 6);
let late = lost.replies.remove(0);
let mut third = host_instance(3);
client.connection_lost();
client.sync(&mut Flaky::new(&mut third), &shape(), 6);
let Reply::Correction(late) = late else {
panic!("the lost reply was a correction")
};
client.apply(&late);
client.connection_lost();
let (state, _) = client.sync(&mut Flaky::new(&mut third), &shape(), 6);
assert_eq!(
state,
Synced::Settled,
"the live host's answers must be taken"
);
assert_eq!(client.held(&shape()), Some(&drawn(&third, &window)));
}
// ── the whole thing, at random ─────────────────────────────────────────────
mod random {
@@ -1265,6 +1304,39 @@ mod random {
deliver_all(client, &notices);
}
/// Epochs the client has applied a correction from, oldest first.
fn note_epoch(client: &EdgeClient, epochs: &mut Vec<(u64, u64)>) {
if let Some(w) = client.watermark(&shape()) {
let now = (w.host, w.generation);
if !epochs.contains(&now) {
epochs.push(now);
}
}
}
/// A late reply arrives out of band. One from an epoch the client was in and has left is refused;
/// one from an epoch it never joined may legitimately move it, and the next answer brings it back.
fn replay(
client: &mut EdgeClient,
c: &improvise_protocol::delivery::Correction,
epochs: &mut Vec<(u64, u64)>,
) -> Result<(), TestCaseError> {
let theirs = (c.through.host, c.through.generation);
let current = client.watermark(&shape()).map(|w| (w.host, w.generation));
let left = epochs.contains(&theirs) && current != Some(theirs);
let (outcome, _) = client.apply(c);
if left {
prop_assert_eq!(
outcome,
Outcome::Duplicate,
"a reply from the left epoch {:?}",
theirs
);
}
note_epoch(client, epochs);
Ok(())
}
proptest! {
#[test]
fn a_corrected_state_is_the_target_and_correcting_it_again_is_the_zero_zset(
@@ -1306,9 +1378,7 @@ mod random {
}
if let Some(i) = weather.replay.filter(|_| !old.is_empty()) {
match &old[i % old.len()] {
Reply::Correction(c) => {
client.apply(c);
}
Reply::Correction(c) => replay(&mut client, c, &mut epochs)?,
Reply::Notices(list) => deliver_all(&mut client, list),
}
}
@@ -1316,20 +1386,10 @@ mod random {
let mut edge = Flaky::scripted(&mut host, script);
client.sync(&mut edge, &shape(), 6);
old.append(&mut edge.replies);
note_epoch(&client, &mut epochs);
let moved_host = matches!(op, Op::Restart | Op::Swap);
if let (true, Some(Reply::Correction(c))) = (moved_host, old.get(step % old.len().max(1))) {
client.apply(c);
}
if let Some(w) = client.watermark(&shape()) {
let now = (w.host, w.generation);
let seen_at = epochs.iter().position(|e| *e == now);
prop_assert!(
seen_at.is_none_or(|i| i + 1 == epochs.len()),
"the client went back to a superseded epoch {now:?} after {epochs:?}"
);
if seen_at.is_none() {
epochs.push(now);
}
replay(&mut client, c, &mut epochs)?;
}
}
crates/improvise-protocol/src/delivery/client.rs
+18 -3
View File
@@ -243,11 +243,26 @@ impl EdgeClient {
}
}
/// A correction. It applies only at the watermark it starts from: `held + (target - held)`.
/// A correction that arrives on its own: it applies only at the watermark it starts from,
/// as `held + (target - held)`, and never from an epoch the client has left.
pub fn apply(&mut self, c: &Correction) -> (Outcome, Vec<Event>) {
self.apply_with(c, false)
}
/// The reply to a request this client just made. The host that answers is the live one, so
/// an epoch the client had written off is live again (`context/code/delivery.md` section 4).
pub fn apply_answer(&mut self, c: &Correction) -> (Outcome, Vec<Event>) {
self.apply_with(c, true)
}
fn apply_with(&mut self, c: &Correction, answer: bool) -> (Outcome, Vec<Event>) {
let Some(t) = self.shapes.get_mut(&c.shape) else {
return (Outcome::Untracked, Vec::new());
};
if answer {
t.left
.retain(|e| *e != (c.through.host, c.through.generation));
}
let target = decode(&c.target);
let older = |s: Option<Stamp>| {
s.is_some_and(|s| s.host == c.through.host && c.through.generation < s.generation)
@@ -347,7 +362,7 @@ impl EdgeClient {
return (Synced::Settled, events);
};
match edge.call(request) {
Ok(Reply::Correction(c)) => events.extend(self.apply(&c).1),
Ok(Reply::Correction(c)) => events.extend(self.apply_answer(&c).1),
Ok(Reply::Notices(list)) => {
list.iter()
.for_each(|n| events.extend(self.accept(n, Some(shape))));
@@ -387,7 +402,7 @@ impl EdgeClient {
Ok(Reply::Notices(list)) => list
.iter()
.for_each(|n| events.extend(self.accept(n, Some(shape)))),
Ok(Reply::Correction(c)) => events.extend(self.apply(&c).1),
Ok(Reply::Correction(c)) => events.extend(self.apply_answer(&c).1),
Err(EdgeError::Failed) => {
self.request_failed(shape);
return (Synced::Failed, events);