From 075522256201b83f99d735f6c10a98b8f251da9c Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-circuit)" Date: Sat, 10 Oct 2026 11:53:08 -0700 Subject: [PATCH 01/10] feat(zset-ffi): a C ABI over zset for CFFI callers (improvise-bw4p.1) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- Cargo.lock | 18 ++ Cargo.toml | 2 +- crates/zset-ffi/Cargo.toml | 18 ++ crates/zset-ffi/include/zset_ffi.h | 121 ++++++++ crates/zset-ffi/src/engine.rs | 421 ++++++++++++++++++++++++++ crates/zset-ffi/src/lib.rs | 373 +++++++++++++++++++++++ crates/zset-ffi/tests/abi.rs | 462 +++++++++++++++++++++++++++++ crates/zset-ffi/tests/dylib.rs | 63 ++++ crates/zset-ffi/tests/header.rs | 141 +++++++++ 9 files changed, 1618 insertions(+), 1 deletion(-) create mode 100644 crates/zset-ffi/Cargo.toml create mode 100644 crates/zset-ffi/include/zset_ffi.h create mode 100644 crates/zset-ffi/src/engine.rs create mode 100644 crates/zset-ffi/src/lib.rs create mode 100644 crates/zset-ffi/tests/abi.rs create mode 100644 crates/zset-ffi/tests/dylib.rs create mode 100644 crates/zset-ffi/tests/header.rs diff --git a/Cargo.lock b/Cargo.lock index d09a5591..0cc7278b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1124,6 +1124,16 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -2732,3 +2742,11 @@ version = "0.1.0" dependencies = [ "proptest", ] + +[[package]] +name = "zset-ffi" +version = "0.1.0" +dependencies = [ + "libloading", + "zset", +] diff --git a/Cargo.toml b/Cargo.toml index 1f99d257..e0db5c95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/improvise-formula", "crates/improvise-core", "crates/improvise-protocol", "crates/improvise-io", "crates/improvise-command", "crates/improvise-wasm-client", "crates/improvise-ws-server", "crates/improvise-worker-server", "crates/gp-tree", "crates/improvise-gp", "crates/numtower", "crates/zset"] +members = [".", "crates/improvise-formula", "crates/improvise-core", "crates/improvise-protocol", "crates/improvise-io", "crates/improvise-command", "crates/improvise-wasm-client", "crates/improvise-ws-server", "crates/improvise-worker-server", "crates/gp-tree", "crates/improvise-gp", "crates/numtower", "crates/zset", "crates/zset-ffi"] [package] name = "improvise" diff --git a/crates/zset-ffi/Cargo.toml b/crates/zset-ffi/Cargo.toml new file mode 100644 index 00000000..96465bfd --- /dev/null +++ b/crates/zset-ffi/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "zset-ffi" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +description = "A C ABI over zset (byte-string keys, i64 weights) for CFFI callers: cdylib, staticlib and rlib." +publish = false + +# Native only, and deliberately not in the wasm bundle: `zset` stays std-only. The +# library is named zset_ffi, so the artifacts are libzset_ffi.{so,dylib,a}. +[lib] +crate-type = ["cdylib", "staticlib", "rlib"] + +[dependencies] +zset = { path = "../zset" } + +[dev-dependencies] +libloading = "0.8" diff --git a/crates/zset-ffi/include/zset_ffi.h b/crates/zset-ffi/include/zset_ffi.h new file mode 100644 index 00000000..a320c605 --- /dev/null +++ b/crates/zset-ffi/include/zset_ffi.h @@ -0,0 +1,121 @@ +/* zset_ffi.h: the C ABI of crates/zset-ffi (libzset_ffi.{so,dylib,a}). + * Hand-written; tests/header.rs checks it against src/lib.rs. + * Reasoning: context/code/zset-ffi.md. + * + * Conventions + * - Every function returns int: 0 is success, anything else is a ZS_ERR_* code. + * Results come back through out-pointers, which are written only on success. + * On failure zs_last_error() describes it (per thread, valid until the next failure). + * - Handles (ZsZSet, ZsCircuit, ZsRebuild) are opaque. Each is created by a function that + * takes ZsX **out and is released by exactly one zs_X_free. Freeing NULL is a no-op; + * freeing twice or using a freed handle is undefined. + * - Keys are byte strings: (pointer, length). The library copies them; the caller keeps + * ownership of every buffer it passes, and may reuse it after the call returns. + * A zero length accepts a NULL pointer. + * - The library never returns memory the caller must free, except handles. + * - Weights are int64_t. A weight that would leave int64_t is ZS_ERR_OVERFLOW, never a + * wrong value, and never an unwinding exception. + * - Not thread-safe: use a handle from one thread at a time. Streams, inputs and sinks of + * a circuit are uint32_t ids local to that circuit. + */ +#ifndef ZSET_FFI_H +#define ZSET_FFI_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define ZS_ABI_VERSION 1 + +#define ZS_OK 0 +#define ZS_ERR_NULL_ARGUMENT 1 +#define ZS_ERR_OVERFLOW 2 +#define ZS_ERR_PANIC 3 +#define ZS_ERR_BAD_HANDLE 4 +#define ZS_ERR_BUFFER_TOO_SMALL 5 +#define ZS_ERR_BAD_VALUE 6 +#define ZS_ERR_POISONED 7 + +#define ZS_AGG_COUNT 0 +#define ZS_AGG_SUM 1 +#define ZS_AGG_MIN 2 +#define ZS_AGG_MAX 3 + +typedef struct ZsZSet ZsZSet; +typedef struct ZsCircuit ZsCircuit; +typedef struct ZsRebuild ZsRebuild; + +/* A callback may emit images of a key by calling emit(emit_ctx, bytes, len); the bytes are + * copied during the call. A callback must not call back into the library. */ +typedef void (*ZsEmit)(void *emit_ctx, const uint8_t *bytes, size_t len); +typedef void (*ZsFlatMapFn)(void *ctx, const uint8_t *key, size_t len, ZsEmit emit, void *emit_ctx); +typedef int32_t (*ZsKeepFn)(void *ctx, const uint8_t *key, size_t len); +typedef int32_t (*ZsExpiredFn)(void *ctx); + +uint32_t zs_abi_version(void); +const char *zs_last_error(void); + +/* Z-sets. add/sub/neg/scale/decode_add leave the set unchanged when they fail. */ +int32_t zs_zset_new(ZsZSet **out); +void zs_zset_free(ZsZSet *z); +int32_t zs_zset_clone(const ZsZSet *z, ZsZSet **out); +int32_t zs_zset_insert(ZsZSet *z, const uint8_t *key, size_t len, int64_t weight); +int32_t zs_zset_weight(const ZsZSet *z, const uint8_t *key, size_t len, int64_t *out); +int32_t zs_zset_len(const ZsZSet *z, size_t *out); +int32_t zs_zset_add(ZsZSet *a, const ZsZSet *b); +int32_t zs_zset_sub(ZsZSet *a, const ZsZSet *b); +int32_t zs_zset_neg(ZsZSet *a); +int32_t zs_zset_scale(ZsZSet *a, int64_t n); +int32_t zs_zset_distinct(const ZsZSet *z, ZsZSet **out); + +/* Encoding: entries ascending by key, each [u32 LE key length][key][i64 LE weight]. + * *needed is always set; the bytes are written only if cap >= *needed, else + * ZS_ERR_BUFFER_TOO_SMALL (call again with a larger buffer). */ +int32_t zs_zset_encode(const ZsZSet *z, uint8_t *buf, size_t cap, size_t *needed); +int32_t zs_zset_decode_add(ZsZSet *z, const uint8_t *buf, size_t len); + +/* Circuits. Every stream carries a Z-set of byte-string keys. */ +int32_t zs_circuit_new(ZsCircuit **out); +void zs_circuit_free(ZsCircuit *c); +int32_t zs_circuit_input(ZsCircuit *c, uint32_t *out); +int32_t zs_circuit_output(ZsCircuit *c, uint32_t stream, uint32_t *out); +int32_t zs_circuit_delay(ZsCircuit *c, uint32_t stream, uint32_t *out); +int32_t zs_circuit_integrate(ZsCircuit *c, uint32_t stream, uint32_t *out); +int32_t zs_circuit_differentiate(ZsCircuit *c, uint32_t stream, uint32_t *out); +int32_t zs_circuit_distinct(ZsCircuit *c, uint32_t stream, uint32_t *out); +int32_t zs_circuit_plus(ZsCircuit *c, uint32_t a, uint32_t b, uint32_t *out); +int32_t zs_circuit_filter(ZsCircuit *c, uint32_t stream, ZsKeepFn keep, void *ctx, uint32_t *out); +int32_t zs_circuit_flat_map(ZsCircuit *c, uint32_t stream, ZsFlatMapFn f, void *ctx, uint32_t *out); + +/* index_by: f emits exactly one group key g per element; the output element is + * pair(g, element), pair(k, v) = [u32 LE length of k][k][v]. */ +int32_t zs_circuit_index_by(ZsCircuit *c, uint32_t stream, ZsFlatMapFn f, void *ctx, uint32_t *out); + +/* join: both inputs carry pair(k, v); the output element is pair(k, pair(a, b)). */ +int32_t zs_circuit_join(ZsCircuit *c, uint32_t a, uint32_t b, uint32_t *out); + +/* aggregate: the input carries pair(k, v) with v an 8-byte little-endian int64 (any bytes + * for COUNT); the output is pair(k, 8-byte little-endian int64). COUNT is the total weight, + * SUM is the weighted sum, MIN and MAX read the values of positive weight. */ +int32_t zs_circuit_aggregate(ZsCircuit *c, uint32_t stream, int32_t kind, uint32_t *out); + +int32_t zs_circuit_push(ZsCircuit *c, uint32_t input, const ZsZSet *z); + +/* One clock tick. After a failure in a step the circuit is poisoned: every later call that + * uses it returns ZS_ERR_POISONED, except the free. */ +int32_t zs_circuit_step(ZsCircuit *c); +int32_t zs_circuit_take(ZsCircuit *c, uint32_t sink, ZsZSet **out); + +/* Resumable snapshot load: chunk keys per circuit step (0 reads as 1). */ +int32_t zs_rebuild_new(ZsCircuit *c, uint32_t input, const ZsZSet *snapshot, size_t chunk, ZsRebuild **out); +int32_t zs_rebuild_step(ZsRebuild *job, ZsCircuit *c, ZsExpiredFn expired, void *ctx, int32_t *done); +void zs_rebuild_free(ZsRebuild *job); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/crates/zset-ffi/src/engine.rs b/crates/zset-ffi/src/engine.rs new file mode 100644 index 00000000..b8c23740 --- /dev/null +++ b/crates/zset-ffi/src/engine.rs @@ -0,0 +1,421 @@ +//! The byte-keyed circuit behind the C ABI, in plain Rust: every fallible operation +//! returns a `Result`, and a panic from `zset` is caught and classified here. +//! The reasoning is in `context/code/zset-ffi.md`. + +use std::any::Any; +use std::cell::Cell; +use std::ffi::c_void; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::rc::Rc; +use zset::{Circuit, Input, Output, Progress, Rebuild, Stream, ZSet}; + +pub type Bytes = Vec; +pub type BZ = ZSet; + +/// The error codes of the ABI; 0 is success and is not a variant. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(i32)] +pub enum Code { + NullArgument = 1, + Overflow = 2, + Panic = 3, + BadHandle = 4, + BufferTooSmall = 5, + BadValue = 6, + Poisoned = 7, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Fail { + pub code: Code, + pub msg: String, +} + +impl Fail { + pub fn new(code: Code, msg: &str) -> Fail { + Fail { + code, + msg: msg.to_string(), + } + } +} + +type Res = Result; + +const OVERFLOW_PREFIX: &str = "zset: weight overflow"; + +/// Turns a caught panic into an error: weight overflow is `Overflow`, anything else `Panic`. +pub fn classify(payload: Box) -> Fail { + let msg = payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_else(|| "panic with a non-string payload".to_string()); + let code = if msg.starts_with(OVERFLOW_PREFIX) { + Code::Overflow + } else { + Code::Panic + }; + Fail { code, msg } +} + +/// `(key, value)` as one byte string: a little-endian `u32` key length, the key, the value. +pub fn pair(k: &[u8], v: &[u8]) -> Bytes { + let mut out = Vec::with_capacity(4 + k.len() + v.len()); + out.extend_from_slice(&(k.len() as u32).to_le_bytes()); + out.extend_from_slice(k); + out.extend_from_slice(v); + out +} + +pub fn unpair(b: &[u8]) -> Option<(&[u8], &[u8])> { + let n = u32::from_le_bytes(b.get(..4)?.try_into().ok()?) as usize; + let rest = &b[4..]; + (rest.len() >= n).then(|| rest.split_at(n)) +} + +/// Entries as `[u32 LE key length][key][i64 LE weight]`, ascending by key. +pub fn encode(z: &BZ) -> Bytes { + let mut out = Vec::new(); + for (k, w) in z.iter() { + out.extend_from_slice(&(k.len() as u32).to_le_bytes()); + out.extend_from_slice(k); + out.extend_from_slice(&w.to_le_bytes()); + } + out +} + +/// The inverse of `encode`; equal keys sum, and an overflow panics like any addition. +pub fn decode(mut b: &[u8]) -> Res { + let bad = || Fail::new(Code::BadValue, "malformed zset encoding"); + let mut z = ZSet::new(); + while !b.is_empty() { + let n = u32::from_le_bytes(b.get(..4).ok_or_else(bad)?.try_into().unwrap()) as usize; + let key = b.get(4..4 + n).ok_or_else(bad)?; + let w = b.get(4 + n..12 + n).ok_or_else(bad)?; + z.insert(key.to_vec(), i64::from_le_bytes(w.try_into().unwrap())); + b = &b[12 + n..]; + } + Ok(z) +} + +pub type Emit = extern "C" fn(*mut c_void, *const u8, usize); +pub type FlatMapFn = extern "C" fn(*mut c_void, *const u8, usize, Emit, *mut c_void); +pub type KeepFn = extern "C" fn(*mut c_void, *const u8, usize) -> i32; +pub type ExpiredFn = extern "C" fn(*mut c_void) -> i32; + +/// A caller's context pointer; the circuit never dereferences it. +#[derive(Clone, Copy)] +struct Ctx(*mut c_void); + +extern "C" fn collect(ctx: *mut c_void, p: *const u8, n: usize) { + // SAFETY: `ctx` is the `Vec` `images` passes below, alive for the callback's duration. + let out = unsafe { &mut *(ctx as *mut Vec) }; + let bytes = if n == 0 || p.is_null() { + Vec::new() + } else { + // SAFETY: the caller of `emit` promises `n` readable bytes at `p`. + unsafe { std::slice::from_raw_parts(p, n) }.to_vec() + }; + out.push(bytes); +} + +fn images(f: FlatMapFn, ctx: Ctx, key: &Bytes) -> Vec { + let mut out: Vec = Vec::new(); + f( + ctx.0, + key.as_ptr(), + key.len(), + collect, + &mut out as *mut Vec as *mut c_void, + ); + out +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Agg { + Count, + Sum, + Min, + Max, +} + +impl Agg { + pub fn from_code(n: i32) -> Res { + Ok(match n { + 0 => Agg::Count, + 1 => Agg::Sum, + 2 => Agg::Min, + 3 => Agg::Max, + _ => return Err(Fail::new(Code::BadValue, "unknown aggregate kind")), + }) + } +} + +fn checked(v: Option) -> i64 { + v.unwrap_or_else(|| panic!("{OVERFLOW_PREFIX} in aggregate")) +} + +/// The circuit, its streams by id, and the first fault a closure recorded. +pub struct Engine { + circuit: Circuit, + streams: Vec>, + inputs: Vec>>, + sinks: Vec>, + fault: Rc>>, + poisoned: bool, +} + +impl Default for Engine { + fn default() -> Self { + Self::new() + } +} + +impl Engine { + pub fn new() -> Engine { + Engine { + circuit: Circuit::new(), + streams: Vec::new(), + inputs: Vec::new(), + sinks: Vec::new(), + fault: Rc::default(), + poisoned: false, + } + } + + fn stream(&self, id: u32) -> Res> { + self.streams + .get(id as usize) + .cloned() + .ok_or_else(|| Fail::new(Code::BadHandle, "no such stream")) + } + + fn register(&mut self, s: Stream, input: Option>) -> u32 { + self.streams.push(s); + self.inputs.push(input); + (self.streams.len() - 1) as u32 + } + + fn alive(&self) -> Res<()> { + if self.poisoned { + return Err(Fail::new( + Code::Poisoned, + "the circuit failed in a step and cannot be used again", + )); + } + Ok(()) + } + + pub fn input(&mut self) -> Res { + self.alive()?; + let (i, s) = self.circuit.input::(); + Ok(self.register(s, Some(i))) + } + + pub fn output(&mut self, s: u32) -> Res { + self.alive()?; + let s = self.stream(s)?; + self.sinks.push(self.circuit.output(&s)); + Ok((self.sinks.len() - 1) as u32) + } + + pub fn delay(&mut self, s: u32) -> Res { + self.alive()?; + let s = self.stream(s)?; + let out = self.circuit.delay(&s); + Ok(self.register(out, None)) + } + + pub fn integrate(&mut self, s: u32) -> Res { + self.alive()?; + let s = self.stream(s)?; + let out = self.circuit.integrate(&s); + Ok(self.register(out, None)) + } + + pub fn differentiate(&mut self, s: u32) -> Res { + self.alive()?; + let s = self.stream(s)?; + let out = self.circuit.differentiate(&s); + Ok(self.register(out, None)) + } + + pub fn distinct(&mut self, s: u32) -> Res { + self.alive()?; + let s = self.stream(s)?; + let out = self.circuit.distinct(&s); + Ok(self.register(out, None)) + } + + pub fn plus(&mut self, a: u32, b: u32) -> Res { + self.alive()?; + let (a, b) = (self.stream(a)?, self.stream(b)?); + let out = self.circuit.plus(&a, &b); + Ok(self.register(out, None)) + } + + pub fn filter(&mut self, s: u32, keep: KeepFn, ctx: *mut c_void) -> Res { + self.alive()?; + let s = self.stream(s)?; + let ctx = Ctx(ctx); + let out = self + .circuit + .filter(&s, move |k: &Bytes| keep(ctx.0, k.as_ptr(), k.len()) != 0); + Ok(self.register(out, None)) + } + + pub fn flat_map(&mut self, s: u32, f: FlatMapFn, ctx: *mut c_void) -> Res { + self.alive()?; + let s = self.stream(s)?; + let ctx = Ctx(ctx); + let out = self + .circuit + .flat_map(&s, move |k: &Bytes| images(f, ctx, k)); + Ok(self.register(out, None)) + } + + /// Each element becomes `pair(g, element)` where the callback emits exactly one `g`. + pub fn index_by(&mut self, s: u32, f: FlatMapFn, ctx: *mut c_void) -> Res { + self.alive()?; + let s = self.stream(s)?; + let (ctx, fault) = (Ctx(ctx), Rc::clone(&self.fault)); + let out = self.circuit.flat_map(&s, move |k: &Bytes| { + let mut gs = images(f, ctx, k); + if gs.len() != 1 { + fault.set(Some("index_by: the key callback must emit exactly one key")); + return Vec::new(); + } + vec![pair(&gs.remove(0), k)] + }); + Ok(self.register(out, None)) + } + + /// Splits `pair(k, v)` elements; a malformed one records a fault and is dropped. + fn keyed(&mut self, s: &Stream) -> Stream> { + let fault = Rc::clone(&self.fault); + self.circuit.flat_map(s, move |b: &Bytes| match unpair(b) { + Some((k, v)) => vec![(k.to_vec(), v.to_vec())], + None => { + fault.set(Some("a keyed stream carried an element that is not a pair")); + Vec::new() + } + }) + } + + /// Joins two streams of `pair(k, v)`; the output is `pair(k, pair(a, b))`. + pub fn join(&mut self, a: u32, b: u32) -> Res { + self.alive()?; + let (a, b) = (self.stream(a)?, self.stream(b)?); + let (ka, kb) = (self.keyed(&a), self.keyed(&b)); + let j = self + .circuit + .join_indexed(&ka, &kb, |k: &Bytes, x: &Bytes, y: &Bytes| { + pair(k, &pair(x, y)) + }); + Ok(self.register(j, None)) + } + + /// Groups a stream of `pair(k, v)` (v is 8 bytes, little-endian `i64`, except for + /// `Count`); the output is `pair(k, 8 bytes)`. + pub fn aggregate(&mut self, s: u32, kind: Agg) -> Res { + self.alive()?; + let s = self.stream(s)?; + let keyed = self.keyed(&s); + let fault = Rc::clone(&self.fault); + let folded = self + .circuit + .aggregate(&keyed, move |_, vals: &[(Bytes, i64)]| { + if kind == Agg::Count { + let n = vals.iter().try_fold(0i64, |a, (_, w)| a.checked_add(*w)); + return Some(checked(n)).filter(|n| *n != 0); + } + let mut nums = Vec::with_capacity(vals.len()); + for (v, w) in vals { + match <[u8; 8]>::try_from(v.as_slice()) { + Ok(b) => nums.push((i64::from_le_bytes(b), *w)), + Err(_) => { + fault.set(Some("an aggregated value is not 8 bytes")); + return None; + } + } + } + match kind { + Agg::Sum => { + let t = nums.iter().try_fold(0i64, |a, (v, w)| { + v.checked_mul(*w).and_then(|p| a.checked_add(p)) + }); + Some(checked(t)).filter(|t| *t != 0) + } + Agg::Min => nums.iter().filter(|(_, w)| *w > 0).map(|(v, _)| *v).min(), + Agg::Max | Agg::Count => { + nums.iter().filter(|(_, w)| *w > 0).map(|(v, _)| *v).max() + } + } + }); + let out = self + .circuit + .map(&folded, |(k, r): &(Bytes, i64)| pair(k, &r.to_le_bytes())); + Ok(self.register(out, None)) + } + + fn input_of(&self, id: u32) -> Res> { + self.inputs + .get(id as usize) + .and_then(|i| i.clone()) + .ok_or_else(|| Fail::new(Code::BadHandle, "that stream is not an input")) + } + + pub fn push(&mut self, input: u32, z: &BZ) -> Res<()> { + self.alive()?; + self.input_of(input)?.push(z.clone()); + Ok(()) + } + + /// Runs `f` on the circuit; a panic or a recorded fault poisons the engine. + fn guarded(&mut self, f: impl FnOnce(&mut Circuit) -> T) -> Res { + self.alive()?; + self.fault.set(None); + match catch_unwind(AssertUnwindSafe(|| f(&mut self.circuit))) { + Err(payload) => { + self.poisoned = true; + Err(classify(payload)) + } + Ok(v) => match self.fault.take() { + Some(msg) => { + self.poisoned = true; + Err(Fail::new(Code::BadValue, msg)) + } + None => Ok(v), + }, + } + } + + pub fn step(&mut self) -> Res<()> { + self.guarded(|c| c.step()) + } + + pub fn take(&mut self, sink: u32) -> Res { + self.alive()?; + self.sinks + .get(sink as usize) + .map(|s| s.take()) + .ok_or_else(|| Fail::new(Code::BadHandle, "no such output")) + } + + pub fn start_rebuild(&mut self, input: u32, snapshot: BZ, chunk: usize) -> Res> { + self.alive()?; + Ok(Rebuild::new(self.input_of(input)?, snapshot, chunk)) + } + + /// `true` when the snapshot is consumed. With no `expired`, runs to completion. + pub fn rebuild_step( + &mut self, + job: &mut Rebuild, + expired: Option<(ExpiredFn, *mut c_void)>, + ) -> Res { + let progress = + self.guarded(|c| job.step(c, || expired.is_some_and(|(f, ctx)| f(ctx) != 0)))?; + Ok(progress == Progress::Done) + } +} diff --git a/crates/zset-ffi/src/lib.rs b/crates/zset-ffi/src/lib.rs new file mode 100644 index 00000000..00c92c21 --- /dev/null +++ b/crates/zset-ffi/src/lib.rs @@ -0,0 +1,373 @@ +//! The C ABI over `zset`: byte-string keys, `i64` weights, opaque handles, an `i32` code +//! from every function. Ownership, lifetimes and the error model: `context/code/zset-ffi.md` +//! and `include/zset_ffi.h`. + +pub mod engine; + +use engine::{ + Agg, BZ, Bytes, Code, Engine, ExpiredFn, Fail, FlatMapFn, KeepFn, classify, decode, encode, +}; +use std::cell::RefCell; +use std::ffi::{CString, c_char, c_void}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use zset::{Rebuild, ZSet}; + +pub const ZS_ABI_VERSION: u32 = 1; + +/// An opaque Z-set of byte-string keys. +pub struct ZsZSet(BZ); +/// An opaque circuit over such Z-sets. +pub struct ZsCircuit(Engine); +/// An opaque resumable snapshot load. +pub struct ZsRebuild(Rebuild); + +thread_local! { + static LAST_ERROR: RefCell = RefCell::new(CString::default()); +} + +fn remember(msg: &str) { + let clean: String = msg.chars().filter(|c| *c != '\0').collect(); + LAST_ERROR.with(|e| *e.borrow_mut() = CString::new(clean).unwrap_or_default()); +} + +/// Runs an entry point body: no panic crosses the ABI, and every failure is a code. +fn guard(f: impl FnOnce() -> Result<(), Fail>) -> i32 { + let fail = match catch_unwind(AssertUnwindSafe(f)) { + Ok(Ok(())) => return 0, + Ok(Err(fail)) => fail, + Err(payload) => classify(payload), + }; + remember(&fail.msg); + fail.code as i32 +} + +fn null() -> Fail { + Fail::new(Code::NullArgument, "a required argument was null") +} + +fn as_mut<'a, T>(p: *mut T) -> Result<&'a mut T, Fail> { + // SAFETY: the ABI contract says a non-null handle came from this library and is live. + unsafe { p.as_mut() }.ok_or_else(null) +} + +fn as_ref<'a, T>(p: *const T) -> Result<&'a T, Fail> { + // SAFETY: as `as_mut`. + unsafe { p.as_ref() }.ok_or_else(null) +} + +fn bytes<'a>(p: *const u8, n: usize) -> Result<&'a [u8], Fail> { + match (p.is_null(), n) { + (_, 0) => Ok(&[]), + (true, _) => Err(null()), + // SAFETY: the ABI contract says `p` has `n` readable bytes. + (false, _) => Ok(unsafe { std::slice::from_raw_parts(p, n) }), + } +} + +fn put(out: *mut T, v: T) -> Result<(), Fail> { + *as_mut(out)? = v; + Ok(()) +} + +fn hand_out(out: *mut *mut T, v: T) -> Result<(), Fail> { + put(out, Box::into_raw(Box::new(v))) +} + +/// Frees a handle; null is accepted and ignored. +fn release(p: *mut T) { + if !p.is_null() { + // SAFETY: a non-null handle was made by `Box::into_raw` here and is freed once. + drop(unsafe { Box::from_raw(p) }); + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_abi_version() -> u32 { + ZS_ABI_VERSION +} + +/// The message of the last failure on this thread; never null, valid until the next failure. +#[unsafe(no_mangle)] +pub extern "C" fn zs_last_error() -> *const c_char { + LAST_ERROR.with(|e| e.borrow().as_ptr()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_new(out: *mut *mut ZsZSet) -> i32 { + guard(|| hand_out(out, ZsZSet(ZSet::new()))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_free(z: *mut ZsZSet) { + release(z) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_clone(z: *const ZsZSet, out: *mut *mut ZsZSet) -> i32 { + guard(|| hand_out(out, ZsZSet(as_ref(z)?.0.clone()))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_insert(z: *mut ZsZSet, key: *const u8, len: usize, weight: i64) -> i32 { + guard(|| { + let (z, key) = (as_mut(z)?, bytes(key, len)?); + z.0.insert(key.to_vec(), weight); + Ok(()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_weight( + z: *const ZsZSet, + key: *const u8, + len: usize, + out: *mut i64, +) -> i32 { + guard(|| { + let w = as_ref(z)?.0.weight(&bytes(key, len)?.to_vec()); + put(out, w) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_len(z: *const ZsZSet, out: *mut usize) -> i32 { + guard(|| put(out, as_ref(z)?.0.len())) +} + +/// The binary operations compute a new value first, so a failure leaves `a` unchanged. +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_add(a: *mut ZsZSet, b: *const ZsZSet) -> i32 { + guard(|| { + let (a, b) = (as_mut(a)?, as_ref(b)?); + a.0 = &a.0 + &b.0; + Ok(()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_sub(a: *mut ZsZSet, b: *const ZsZSet) -> i32 { + guard(|| { + let (a, b) = (as_mut(a)?, as_ref(b)?); + a.0 = &a.0 - &b.0; + Ok(()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_neg(a: *mut ZsZSet) -> i32 { + guard(|| { + let a = as_mut(a)?; + a.0 = -&a.0; + Ok(()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_scale(a: *mut ZsZSet, n: i64) -> i32 { + guard(|| { + let a = as_mut(a)?; + a.0 = a.0.scale(n); + Ok(()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_distinct(z: *const ZsZSet, out: *mut *mut ZsZSet) -> i32 { + guard(|| hand_out(out, ZsZSet(as_ref(z)?.0.distinct()))) +} + +/// Writes the encoding to `buf` if it fits. `*needed` is always set to the full size. +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_encode( + z: *const ZsZSet, + buf: *mut u8, + cap: usize, + needed: *mut usize, +) -> i32 { + guard(|| { + let enc = encode(&as_ref(z)?.0); + put(needed, enc.len())?; + if enc.len() > cap { + return Err(Fail::new( + Code::BufferTooSmall, + "the buffer is smaller than *needed", + )); + } + if !enc.is_empty() { + // SAFETY: the contract says `buf` has `cap >= enc.len()` writable bytes. + unsafe { std::ptr::copy_nonoverlapping(enc.as_ptr(), buf, enc.len()) }; + } + Ok(()) + }) +} + +/// Adds the decoded entries to `z`; nothing is added if the encoding is bad or overflows. +#[unsafe(no_mangle)] +pub extern "C" fn zs_zset_decode_add(z: *mut ZsZSet, buf: *const u8, len: usize) -> i32 { + guard(|| { + let (z, buf) = (as_mut(z)?, bytes(buf, len)?); + z.0 = &z.0 + &decode(buf)?; + Ok(()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_new(out: *mut *mut ZsCircuit) -> i32 { + guard(|| hand_out(out, ZsCircuit(Engine::new()))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_free(c: *mut ZsCircuit) { + release(c) +} + +/// A new input stream; its id is both the stream to read and the input to push to. +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_input(c: *mut ZsCircuit, out: *mut u32) -> i32 { + guard(|| put(out, as_mut(c)?.0.input()?)) +} + +/// A sink that sums the stream's deltas until `zs_circuit_take`. +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_output(c: *mut ZsCircuit, stream: u32, out: *mut u32) -> i32 { + guard(|| put(out, as_mut(c)?.0.output(stream)?)) +} + +macro_rules! unary_op { + ($name:ident, $method:ident) => { + #[unsafe(no_mangle)] + pub extern "C" fn $name(c: *mut ZsCircuit, stream: u32, out: *mut u32) -> i32 { + guard(|| put(out, as_mut(c)?.0.$method(stream)?)) + } + }; +} + +unary_op!(zs_circuit_delay, delay); +unary_op!(zs_circuit_integrate, integrate); +unary_op!(zs_circuit_differentiate, differentiate); +unary_op!(zs_circuit_distinct, distinct); + +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_plus(c: *mut ZsCircuit, a: u32, b: u32, out: *mut u32) -> i32 { + guard(|| put(out, as_mut(c)?.0.plus(a, b)?)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_filter( + c: *mut ZsCircuit, + stream: u32, + keep: Option, + ctx: *mut c_void, + out: *mut u32, +) -> i32 { + guard(|| { + put( + out, + as_mut(c)?.0.filter(stream, keep.ok_or_else(null)?, ctx)?, + ) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_flat_map( + c: *mut ZsCircuit, + stream: u32, + f: Option, + ctx: *mut c_void, + out: *mut u32, +) -> i32 { + guard(|| { + put( + out, + as_mut(c)?.0.flat_map(stream, f.ok_or_else(null)?, ctx)?, + ) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_index_by( + c: *mut ZsCircuit, + stream: u32, + f: Option, + ctx: *mut c_void, + out: *mut u32, +) -> i32 { + guard(|| { + put( + out, + as_mut(c)?.0.index_by(stream, f.ok_or_else(null)?, ctx)?, + ) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_join(c: *mut ZsCircuit, a: u32, b: u32, out: *mut u32) -> i32 { + guard(|| put(out, as_mut(c)?.0.join(a, b)?)) +} + +/// `kind`: 0 count, 1 sum, 2 min, 3 max. +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_aggregate( + c: *mut ZsCircuit, + stream: u32, + kind: i32, + out: *mut u32, +) -> i32 { + guard(|| put(out, as_mut(c)?.0.aggregate(stream, Agg::from_code(kind)?)?)) +} + +/// Adds a copy of `z` to the input's pending delta. +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_push(c: *mut ZsCircuit, input: u32, z: *const ZsZSet) -> i32 { + guard(|| as_mut(c)?.0.push(input, &as_ref(z)?.0)) +} + +/// One clock tick. On any error but a null or bad id the circuit is poisoned. +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_step(c: *mut ZsCircuit) -> i32 { + guard(|| as_mut(c)?.0.step()) +} + +/// Takes the sink's accumulated delta as a new Z-set handle, and clears the sink. +#[unsafe(no_mangle)] +pub extern "C" fn zs_circuit_take(c: *mut ZsCircuit, sink: u32, out: *mut *mut ZsZSet) -> i32 { + guard(|| hand_out(out, ZsZSet(as_mut(c)?.0.take(sink)?))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_rebuild_new( + c: *mut ZsCircuit, + input: u32, + snapshot: *const ZsZSet, + chunk: usize, + out: *mut *mut ZsRebuild, +) -> i32 { + guard(|| { + let job = as_mut(c)? + .0 + .start_rebuild(input, as_ref(snapshot)?.0.clone(), chunk)?; + hand_out(out, ZsRebuild(job)) + }) +} + +/// Runs chunks until done or `expired(ctx)` returns non-zero; `*done` is 1 when finished. +/// A null `expired` runs to completion. +#[unsafe(no_mangle)] +pub extern "C" fn zs_rebuild_step( + job: *mut ZsRebuild, + c: *mut ZsCircuit, + expired: Option, + ctx: *mut c_void, + done: *mut i32, +) -> i32 { + guard(|| { + let (job, c) = (as_mut(job)?, as_mut(c)?); + let finished = c.0.rebuild_step(&mut job.0, expired.map(|f| (f, ctx)))?; + put(done, i32::from(finished)) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn zs_rebuild_free(job: *mut ZsRebuild) { + release(job) +} diff --git a/crates/zset-ffi/tests/abi.rs b/crates/zset-ffi/tests/abi.rs new file mode 100644 index 00000000..91f7c765 --- /dev/null +++ b/crates/zset-ffi/tests/abi.rs @@ -0,0 +1,462 @@ +//! The ABI exercised through the `extern "C"` functions themselves, as a C caller would. +//! Evidence: `context/test-evidence/zset-ffi.md`. + +use std::ffi::{CStr, c_void}; +use std::ptr::{null, null_mut}; +use zset_ffi::engine::{Code, pair}; +use zset_ffi::*; + +const OK: i32 = 0; + +fn zset(entries: &[(&[u8], i64)]) -> *mut ZsZSet { + let mut z = null_mut(); + assert_eq!(zs_zset_new(&mut z), OK); + for (k, w) in entries { + assert_eq!(zs_zset_insert(z, k.as_ptr(), k.len(), *w), OK); + } + z +} + +fn weight(z: *const ZsZSet, k: &[u8]) -> i64 { + let mut w = 0; + assert_eq!(zs_zset_weight(z, k.as_ptr(), k.len(), &mut w), OK); + w +} + +fn len(z: *const ZsZSet) -> usize { + let mut n = 0; + assert_eq!(zs_zset_len(z, &mut n), OK); + n +} + +fn encoded(z: *const ZsZSet) -> Vec { + let mut needed = 0; + assert_eq!( + zs_zset_encode(z, null_mut(), 0, &mut needed), + if needed == 0 { + OK + } else { + Code::BufferTooSmall as i32 + } + ); + let mut buf = vec![0u8; needed]; + assert_eq!( + zs_zset_encode(z, buf.as_mut_ptr(), buf.len(), &mut needed), + OK + ); + buf +} + +fn last_error() -> String { + unsafe { CStr::from_ptr(zs_last_error()) } + .to_string_lossy() + .into_owned() +} + +struct C(*mut ZsCircuit); + +impl C { + fn new() -> C { + let mut c = null_mut(); + assert_eq!(zs_circuit_new(&mut c), OK); + C(c) + } + fn input(&self) -> u32 { + let mut s = 0; + assert_eq!(zs_circuit_input(self.0, &mut s), OK); + s + } + fn sink(&self, s: u32) -> u32 { + let mut o = 0; + assert_eq!(zs_circuit_output(self.0, s, &mut o), OK); + o + } + fn push(&self, input: u32, entries: &[(&[u8], i64)]) { + let z = zset(entries); + assert_eq!(zs_circuit_push(self.0, input, z), OK); + zs_zset_free(z); + } + /// Steps and takes the sink as sorted `(key, weight)` pairs. + fn run(&self, sink: u32) -> Vec<(Vec, i64)> { + assert_eq!(zs_circuit_step(self.0), OK); + self.take(sink) + } + fn take(&self, sink: u32) -> Vec<(Vec, i64)> { + let mut z = null_mut(); + assert_eq!(zs_circuit_take(self.0, sink, &mut z), OK); + let enc = encoded(z); + zs_zset_free(z); + decode_pairs(&enc) + } +} + +impl Drop for C { + fn drop(&mut self) { + zs_circuit_free(self.0) + } +} + +fn decode_pairs(mut b: &[u8]) -> Vec<(Vec, i64)> { + let mut out = Vec::new(); + while !b.is_empty() { + let n = u32::from_le_bytes(b[..4].try_into().unwrap()) as usize; + let w = i64::from_le_bytes(b[4 + n..12 + n].try_into().unwrap()); + out.push((b[4..4 + n].to_vec(), w)); + b = &b[12 + n..]; + } + out +} + +fn one(entries: &[(&[u8], i64)]) -> Vec<(Vec, i64)> { + entries.iter().map(|(k, w)| (k.to_vec(), *w)).collect() +} + +fn le(n: i64) -> [u8; 8] { + n.to_le_bytes() +} + +#[test] +fn a_zset_adds_queries_and_frees() { + let (a, b) = ( + zset(&[(b"x", 2), (b"y", -1)]), + zset(&[(b"x", -2), (b"z", 5)]), + ); + assert_eq!(zs_zset_add(a, b), OK); + assert_eq!( + (weight(a, b"x"), weight(a, b"y"), weight(a, b"z"), len(a)), + (0, -1, 5, 2) + ); + assert_eq!(zs_zset_neg(a), OK); + assert_eq!(zs_zset_scale(a, 3), OK); + assert_eq!((weight(a, b"y"), weight(a, b"z")), (3, -15)); + assert_eq!(zs_zset_sub(a, a), OK); + assert_eq!(len(a), 0); + for z in [a, b] { + zs_zset_free(z); + } + zs_zset_free(null_mut()); +} + +#[test] +fn keys_are_byte_strings_including_nul_and_empty() { + let z = zset(&[(b"a\0b", 1), (b"", 2), (b"a", 3)]); + assert_eq!( + (weight(z, b"a\0b"), weight(z, b""), weight(z, b"a"), len(z)), + (1, 2, 3, 3) + ); + assert_eq!(zs_zset_insert(z, null(), 0, 4), OK); + assert_eq!(weight(z, b""), 6); + zs_zset_free(z); +} + +#[test] +fn distinct_clone_and_encoding_round_trip() { + let z = zset(&[(b"k1", 5), (b"k2", -2), (b"k3", 1)]); + let mut d = null_mut(); + assert_eq!(zs_zset_distinct(z, &mut d), OK); + assert_eq!( + (weight(d, b"k1"), weight(d, b"k2"), weight(d, b"k3")), + (1, 0, 1) + ); + let mut copy = null_mut(); + assert_eq!(zs_zset_clone(z, &mut copy), OK); + let enc = encoded(z); + let mut back = null_mut(); + assert_eq!(zs_zset_new(&mut back), OK); + assert_eq!(zs_zset_decode_add(back, enc.as_ptr(), enc.len()), OK); + assert_eq!(encoded(back), encoded(copy)); + assert_eq!(encoded(back), enc); + for h in [z, d, copy, back] { + zs_zset_free(h); + } +} + +#[test] +fn a_small_buffer_reports_the_size_it_needs() { + let z = zset(&[(b"abc", 1)]); + let (mut needed, mut small) = (0, [0u8; 3]); + assert_eq!( + zs_zset_encode(z, small.as_mut_ptr(), small.len(), &mut needed), + Code::BufferTooSmall as i32 + ); + assert_eq!(needed, 4 + 3 + 8); + assert_eq!(small, [0; 3], "nothing written on failure"); + zs_zset_free(z); +} + +#[test] +fn a_bad_encoding_is_refused_and_adds_nothing() { + let z = zset(&[(b"a", 1)]); + let mut enc = encoded(z); + enc.push(7); + assert_eq!( + zs_zset_decode_add(z, enc.as_ptr(), enc.len()), + Code::BadValue as i32 + ); + assert_eq!((weight(z, b"a"), len(z)), (1, 1)); + zs_zset_free(z); +} + +#[test] +fn overflow_is_an_error_code_and_leaves_the_set_unchanged() { + let (a, b) = ( + zset(&[(b"k", i64::MAX), (b"j", 1)]), + zset(&[(b"j", 1), (b"k", 1)]), + ); + assert_eq!(zs_zset_add(a, b), Code::Overflow as i32); + assert_eq!( + (weight(a, b"k"), weight(a, b"j")), + (i64::MAX, 1), + "a is unchanged, even for the key before the overflow" + ); + assert!(last_error().contains("weight overflow"), "{}", last_error()); + assert_eq!(zs_zset_scale(a, 2), Code::Overflow as i32); + assert_eq!( + zs_zset_insert(a, b"k".as_ptr(), 1, 1), + Code::Overflow as i32 + ); + let m = zset(&[(b"k", i64::MIN)]); + assert_eq!(zs_zset_neg(m), Code::Overflow as i32); + assert_eq!(weight(m, b"k"), i64::MIN); + assert_eq!(zs_zset_sub(a, a), OK); + for z in [a, b, m] { + zs_zset_free(z); + } +} + +#[test] +fn a_null_argument_is_a_code_not_a_crash() { + assert_eq!(zs_zset_new(null_mut()), Code::NullArgument as i32); + assert_eq!(zs_zset_len(null(), &mut 0), Code::NullArgument as i32); + let z = zset(&[]); + assert_eq!(zs_zset_insert(z, null(), 3, 1), Code::NullArgument as i32); + assert_eq!(zs_zset_add(z, null()), Code::NullArgument as i32); + assert_eq!( + zs_circuit_filter(null_mut(), 0, None, null_mut(), &mut 0), + Code::NullArgument as i32 + ); + zs_zset_free(z); + assert_ne!(zs_abi_version(), 0); +} + +#[test] +fn a_circuit_runs_distinct_and_integrate() { + let c = C::new(); + let i = c.input(); + let (mut d, mut int) = (0, 0); + assert_eq!(zs_circuit_distinct(c.0, i, &mut d), OK); + assert_eq!(zs_circuit_integrate(c.0, d, &mut int), OK); + let (sd, sint) = (c.sink(d), c.sink(int)); + c.push(i, &[(b"a", 3), (b"b", -1)]); + assert_eq!(zs_circuit_step(c.0), OK); + assert_eq!(c.take(sd), one(&[(b"a", 1)])); + assert_eq!(c.take(sint), one(&[(b"a", 1)])); + c.push(i, &[(b"a", -3), (b"b", 1)]); + assert_eq!(c.run(sd), one(&[(b"a", -1)])); +} + +extern "C" fn upper(_: *mut c_void, k: *const u8, n: usize, emit: engine::Emit, e: *mut c_void) { + let up: Vec = unsafe { std::slice::from_raw_parts(k, n) }.to_ascii_uppercase(); + emit(e, up.as_ptr(), up.len()); + emit(e, b"!".as_ptr(), 1); +} + +extern "C" fn even_len(_: *mut c_void, _: *const u8, n: usize) -> i32 { + i32::from(n % 2 == 0) +} + +extern "C" fn first_byte( + _: *mut c_void, + k: *const u8, + n: usize, + emit: engine::Emit, + e: *mut c_void, +) { + emit(e, k, n.min(1)); +} + +#[test] +fn callbacks_filter_and_flat_map() { + let c = C::new(); + let i = c.input(); + let (mut f, mut m) = (0, 0); + assert_eq!( + zs_circuit_filter(c.0, i, Some(even_len), null_mut(), &mut f), + OK + ); + assert_eq!( + zs_circuit_flat_map(c.0, f, Some(upper), null_mut(), &mut m), + OK + ); + let s = c.sink(m); + c.push(i, &[(b"ab", 2), (b"abc", 5), (b"cd", -1)]); + assert_eq!(c.run(s), one(&[(b"!", 1), (b"AB", 2), (b"CD", -1)])); +} + +#[test] +fn index_by_join_and_aggregate_follow_the_pair_encoding() { + let c = C::new(); + let (orders, prices) = (c.input(), c.input()); + let (mut by_item, mut joined) = (0, 0); + assert_eq!( + zs_circuit_index_by(c.0, orders, Some(first_byte), null_mut(), &mut by_item), + OK + ); + assert_eq!(zs_circuit_join(c.0, by_item, prices, &mut joined), OK); + let sink = c.sink(joined); + c.push(orders, &[(b"apple", 1)]); + c.push(prices, &[(&pair(b"a", b"3"), 1)]); + let want = pair(b"a", &pair(b"apple", b"3")); + assert_eq!(c.run(sink), one(&[(&want, 1)])); + c.push(prices, &[(&pair(b"a", b"3"), -1)]); + assert_eq!(c.run(sink), one(&[(&want, -1)])); +} + +#[test] +fn aggregates_count_sum_min_and_max() { + let c = C::new(); + let i = c.input(); + let outs: Vec = (0..4) + .map(|kind| { + let mut o = 0; + assert_eq!(zs_circuit_aggregate(c.0, i, kind, &mut o), OK); + c.sink(o) + }) + .collect(); + c.push( + i, + &[ + (&pair(b"g", &le(5)), 1), + (&pair(b"g", &le(2)), 2), + (&pair(b"h", &le(9)), 1), + ], + ); + assert_eq!(zs_circuit_step(c.0), OK); + let val = |k: &[u8], n: i64| (pair(k, &le(n)), 1); + let want = [ + [val(b"g", 3), val(b"h", 1)], + [val(b"g", 9), val(b"h", 9)], + [val(b"g", 2), val(b"h", 9)], + [val(b"g", 5), val(b"h", 9)], + ]; + for (sink, want) in outs.iter().zip(want) { + let got = c.take(*sink); + let want: Vec<(Vec, i64)> = want.into_iter().collect(); + let mut sorted = want.clone(); + sorted.sort(); + assert_eq!(got, sorted, "kind of sink {sink}"); + } +} + +#[test] +fn a_weight_overflow_in_a_step_is_a_code_and_poisons_the_circuit() { + let c = C::new(); + let i = c.input(); + let (mut int, mut again) = (0, 0); + assert_eq!(zs_circuit_integrate(c.0, i, &mut int), OK); + c.push(i, &[(b"k", i64::MAX)]); + assert_eq!(zs_circuit_step(c.0), OK); + c.push(i, &[(b"k", 1)]); + assert_eq!(zs_circuit_step(c.0), Code::Overflow as i32); + assert!(last_error().contains("weight overflow")); + assert_eq!(zs_circuit_step(c.0), Code::Poisoned as i32); + assert_eq!( + zs_circuit_distinct(c.0, i, &mut again), + Code::Poisoned as i32 + ); +} + +#[test] +fn a_malformed_pair_and_an_unknown_id_are_codes() { + let c = C::new(); + let i = c.input(); + let mut j = 0; + assert_eq!(zs_circuit_join(c.0, i, 99, &mut j), Code::BadHandle as i32); + assert_eq!(zs_circuit_join(c.0, i, i, &mut j), OK); + c.push(i, &[(b"\xff\xff\xff\xff", 1)]); + assert_eq!(zs_circuit_step(c.0), Code::BadValue as i32); + assert_eq!(zs_circuit_step(c.0), Code::Poisoned as i32); + let mut a = 0; + let c2 = C::new(); + assert_eq!( + zs_circuit_aggregate(c2.0, 0, 9, &mut a), + Code::BadValue as i32 + ); + let mut z = null_mut(); + assert_eq!(zs_circuit_take(c2.0, 5, &mut z), Code::BadHandle as i32); + assert_eq!( + zs_circuit_push(c2.0, 0, zset(&[]).cast_const()), + Code::BadHandle as i32 + ); +} + +extern "C" fn out_of_time(ctx: *mut c_void) -> i32 { + let calls = unsafe { &mut *(ctx as *mut u32) }; + *calls += 1; + 1 +} + +#[test] +fn a_rebuild_resumes_across_calls_and_matches_one_step() { + let keys: Vec> = (0u8..9).map(|n| vec![b'k', n]).collect(); + let entries: Vec<(&[u8], i64)> = keys.iter().map(|k| (k.as_slice(), 1)).collect(); + let snapshot = zset(&entries); + + let whole = C::new(); + let wi = whole.input(); + let mut wd = 0; + assert_eq!(zs_circuit_distinct(whole.0, wi, &mut wd), OK); + let ws = whole.sink(wd); + assert_eq!(zs_circuit_push(whole.0, wi, snapshot), OK); + let want = whole.run(ws); + + let c = C::new(); + let i = c.input(); + let mut d = 0; + assert_eq!(zs_circuit_distinct(c.0, i, &mut d), OK); + let s = c.sink(d); + let mut job = null_mut(); + assert_eq!(zs_rebuild_new(c.0, i, snapshot, 2, &mut job), OK); + let (mut calls, mut got, mut done) = (0u32, Vec::new(), 0); + for _ in 0..20 { + assert_eq!( + zs_rebuild_step( + job, + c.0, + Some(out_of_time), + (&mut calls as *mut u32).cast(), + &mut done + ), + OK + ); + got.extend(c.take(s)); + if done == 1 { + break; + } + } + assert_eq!(done, 1, "the job finished within its bound"); + assert_eq!( + calls, 5, + "ceil(9 / 2) chunks, each followed by one deadline check" + ); + got.sort(); + assert_eq!(got, want); + zs_rebuild_free(job); + zs_zset_free(snapshot); +} + +#[test] +fn a_rebuild_without_a_deadline_runs_to_completion() { + let snapshot = zset(&[(b"a", 1), (b"b", 1), (b"c", 1)]); + let c = C::new(); + let i = c.input(); + let s = c.sink(i); + let mut job = null_mut(); + let mut done = 0; + assert_eq!(zs_rebuild_new(c.0, i, snapshot, 1, &mut job), OK); + assert_eq!(zs_rebuild_step(job, c.0, None, null_mut(), &mut done), OK); + assert_eq!(done, 1); + assert_eq!(c.take(s).len(), 3); + zs_rebuild_free(job); + zs_zset_free(snapshot); +} diff --git a/crates/zset-ffi/tests/dylib.rs b/crates/zset-ffi/tests/dylib.rs new file mode 100644 index 00000000..03df592e --- /dev/null +++ b/crates/zset-ffi/tests/dylib.rs @@ -0,0 +1,63 @@ +//! The built shared library, loaded the way a CFFI caller loads it. +//! Evidence: `context/test-evidence/zset-ffi.md`. + +use libloading::{Library, Symbol}; +use std::path::PathBuf; +use std::ptr::null_mut; + +type Opaque = std::ffi::c_void; + +/// Cargo puts the cdylib in `target//deps` beside this test binary's own directory. +fn library() -> PathBuf { + let exe = std::env::current_exe().unwrap(); + let deps = exe.parent().unwrap(); + let ext = if cfg!(target_os = "macos") { + "dylib" + } else { + "so" + }; + let name = format!("{}zset_ffi.{ext}", if cfg!(windows) { "" } else { "lib" }); + [deps.join(&name), deps.parent().unwrap().join(&name)] + .into_iter() + .find(|p| p.exists()) + .unwrap_or_else(|| panic!("{name} was not built next to {}", deps.display())) +} + +#[test] +fn a_loaded_library_adds_queries_frees_and_reports_overflow() { + // SAFETY: loads this workspace's own library; the symbols below have the header's types. + unsafe { + let lib = Library::new(library()).unwrap(); + let version: Symbol u32> = lib.get(b"zs_abi_version").unwrap(); + let new: Symbol i32> = lib.get(b"zs_zset_new").unwrap(); + let free: Symbol = lib.get(b"zs_zset_free").unwrap(); + let insert: Symbol i32> = + lib.get(b"zs_zset_insert").unwrap(); + let weight: Symbol i32> = + lib.get(b"zs_zset_weight").unwrap(); + let last: Symbol *const std::ffi::c_char> = + lib.get(b"zs_last_error").unwrap(); + assert_eq!(version(), 1); + let mut z = null_mut(); + assert_eq!(new(&mut z), 0); + assert_eq!(insert(z, b"k".as_ptr(), 1, 40), 0); + assert_eq!(insert(z, b"k".as_ptr(), 1, 2), 0); + let mut w = 0; + assert_eq!(weight(z, b"k".as_ptr(), 1, &mut w), 0); + assert_eq!(w, 42); + assert_eq!( + insert(z, b"k".as_ptr(), 1, i64::MAX), + 2, + "overflow is ZS_ERR_OVERFLOW" + ); + assert!( + std::ffi::CStr::from_ptr(last()) + .to_str() + .unwrap() + .contains("overflow") + ); + assert_eq!(weight(z, b"k".as_ptr(), 1, &mut w), 0); + assert_eq!(w, 42, "the failed insert changed nothing"); + free(z); + } +} diff --git a/crates/zset-ffi/tests/header.rs b/crates/zset-ffi/tests/header.rs new file mode 100644 index 00000000..fd1b43f8 --- /dev/null +++ b/crates/zset-ffi/tests/header.rs @@ -0,0 +1,141 @@ +//! The checked-in C header matches the exported functions and constants. +//! Evidence: `context/test-evidence/zset-ffi.md`. + +use std::collections::BTreeMap; +use zset_ffi::ZS_ABI_VERSION; +use zset_ffi::engine::Code; + +const SRC: &str = include_str!("../src/lib.rs"); +const HEADER: &str = include_str!("../include/zset_ffi.h"); + +/// The comma-separated parameter count of the text between a function's parentheses. +fn arity(params: &str) -> usize { + let (mut depth, mut n, mut any) = (0i32, 1, false); + for c in params.chars() { + match c { + '(' | '<' => depth += 1, + ')' | '>' => depth -= 1, + ',' if depth == 0 => n += 1, + c if !c.is_whitespace() => any = true, + _ => {} + } + } + if any { + n - usize::from(params.trim_end().ends_with(',')) + } else { + 0 + } +} + +/// Every `pub extern "C" fn zs_*` in the source, with its parameter count; the +/// `unary_op!` expansions all take (circuit, stream, out). +fn exported() -> BTreeMap { + let mut out = BTreeMap::new(); + for part in SRC.split("pub extern \"C\" fn ").skip(1) { + if part.starts_with('$') { + continue; + } + let name = part.split('(').next().unwrap().to_string(); + let params = &part[name.len() + 1..]; + let mut depth = 1; + let end = params + .char_indices() + .find(|(_, c)| { + depth += i32::from(*c == '(') - i32::from(*c == ')'); + depth == 0 + }) + .unwrap() + .0; + out.insert(name, arity(¶ms[..end])); + } + for line in SRC.lines().filter(|l| l.starts_with("unary_op!(")) { + let name = line["unary_op!(".len()..].split(',').next().unwrap(); + out.insert(name.to_string(), 3); + } + out +} + +fn declared() -> BTreeMap { + let mut flat = HEADER.replace('\n', " "); + while let Some(open) = flat.find("/*") { + let close = flat[open..].find("*/").expect("an unterminated comment") + open + 2; + flat.replace_range(open..close, " "); + } + let mut out = BTreeMap::new(); + for decl in flat.split(';') { + let Some(open) = decl.find("(") else { continue }; + let before = &decl[..open]; + let name = before + .rsplit(|c: char| !(c.is_alphanumeric() || c == '_')) + .next() + .unwrap(); + if !name.starts_with("zs_") || before.contains("typedef") || before.contains("(*") { + continue; + } + let params = decl[open + 1..].rsplit_once(')').unwrap().0; + out.insert( + name.to_string(), + if params.trim() == "void" { + 0 + } else { + arity(params) + }, + ); + } + out +} + +fn defines() -> BTreeMap { + HEADER + .lines() + .filter_map(|l| l.strip_prefix("#define ")) + .filter_map(|l| { + let (k, v) = l.split_once(' ')?; + Some((k.to_string(), v.trim().parse().ok()?)) + }) + .collect() +} + +#[test] +fn the_header_declares_exactly_the_exported_functions_with_their_arity() { + let (src, hdr) = (exported(), declared()); + assert!( + src.len() > 30, + "the scanner saw {} functions, it is not reading the source", + src.len() + ); + assert_eq!(src, hdr); +} + +#[test] +fn the_header_constants_are_the_rust_constants() { + let d = defines(); + let codes = [ + ("ZS_ERR_NULL_ARGUMENT", Code::NullArgument), + ("ZS_ERR_OVERFLOW", Code::Overflow), + ("ZS_ERR_PANIC", Code::Panic), + ("ZS_ERR_BAD_HANDLE", Code::BadHandle), + ("ZS_ERR_BUFFER_TOO_SMALL", Code::BufferTooSmall), + ("ZS_ERR_BAD_VALUE", Code::BadValue), + ("ZS_ERR_POISONED", Code::Poisoned), + ]; + for (name, code) in codes { + assert_eq!(d[name], code as i64, "{name}"); + } + assert_eq!( + d["ZS_ERR_POISONED"] as usize, + codes.len(), + "a new code needs a header line" + ); + assert_eq!(d["ZS_ABI_VERSION"], i64::from(ZS_ABI_VERSION)); + assert_eq!(d["ZS_OK"], 0); + assert_eq!( + [ + d["ZS_AGG_COUNT"], + d["ZS_AGG_SUM"], + d["ZS_AGG_MIN"], + d["ZS_AGG_MAX"] + ], + [0, 1, 2, 3] + ); +} From 8a76469bcb5556078dc213798e4e5ceb613439d8 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-circuit)" Date: Sat, 10 Oct 2026 11:56:17 -0700 Subject: [PATCH 02/10] docs(zset-ffi): the ABI's reasoning, a CFFI smoke test and test evidence (improvise-bw4p.1) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/code/zset-ffi.md | 90 +++++++++++++++++++++++++++++++ context/test-evidence/zset-ffi.md | 60 +++++++++++++++++++++ crates/zset-ffi/src/engine.rs | 5 +- crates/zset-ffi/src/lib.rs | 17 +++--- crates/zset-ffi/tests/abi.rs | 39 +++++++++++++- crates/zset-ffi/tests/smoke.lisp | 42 +++++++++++++++ 6 files changed, 242 insertions(+), 11 deletions(-) create mode 100644 context/code/zset-ffi.md create mode 100644 context/test-evidence/zset-ffi.md create mode 100644 crates/zset-ffi/tests/smoke.lisp diff --git a/context/code/zset-ffi.md b/context/code/zset-ffi.md new file mode 100644 index 00000000..e0e56a8a --- /dev/null +++ b/context/code/zset-ffi.md @@ -0,0 +1,90 @@ +# `zset-ffi`: the reasoning + +`crates/zset-ffi` is a C ABI over `zset` for CFFI callers (the first is Common Lisp). It is a +separate crate so `zset` stays std-only, serde-free and wasm32-clean (ADR 009 clause 1). It is +native only: it is **not** in `nix/ci.nix`'s `wasmCrates`, and the workspace-wide native +build (`--workspace`) already compiles it, so nothing in the nix files changes. The contract +is `crates/zset-ffi/include/zset_ffi.h`; this file records the decisions behind it. The +evidence is in `context/test-evidence/zset-ffi.md`. + +## Build + +`cargo build -p zset-ffi --release` writes `target/release/libzset_ffi.{so,dylib}` (the +`cdylib`), `libzset_ffi.a` (the `staticlib`) and the rlib. The `rlib` is what the Rust tests +link, so they call the `extern "C"` functions as plain functions. + +## Decisions + +1. **Monomorphic.** Keys are byte strings `(ptr, len)`, weights are `int64_t`. The generic + `ZSet` is instantiated once, at `Vec`. A caller encodes whatever it keys by. +2. **Opaque handles, explicit create and free.** `ZsZSet`, `ZsCircuit` and `ZsRebuild` are + `Box::into_raw` pointers; `zs_X_new(&out)` makes one and `zs_X_free` releases it. Freeing + null is a no-op. A double free or a use after free is undefined, as in C. Circuit streams, + inputs and sinks are `uint32_t` ids local to their circuit, not handles: they own nothing. +3. **Error codes, never unwinding.** Every function but the frees and two accessors returns + `int32_t`: 0, or a `ZS_ERR_*`. Every entry point body runs in `catch_unwind` (`guard`). + A `zset` weight-overflow panic is recognised by its message prefix and becomes + `ZS_ERR_OVERFLOW`; any other panic is `ZS_ERR_PANIC`. Results come back through + out-pointers written only on success. `zs_last_error()` is a thread-local, NUL-terminated + message, never null, valid until the next failure on that thread. + The Rust panic hook still prints the overflow message to stderr before the code returns, + because replacing a process-wide hook from a library is not this library's call. A host that + wants silence installs its own hook, or avoids the overflow. +4. **Failure leaves the Z-set unchanged.** `add`, `sub`, `neg`, `scale` and `decode_add` + compute a new value and then assign it, so an overflow half way through cannot leave a + half-applied update (`zs_zset_insert` is one addition, so it has no half). +5. **A failed step poisons the circuit.** A panic in `step` can leave operator state half + updated, and the circuit has no rollback (ADR 009 clause 11 is the host's). So the first + failure in a step, or in a rebuild step, marks the circuit poisoned, and every later call + that uses it returns `ZS_ERR_POISONED`. A caller drops it and builds another. Ids that do + not exist, and malformed arguments, are refused before any state changes and do not poison. +6. **Buffers.** The caller owns every buffer it passes and may reuse it once the call + returns; the library copies. The library hands out no memory except handles. Output of + variable size uses a two-call protocol: `zs_zset_encode(z, buf, cap, &needed)` always sets + `needed`, and writes only if `cap >= needed` (else `ZS_ERR_BUFFER_TOO_SMALL`). +7. **Wire shape.** A Z-set encodes as entries ascending by key, each `[u32 LE key length][key] + [i64 LE weight]`. A keyed element is `pair(k, v) = [u32 LE length of k][k][v]`. These are + this crate's own, not clause 9's wire form (that lives in `improvise-protocol`). +8. **What the circuit offers.** Input, output sinks, `delay`, `integrate`, `differentiate`, + `distinct`, `plus`, `filter` and `flat_map` (C callbacks), `index_by` (a callback emits the + one group key), `join` over two pair streams, and `aggregate` with count, sum, min and max + over `int64` values. `map` is `flat_map` that emits once. `aggregate`'s fold is fixed to + those four because a C callback cannot return an `Option` of a group result without a + second encoding; richer folds belong to a caller that wants one. The `Trace` interface and + the `_in` constructors are Rust-side seams and are not exposed. +9. **Callbacks** are `extern "C"` function pointers with a `void *ctx`; the library never + dereferences `ctx`. A callback emits images by calling the `emit` it is given, which copies + the bytes during the call. A callback must not call back into the library and must not + unwind (a Rust panic across an `extern "C"` boundary aborts; a CL callback should not + non-locally exit through the library). `index_by`'s callback must emit exactly one key and a + keyed stream must carry valid pairs; otherwise the step fails with `ZS_ERR_BAD_VALUE` + (poisoning). The engine records that fault rather than panicking. +10. **Not thread-safe.** A handle is used from one thread at a time (the circuit is `Rc` + inside); different handles are independent. The error message is per thread. +11. **The header is hand-written** (cbindgen is not a dependency of this repo). `tests/header.rs` + keeps it honest: the set of function names and each function's parameter count must match + the exported `extern "C"` functions in `src/lib.rs` (including the `unary_op!` expansions), + and every `#define` code must equal the Rust enum. Parameter *types* are checked by review + and by the ABI tests, which call each function with the header's types, not by the scanner. +12. **Versioning.** `zs_abi_version()` and `ZS_ABI_VERSION` are 1. A change that breaks a + signature or a code bumps it. + +## Calling it from Common Lisp + +`crates/zset-ffi/tests/smoke.lisp` is a runnable CFFI program (`defcfun` for each call, a +foreign string for keys, `with-foreign-object` for out-parameters, the overflow code checked +and the message read back with `zs-last-error`). It loads the built library and passes. It is +**not a CI gate**: `sbcl` and `cffi` come from the user's profile and Quicklisp here, and are +not in the nix dev shell. The command to run it is in the file's header. Sketch of the +definitions it uses: + +```lisp +(defcfun "zs_zset_new" :int32 (out :pointer)) +(defcfun "zs_zset_insert" :int32 (z :pointer) (key :pointer) (len :size) (weight :int64)) +(defcfun "zs_zset_weight" :int32 (z :pointer) (key :pointer) (len :size) (out :pointer)) +(defcfun "zs_last_error" :string) +(defcfun "zs_zset_free" :void (z :pointer)) +``` + +A callback (`filter`, `flat_map`) is a `defcallback` passed with `(callback name)`; its +`emit` argument is a foreign function pointer to call with `foreign-funcall-pointer`. diff --git a/context/test-evidence/zset-ffi.md b/context/test-evidence/zset-ffi.md new file mode 100644 index 00000000..fcb5a9eb --- /dev/null +++ b/context/test-evidence/zset-ffi.md @@ -0,0 +1,60 @@ +# Test evidence: `zset-ffi` + +What each test guards and where a rival failed. Keyed by test name; the reasoning is in +`context/code/zset-ffi.md`. The tests are `crates/zset-ffi/tests/{abi,header,dylib}.rs`. + +**Injected in the source**, one at a time, each reverted and the suite rerun (a build that +printed `Running` was confirmed each time). The failing tests, by name: +- `guard` without `catch_unwind`: the overflow tests abort the test binary (`SIGABRT`, a panic + across `extern "C"`), which is the failure this crate exists to prevent; +- `zs_zset_add` that adds in place: `overflow_is_an_error_code_and_leaves_the_set_unchanged`; +- every panic classed `ZS_ERR_PANIC`: `overflow_is_an_error_code_and_leaves_the_set_unchanged`, + `a_weight_overflow_in_a_step_is_a_code_and_poisons_the_circuit`, + `a_loaded_library_adds_queries_frees_and_reports_overflow`; +- a failed step that does not poison: `a_weight_overflow_in_a_step_is_a_code_and_poisons_the_circuit`; +- a recorded fault that does not poison: `a_malformed_pair_and_an_unknown_id_are_codes`; +- `rebuild_step` that ignores the deadline: `a_rebuild_resumes_across_calls_and_matches_one_step`; +- `Min` that reads non-positive weights: `min_and_max_read_only_values_of_positive_weight` + (the first version of the suite did not catch this; the test was added for it); +- a header declaration with a parameter dropped: + `the_header_declares_exactly_the_exported_functions_with_their_arity`; +- a header code renumbered: `the_header_constants_are_the_rust_constants`; +- `zs_zset_encode` that reports the wrong size: eight ABI tests, among them + `a_small_buffer_reports_the_size_it_needs`. + +### overflow_is_an_error_code_and_leaves_the_set_unchanged + +`a` holds `k = MAX` and `j = 1`; adding a set that overflows `k` must return the overflow code +and leave **both** keys as they were. The key `j` sorts before `k`, so a partial in-place add +changes `j` first; a fixture where the overflowing key came first would pass that rival. + +### a_weight_overflow_in_a_step_is_a_code_and_poisons_the_circuit + +A weight reaches `MAX` in an `integrate`, then one more overflows inside `step`. The step +returns the overflow code, the next step and a new operator return the poisoned code. + +### a_rebuild_resumes_across_calls_and_matches_one_step + +Nine keys, chunks of two, a deadline callback that is always expired: the job needs five +calls' worth of chunks (the callback is counted: five, one per chunk), and the sum of what +the sink yields equals one uninterrupted step on a second circuit. The loop is bounded at 20. + +### min_and_max_read_only_values_of_positive_weight + +Weights of mixed sign in one group, where the smallest value has a negative weight, so +`> 0` and `!= 0` give different minima. + +### a_loaded_library_adds_queries_frees_and_reports_overflow + +The built `libzset_ffi.{so,dylib}` is opened with `libloading` and driven through symbol +lookups: the contract a CFFI caller meets, including the exported names, not the rlib. + +### the_header_declares_exactly_the_exported_functions_with_their_arity + +Parses `src/lib.rs` and the header (comments stripped) and compares names and parameter +counts; asserts the scanner saw more than 30 functions, so an empty parse cannot pass. + +### the_header_constants_are_the_rust_constants + +Every `ZS_ERR_*`, `ZS_ABI_VERSION`, `ZS_OK` and the four `ZS_AGG_*` equal the Rust values; the +last code's number equals the number of codes, so a new code without a header line fails. diff --git a/crates/zset-ffi/src/engine.rs b/crates/zset-ffi/src/engine.rs index b8c23740..13280049 100644 --- a/crates/zset-ffi/src/engine.rs +++ b/crates/zset-ffi/src/engine.rs @@ -1,6 +1,5 @@ -//! The byte-keyed circuit behind the C ABI, in plain Rust: every fallible operation -//! returns a `Result`, and a panic from `zset` is caught and classified here. -//! The reasoning is in `context/code/zset-ffi.md`. +//! The byte-keyed circuit behind the C ABI, as plain `Result`-returning Rust; panics from +//! `zset` are caught and classified here. See `context/code/zset-ffi.md`. use std::any::Any; use std::cell::Cell; diff --git a/crates/zset-ffi/src/lib.rs b/crates/zset-ffi/src/lib.rs index 00c92c21..51b451d4 100644 --- a/crates/zset-ffi/src/lib.rs +++ b/crates/zset-ffi/src/lib.rs @@ -1,6 +1,5 @@ -//! The C ABI over `zset`: byte-string keys, `i64` weights, opaque handles, an `i32` code -//! from every function. Ownership, lifetimes and the error model: `context/code/zset-ffi.md` -//! and `include/zset_ffi.h`. +//! The C ABI over `zset`: byte keys, `i64` weights, opaque handles, `i32` codes. +//! Ownership and errors: `context/code/zset-ffi.md` and `include/zset_ffi.h`. pub mod engine; @@ -64,6 +63,13 @@ fn bytes<'a>(p: *const u8, n: usize) -> Result<&'a [u8], Fail> { } } +fn write_out(buf: *mut u8, data: &[u8]) { + if !data.is_empty() { + // SAFETY: the ABI contract says `buf` has room for `data`; the caller checked `cap`. + unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), buf, data.len()) }; + } +} + fn put(out: *mut T, v: T) -> Result<(), Fail> { *as_mut(out)? = v; Ok(()) @@ -193,10 +199,7 @@ pub extern "C" fn zs_zset_encode( "the buffer is smaller than *needed", )); } - if !enc.is_empty() { - // SAFETY: the contract says `buf` has `cap >= enc.len()` writable bytes. - unsafe { std::ptr::copy_nonoverlapping(enc.as_ptr(), buf, enc.len()) }; - } + write_out(buf, &enc); Ok(()) }) } diff --git a/crates/zset-ffi/tests/abi.rs b/crates/zset-ffi/tests/abi.rs index 91f7c765..7c358819 100644 --- a/crates/zset-ffi/tests/abi.rs +++ b/crates/zset-ffi/tests/abi.rs @@ -262,7 +262,7 @@ extern "C" fn upper(_: *mut c_void, k: *const u8, n: usize, emit: engine::Emit, } extern "C" fn even_len(_: *mut c_void, _: *const u8, n: usize) -> i32 { - i32::from(n % 2 == 0) + i32::from(n.is_multiple_of(2)) } extern "C" fn first_byte( @@ -460,3 +460,40 @@ fn a_rebuild_without_a_deadline_runs_to_completion() { zs_rebuild_free(job); zs_zset_free(snapshot); } + +#[test] +fn min_and_max_read_only_values_of_positive_weight() { + let c = C::new(); + let i = c.input(); + let (mut lo, mut hi) = (0, 0); + assert_eq!(zs_circuit_aggregate(c.0, i, 2, &mut lo), OK); + assert_eq!(zs_circuit_aggregate(c.0, i, 3, &mut hi), OK); + let (slo, shi) = (c.sink(lo), c.sink(hi)); + c.push( + i, + &[ + (&pair(b"g", &le(5)), 1), + (&pair(b"g", &le(1)), -1), + (&pair(b"g", &le(9)), -1), + ], + ); + assert_eq!(zs_circuit_step(c.0), OK); + assert_eq!(c.take(slo), one(&[(&pair(b"g", &le(5)), 1)])); + assert_eq!(c.take(shi), one(&[(&pair(b"g", &le(5)), 1)])); +} + +#[test] +fn an_index_by_callback_that_emits_nothing_is_a_bad_value() { + extern "C" fn nothing(_: *mut c_void, _: *const u8, _: usize, _: engine::Emit, _: *mut c_void) { + } + let c = C::new(); + let i = c.input(); + let mut o = 0; + assert_eq!( + zs_circuit_index_by(c.0, i, Some(nothing), null_mut(), &mut o), + OK + ); + c.push(i, &[(b"x", 1)]); + assert_eq!(zs_circuit_step(c.0), Code::BadValue as i32); + assert!(last_error().contains("exactly one key"), "{}", last_error()); +} diff --git a/crates/zset-ffi/tests/smoke.lisp b/crates/zset-ffi/tests/smoke.lisp new file mode 100644 index 00000000..15111673 --- /dev/null +++ b/crates/zset-ffi/tests/smoke.lisp @@ -0,0 +1,42 @@ +;;; A CFFI smoke test: load libzset_ffi, add weights, query, hit the overflow code, free. +;;; Run: sbcl --non-interactive --load ~/quicklisp/setup.lisp \ +;;; --eval '(defvar cl-user::*zset-lib* "target/release/libzset_ffi.dylib")' \ +;;; --load crates/zset-ffi/tests/smoke.lisp +;;; Not a CI gate: sbcl and cffi are not in the dev shell. See context/code/zset-ffi.md. +(ql:quickload :cffi :silent t) +(defpackage :zset-smoke (:use :cl :cffi)) +(in-package :zset-smoke) + +(load-foreign-library cl-user::*zset-lib*) + +(defcfun "zs_abi_version" :uint32) +(defcfun "zs_last_error" :string) +(defcfun "zs_zset_new" :int32 (out :pointer)) +(defcfun "zs_zset_free" :void (z :pointer)) +(defcfun "zs_zset_insert" :int32 (z :pointer) (key :pointer) (len :size) (weight :int64)) +(defcfun "zs_zset_weight" :int32 (z :pointer) (key :pointer) (len :size) (out :pointer)) +(defcfun "zs_zset_len" :int32 (z :pointer) (out :pointer)) + +(defconstant +overflow+ 2) + +(defun check (what got want) + (unless (equal got want) (error "~a: got ~s, wanted ~s" what got want)) + (format t "ok ~a~%" what)) + +(defun key-call (fn z key &rest more) + "Calls FN with Z, a foreign copy of the string KEY's UTF-8 bytes, its length, MORE." + (with-foreign-string ((ptr len) key :null-terminated-p nil) + (apply fn z ptr len more))) + +(with-foreign-object (out :pointer) + (check "new" (zs-zset-new out) 0) + (let ((z (mem-ref out :pointer))) + (check "insert" (key-call #'zs-zset-insert z "k" 40) 0) + (check "insert again" (key-call #'zs-zset-insert z "k" 2) 0) + (with-foreign-object (w :int64) + (check "weight" (progn (key-call #'zs-zset-weight z "k" w) (mem-ref w :int64)) 42) + (check "overflow code" (key-call #'zs-zset-insert z "k" (1- (expt 2 63))) +overflow+) + (check "error text" (and (search "overflow" (zs-last-error)) t) t) + (check "unchanged" (progn (key-call #'zs-zset-weight z "k" w) (mem-ref w :int64)) 42)) + (zs-zset-free z))) +(format t "zset smoke: all ok~%") From b3f5cedff0f715d17329ecf924b170307f750b08 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-circuit)" Date: Sat, 10 Oct 2026 12:10:29 -0700 Subject: [PATCH 03/10] test(zset-ffi): the rebuild is one tick and its last chunk is Done (improvise-bw4p.1) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/code/zset-ffi.md | 8 ++++++-- context/test-evidence/zset-ffi.md | 7 +++++++ crates/zset-ffi/tests/abi.rs | 33 +++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/context/code/zset-ffi.md b/context/code/zset-ffi.md index e0e56a8a..f3373e9d 100644 --- a/context/code/zset-ffi.md +++ b/context/code/zset-ffi.md @@ -33,8 +33,8 @@ link, so they call the `extern "C"` functions as plain functions. 4. **Failure leaves the Z-set unchanged.** `add`, `sub`, `neg`, `scale` and `decode_add` compute a new value and then assign it, so an overflow half way through cannot leave a half-applied update (`zs_zset_insert` is one addition, so it has no half). -5. **A failed step poisons the circuit.** A panic in `step` can leave operator state half - updated, and the circuit has no rollback (ADR 009 clause 11 is the host's). So the first +5. **A failed step poisons the circuit.** A panic in `step` leaves operator state half + updated, and the circuit has no rollback (ADR 009 clause 11 is the host's; `zset.md`). So the first failure in a step, or in a rebuild step, marks the circuit poisoned, and every later call that uses it returns `ZS_ERR_POISONED`. A caller drops it and builds another. Ids that do not exist, and malformed arguments, are refused before any state changes and do not poison. @@ -69,6 +69,10 @@ link, so they call the `extern "C"` functions as plain functions. 12. **Versioning.** `zs_abi_version()` and `ZS_ABI_VERSION` are 1. A change that breaks a signature or a code bumps it. +A rebuild is one logical tick however many `zs_rebuild_step` calls it takes (`zset.md`, +decision 8): a `delay` fed by it shows the previous tick once, and `done` is 1 as soon as the +last chunk ran, whatever the deadline callback says. + ## Calling it from Common Lisp `crates/zset-ffi/tests/smoke.lisp` is a runnable CFFI program (`defcfun` for each call, a diff --git a/context/test-evidence/zset-ffi.md b/context/test-evidence/zset-ffi.md index fcb5a9eb..e74b241f 100644 --- a/context/test-evidence/zset-ffi.md +++ b/context/test-evidence/zset-ffi.md @@ -58,3 +58,10 @@ counts; asserts the scanner saw more than 30 functions, so an empty parse cannot Every `ZS_ERR_*`, `ZS_ABI_VERSION`, `ZS_OK` and the four `ZS_AGG_*` equal the Rust values; the last code's number equals the number of codes, so a new code without a header line fails. + +### a_chunked_rebuild_through_delay_is_one_tick + +After #345's review the rebuild is one logical tick. A snapshot of two keys, chunks of one, +an always-expired deadline: the first call is `done == 0`, the second `done == 1` (the last +chunk is Done despite the deadline), and a `delay` sink shows the previous tick's key once. +Under the old per-chunk ticking the delay showed the first chunk instead. diff --git a/crates/zset-ffi/tests/abi.rs b/crates/zset-ffi/tests/abi.rs index 7c358819..730026be 100644 --- a/crates/zset-ffi/tests/abi.rs +++ b/crates/zset-ffi/tests/abi.rs @@ -497,3 +497,36 @@ fn an_index_by_callback_that_emits_nothing_is_a_bad_value() { assert_eq!(zs_circuit_step(c.0), Code::BadValue as i32); assert!(last_error().contains("exactly one key"), "{}", last_error()); } + +#[test] +fn a_chunked_rebuild_through_delay_is_one_tick() { + let snapshot = zset(&[(b"a", 1), (b"b", 1)]); + let c = C::new(); + let i = c.input(); + let mut d = 0; + assert_eq!(zs_circuit_delay(c.0, i, &mut d), OK); + let s = c.sink(d); + c.push(i, &[(b"p", 1)]); + assert_eq!(zs_circuit_step(c.0), OK); + c.take(s); + let mut job = null_mut(); + assert_eq!(zs_rebuild_new(c.0, i, snapshot, 1, &mut job), OK); + let (mut calls, mut done) = (0u32, 0); + let ctx = (&mut calls as *mut u32).cast(); + assert_eq!( + zs_rebuild_step(job, c.0, Some(out_of_time), ctx, &mut done), + OK + ); + assert_eq!(done, 0); + assert_eq!( + zs_rebuild_step(job, c.0, Some(out_of_time), ctx, &mut done), + OK + ); + assert_eq!( + done, 1, + "the last chunk is Done though the deadline expired" + ); + assert_eq!(c.take(s), one(&[(b"p", 1)])); + zs_rebuild_free(job); + zs_zset_free(snapshot); +} From 9bcfa9d10e685eb82937787099b3be99426ada80 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-circuit)" Date: Sat, 10 Oct 2026 12:19:02 -0700 Subject: [PATCH 04/10] fix(zset-ffi): answer the #347 reviews: aliasing, callbacks that fail, out-pointers, unsafe exports, typed header (improvise-bw4p.1) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/code/zset-ffi.md | 61 +++- context/test-evidence/zset-ffi.md | 81 ++++- crates/zset-ffi/include/zset_ffi.h | 47 ++- crates/zset-ffi/src/engine.rs | 169 +++++++--- crates/zset-ffi/src/lib.rs | 443 ++++++++++++++++++------- crates/zset-ffi/tests/abi.rs | 506 +++++++++++++++++++++++------ crates/zset-ffi/tests/header.rs | 169 +++++++--- crates/zset-ffi/tests/smoke.lisp | 27 ++ 8 files changed, 1156 insertions(+), 347 deletions(-) diff --git a/context/code/zset-ffi.md b/context/code/zset-ffi.md index f3373e9d..649bb2af 100644 --- a/context/code/zset-ffi.md +++ b/context/code/zset-ffi.md @@ -53,19 +53,28 @@ link, so they call the `extern "C"` functions as plain functions. second encoding; richer folds belong to a caller that wants one. The `Trace` interface and the `_in` constructors are Rust-side seams and are not exposed. 9. **Callbacks** are `extern "C"` function pointers with a `void *ctx`; the library never - dereferences `ctx`. A callback emits images by calling the `emit` it is given, which copies - the bytes during the call. A callback must not call back into the library and must not - unwind (a Rust panic across an `extern "C"` boundary aborts; a CL callback should not - non-locally exit through the library). `index_by`'s callback must emit exactly one key and a - keyed stream must carry valid pairs; otherwise the step fails with `ZS_ERR_BAD_VALUE` - (poisoning). The engine records that fault rather than panicking. -10. **Not thread-safe.** A handle is used from one thread at a time (the circuit is `Rc` - inside); different handles are independent. The error message is per thread. + dereferences `ctx`, and keeps both for the circuit's life. A callback reports failure by + its return value (`flat_map`/`index_by` non-zero, `keep` and `expired` negative), and the + step then fails with `ZS_ERR_CALLBACK`. A callback must return normally. A Rust panic across + `extern "C"` aborts, and a Lisp `handler-case` outside `zs_circuit_step` that unwinds + through the Rust frames skips destructors and leaves a `RefCell` inside `zset` borrowed. + The ABI cannot stop the unwinding, so it detects it: `ZsCircuit::in_step` is set for the + duration of `step` and `rebuild_step`, and cleared only when they return. Every entry + point first reads it, so a circuit whose frames were skipped answers `ZS_ERR_POISONED` + forever, and a callback that re-enters its own circuit gets `ZS_ERR_POISONED` without the + library touching the engine. The CL rule is to put `handler-case` *inside* the callback and + return the failure value (`tests/smoke.lisp` does). `index_by` needs exactly one emitted + key and a keyed stream valid pairs, else `ZS_ERR_BAD_VALUE`; these faults are recorded in a + shared cell rather than panicking. +10. **Thread confinement.** A circuit and every rebuild job made from it share an `Rc` + (`Input`), so they are one group: keep the group on one thread at a time. Z-sets are + separate handles. The error message is per thread. 11. **The header is hand-written** (cbindgen is not a dependency of this repo). `tests/header.rs` - keeps it honest: the set of function names and each function's parameter count must match - the exported `extern "C"` functions in `src/lib.rs` (including the `unary_op!` expansions), - and every `#define` code must equal the Rust enum. Parameter *types* are checked by review - and by the ABI tests, which call each function with the header's types, not by the scanner. + keeps it honest. It maps each Rust parameter and return type of every exported function + (including the `unary_op!` expansions) to its C spelling, through a table that panics on + an unlisted type, and compares them with the header's declarations. Every `Code` is + declared by the `codes!` macro, which generates the enum, `Code::ALL` and the header + name, so a new code cannot exist without a `#define` line and a matching value. 12. **Versioning.** `zs_abi_version()` and `ZS_ABI_VERSION` are 1. A change that breaks a signature or a code bumps it. @@ -73,6 +82,29 @@ A rebuild is one logical tick however many `zs_rebuild_step` calls it takes (`zs decision 8): a `delay` fed by it shows the previous tick once, and `done` is 1 as soon as the last chunk ran, whatever the deadline callback says. +## Ids, handles and unsafety + +All exported functions are `pub unsafe extern "C" fn`: the rlib is public API, and a safe +Rust caller could otherwise pass any pointer. Output pointers are checked before any work +(`with_out`): a refused call changes nothing and leaks nothing. Stream ids and sink ids are +different spaces (sink ids carry the top bit). A `ZsRebuild` records its circuit and is +refused on another. An id from another circuit is not detected. `zs_zset_add(a, a)` is +defined: the sum is computed through shared references and then stored. Keys of 4 GiB or +longer are `ZS_ERR_BAD_VALUE`, not truncated. A non-empty aggregate group always has a row, +even when its count or sum is zero. + +## Miri + +Miri checks the aliasing, leak and null rules the C compiler cannot. It runs `tests/abi.rs` +on a nightly built through the flake's `rust-overlay` input (passes at this commit, 12s): + +``` +nix shell --impure --expr 'let p = import { overlays = [ (builtins.getFlake "github:oxalica/rust-overlay").overlays.default ]; }; in p.rust-bin.nightly."2026-09-24".default.override { extensions = [ "miri" "rust-src" ]; }' --command cargo miri test -p zset-ffi --test abi +``` + +It is not a CI job. A proposed one: a `miri` job in `.github/workflows/ci.yml` running that +command in `nix develop .#ci`. + ## Calling it from Common Lisp `crates/zset-ffi/tests/smoke.lisp` is a runnable CFFI program (`defcfun` for each call, a @@ -90,5 +122,6 @@ definitions it uses: (defcfun "zs_zset_free" :void (z :pointer)) ``` -A callback (`filter`, `flat_map`) is a `defcallback` passed with `(callback name)`; its -`emit` argument is a foreign function pointer to call with `foreign-funcall-pointer`. +A callback (`filter`, `flat_map`) is a `defcallback` passed with `(callback name)`, with a +`handler-case` inside it that returns the failure value; its `emit` argument is a foreign +function pointer to call with `foreign-funcall-pointer`. diff --git a/context/test-evidence/zset-ffi.md b/context/test-evidence/zset-ffi.md index e74b241f..3f9a7031 100644 --- a/context/test-evidence/zset-ffi.md +++ b/context/test-evidence/zset-ffi.md @@ -16,9 +16,10 @@ printed `Running` was confirmed each time). The failing tests, by name: - `rebuild_step` that ignores the deadline: `a_rebuild_resumes_across_calls_and_matches_one_step`; - `Min` that reads non-positive weights: `min_and_max_read_only_values_of_positive_weight` (the first version of the suite did not catch this; the test was added for it); -- a header declaration with a parameter dropped: - `the_header_declares_exactly_the_exported_functions_with_their_arity`; -- a header code renumbered: `the_header_constants_are_the_rust_constants`; +- a header declaration with a parameter dropped or retyped (`int64_t weight` as `int32_t`): + `the_header_declares_exactly_the_exported_functions_with_their_types`; +- a header code renumbered, and a new `Code` variant with no header line: + `the_header_constants_are_the_rust_constants`; - `zs_zset_encode` that reports the wrong size: eight ABI tests, among them `a_small_buffer_reports_the_size_it_needs`. @@ -49,15 +50,18 @@ Weights of mixed sign in one group, where the smallest value has a negative weig The built `libzset_ffi.{so,dylib}` is opened with `libloading` and driven through symbol lookups: the contract a CFFI caller meets, including the exported names, not the rlib. -### the_header_declares_exactly_the_exported_functions_with_their_arity +### the_header_declares_exactly_the_exported_functions_with_their_types -Parses `src/lib.rs` and the header (comments stripped) and compares names and parameter -counts; asserts the scanner saw more than 30 functions, so an empty parse cannot pass. +Parses `src/lib.rs` and the header (comments stripped), maps each Rust type to its C spelling +and compares names, parameter types and return types; asserts the scanner saw more than 30 +functions, so an empty parse cannot pass. The first version compared only arity and passed a +retyped parameter. ### the_header_constants_are_the_rust_constants Every `ZS_ERR_*`, `ZS_ABI_VERSION`, `ZS_OK` and the four `ZS_AGG_*` equal the Rust values; the -last code's number equals the number of codes, so a new code without a header line fails. +the header's `ZS_ERR_*` lines are counted against `Code::ALL`, which the `codes!` macro +generates, so a new variant without a line fails (it did, injected as `Novel = 9`). ### a_chunked_rebuild_through_delay_is_one_tick @@ -65,3 +69,66 @@ After #345's review the rebuild is one logical tick. A snapshot of two keys, chu an always-expired deadline: the first call is `done == 0`, the second `done == 1` (the last chunk is Done despite the deadline), and a `delay` sink shows the previous tick's key once. Under the old per-chunk ticking the delay showed the first chunk instead. + +## Review of #347 (8a76469b) + +**Injected afterwards**, one at a time, each failing by name: a null buffer written through +(the test binary aborts, which is the failure); `zs_circuit_input` adding the input before +checking `out`, and `zs_circuit_take` clearing the sink first: +`a_null_out_pointer_is_refused_before_any_state_changes`; a rebuild stepped on another circuit: +`a_rebuild_is_stepped_only_on_its_own_circuit`; the sink bit unchecked in `take`: +`stream_ids_and_sink_ids_are_not_interchangeable`; a `flat_map` or `keep` failure ignored: +`a_callback_that_reports_failure_fails_the_step_and_kills_the_circuit`; re-entry undetected, or +`step` not setting `in_step`: +`a_callback_that_calls_back_into_its_circuit_gets_poisoned_and_changes_nothing` +(and the unit test below for the first); a count that drops its zero row: +`a_count_or_sum_that_cancels_to_zero_keeps_its_row`; `len32` truncating: +`a_length_of_4_gib_or_more_is_refused_not_truncated`. +The aliasing finding (`zs_zset_add(a, a)`) is not an assertion failure on a native build: +only Miri finds it, and `a_set_added_to_or_subtracted_from_itself_is_defined` is the test +Miri failed before the fix and passes after (recipe in `zset-ffi.md`). + +### a_set_added_to_or_subtracted_from_itself_is_defined + +Self-add doubles, self-sub empties, and a self-add that overflows leaves the set unchanged. + +### a_null_buffer_is_refused_before_it_is_written + +`cap >= needed` with a null buffer is `ZS_ERR_NULL_ARGUMENT`; a null buffer with too small a +`cap` still probes the size. + +### a_null_out_pointer_is_refused_before_any_state_changes + +The next input id, the sink's contents and the job's progress are observed after a refused +call with a null out-pointer; each is as before. + +### a_rebuild_is_stepped_only_on_its_own_circuit + +A job made from one circuit stepped on another is `ZS_ERR_BAD_HANDLE` and neither sink moves. + +### stream_ids_and_sink_ids_are_not_interchangeable + +A stream id given to `take`, and a sink id given to `delay`, `output` and `push`, are refused. + +### a_callback_that_reports_failure_fails_the_step_and_kills_the_circuit + +`keep`, `flat_map` and `expired` callbacks each return their failure value: `ZS_ERR_CALLBACK`, +then `ZS_ERR_POISONED`. + +### a_callback_that_calls_back_into_its_circuit_gets_poisoned_and_changes_nothing + +A filter callback calls `zs_circuit_input` on its own circuit: it gets `ZS_ERR_POISONED`, the +outer step completes, and no input was added. + +### tests::a_circuit_left_mid_step_by_a_skipped_frame_answers_poisoned_to_everything + +What a Lisp non-local exit through `zs_circuit_step` leaves behind is the `in_step` flag set; +every later call is `ZS_ERR_POISONED` and writes nothing. + +### a_count_or_sum_that_cancels_to_zero_keeps_its_row + +Codex's finding: `Some(0)` was filtered away for a non-empty group. + +### a_length_of_4_gib_or_more_is_refused_not_truncated + +`len32` refuses `1 << 32` and accepts `u32::MAX`. diff --git a/crates/zset-ffi/include/zset_ffi.h b/crates/zset-ffi/include/zset_ffi.h index a320c605..bcb61b20 100644 --- a/crates/zset-ffi/include/zset_ffi.h +++ b/crates/zset-ffi/include/zset_ffi.h @@ -3,8 +3,10 @@ * Reasoning: context/code/zset-ffi.md. * * Conventions - * - Every function returns int: 0 is success, anything else is a ZS_ERR_* code. - * Results come back through out-pointers, which are written only on success. + * - Every function but the frees, zs_abi_version and zs_last_error returns int32_t: 0 is + * success, anything else is a ZS_ERR_* code. + * Results come back through out-pointers, which are checked FIRST (a NULL out is + * ZS_ERR_NULL_ARGUMENT with no state changed) and written only on success. * On failure zs_last_error() describes it (per thread, valid until the next failure). * - Handles (ZsZSet, ZsCircuit, ZsRebuild) are opaque. Each is created by a function that * takes ZsX **out and is released by exactly one zs_X_free. Freeing NULL is a no-op; @@ -15,8 +17,13 @@ * - The library never returns memory the caller must free, except handles. * - Weights are int64_t. A weight that would leave int64_t is ZS_ERR_OVERFLOW, never a * wrong value, and never an unwinding exception. - * - Not thread-safe: use a handle from one thread at a time. Streams, inputs and sinks of - * a circuit are uint32_t ids local to that circuit. + * - Not thread-safe. A ZsCircuit and every ZsRebuild made from it form ONE group (they share + * a reference count): keep the whole group on one thread at a time. Z-sets are separate + * handles. Streams, inputs and sinks are uint32_t ids local to their circuit; a sink id is + * never a stream id (they are refused as ZS_ERR_BAD_HANDLE), but an id from another circuit + * is not detected. A ZsRebuild is stepped only on its own circuit (else ZS_ERR_BAD_HANDLE); + * free the jobs before the circuit. + * - Keys of 4 GiB or longer are refused with ZS_ERR_BAD_VALUE: the wire form has u32 lengths. */ #ifndef ZSET_FFI_H #define ZSET_FFI_H @@ -38,6 +45,7 @@ extern "C" { #define ZS_ERR_BUFFER_TOO_SMALL 5 #define ZS_ERR_BAD_VALUE 6 #define ZS_ERR_POISONED 7 +#define ZS_ERR_CALLBACK 8 #define ZS_AGG_COUNT 0 #define ZS_AGG_SUM 1 @@ -48,17 +56,30 @@ typedef struct ZsZSet ZsZSet; typedef struct ZsCircuit ZsCircuit; typedef struct ZsRebuild ZsRebuild; -/* A callback may emit images of a key by calling emit(emit_ctx, bytes, len); the bytes are - * copied during the call. A callback must not call back into the library. */ +/* Callbacks run inside zs_circuit_step and zs_rebuild_step. + * - The function pointer and its ctx are kept by the circuit and must stay valid until + * zs_circuit_free. emit and emit_ctx are valid only during the callback that received them. + * - A callback emits images of a key by calling emit(emit_ctx, bytes, len); the bytes are + * copied during the call. + * - A callback reports failure by its return value: ZsFlatMapFn non-zero, ZsKeepFn and + * ZsExpiredFn negative. The step then fails with ZS_ERR_CALLBACK and the circuit is dead. + * - A callback MUST return normally: no longjmp, no C++ exception, no Lisp non-local exit + * through it (wrap the body in handler-case and return the failure value). If one does, + * the circuit is left marked mid-step and every later call on it is ZS_ERR_POISONED. + * - A callback must not call back into the library for its own circuit: that call returns + * ZS_ERR_POISONED. */ typedef void (*ZsEmit)(void *emit_ctx, const uint8_t *bytes, size_t len); -typedef void (*ZsFlatMapFn)(void *ctx, const uint8_t *key, size_t len, ZsEmit emit, void *emit_ctx); +typedef int32_t (*ZsFlatMapFn)(void *ctx, const uint8_t *key, size_t len, ZsEmit emit, void *emit_ctx); +/* 1 keep, 0 drop, negative fail. */ typedef int32_t (*ZsKeepFn)(void *ctx, const uint8_t *key, size_t len); +/* 1 expired, 0 not yet, negative fail. */ typedef int32_t (*ZsExpiredFn)(void *ctx); uint32_t zs_abi_version(void); const char *zs_last_error(void); -/* Z-sets. add/sub/neg/scale/decode_add leave the set unchanged when they fail. */ +/* Z-sets. add/sub/neg/scale/decode_add leave the set unchanged when they fail. add and sub + * accept the same handle as both arguments. */ int32_t zs_zset_new(ZsZSet **out); void zs_zset_free(ZsZSet *z); int32_t zs_zset_clone(const ZsZSet *z, ZsZSet **out); @@ -72,8 +93,9 @@ int32_t zs_zset_scale(ZsZSet *a, int64_t n); int32_t zs_zset_distinct(const ZsZSet *z, ZsZSet **out); /* Encoding: entries ascending by key, each [u32 LE key length][key][i64 LE weight]. - * *needed is always set; the bytes are written only if cap >= *needed, else - * ZS_ERR_BUFFER_TOO_SMALL (call again with a larger buffer). */ + * *needed is set whenever z is valid; the bytes are written only if cap >= *needed, else + * ZS_ERR_BUFFER_TOO_SMALL (call again with a larger buffer). buf may be NULL only when + * *needed is zero or to probe with cap too small; otherwise ZS_ERR_NULL_ARGUMENT. */ int32_t zs_zset_encode(const ZsZSet *z, uint8_t *buf, size_t cap, size_t *needed); int32_t zs_zset_decode_add(ZsZSet *z, const uint8_t *buf, size_t len); @@ -99,7 +121,7 @@ int32_t zs_circuit_join(ZsCircuit *c, uint32_t a, uint32_t b, uint32_t *out); /* aggregate: the input carries pair(k, v) with v an 8-byte little-endian int64 (any bytes * for COUNT); the output is pair(k, 8-byte little-endian int64). COUNT is the total weight, - * SUM is the weighted sum, MIN and MAX read the values of positive weight. */ + * SUM is the weighted sum (a non-empty group always has a row, even at zero), MIN and MAX read the values of positive weight. */ int32_t zs_circuit_aggregate(ZsCircuit *c, uint32_t stream, int32_t kind, uint32_t *out); int32_t zs_circuit_push(ZsCircuit *c, uint32_t input, const ZsZSet *z); @@ -109,7 +131,8 @@ int32_t zs_circuit_push(ZsCircuit *c, uint32_t input, const ZsZSet *z); int32_t zs_circuit_step(ZsCircuit *c); int32_t zs_circuit_take(ZsCircuit *c, uint32_t sink, ZsZSet **out); -/* Resumable snapshot load: chunk keys per circuit step (0 reads as 1). */ +/* Resumable snapshot load: chunk keys per piece (0 reads as 1). The whole load is one + * logical tick. done is 1 as soon as the last chunk ran. */ int32_t zs_rebuild_new(ZsCircuit *c, uint32_t input, const ZsZSet *snapshot, size_t chunk, ZsRebuild **out); int32_t zs_rebuild_step(ZsRebuild *job, ZsCircuit *c, ZsExpiredFn expired, void *ctx, int32_t *done); void zs_rebuild_free(ZsRebuild *job); diff --git a/crates/zset-ffi/src/engine.rs b/crates/zset-ffi/src/engine.rs index 13280049..5655f8c5 100644 --- a/crates/zset-ffi/src/engine.rs +++ b/crates/zset-ffi/src/engine.rs @@ -11,17 +11,34 @@ use zset::{Circuit, Input, Output, Progress, Rebuild, Stream, ZSet}; pub type Bytes = Vec; pub type BZ = ZSet; -/// The error codes of the ABI; 0 is success and is not a variant. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -#[repr(i32)] -pub enum Code { - NullArgument = 1, - Overflow = 2, - Panic = 3, - BadHandle = 4, - BufferTooSmall = 5, - BadValue = 6, - Poisoned = 7, +/// Declares the error codes once: the enum, the list of all of them and the header's name. +macro_rules! codes { + ($($variant:ident = $n:literal => $name:literal,)*) => { + /// The error codes of the ABI; 0 is success and is not a variant. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + #[repr(i32)] + pub enum Code { $($variant = $n,)* } + + impl Code { + pub const ALL: &[Code] = &[$(Code::$variant,)*]; + + /// The `#define` this code has in the header. + pub fn header_name(self) -> &'static str { + match self { $(Code::$variant => $name,)* } + } + } + }; +} + +codes! { + NullArgument = 1 => "ZS_ERR_NULL_ARGUMENT", + Overflow = 2 => "ZS_ERR_OVERFLOW", + Panic = 3 => "ZS_ERR_PANIC", + BadHandle = 4 => "ZS_ERR_BAD_HANDLE", + BufferTooSmall = 5 => "ZS_ERR_BUFFER_TOO_SMALL", + BadValue = 6 => "ZS_ERR_BAD_VALUE", + Poisoned = 7 => "ZS_ERR_POISONED", + Callback = 8 => "ZS_ERR_CALLBACK", } #[derive(Clone, Debug, PartialEq, Eq)] @@ -58,13 +75,18 @@ pub fn classify(payload: Box) -> Fail { Fail { code, msg } } +/// A length as the wire's `u32`; a longer one is refused rather than truncated. +pub fn len32(n: usize) -> Res { + u32::try_from(n).map_err(|_| Fail::new(Code::BadValue, "a key is 4 GiB or longer")) +} + /// `(key, value)` as one byte string: a little-endian `u32` key length, the key, the value. -pub fn pair(k: &[u8], v: &[u8]) -> Bytes { +pub fn pair(k: &[u8], v: &[u8]) -> Res { let mut out = Vec::with_capacity(4 + k.len() + v.len()); - out.extend_from_slice(&(k.len() as u32).to_le_bytes()); + out.extend_from_slice(&len32(k.len())?.to_le_bytes()); out.extend_from_slice(k); out.extend_from_slice(v); - out + Ok(out) } pub fn unpair(b: &[u8]) -> Option<(&[u8], &[u8])> { @@ -74,14 +96,14 @@ pub fn unpair(b: &[u8]) -> Option<(&[u8], &[u8])> { } /// Entries as `[u32 LE key length][key][i64 LE weight]`, ascending by key. -pub fn encode(z: &BZ) -> Bytes { +pub fn encode(z: &BZ) -> Res { let mut out = Vec::new(); for (k, w) in z.iter() { - out.extend_from_slice(&(k.len() as u32).to_le_bytes()); + out.extend_from_slice(&len32(k.len())?.to_le_bytes()); out.extend_from_slice(k); out.extend_from_slice(&w.to_le_bytes()); } - out + Ok(out) } /// The inverse of `encode`; equal keys sum, and an overflow panics like any addition. @@ -99,14 +121,19 @@ pub fn decode(mut b: &[u8]) -> Res { } pub type Emit = extern "C" fn(*mut c_void, *const u8, usize); -pub type FlatMapFn = extern "C" fn(*mut c_void, *const u8, usize, Emit, *mut c_void); +/// Returns 0, or non-zero to fail the step. +pub type FlatMapFn = extern "C" fn(*mut c_void, *const u8, usize, Emit, *mut c_void) -> i32; +/// Returns 0 (drop), 1 (keep), or negative to fail the step. pub type KeepFn = extern "C" fn(*mut c_void, *const u8, usize) -> i32; +/// Returns 0 (still time), 1 (expired), or negative to fail the step. pub type ExpiredFn = extern "C" fn(*mut c_void) -> i32; /// A caller's context pointer; the circuit never dereferences it. #[derive(Clone, Copy)] struct Ctx(*mut c_void); +type Fault = Rc>>; + extern "C" fn collect(ctx: *mut c_void, p: *const u8, n: usize) { // SAFETY: `ctx` is the `Vec` `images` passes below, alive for the callback's duration. let out = unsafe { &mut *(ctx as *mut Vec) }; @@ -119,15 +146,20 @@ extern "C" fn collect(ctx: *mut c_void, p: *const u8, n: usize) { out.push(bytes); } -fn images(f: FlatMapFn, ctx: Ctx, key: &Bytes) -> Vec { +/// The images a callback emits for `key`; a failing callback records a fault and emits none. +fn images(f: FlatMapFn, ctx: Ctx, key: &Bytes, fault: &Fault) -> Vec { let mut out: Vec = Vec::new(); - f( + let rc = f( ctx.0, key.as_ptr(), key.len(), collect, &mut out as *mut Vec as *mut c_void, ); + if rc != 0 { + fault.set(Some((Code::Callback, "a callback reported failure"))); + out.clear(); + } out } @@ -155,13 +187,17 @@ fn checked(v: Option) -> i64 { v.unwrap_or_else(|| panic!("{OVERFLOW_PREFIX} in aggregate")) } +/// Sink ids carry this bit; a stream id is an index far below it, so `streams.get` refuses a +/// sink id and `take` refuses a stream id. +const SINK_BIT: u32 = 1 << 31; + /// The circuit, its streams by id, and the first fault a closure recorded. pub struct Engine { circuit: Circuit, streams: Vec>, inputs: Vec>>, sinks: Vec>, - fault: Rc>>, + fault: Fault, poisoned: bool, } @@ -216,7 +252,7 @@ impl Engine { self.alive()?; let s = self.stream(s)?; self.sinks.push(self.circuit.output(&s)); - Ok((self.sinks.len() - 1) as u32) + Ok((self.sinks.len() - 1) as u32 | SINK_BIT) } pub fn delay(&mut self, s: u32) -> Res { @@ -257,20 +293,27 @@ impl Engine { pub fn filter(&mut self, s: u32, keep: KeepFn, ctx: *mut c_void) -> Res { self.alive()?; let s = self.stream(s)?; - let ctx = Ctx(ctx); - let out = self - .circuit - .filter(&s, move |k: &Bytes| keep(ctx.0, k.as_ptr(), k.len()) != 0); + let (ctx, fault) = (Ctx(ctx), Rc::clone(&self.fault)); + let out = self.circuit.filter(&s, move |k: &Bytes| { + match keep(ctx.0, k.as_ptr(), k.len()) { + 0 => false, + n if n > 0 => true, + _ => { + fault.set(Some((Code::Callback, "a callback reported failure"))); + false + } + } + }); Ok(self.register(out, None)) } pub fn flat_map(&mut self, s: u32, f: FlatMapFn, ctx: *mut c_void) -> Res { self.alive()?; let s = self.stream(s)?; - let ctx = Ctx(ctx); + let (ctx, fault) = (Ctx(ctx), Rc::clone(&self.fault)); let out = self .circuit - .flat_map(&s, move |k: &Bytes| images(f, ctx, k)); + .flat_map(&s, move |k: &Bytes| images(f, ctx, k, &fault)); Ok(self.register(out, None)) } @@ -280,12 +323,21 @@ impl Engine { let s = self.stream(s)?; let (ctx, fault) = (Ctx(ctx), Rc::clone(&self.fault)); let out = self.circuit.flat_map(&s, move |k: &Bytes| { - let mut gs = images(f, ctx, k); + let mut gs = images(f, ctx, k, &fault); if gs.len() != 1 { - fault.set(Some("index_by: the key callback must emit exactly one key")); + fault.set(Some(( + Code::BadValue, + "index_by: the key callback must emit exactly one key", + ))); return Vec::new(); } - vec![pair(&gs.remove(0), k)] + match pair(&gs.remove(0), k) { + Ok(p) => vec![p], + Err(_) => { + fault.set(Some((Code::BadValue, "a key is 4 GiB or longer"))); + Vec::new() + } + } }); Ok(self.register(out, None)) } @@ -296,7 +348,10 @@ impl Engine { self.circuit.flat_map(s, move |b: &Bytes| match unpair(b) { Some((k, v)) => vec![(k.to_vec(), v.to_vec())], None => { - fault.set(Some("a keyed stream carried an element that is not a pair")); + fault.set(Some(( + Code::BadValue, + "a keyed stream carried an element that is not a pair", + ))); Vec::new() } }) @@ -307,16 +362,20 @@ impl Engine { self.alive()?; let (a, b) = (self.stream(a)?, self.stream(b)?); let (ka, kb) = (self.keyed(&a), self.keyed(&b)); + let fault = Rc::clone(&self.fault); let j = self .circuit - .join_indexed(&ka, &kb, |k: &Bytes, x: &Bytes, y: &Bytes| { - pair(k, &pair(x, y)) + .join_indexed(&ka, &kb, move |k: &Bytes, x: &Bytes, y: &Bytes| { + pair(x, y).and_then(|xy| pair(k, &xy)).unwrap_or_else(|_| { + fault.set(Some((Code::BadValue, "a key is 4 GiB or longer"))); + Vec::new() + }) }); Ok(self.register(j, None)) } /// Groups a stream of `pair(k, v)` (v is 8 bytes, little-endian `i64`, except for - /// `Count`); the output is `pair(k, 8 bytes)`. + /// `Count`); the output is `pair(k, 8 bytes)`. A non-empty group always has a row. pub fn aggregate(&mut self, s: u32, kind: Agg) -> Res { self.alive()?; let s = self.stream(s)?; @@ -327,14 +386,14 @@ impl Engine { .aggregate(&keyed, move |_, vals: &[(Bytes, i64)]| { if kind == Agg::Count { let n = vals.iter().try_fold(0i64, |a, (_, w)| a.checked_add(*w)); - return Some(checked(n)).filter(|n| *n != 0); + return Some(checked(n)); } let mut nums = Vec::with_capacity(vals.len()); for (v, w) in vals { match <[u8; 8]>::try_from(v.as_slice()) { Ok(b) => nums.push((i64::from_le_bytes(b), *w)), Err(_) => { - fault.set(Some("an aggregated value is not 8 bytes")); + fault.set(Some((Code::BadValue, "an aggregated value is not 8 bytes"))); return None; } } @@ -344,7 +403,7 @@ impl Engine { let t = nums.iter().try_fold(0i64, |a, (v, w)| { v.checked_mul(*w).and_then(|p| a.checked_add(p)) }); - Some(checked(t)).filter(|t| *t != 0) + Some(checked(t)) } Agg::Min => nums.iter().filter(|(_, w)| *w > 0).map(|(v, _)| *v).min(), Agg::Max | Agg::Count => { @@ -352,9 +411,18 @@ impl Engine { } } }); + let fault = Rc::clone(&self.fault); let out = self .circuit - .map(&folded, |(k, r): &(Bytes, i64)| pair(k, &r.to_le_bytes())); + .flat_map(&folded, move |(k, r): &(Bytes, i64)| { + match pair(k, &r.to_le_bytes()) { + Ok(p) => vec![p], + Err(_) => { + fault.set(Some((Code::BadValue, "a key is 4 GiB or longer"))); + Vec::new() + } + } + }); Ok(self.register(out, None)) } @@ -381,9 +449,9 @@ impl Engine { Err(classify(payload)) } Ok(v) => match self.fault.take() { - Some(msg) => { + Some((code, msg)) => { self.poisoned = true; - Err(Fail::new(Code::BadValue, msg)) + Err(Fail::new(code, msg)) } None => Ok(v), }, @@ -397,7 +465,8 @@ impl Engine { pub fn take(&mut self, sink: u32) -> Res { self.alive()?; self.sinks - .get(sink as usize) + .get((sink & !SINK_BIT) as usize) + .filter(|_| sink & SINK_BIT != 0) .map(|s| s.take()) .ok_or_else(|| Fail::new(Code::BadHandle, "no such output")) } @@ -413,8 +482,20 @@ impl Engine { job: &mut Rebuild, expired: Option<(ExpiredFn, *mut c_void)>, ) -> Res { - let progress = - self.guarded(|c| job.step(c, || expired.is_some_and(|(f, ctx)| f(ctx) != 0)))?; + let fault = Rc::clone(&self.fault); + let progress = self.guarded(|c| { + job.step(c, || match expired { + None => false, + Some((f, ctx)) => match f(ctx) { + 0 => false, + n if n > 0 => true, + _ => { + fault.set(Some((Code::Callback, "a callback reported failure"))); + true + } + }, + }) + })?; Ok(progress == Progress::Done) } } diff --git a/crates/zset-ffi/src/lib.rs b/crates/zset-ffi/src/lib.rs index 51b451d4..38c29ab5 100644 --- a/crates/zset-ffi/src/lib.rs +++ b/crates/zset-ffi/src/lib.rs @@ -6,7 +6,7 @@ pub mod engine; use engine::{ Agg, BZ, Bytes, Code, Engine, ExpiredFn, Fail, FlatMapFn, KeepFn, classify, decode, encode, }; -use std::cell::RefCell; +use std::cell::{Cell, RefCell, UnsafeCell}; use std::ffi::{CString, c_char, c_void}; use std::panic::{AssertUnwindSafe, catch_unwind}; use zset::{Rebuild, ZSet}; @@ -15,10 +15,19 @@ pub const ZS_ABI_VERSION: u32 = 1; /// An opaque Z-set of byte-string keys. pub struct ZsZSet(BZ); -/// An opaque circuit over such Z-sets. -pub struct ZsCircuit(Engine); -/// An opaque resumable snapshot load. -pub struct ZsRebuild(Rebuild); + +/// An opaque circuit. `in_step` is set while the engine may be running callbacks; it is +/// still set if a caller's non-local exit skipped our frames, and then the circuit is dead. +pub struct ZsCircuit { + engine: UnsafeCell, + in_step: Cell, +} + +/// An opaque resumable snapshot load, bound to the circuit that made it. +pub struct ZsRebuild { + job: Rebuild, + owner: *const ZsCircuit, +} thread_local! { static LAST_ERROR: RefCell = RefCell::new(CString::default()); @@ -44,49 +53,85 @@ fn null() -> Fail { Fail::new(Code::NullArgument, "a required argument was null") } -fn as_mut<'a, T>(p: *mut T) -> Result<&'a mut T, Fail> { - // SAFETY: the ABI contract says a non-null handle came from this library and is live. +/// # Safety +/// `p` is null or points to a live `T` that nothing else is using. +unsafe fn as_mut<'a, T>(p: *mut T) -> Result<&'a mut T, Fail> { + // SAFETY: the caller's contract. unsafe { p.as_mut() }.ok_or_else(null) } -fn as_ref<'a, T>(p: *const T) -> Result<&'a T, Fail> { - // SAFETY: as `as_mut`. +/// # Safety +/// `p` is null or points to a live `T` that nothing is mutating. +unsafe fn as_ref<'a, T>(p: *const T) -> Result<&'a T, Fail> { + // SAFETY: the caller's contract. unsafe { p.as_ref() }.ok_or_else(null) } -fn bytes<'a>(p: *const u8, n: usize) -> Result<&'a [u8], Fail> { +/// # Safety +/// `p` is null (with `n` zero) or has `n` readable bytes. +unsafe fn bytes<'a>(p: *const u8, n: usize) -> Result<&'a [u8], Fail> { match (p.is_null(), n) { (_, 0) => Ok(&[]), (true, _) => Err(null()), - // SAFETY: the ABI contract says `p` has `n` readable bytes. + // SAFETY: the caller's contract. (false, _) => Ok(unsafe { std::slice::from_raw_parts(p, n) }), } } -fn write_out(buf: *mut u8, data: &[u8]) { - if !data.is_empty() { - // SAFETY: the ABI contract says `buf` has room for `data`; the caller checked `cap`. - unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), buf, data.len()) }; - } -} - -fn put(out: *mut T, v: T) -> Result<(), Fail> { - *as_mut(out)? = v; +/// # Safety +/// `out` is null or valid for a write of `T`. +unsafe fn with_out(out: *mut T, f: impl FnOnce() -> Result) -> Result<(), Fail> { + // SAFETY: the caller's contract. + let out = unsafe { as_mut(out)? }; + *out = f()?; Ok(()) } -fn hand_out(out: *mut *mut T, v: T) -> Result<(), Fail> { - put(out, Box::into_raw(Box::new(v))) +/// # Safety +/// `c` is null or a live circuit handle. Callbacks may call back in: this only reads `in_step`. +unsafe fn circuit<'a>(c: *const ZsCircuit) -> Result<&'a ZsCircuit, Fail> { + // SAFETY: the caller's contract; only a shared reference is made. + let c = unsafe { as_ref(c)? }; + if c.in_step.get() { + return Err(Fail::new( + Code::Poisoned, + "the circuit is mid-step: a callback re-entered it, or left it without returning", + )); + } + Ok(c) } -/// Frees a handle; null is accepted and ignored. -fn release(p: *mut T) { +impl ZsCircuit { + /// Only `circuit` hands out `&ZsCircuit` while no step is running, and the handle is used + /// from one thread, so nothing else holds the engine. + #[allow(clippy::mut_from_ref)] + fn engine(&self) -> &mut Engine { + // SAFETY: see above. + unsafe { &mut *self.engine.get() } + } + + /// Runs `f` with `in_step` set; a frame skipped by a caller's non-local exit leaves it set. + fn stepping(&self, f: impl FnOnce(&mut Engine) -> T) -> T { + self.in_step.set(true); + let v = f(self.engine()); + self.in_step.set(false); + v + } +} + +/// # Safety +/// `p` is null, or came from this library's constructor and is freed once. +unsafe fn release(p: *mut T) { if !p.is_null() { - // SAFETY: a non-null handle was made by `Box::into_raw` here and is freed once. + // SAFETY: the caller's contract. drop(unsafe { Box::from_raw(p) }); } } +fn boxed(v: T) -> Result<*mut T, Fail> { + Ok(Box::into_raw(Box::new(v))) +} + #[unsafe(no_mangle)] pub extern "C" fn zs_abi_version() -> u32 { ZS_ABI_VERSION @@ -98,149 +143,219 @@ pub extern "C" fn zs_last_error() -> *const c_char { LAST_ERROR.with(|e| e.borrow().as_ptr()) } +/// # Safety +/// `out` is null or valid for a write. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_new(out: *mut *mut ZsZSet) -> i32 { - guard(|| hand_out(out, ZsZSet(ZSet::new()))) +pub unsafe extern "C" fn zs_zset_new(out: *mut *mut ZsZSet) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || boxed(ZsZSet(ZSet::new()))) }) } +/// # Safety +/// `z` is null or a live Z-set handle, freed once. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_free(z: *mut ZsZSet) { - release(z) +pub unsafe extern "C" fn zs_zset_free(z: *mut ZsZSet) { + // SAFETY: the caller's contract. + unsafe { release(z) } } +/// # Safety +/// `z` is a live handle or null; `out` is null or valid for a write. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_clone(z: *const ZsZSet, out: *mut *mut ZsZSet) -> i32 { - guard(|| hand_out(out, ZsZSet(as_ref(z)?.0.clone()))) +pub unsafe extern "C" fn zs_zset_clone(z: *const ZsZSet, out: *mut *mut ZsZSet) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || boxed(ZsZSet(as_ref(z)?.0.clone()))) }) } +/// # Safety +/// `z` is a live handle or null; `key` has `len` readable bytes. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_insert(z: *mut ZsZSet, key: *const u8, len: usize, weight: i64) -> i32 { +pub unsafe extern "C" fn zs_zset_insert( + z: *mut ZsZSet, + key: *const u8, + len: usize, + weight: i64, +) -> i32 { guard(|| { - let (z, key) = (as_mut(z)?, bytes(key, len)?); + // SAFETY: the caller's contract. + let (z, key) = unsafe { (as_mut(z)?, bytes(key, len)?) }; z.0.insert(key.to_vec(), weight); Ok(()) }) } +/// # Safety +/// `z` is a live handle or null; `key` has `len` readable bytes; `out` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_weight( +pub unsafe extern "C" fn zs_zset_weight( z: *const ZsZSet, key: *const u8, len: usize, out: *mut i64, ) -> i32 { guard(|| { - let w = as_ref(z)?.0.weight(&bytes(key, len)?.to_vec()); - put(out, w) + // SAFETY: the caller's contract. + unsafe { with_out(out, || Ok(as_ref(z)?.0.weight(&bytes(key, len)?.to_vec()))) } }) } +/// # Safety +/// `z` is a live handle or null; `out` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_len(z: *const ZsZSet, out: *mut usize) -> i32 { - guard(|| put(out, as_ref(z)?.0.len())) +pub unsafe extern "C" fn zs_zset_len(z: *const ZsZSet, out: *mut usize) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || Ok(as_ref(z)?.0.len())) }) } -/// The binary operations compute a new value first, so a failure leaves `a` unchanged. +/// # Safety +/// Both are live handles or null. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_add(a: *mut ZsZSet, b: *const ZsZSet) -> i32 { +pub unsafe extern "C" fn zs_zset_add(a: *mut ZsZSet, b: *const ZsZSet) -> i32 { guard(|| { - let (a, b) = (as_mut(a)?, as_ref(b)?); - a.0 = &a.0 + &b.0; + // SAFETY: the caller's contract; the shared borrows end before the mutable one. + let sum = unsafe { &as_ref(a)?.0 + &as_ref(b)?.0 }; + unsafe { as_mut(a)? }.0 = sum; Ok(()) }) } +/// # Safety +/// Both are live handles or null. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_sub(a: *mut ZsZSet, b: *const ZsZSet) -> i32 { +pub unsafe extern "C" fn zs_zset_sub(a: *mut ZsZSet, b: *const ZsZSet) -> i32 { guard(|| { - let (a, b) = (as_mut(a)?, as_ref(b)?); - a.0 = &a.0 - &b.0; + // SAFETY: as `zs_zset_add`. + let diff = unsafe { &as_ref(a)?.0 - &as_ref(b)?.0 }; + unsafe { as_mut(a)? }.0 = diff; Ok(()) }) } +/// # Safety +/// `a` is a live handle or null. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_neg(a: *mut ZsZSet) -> i32 { +pub unsafe extern "C" fn zs_zset_neg(a: *mut ZsZSet) -> i32 { guard(|| { - let a = as_mut(a)?; + // SAFETY: the caller's contract. + let a = unsafe { as_mut(a)? }; a.0 = -&a.0; Ok(()) }) } +/// # Safety +/// `a` is a live handle or null. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_scale(a: *mut ZsZSet, n: i64) -> i32 { +pub unsafe extern "C" fn zs_zset_scale(a: *mut ZsZSet, n: i64) -> i32 { guard(|| { - let a = as_mut(a)?; + // SAFETY: the caller's contract. + let a = unsafe { as_mut(a)? }; a.0 = a.0.scale(n); Ok(()) }) } +/// # Safety +/// `z` is a live handle or null; `out` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_distinct(z: *const ZsZSet, out: *mut *mut ZsZSet) -> i32 { - guard(|| hand_out(out, ZsZSet(as_ref(z)?.0.distinct()))) +pub unsafe extern "C" fn zs_zset_distinct(z: *const ZsZSet, out: *mut *mut ZsZSet) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || boxed(ZsZSet(as_ref(z)?.0.distinct()))) }) } -/// Writes the encoding to `buf` if it fits. `*needed` is always set to the full size. +/// # Safety +/// `z` is a live handle or null; `buf` is null or has `cap` writable bytes; `needed` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_encode( +pub unsafe extern "C" fn zs_zset_encode( z: *const ZsZSet, buf: *mut u8, cap: usize, needed: *mut usize, ) -> i32 { guard(|| { - let enc = encode(&as_ref(z)?.0); - put(needed, enc.len())?; + // SAFETY: the caller's contract. + let (z, needed) = unsafe { (as_ref(z)?, as_mut(needed)?) }; + let enc = encode(&z.0)?; + *needed = enc.len(); if enc.len() > cap { return Err(Fail::new( Code::BufferTooSmall, "the buffer is smaller than *needed", )); } - write_out(buf, &enc); + if !enc.is_empty() { + if buf.is_null() { + return Err(null()); + } + // SAFETY: `buf` is non-null and has `cap >= enc.len()` writable bytes. + unsafe { std::ptr::copy_nonoverlapping(enc.as_ptr(), buf, enc.len()) }; + } Ok(()) }) } -/// Adds the decoded entries to `z`; nothing is added if the encoding is bad or overflows. +/// # Safety +/// `z` is a live handle or null; `buf` has `len` readable bytes. #[unsafe(no_mangle)] -pub extern "C" fn zs_zset_decode_add(z: *mut ZsZSet, buf: *const u8, len: usize) -> i32 { +pub unsafe extern "C" fn zs_zset_decode_add(z: *mut ZsZSet, buf: *const u8, len: usize) -> i32 { guard(|| { - let (z, buf) = (as_mut(z)?, bytes(buf, len)?); - z.0 = &z.0 + &decode(buf)?; + // SAFETY: the caller's contract. + let (zr, buf) = unsafe { (as_ref(z)?, bytes(buf, len)?) }; + let sum = &zr.0 + &decode(buf)?; + unsafe { as_mut(z)? }.0 = sum; Ok(()) }) } +/// # Safety +/// `out` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_new(out: *mut *mut ZsCircuit) -> i32 { - guard(|| hand_out(out, ZsCircuit(Engine::new()))) +pub unsafe extern "C" fn zs_circuit_new(out: *mut *mut ZsCircuit) -> i32 { + guard(|| { + // SAFETY: the caller's contract. + unsafe { + with_out(out, || { + boxed(ZsCircuit { + engine: UnsafeCell::new(Engine::new()), + in_step: Cell::new(false), + }) + }) + } + }) } +/// # Safety +/// `c` is null, or a live circuit freed once and not mid-step. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_free(c: *mut ZsCircuit) { - release(c) +pub unsafe extern "C" fn zs_circuit_free(c: *mut ZsCircuit) { + // SAFETY: the caller's contract. + unsafe { release(c) } } -/// A new input stream; its id is both the stream to read and the input to push to. +/// # Safety +/// `c` is a live circuit or null; `out` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_input(c: *mut ZsCircuit, out: *mut u32) -> i32 { - guard(|| put(out, as_mut(c)?.0.input()?)) +pub unsafe extern "C" fn zs_circuit_input(c: *mut ZsCircuit, out: *mut u32) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || circuit(c)?.engine().input()) }) } -/// A sink that sums the stream's deltas until `zs_circuit_take`. +/// # Safety +/// As `zs_circuit_input`. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_output(c: *mut ZsCircuit, stream: u32, out: *mut u32) -> i32 { - guard(|| put(out, as_mut(c)?.0.output(stream)?)) +pub unsafe extern "C" fn zs_circuit_output(c: *mut ZsCircuit, stream: u32, out: *mut u32) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || circuit(c)?.engine().output(stream)) }) } macro_rules! unary_op { ($name:ident, $method:ident) => { + /// # Safety + /// As `zs_circuit_input`. #[unsafe(no_mangle)] - pub extern "C" fn $name(c: *mut ZsCircuit, stream: u32, out: *mut u32) -> i32 { - guard(|| put(out, as_mut(c)?.0.$method(stream)?)) + pub unsafe extern "C" fn $name(c: *mut ZsCircuit, stream: u32, out: *mut u32) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || circuit(c)?.engine().$method(stream)) }) } }; } @@ -250,13 +365,18 @@ unary_op!(zs_circuit_integrate, integrate); unary_op!(zs_circuit_differentiate, differentiate); unary_op!(zs_circuit_distinct, distinct); +/// # Safety +/// As `zs_circuit_input`. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_plus(c: *mut ZsCircuit, a: u32, b: u32, out: *mut u32) -> i32 { - guard(|| put(out, as_mut(c)?.0.plus(a, b)?)) +pub unsafe extern "C" fn zs_circuit_plus(c: *mut ZsCircuit, a: u32, b: u32, out: *mut u32) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || circuit(c)?.engine().plus(a, b)) }) } +/// # Safety +/// As `zs_circuit_input`. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_filter( +pub unsafe extern "C" fn zs_circuit_filter( c: *mut ZsCircuit, stream: u32, keep: Option, @@ -264,15 +384,21 @@ pub extern "C" fn zs_circuit_filter( out: *mut u32, ) -> i32 { guard(|| { - put( - out, - as_mut(c)?.0.filter(stream, keep.ok_or_else(null)?, ctx)?, - ) + // SAFETY: the caller's contract. + unsafe { + with_out(out, || { + circuit(c)? + .engine() + .filter(stream, keep.ok_or_else(null)?, ctx) + }) + } }) } +/// # Safety +/// As `zs_circuit_input`. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_flat_map( +pub unsafe extern "C" fn zs_circuit_flat_map( c: *mut ZsCircuit, stream: u32, f: Option, @@ -280,15 +406,21 @@ pub extern "C" fn zs_circuit_flat_map( out: *mut u32, ) -> i32 { guard(|| { - put( - out, - as_mut(c)?.0.flat_map(stream, f.ok_or_else(null)?, ctx)?, - ) + // SAFETY: the caller's contract. + unsafe { + with_out(out, || { + circuit(c)? + .engine() + .flat_map(stream, f.ok_or_else(null)?, ctx) + }) + } }) } +/// # Safety +/// As `zs_circuit_input`. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_index_by( +pub unsafe extern "C" fn zs_circuit_index_by( c: *mut ZsCircuit, stream: u32, f: Option, @@ -296,49 +428,86 @@ pub extern "C" fn zs_circuit_index_by( out: *mut u32, ) -> i32 { guard(|| { - put( - out, - as_mut(c)?.0.index_by(stream, f.ok_or_else(null)?, ctx)?, - ) + // SAFETY: the caller's contract. + unsafe { + with_out(out, || { + circuit(c)? + .engine() + .index_by(stream, f.ok_or_else(null)?, ctx) + }) + } }) } +/// # Safety +/// As `zs_circuit_input`. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_join(c: *mut ZsCircuit, a: u32, b: u32, out: *mut u32) -> i32 { - guard(|| put(out, as_mut(c)?.0.join(a, b)?)) +pub unsafe extern "C" fn zs_circuit_join(c: *mut ZsCircuit, a: u32, b: u32, out: *mut u32) -> i32 { + // SAFETY: the caller's contract. + guard(|| unsafe { with_out(out, || circuit(c)?.engine().join(a, b)) }) } -/// `kind`: 0 count, 1 sum, 2 min, 3 max. +/// # Safety +/// As `zs_circuit_input`. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_aggregate( +pub unsafe extern "C" fn zs_circuit_aggregate( c: *mut ZsCircuit, stream: u32, kind: i32, out: *mut u32, ) -> i32 { - guard(|| put(out, as_mut(c)?.0.aggregate(stream, Agg::from_code(kind)?)?)) + guard(|| { + // SAFETY: the caller's contract. + unsafe { + with_out(out, || { + circuit(c)? + .engine() + .aggregate(stream, Agg::from_code(kind)?) + }) + } + }) } -/// Adds a copy of `z` to the input's pending delta. +/// # Safety +/// `c` and `z` are live handles or null. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_push(c: *mut ZsCircuit, input: u32, z: *const ZsZSet) -> i32 { - guard(|| as_mut(c)?.0.push(input, &as_ref(z)?.0)) +pub unsafe extern "C" fn zs_circuit_push(c: *mut ZsCircuit, input: u32, z: *const ZsZSet) -> i32 { + guard(|| { + // SAFETY: the caller's contract. + let (c, z) = unsafe { (circuit(c)?, as_ref(z)?) }; + c.engine().push(input, &z.0) + }) } -/// One clock tick. On any error but a null or bad id the circuit is poisoned. +/// # Safety +/// `c` is a live circuit or null. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_step(c: *mut ZsCircuit) -> i32 { - guard(|| as_mut(c)?.0.step()) +pub unsafe extern "C" fn zs_circuit_step(c: *mut ZsCircuit) -> i32 { + guard(|| { + // SAFETY: the caller's contract. + let c = unsafe { circuit(c)? }; + c.stepping(|e| e.step()) + }) } -/// Takes the sink's accumulated delta as a new Z-set handle, and clears the sink. +/// # Safety +/// `c` is a live circuit or null; `out` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_circuit_take(c: *mut ZsCircuit, sink: u32, out: *mut *mut ZsZSet) -> i32 { - guard(|| hand_out(out, ZsZSet(as_mut(c)?.0.take(sink)?))) +pub unsafe extern "C" fn zs_circuit_take( + c: *mut ZsCircuit, + sink: u32, + out: *mut *mut ZsZSet, +) -> i32 { + guard(|| { + // SAFETY: the caller's contract. + unsafe { with_out(out, || boxed(ZsZSet(circuit(c)?.engine().take(sink)?))) } + }) } +/// # Safety +/// `c` and `snapshot` are live handles or null; `out` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_rebuild_new( +pub unsafe extern "C" fn zs_rebuild_new( c: *mut ZsCircuit, input: u32, snapshot: *const ZsZSet, @@ -346,17 +515,21 @@ pub extern "C" fn zs_rebuild_new( out: *mut *mut ZsRebuild, ) -> i32 { guard(|| { - let job = as_mut(c)? - .0 - .start_rebuild(input, as_ref(snapshot)?.0.clone(), chunk)?; - hand_out(out, ZsRebuild(job)) + // SAFETY: the caller's contract. + unsafe { + with_out(out, || { + let (cr, snap) = (circuit(c)?, as_ref(snapshot)?); + let job = cr.engine().start_rebuild(input, snap.0.clone(), chunk)?; + boxed(ZsRebuild { job, owner: c }) + }) + } }) } -/// Runs chunks until done or `expired(ctx)` returns non-zero; `*done` is 1 when finished. -/// A null `expired` runs to completion. +/// # Safety +/// `job` and `c` are live handles or null; `done` is null or writable. #[unsafe(no_mangle)] -pub extern "C" fn zs_rebuild_step( +pub unsafe extern "C" fn zs_rebuild_step( job: *mut ZsRebuild, c: *mut ZsCircuit, expired: Option, @@ -364,13 +537,51 @@ pub extern "C" fn zs_rebuild_step( done: *mut i32, ) -> i32 { guard(|| { - let (job, c) = (as_mut(job)?, as_mut(c)?); - let finished = c.0.rebuild_step(&mut job.0, expired.map(|f| (f, ctx)))?; - put(done, i32::from(finished)) + // SAFETY: the caller's contract. + unsafe { + with_out(done, || { + let (job, cr) = (as_mut(job)?, circuit(c)?); + if !std::ptr::eq(job.owner, c) { + return Err(Fail::new( + Code::BadHandle, + "the rebuild belongs to another circuit", + )); + } + let finished = + cr.stepping(|e| e.rebuild_step(&mut job.job, expired.map(|f| (f, ctx))))?; + Ok(i32::from(finished)) + }) + } }) } +/// # Safety +/// `job` is null, or a live job freed once. #[unsafe(no_mangle)] -pub extern "C" fn zs_rebuild_free(job: *mut ZsRebuild) { - release(job) +pub unsafe extern "C" fn zs_rebuild_free(job: *mut ZsRebuild) { + // SAFETY: the caller's contract. + unsafe { release(job) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_circuit_left_mid_step_by_a_skipped_frame_answers_poisoned_to_everything() { + let mut c = std::ptr::null_mut(); + assert_eq!(unsafe { zs_circuit_new(&mut c) }, 0); + let mut id = 0; + assert_eq!(unsafe { zs_circuit_input(c, &mut id) }, 0); + // What a non-local exit out of a callback leaves behind: the flag never cleared. + unsafe { &*c }.in_step.set(true); + assert_eq!(unsafe { zs_circuit_step(c) }, Code::Poisoned as i32); + id = 77; + assert_eq!( + unsafe { zs_circuit_input(c, &mut id) }, + Code::Poisoned as i32 + ); + assert_eq!(id, 77, "nothing was written"); + unsafe { zs_circuit_free(c) }; + } } diff --git a/crates/zset-ffi/tests/abi.rs b/crates/zset-ffi/tests/abi.rs index 730026be..c5d4e66a 100644 --- a/crates/zset-ffi/tests/abi.rs +++ b/crates/zset-ffi/tests/abi.rs @@ -3,36 +3,43 @@ use std::ffi::{CStr, c_void}; use std::ptr::{null, null_mut}; -use zset_ffi::engine::{Code, pair}; +use zset_ffi::engine::Code; use zset_ffi::*; const OK: i32 = 0; +fn pair(k: &[u8], v: &[u8]) -> Vec { + zset_ffi::engine::pair(k, v).unwrap() +} + fn zset(entries: &[(&[u8], i64)]) -> *mut ZsZSet { let mut z = null_mut(); - assert_eq!(zs_zset_new(&mut z), OK); + assert_eq!(unsafe { zs_zset_new(&mut z) }, OK); for (k, w) in entries { - assert_eq!(zs_zset_insert(z, k.as_ptr(), k.len(), *w), OK); + assert_eq!(unsafe { zs_zset_insert(z, k.as_ptr(), k.len(), *w) }, OK); } z } fn weight(z: *const ZsZSet, k: &[u8]) -> i64 { let mut w = 0; - assert_eq!(zs_zset_weight(z, k.as_ptr(), k.len(), &mut w), OK); + assert_eq!( + unsafe { zs_zset_weight(z, k.as_ptr(), k.len(), &mut w) }, + OK + ); w } fn len(z: *const ZsZSet) -> usize { let mut n = 0; - assert_eq!(zs_zset_len(z, &mut n), OK); + assert_eq!(unsafe { zs_zset_len(z, &mut n) }, OK); n } fn encoded(z: *const ZsZSet) -> Vec { let mut needed = 0; assert_eq!( - zs_zset_encode(z, null_mut(), 0, &mut needed), + unsafe { zs_zset_encode(z, null_mut(), 0, &mut needed) }, if needed == 0 { OK } else { @@ -41,7 +48,7 @@ fn encoded(z: *const ZsZSet) -> Vec { ); let mut buf = vec![0u8; needed]; assert_eq!( - zs_zset_encode(z, buf.as_mut_ptr(), buf.len(), &mut needed), + unsafe { zs_zset_encode(z, buf.as_mut_ptr(), buf.len(), &mut needed) }, OK ); buf @@ -58,41 +65,41 @@ struct C(*mut ZsCircuit); impl C { fn new() -> C { let mut c = null_mut(); - assert_eq!(zs_circuit_new(&mut c), OK); + assert_eq!(unsafe { zs_circuit_new(&mut c) }, OK); C(c) } fn input(&self) -> u32 { let mut s = 0; - assert_eq!(zs_circuit_input(self.0, &mut s), OK); + assert_eq!(unsafe { zs_circuit_input(self.0, &mut s) }, OK); s } fn sink(&self, s: u32) -> u32 { let mut o = 0; - assert_eq!(zs_circuit_output(self.0, s, &mut o), OK); + assert_eq!(unsafe { zs_circuit_output(self.0, s, &mut o) }, OK); o } fn push(&self, input: u32, entries: &[(&[u8], i64)]) { let z = zset(entries); - assert_eq!(zs_circuit_push(self.0, input, z), OK); - zs_zset_free(z); + assert_eq!(unsafe { zs_circuit_push(self.0, input, z) }, OK); + unsafe { zs_zset_free(z) }; } /// Steps and takes the sink as sorted `(key, weight)` pairs. fn run(&self, sink: u32) -> Vec<(Vec, i64)> { - assert_eq!(zs_circuit_step(self.0), OK); + assert_eq!(unsafe { zs_circuit_step(self.0) }, OK); self.take(sink) } fn take(&self, sink: u32) -> Vec<(Vec, i64)> { let mut z = null_mut(); - assert_eq!(zs_circuit_take(self.0, sink, &mut z), OK); + assert_eq!(unsafe { zs_circuit_take(self.0, sink, &mut z) }, OK); let enc = encoded(z); - zs_zset_free(z); + unsafe { zs_zset_free(z) }; decode_pairs(&enc) } } impl Drop for C { fn drop(&mut self) { - zs_circuit_free(self.0) + unsafe { zs_circuit_free(self.0) } } } @@ -121,20 +128,20 @@ fn a_zset_adds_queries_and_frees() { zset(&[(b"x", 2), (b"y", -1)]), zset(&[(b"x", -2), (b"z", 5)]), ); - assert_eq!(zs_zset_add(a, b), OK); + assert_eq!(unsafe { zs_zset_add(a, b) }, OK); assert_eq!( (weight(a, b"x"), weight(a, b"y"), weight(a, b"z"), len(a)), (0, -1, 5, 2) ); - assert_eq!(zs_zset_neg(a), OK); - assert_eq!(zs_zset_scale(a, 3), OK); + assert_eq!(unsafe { zs_zset_neg(a) }, OK); + assert_eq!(unsafe { zs_zset_scale(a, 3) }, OK); assert_eq!((weight(a, b"y"), weight(a, b"z")), (3, -15)); - assert_eq!(zs_zset_sub(a, a), OK); + assert_eq!(unsafe { zs_zset_sub(a, a) }, OK); assert_eq!(len(a), 0); for z in [a, b] { - zs_zset_free(z); + unsafe { zs_zset_free(z) }; } - zs_zset_free(null_mut()); + unsafe { zs_zset_free(null_mut()) }; } #[test] @@ -144,30 +151,33 @@ fn keys_are_byte_strings_including_nul_and_empty() { (weight(z, b"a\0b"), weight(z, b""), weight(z, b"a"), len(z)), (1, 2, 3, 3) ); - assert_eq!(zs_zset_insert(z, null(), 0, 4), OK); + assert_eq!(unsafe { zs_zset_insert(z, null(), 0, 4) }, OK); assert_eq!(weight(z, b""), 6); - zs_zset_free(z); + unsafe { zs_zset_free(z) }; } #[test] fn distinct_clone_and_encoding_round_trip() { let z = zset(&[(b"k1", 5), (b"k2", -2), (b"k3", 1)]); let mut d = null_mut(); - assert_eq!(zs_zset_distinct(z, &mut d), OK); + assert_eq!(unsafe { zs_zset_distinct(z, &mut d) }, OK); assert_eq!( (weight(d, b"k1"), weight(d, b"k2"), weight(d, b"k3")), (1, 0, 1) ); let mut copy = null_mut(); - assert_eq!(zs_zset_clone(z, &mut copy), OK); + assert_eq!(unsafe { zs_zset_clone(z, &mut copy) }, OK); let enc = encoded(z); let mut back = null_mut(); - assert_eq!(zs_zset_new(&mut back), OK); - assert_eq!(zs_zset_decode_add(back, enc.as_ptr(), enc.len()), OK); + assert_eq!(unsafe { zs_zset_new(&mut back) }, OK); + assert_eq!( + unsafe { zs_zset_decode_add(back, enc.as_ptr(), enc.len()) }, + OK + ); assert_eq!(encoded(back), encoded(copy)); assert_eq!(encoded(back), enc); for h in [z, d, copy, back] { - zs_zset_free(h); + unsafe { zs_zset_free(h) }; } } @@ -176,12 +186,12 @@ fn a_small_buffer_reports_the_size_it_needs() { let z = zset(&[(b"abc", 1)]); let (mut needed, mut small) = (0, [0u8; 3]); assert_eq!( - zs_zset_encode(z, small.as_mut_ptr(), small.len(), &mut needed), + unsafe { zs_zset_encode(z, small.as_mut_ptr(), small.len(), &mut needed) }, Code::BufferTooSmall as i32 ); assert_eq!(needed, 4 + 3 + 8); assert_eq!(small, [0; 3], "nothing written on failure"); - zs_zset_free(z); + unsafe { zs_zset_free(z) }; } #[test] @@ -190,11 +200,11 @@ fn a_bad_encoding_is_refused_and_adds_nothing() { let mut enc = encoded(z); enc.push(7); assert_eq!( - zs_zset_decode_add(z, enc.as_ptr(), enc.len()), + unsafe { zs_zset_decode_add(z, enc.as_ptr(), enc.len()) }, Code::BadValue as i32 ); assert_eq!((weight(z, b"a"), len(z)), (1, 1)); - zs_zset_free(z); + unsafe { zs_zset_free(z) }; } #[test] @@ -203,39 +213,48 @@ fn overflow_is_an_error_code_and_leaves_the_set_unchanged() { zset(&[(b"k", i64::MAX), (b"j", 1)]), zset(&[(b"j", 1), (b"k", 1)]), ); - assert_eq!(zs_zset_add(a, b), Code::Overflow as i32); + assert_eq!(unsafe { zs_zset_add(a, b) }, Code::Overflow as i32); assert_eq!( (weight(a, b"k"), weight(a, b"j")), (i64::MAX, 1), "a is unchanged, even for the key before the overflow" ); assert!(last_error().contains("weight overflow"), "{}", last_error()); - assert_eq!(zs_zset_scale(a, 2), Code::Overflow as i32); + assert_eq!(unsafe { zs_zset_scale(a, 2) }, Code::Overflow as i32); assert_eq!( - zs_zset_insert(a, b"k".as_ptr(), 1, 1), + unsafe { zs_zset_insert(a, b"k".as_ptr(), 1, 1) }, Code::Overflow as i32 ); let m = zset(&[(b"k", i64::MIN)]); - assert_eq!(zs_zset_neg(m), Code::Overflow as i32); + assert_eq!(unsafe { zs_zset_neg(m) }, Code::Overflow as i32); assert_eq!(weight(m, b"k"), i64::MIN); - assert_eq!(zs_zset_sub(a, a), OK); + assert_eq!(unsafe { zs_zset_sub(a, a) }, OK); for z in [a, b, m] { - zs_zset_free(z); + unsafe { zs_zset_free(z) }; } } #[test] fn a_null_argument_is_a_code_not_a_crash() { - assert_eq!(zs_zset_new(null_mut()), Code::NullArgument as i32); - assert_eq!(zs_zset_len(null(), &mut 0), Code::NullArgument as i32); - let z = zset(&[]); - assert_eq!(zs_zset_insert(z, null(), 3, 1), Code::NullArgument as i32); - assert_eq!(zs_zset_add(z, null()), Code::NullArgument as i32); assert_eq!( - zs_circuit_filter(null_mut(), 0, None, null_mut(), &mut 0), + unsafe { zs_zset_new(null_mut()) }, Code::NullArgument as i32 ); - zs_zset_free(z); + assert_eq!( + unsafe { zs_zset_len(null(), &mut 0) }, + Code::NullArgument as i32 + ); + let z = zset(&[]); + assert_eq!( + unsafe { zs_zset_insert(z, null(), 3, 1) }, + Code::NullArgument as i32 + ); + assert_eq!(unsafe { zs_zset_add(z, null()) }, Code::NullArgument as i32); + assert_eq!( + unsafe { zs_circuit_filter(null_mut(), 0, None, null_mut(), &mut 0) }, + Code::NullArgument as i32 + ); + unsafe { zs_zset_free(z) }; assert_ne!(zs_abi_version(), 0); } @@ -244,21 +263,28 @@ fn a_circuit_runs_distinct_and_integrate() { let c = C::new(); let i = c.input(); let (mut d, mut int) = (0, 0); - assert_eq!(zs_circuit_distinct(c.0, i, &mut d), OK); - assert_eq!(zs_circuit_integrate(c.0, d, &mut int), OK); + assert_eq!(unsafe { zs_circuit_distinct(c.0, i, &mut d) }, OK); + assert_eq!(unsafe { zs_circuit_integrate(c.0, d, &mut int) }, OK); let (sd, sint) = (c.sink(d), c.sink(int)); c.push(i, &[(b"a", 3), (b"b", -1)]); - assert_eq!(zs_circuit_step(c.0), OK); + assert_eq!(unsafe { zs_circuit_step(c.0) }, OK); assert_eq!(c.take(sd), one(&[(b"a", 1)])); assert_eq!(c.take(sint), one(&[(b"a", 1)])); c.push(i, &[(b"a", -3), (b"b", 1)]); assert_eq!(c.run(sd), one(&[(b"a", -1)])); } -extern "C" fn upper(_: *mut c_void, k: *const u8, n: usize, emit: engine::Emit, e: *mut c_void) { +extern "C" fn upper( + _: *mut c_void, + k: *const u8, + n: usize, + emit: engine::Emit, + e: *mut c_void, +) -> i32 { let up: Vec = unsafe { std::slice::from_raw_parts(k, n) }.to_ascii_uppercase(); emit(e, up.as_ptr(), up.len()); emit(e, b"!".as_ptr(), 1); + 0 } extern "C" fn even_len(_: *mut c_void, _: *const u8, n: usize) -> i32 { @@ -271,8 +297,9 @@ extern "C" fn first_byte( n: usize, emit: engine::Emit, e: *mut c_void, -) { +) -> i32 { emit(e, k, n.min(1)); + 0 } #[test] @@ -281,11 +308,11 @@ fn callbacks_filter_and_flat_map() { let i = c.input(); let (mut f, mut m) = (0, 0); assert_eq!( - zs_circuit_filter(c.0, i, Some(even_len), null_mut(), &mut f), + unsafe { zs_circuit_filter(c.0, i, Some(even_len), null_mut(), &mut f) }, OK ); assert_eq!( - zs_circuit_flat_map(c.0, f, Some(upper), null_mut(), &mut m), + unsafe { zs_circuit_flat_map(c.0, f, Some(upper), null_mut(), &mut m) }, OK ); let s = c.sink(m); @@ -299,10 +326,13 @@ fn index_by_join_and_aggregate_follow_the_pair_encoding() { let (orders, prices) = (c.input(), c.input()); let (mut by_item, mut joined) = (0, 0); assert_eq!( - zs_circuit_index_by(c.0, orders, Some(first_byte), null_mut(), &mut by_item), + unsafe { zs_circuit_index_by(c.0, orders, Some(first_byte), null_mut(), &mut by_item) }, + OK + ); + assert_eq!( + unsafe { zs_circuit_join(c.0, by_item, prices, &mut joined) }, OK ); - assert_eq!(zs_circuit_join(c.0, by_item, prices, &mut joined), OK); let sink = c.sink(joined); c.push(orders, &[(b"apple", 1)]); c.push(prices, &[(&pair(b"a", b"3"), 1)]); @@ -319,7 +349,7 @@ fn aggregates_count_sum_min_and_max() { let outs: Vec = (0..4) .map(|kind| { let mut o = 0; - assert_eq!(zs_circuit_aggregate(c.0, i, kind, &mut o), OK); + assert_eq!(unsafe { zs_circuit_aggregate(c.0, i, kind, &mut o) }, OK); c.sink(o) }) .collect(); @@ -331,7 +361,7 @@ fn aggregates_count_sum_min_and_max() { (&pair(b"h", &le(9)), 1), ], ); - assert_eq!(zs_circuit_step(c.0), OK); + assert_eq!(unsafe { zs_circuit_step(c.0) }, OK); let val = |k: &[u8], n: i64| (pair(k, &le(n)), 1); let want = [ [val(b"g", 3), val(b"h", 1)], @@ -353,15 +383,15 @@ fn a_weight_overflow_in_a_step_is_a_code_and_poisons_the_circuit() { let c = C::new(); let i = c.input(); let (mut int, mut again) = (0, 0); - assert_eq!(zs_circuit_integrate(c.0, i, &mut int), OK); + assert_eq!(unsafe { zs_circuit_integrate(c.0, i, &mut int) }, OK); c.push(i, &[(b"k", i64::MAX)]); - assert_eq!(zs_circuit_step(c.0), OK); + assert_eq!(unsafe { zs_circuit_step(c.0) }, OK); c.push(i, &[(b"k", 1)]); - assert_eq!(zs_circuit_step(c.0), Code::Overflow as i32); + assert_eq!(unsafe { zs_circuit_step(c.0) }, Code::Overflow as i32); assert!(last_error().contains("weight overflow")); - assert_eq!(zs_circuit_step(c.0), Code::Poisoned as i32); + assert_eq!(unsafe { zs_circuit_step(c.0) }, Code::Poisoned as i32); assert_eq!( - zs_circuit_distinct(c.0, i, &mut again), + unsafe { zs_circuit_distinct(c.0, i, &mut again) }, Code::Poisoned as i32 ); } @@ -371,23 +401,31 @@ fn a_malformed_pair_and_an_unknown_id_are_codes() { let c = C::new(); let i = c.input(); let mut j = 0; - assert_eq!(zs_circuit_join(c.0, i, 99, &mut j), Code::BadHandle as i32); - assert_eq!(zs_circuit_join(c.0, i, i, &mut j), OK); + assert_eq!( + unsafe { zs_circuit_join(c.0, i, 99, &mut j) }, + Code::BadHandle as i32 + ); + assert_eq!(unsafe { zs_circuit_join(c.0, i, i, &mut j) }, OK); c.push(i, &[(b"\xff\xff\xff\xff", 1)]); - assert_eq!(zs_circuit_step(c.0), Code::BadValue as i32); - assert_eq!(zs_circuit_step(c.0), Code::Poisoned as i32); + assert_eq!(unsafe { zs_circuit_step(c.0) }, Code::BadValue as i32); + assert_eq!(unsafe { zs_circuit_step(c.0) }, Code::Poisoned as i32); let mut a = 0; let c2 = C::new(); assert_eq!( - zs_circuit_aggregate(c2.0, 0, 9, &mut a), + unsafe { zs_circuit_aggregate(c2.0, 0, 9, &mut a) }, Code::BadValue as i32 ); let mut z = null_mut(); - assert_eq!(zs_circuit_take(c2.0, 5, &mut z), Code::BadHandle as i32); assert_eq!( - zs_circuit_push(c2.0, 0, zset(&[]).cast_const()), + unsafe { zs_circuit_take(c2.0, 5, &mut z) }, Code::BadHandle as i32 ); + let empty = zset(&[]); + assert_eq!( + unsafe { zs_circuit_push(c2.0, 0, empty) }, + Code::BadHandle as i32 + ); + unsafe { zs_zset_free(empty) }; } extern "C" fn out_of_time(ctx: *mut c_void) -> i32 { @@ -405,28 +443,30 @@ fn a_rebuild_resumes_across_calls_and_matches_one_step() { let whole = C::new(); let wi = whole.input(); let mut wd = 0; - assert_eq!(zs_circuit_distinct(whole.0, wi, &mut wd), OK); + assert_eq!(unsafe { zs_circuit_distinct(whole.0, wi, &mut wd) }, OK); let ws = whole.sink(wd); - assert_eq!(zs_circuit_push(whole.0, wi, snapshot), OK); + assert_eq!(unsafe { zs_circuit_push(whole.0, wi, snapshot) }, OK); let want = whole.run(ws); let c = C::new(); let i = c.input(); let mut d = 0; - assert_eq!(zs_circuit_distinct(c.0, i, &mut d), OK); + assert_eq!(unsafe { zs_circuit_distinct(c.0, i, &mut d) }, OK); let s = c.sink(d); let mut job = null_mut(); - assert_eq!(zs_rebuild_new(c.0, i, snapshot, 2, &mut job), OK); + assert_eq!(unsafe { zs_rebuild_new(c.0, i, snapshot, 2, &mut job) }, OK); let (mut calls, mut got, mut done) = (0u32, Vec::new(), 0); for _ in 0..20 { assert_eq!( - zs_rebuild_step( - job, - c.0, - Some(out_of_time), - (&mut calls as *mut u32).cast(), - &mut done - ), + unsafe { + zs_rebuild_step( + job, + c.0, + Some(out_of_time), + (&mut calls as *mut u32).cast(), + &mut done, + ) + }, OK ); got.extend(c.take(s)); @@ -436,13 +476,13 @@ fn a_rebuild_resumes_across_calls_and_matches_one_step() { } assert_eq!(done, 1, "the job finished within its bound"); assert_eq!( - calls, 5, - "ceil(9 / 2) chunks, each followed by one deadline check" + calls, 4, + "ceil(9 / 2) = 5 chunks; the last is Done before any check" ); got.sort(); assert_eq!(got, want); - zs_rebuild_free(job); - zs_zset_free(snapshot); + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; } #[test] @@ -453,12 +493,15 @@ fn a_rebuild_without_a_deadline_runs_to_completion() { let s = c.sink(i); let mut job = null_mut(); let mut done = 0; - assert_eq!(zs_rebuild_new(c.0, i, snapshot, 1, &mut job), OK); - assert_eq!(zs_rebuild_step(job, c.0, None, null_mut(), &mut done), OK); + assert_eq!(unsafe { zs_rebuild_new(c.0, i, snapshot, 1, &mut job) }, OK); + assert_eq!( + unsafe { zs_rebuild_step(job, c.0, None, null_mut(), &mut done) }, + OK + ); assert_eq!(done, 1); assert_eq!(c.take(s).len(), 3); - zs_rebuild_free(job); - zs_zset_free(snapshot); + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; } #[test] @@ -466,8 +509,8 @@ fn min_and_max_read_only_values_of_positive_weight() { let c = C::new(); let i = c.input(); let (mut lo, mut hi) = (0, 0); - assert_eq!(zs_circuit_aggregate(c.0, i, 2, &mut lo), OK); - assert_eq!(zs_circuit_aggregate(c.0, i, 3, &mut hi), OK); + assert_eq!(unsafe { zs_circuit_aggregate(c.0, i, 2, &mut lo) }, OK); + assert_eq!(unsafe { zs_circuit_aggregate(c.0, i, 3, &mut hi) }, OK); let (slo, shi) = (c.sink(lo), c.sink(hi)); c.push( i, @@ -477,24 +520,31 @@ fn min_and_max_read_only_values_of_positive_weight() { (&pair(b"g", &le(9)), -1), ], ); - assert_eq!(zs_circuit_step(c.0), OK); + assert_eq!(unsafe { zs_circuit_step(c.0) }, OK); assert_eq!(c.take(slo), one(&[(&pair(b"g", &le(5)), 1)])); assert_eq!(c.take(shi), one(&[(&pair(b"g", &le(5)), 1)])); } #[test] fn an_index_by_callback_that_emits_nothing_is_a_bad_value() { - extern "C" fn nothing(_: *mut c_void, _: *const u8, _: usize, _: engine::Emit, _: *mut c_void) { + extern "C" fn nothing( + _: *mut c_void, + _: *const u8, + _: usize, + _: engine::Emit, + _: *mut c_void, + ) -> i32 { + 0 } let c = C::new(); let i = c.input(); let mut o = 0; assert_eq!( - zs_circuit_index_by(c.0, i, Some(nothing), null_mut(), &mut o), + unsafe { zs_circuit_index_by(c.0, i, Some(nothing), null_mut(), &mut o) }, OK ); c.push(i, &[(b"x", 1)]); - assert_eq!(zs_circuit_step(c.0), Code::BadValue as i32); + assert_eq!(unsafe { zs_circuit_step(c.0) }, Code::BadValue as i32); assert!(last_error().contains("exactly one key"), "{}", last_error()); } @@ -504,22 +554,22 @@ fn a_chunked_rebuild_through_delay_is_one_tick() { let c = C::new(); let i = c.input(); let mut d = 0; - assert_eq!(zs_circuit_delay(c.0, i, &mut d), OK); + assert_eq!(unsafe { zs_circuit_delay(c.0, i, &mut d) }, OK); let s = c.sink(d); c.push(i, &[(b"p", 1)]); - assert_eq!(zs_circuit_step(c.0), OK); + assert_eq!(unsafe { zs_circuit_step(c.0) }, OK); c.take(s); let mut job = null_mut(); - assert_eq!(zs_rebuild_new(c.0, i, snapshot, 1, &mut job), OK); + assert_eq!(unsafe { zs_rebuild_new(c.0, i, snapshot, 1, &mut job) }, OK); let (mut calls, mut done) = (0u32, 0); let ctx = (&mut calls as *mut u32).cast(); assert_eq!( - zs_rebuild_step(job, c.0, Some(out_of_time), ctx, &mut done), + unsafe { zs_rebuild_step(job, c.0, Some(out_of_time), ctx, &mut done) }, OK ); assert_eq!(done, 0); assert_eq!( - zs_rebuild_step(job, c.0, Some(out_of_time), ctx, &mut done), + unsafe { zs_rebuild_step(job, c.0, Some(out_of_time), ctx, &mut done) }, OK ); assert_eq!( @@ -527,6 +577,260 @@ fn a_chunked_rebuild_through_delay_is_one_tick() { "the last chunk is Done though the deadline expired" ); assert_eq!(c.take(s), one(&[(b"p", 1)])); - zs_rebuild_free(job); - zs_zset_free(snapshot); + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; +} + +#[test] +fn a_set_added_to_or_subtracted_from_itself_is_defined() { + let a = zset(&[(b"x", 2), (b"y", -3)]); + assert_eq!(unsafe { zs_zset_add(a, a) }, OK); + assert_eq!((weight(a, b"x"), weight(a, b"y")), (4, -6)); + assert_eq!(unsafe { zs_zset_sub(a, a) }, OK); + assert_eq!(len(a), 0); + let m = zset(&[(b"k", i64::MAX)]); + assert_eq!(unsafe { zs_zset_add(m, m) }, Code::Overflow as i32); + assert_eq!( + weight(m, b"k"), + i64::MAX, + "the failed self-add changed nothing" + ); + unsafe { zs_zset_free(a) }; + unsafe { zs_zset_free(m) }; +} + +#[test] +fn a_null_buffer_is_refused_before_it_is_written() { + let z = zset(&[(b"abc", 1)]); + let mut needed = 0; + assert_eq!( + unsafe { zs_zset_encode(z, null_mut(), 100, &mut needed) }, + Code::NullArgument as i32 + ); + assert_eq!( + unsafe { zs_zset_encode(z, null_mut(), 0, &mut needed) }, + Code::BufferTooSmall as i32, + "a null buffer still probes the size" + ); + assert_eq!(needed, 15); + let empty = zset(&[]); + assert_eq!( + unsafe { zs_zset_encode(empty, null_mut(), 0, &mut needed) }, + OK + ); + unsafe { zs_zset_free(z) }; + unsafe { zs_zset_free(empty) }; +} + +#[test] +fn a_null_out_pointer_is_refused_before_any_state_changes() { + let c = C::new(); + let i = c.input(); + assert_eq!( + unsafe { zs_circuit_input(c.0, null_mut()) }, + Code::NullArgument as i32 + ); + assert_eq!(c.input(), i + 1, "the refused call added no input"); + let s = c.sink(i); + c.push(i, &[(b"k", 1)]); + assert_eq!(unsafe { zs_circuit_step(c.0) }, OK); + assert_eq!( + unsafe { zs_circuit_take(c.0, s, null_mut()) }, + Code::NullArgument as i32 + ); + assert_eq!( + c.take(s), + one(&[(b"k", 1)]), + "the refused take left the sink alone" + ); + assert_eq!( + unsafe { zs_zset_new(null_mut()) }, + Code::NullArgument as i32 + ); + let snapshot = zset(&[(b"a", 1), (b"b", 1)]); + let mut job = null_mut(); + assert_eq!(unsafe { zs_rebuild_new(c.0, i, snapshot, 1, &mut job) }, OK); + assert_eq!( + unsafe { zs_rebuild_step(job, c.0, None, null_mut(), null_mut()) }, + Code::NullArgument as i32 + ); + let mut done = 0; + assert_eq!( + unsafe { zs_rebuild_step(job, c.0, None, null_mut(), &mut done) }, + OK + ); + assert_eq!(c.take(s).len(), 2, "the refused step consumed nothing"); + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; +} + +#[test] +fn a_rebuild_is_stepped_only_on_its_own_circuit() { + let (c1, c2) = (C::new(), C::new()); + let (i1, i2) = (c1.input(), c2.input()); + let (s1, s2) = (c1.sink(i1), c2.sink(i2)); + let snapshot = zset(&[(b"a", 1)]); + let mut job = null_mut(); + assert_eq!( + unsafe { zs_rebuild_new(c1.0, i1, snapshot, 1, &mut job) }, + OK + ); + let mut done = 0; + assert_eq!( + unsafe { zs_rebuild_step(job, c2.0, None, null_mut(), &mut done) }, + Code::BadHandle as i32 + ); + assert_eq!((c2.take(s2).len(), c1.take(s1).len()), (0, 0)); + assert_eq!( + unsafe { zs_rebuild_step(job, c1.0, None, null_mut(), &mut done) }, + OK + ); + assert_eq!(c1.take(s1), one(&[(b"a", 1)])); + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; +} + +#[test] +fn stream_ids_and_sink_ids_are_not_interchangeable() { + let c = C::new(); + let i = c.input(); + let s = c.sink(i); + let mut z = null_mut(); + assert_eq!( + unsafe { zs_circuit_take(c.0, i, &mut z) }, + Code::BadHandle as i32 + ); + let mut o = 0; + assert_eq!( + unsafe { zs_circuit_delay(c.0, s, &mut o) }, + Code::BadHandle as i32 + ); + assert_eq!( + unsafe { zs_circuit_output(c.0, s, &mut o) }, + Code::BadHandle as i32 + ); + let empty = zset(&[]); + assert_eq!( + unsafe { zs_circuit_push(c.0, s, empty) }, + Code::BadHandle as i32 + ); + unsafe { zs_zset_free(empty) }; +} + +extern "C" fn fails(_: *mut c_void, _: *const u8, _: usize) -> i32 { + -1 +} + +extern "C" fn flat_fails( + _: *mut c_void, + _: *const u8, + _: usize, + _: engine::Emit, + _: *mut c_void, +) -> i32 { + 7 +} + +extern "C" fn expire_fails(_: *mut c_void) -> i32 { + -1 +} + +#[test] +fn a_callback_that_reports_failure_fails_the_step_and_kills_the_circuit() { + let c = C::new(); + let i = c.input(); + let (mut f, mut m) = (0, 0); + assert_eq!( + unsafe { zs_circuit_filter(c.0, i, Some(fails), null_mut(), &mut f) }, + OK + ); + c.push(i, &[(b"k", 1)]); + assert_eq!(unsafe { zs_circuit_step(c.0) }, Code::Callback as i32); + assert_eq!(unsafe { zs_circuit_step(c.0) }, Code::Poisoned as i32); + let d = C::new(); + let j = d.input(); + assert_eq!( + unsafe { zs_circuit_flat_map(d.0, j, Some(flat_fails), null_mut(), &mut m) }, + OK + ); + d.push(j, &[(b"k", 1)]); + assert_eq!(unsafe { zs_circuit_step(d.0) }, Code::Callback as i32); + let e = C::new(); + let k = e.input(); + let snapshot = zset(&[(b"a", 1), (b"b", 1)]); + let mut job = null_mut(); + assert_eq!(unsafe { zs_rebuild_new(e.0, k, snapshot, 1, &mut job) }, OK); + let mut done = 0; + assert_eq!( + unsafe { zs_rebuild_step(job, e.0, Some(expire_fails), null_mut(), &mut done) }, + Code::Callback as i32 + ); + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; +} + +extern "C" fn reenters(ctx: *mut c_void, _: *const u8, _: usize) -> i32 { + let (c, seen) = unsafe { &mut *(ctx as *mut (*mut ZsCircuit, i32)) }; + let mut id = 0; + *seen = unsafe { zs_circuit_input(*c, &mut id) }; + 1 +} + +#[test] +fn a_callback_that_calls_back_into_its_circuit_gets_poisoned_and_changes_nothing() { + let c = C::new(); + let i = c.input(); + let mut state: (*mut ZsCircuit, i32) = (c.0, 0); + let mut f = 0; + let ctx = (&mut state as *mut (*mut ZsCircuit, i32)).cast(); + assert_eq!( + unsafe { zs_circuit_filter(c.0, i, Some(reenters), ctx, &mut f) }, + OK + ); + c.push(i, &[(b"k", 1)]); + assert_eq!( + unsafe { zs_circuit_step(c.0) }, + OK, + "the outer step completes" + ); + assert_eq!(state.1, Code::Poisoned as i32); + assert_eq!(c.input(), f + 1, "the re-entrant call added no input"); +} + +#[test] +fn a_count_or_sum_that_cancels_to_zero_keeps_its_row() { + let c = C::new(); + let i = c.input(); + let (mut n, mut t) = (0, 0); + assert_eq!(unsafe { zs_circuit_aggregate(c.0, i, 0, &mut n) }, OK); + assert_eq!(unsafe { zs_circuit_aggregate(c.0, i, 1, &mut t) }, OK); + let (sn, st) = (c.sink(n), c.sink(t)); + c.push( + i, + &[ + (&pair(b"g", &le(5)), 1), + (&pair(b"g", &le(-5)), 1), + (&pair(b"h", &le(1)), -1), + (&pair(b"h", &le(2)), 1), + ], + ); + assert_eq!(unsafe { zs_circuit_step(c.0) }, OK); + assert_eq!( + c.take(st), + one(&[(&pair(b"g", &le(0)), 1), (&pair(b"h", &le(1)), 1)]) + ); + assert_eq!( + c.take(sn), + one(&[(&pair(b"g", &le(2)), 1), (&pair(b"h", &le(0)), 1)]) + ); +} + +#[cfg(target_pointer_width = "64")] +#[test] +fn a_length_of_4_gib_or_more_is_refused_not_truncated() { + assert_eq!(engine::len32(u32::MAX as usize), Ok(u32::MAX)); + assert_eq!( + engine::len32(1usize << 32).unwrap_err().code, + Code::BadValue + ); } diff --git a/crates/zset-ffi/tests/header.rs b/crates/zset-ffi/tests/header.rs index fd1b43f8..6e58076e 100644 --- a/crates/zset-ffi/tests/header.rs +++ b/crates/zset-ffi/tests/header.rs @@ -1,5 +1,5 @@ -//! The checked-in C header matches the exported functions and constants. -//! Evidence: `context/test-evidence/zset-ffi.md`. +//! The checked-in C header matches the exported functions, their parameter and return +//! types, and the constants. Evidence: `context/test-evidence/zset-ffi.md`. use std::collections::BTreeMap; use zset_ffi::ZS_ABI_VERSION; @@ -8,37 +8,75 @@ use zset_ffi::engine::Code; const SRC: &str = include_str!("../src/lib.rs"); const HEADER: &str = include_str!("../include/zset_ffi.h"); -/// The comma-separated parameter count of the text between a function's parentheses. -fn arity(params: &str) -> usize { - let (mut depth, mut n, mut any) = (0i32, 1, false); +fn squash(s: &str) -> String { + s.split_whitespace().collect() +} + +/// Splits at the commas that are not inside brackets. +fn split_params(params: &str) -> Vec { + let (mut depth, mut cur, mut out) = (0i32, String::new(), Vec::new()); for c in params.chars() { match c { '(' | '<' => depth += 1, ')' | '>' => depth -= 1, - ',' if depth == 0 => n += 1, - c if !c.is_whitespace() => any = true, + ',' if depth == 0 => { + out.push(std::mem::take(&mut cur)); + continue; + } _ => {} } + cur.push(c); } - if any { - n - usize::from(params.trim_end().ends_with(',')) - } else { - 0 + if !cur.trim().is_empty() { + out.push(cur); } + out } -/// Every `pub extern "C" fn zs_*` in the source, with its parameter count; the -/// `unary_op!` expansions all take (circuit, stream, out). -fn exported() -> BTreeMap { +/// The C type a Rust ABI type must be declared as. Anything unlisted fails the test, so a +/// new type needs a line here and a conscious choice. +fn c_type(rust: &str) -> String { + let t = squash(rust); + let c = match t.as_str() { + "*mut*mutZsZSet" => "ZsZSet **", + "*mutZsZSet" => "ZsZSet *", + "*constZsZSet" => "const ZsZSet *", + "*mutZsCircuit" => "ZsCircuit *", + "*mut*mutZsCircuit" => "ZsCircuit **", + "*mutZsRebuild" => "ZsRebuild *", + "*mut*mutZsRebuild" => "ZsRebuild **", + "*constu8" => "const uint8_t *", + "*mutu8" => "uint8_t *", + "*mutc_void" => "void *", + "*mutu32" => "uint32_t *", + "*muti64" => "int64_t *", + "*mutusize" => "size_t *", + "*muti32" => "int32_t *", + "*constc_char" => "const char *", + "usize" => "size_t", + "i64" => "int64_t", + "u32" => "uint32_t", + "i32" => "int32_t", + "Option" => "ZsKeepFn", + "Option" => "ZsFlatMapFn", + "Option" => "ZsExpiredFn", + other => panic!("no C type is declared for the Rust type `{other}`"), + }; + squash(c) +} + +/// `name -> (return type, parameter types)` from the Rust source. +fn exported() -> BTreeMap)> { let mut out = BTreeMap::new(); - for part in SRC.split("pub extern \"C\" fn ").skip(1) { - if part.starts_with('$') { - continue; - } + let fns = SRC + .split("extern \"C\" fn ") + .skip(1) + .filter(|p| !p.starts_with('$')); + for part in fns { let name = part.split('(').next().unwrap().to_string(); - let params = &part[name.len() + 1..]; + let rest = &part[name.len() + 1..]; let mut depth = 1; - let end = params + let end = rest .char_indices() .find(|(_, c)| { depth += i32::from(*c == '(') - i32::from(*c == ')'); @@ -46,41 +84,69 @@ fn exported() -> BTreeMap { }) .unwrap() .0; - out.insert(name, arity(¶ms[..end])); + let params: Vec = split_params(&rest[..end]) + .iter() + .map(|p| c_type(p.split_once(':').unwrap().1)) + .collect(); + let ret = rest[end + 1..].split('{').next().unwrap().trim(); + let ret = match ret.strip_prefix("->") { + Some(r) => c_type(r), + None => "void".to_string(), + }; + out.insert(name, (ret, params)); } + // The `unary_op!` expansions all take (circuit, stream, out). for line in SRC.lines().filter(|l| l.starts_with("unary_op!(")) { let name = line["unary_op!(".len()..].split(',').next().unwrap(); - out.insert(name.to_string(), 3); + let params = ["ZsCircuit*", "uint32_t", "uint32_t*"] + .map(String::from) + .to_vec(); + out.insert(name.to_string(), ("int32_t".to_string(), params)); } out } -fn declared() -> BTreeMap { - let mut flat = HEADER.replace('\n', " "); +fn strip_comments(text: &str) -> String { + let mut flat = text.replace('\n', " "); while let Some(open) = flat.find("/*") { let close = flat[open..].find("*/").expect("an unterminated comment") + open + 2; flat.replace_range(open..close, " "); } + flat +} + +/// A header parameter without its name: `const ZsZSet *z` is `const ZsZSet *`. +fn param_type(p: &str) -> String { + let p = p.trim(); + let name_start = p + .rfind(|c: char| !(c.is_alphanumeric() || c == '_')) + .map_or(0, |i| i + 1); + if name_start == 0 { + return squash(p); + } + squash(&p[..name_start]) +} + +fn declared() -> BTreeMap)> { let mut out = BTreeMap::new(); - for decl in flat.split(';') { - let Some(open) = decl.find("(") else { continue }; + for decl in strip_comments(HEADER).split(';') { + let Some(open) = decl.find('(') else { continue }; let before = &decl[..open]; - let name = before - .rsplit(|c: char| !(c.is_alphanumeric() || c == '_')) - .next() - .unwrap(); + let name_at = before + .rfind(|c: char| !(c.is_alphanumeric() || c == '_')) + .map_or(0, |i| i + 1); + let name = &before[name_at..]; if !name.starts_with("zs_") || before.contains("typedef") || before.contains("(*") { continue; } + let ret = squash(&before[..name_at]); let params = decl[open + 1..].rsplit_once(')').unwrap().0; - out.insert( - name.to_string(), - if params.trim() == "void" { - 0 - } else { - arity(params) - }, - ); + let params = if params.trim() == "void" { + Vec::new() + } else { + split_params(params).iter().map(|p| param_type(p)).collect() + }; + out.insert(name.to_string(), (ret, params)); } out } @@ -97,7 +163,7 @@ fn defines() -> BTreeMap { } #[test] -fn the_header_declares_exactly_the_exported_functions_with_their_arity() { +fn the_header_declares_exactly_the_exported_functions_with_their_types() { let (src, hdr) = (exported(), declared()); assert!( src.len() > 30, @@ -110,22 +176,19 @@ fn the_header_declares_exactly_the_exported_functions_with_their_arity() { #[test] fn the_header_constants_are_the_rust_constants() { let d = defines(); - let codes = [ - ("ZS_ERR_NULL_ARGUMENT", Code::NullArgument), - ("ZS_ERR_OVERFLOW", Code::Overflow), - ("ZS_ERR_PANIC", Code::Panic), - ("ZS_ERR_BAD_HANDLE", Code::BadHandle), - ("ZS_ERR_BUFFER_TOO_SMALL", Code::BufferTooSmall), - ("ZS_ERR_BAD_VALUE", Code::BadValue), - ("ZS_ERR_POISONED", Code::Poisoned), - ]; - for (name, code) in codes { - assert_eq!(d[name], code as i64, "{name}"); + for code in Code::ALL { + assert_eq!( + d[code.header_name()], + *code as i64, + "{}", + code.header_name() + ); } + let err_lines = d.keys().filter(|k| k.starts_with("ZS_ERR_")).count(); assert_eq!( - d["ZS_ERR_POISONED"] as usize, - codes.len(), - "a new code needs a header line" + err_lines, + Code::ALL.len(), + "a header code without a Rust variant" ); assert_eq!(d["ZS_ABI_VERSION"], i64::from(ZS_ABI_VERSION)); assert_eq!(d["ZS_OK"], 0); diff --git a/crates/zset-ffi/tests/smoke.lisp b/crates/zset-ffi/tests/smoke.lisp index 15111673..09af9937 100644 --- a/crates/zset-ffi/tests/smoke.lisp +++ b/crates/zset-ffi/tests/smoke.lisp @@ -39,4 +39,31 @@ (check "error text" (and (search "overflow" (zs-last-error)) t) t) (check "unchanged" (progn (key-call #'zs-zset-weight z "k" w) (mem-ref w :int64)) 42)) (zs-zset-free z))) + +;;; A callback: the body is wrapped in handler-case and returns the failure value, because a +;;; non-local exit through the library would leave the circuit dead (ZS_ERR_POISONED). +(defcfun "zs_circuit_new" :int32 (out :pointer)) +(defcfun "zs_circuit_free" :void (c :pointer)) +(defcfun "zs_circuit_input" :int32 (c :pointer) (out :pointer)) +(defcfun "zs_circuit_filter" :int32 (c :pointer) (stream :uint32) (keep :pointer) (ctx :pointer) (out :pointer)) +(defcfun "zs_circuit_push" :int32 (c :pointer) (input :uint32) (z :pointer)) +(defcfun "zs_circuit_step" :int32 (c :pointer)) + +(defcallback keep-or-fail :int32 ((ctx :pointer) (key :pointer) (len :size)) + (declare (ignore ctx key len)) + (handler-case (error "a Lisp error inside a callback") + (error () -1))) + +(with-foreign-objects ((cout :pointer) (id :uint32) (zout :pointer)) + (check "circuit" (zs-circuit-new cout) 0) + (let ((c (mem-ref cout :pointer))) + (check "input" (zs-circuit-input c id) 0) + (check "filter" (zs-circuit-filter c (mem-ref id :uint32) (callback keep-or-fail) (null-pointer) id) 0) + (check "zset" (zs-zset-new zout) 0) + (key-call #'zs-zset-insert (mem-ref zout :pointer) "k" 1) + (check "push" (zs-circuit-push c 0 (mem-ref zout :pointer)) 0) + (check "step reports the callback's failure" (zs-circuit-step c) 8) + (check "the circuit is dead afterwards" (zs-circuit-step c) 7) + (zs-zset-free (mem-ref zout :pointer)) + (zs-circuit-free c))) (format t "zset smoke: all ok~%") From d01599d86cf808e2929ab5df839356b23e06b6d4 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-circuit)" Date: Sat, 10 Oct 2026 12:23:46 -0700 Subject: [PATCH 05/10] build(nix): regenerate Cargo.nix for zset, numtower and zset-ffi (improvise-bw4p.1) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- Cargo.nix | 839 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 749 insertions(+), 90 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index b78b6f31..17f0f403 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -149,6 +149,36 @@ rec { packageId = "improvise-ws-server"; }; + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; + "numtower" = rec { + packageId = "numtower"; + build = internal.buildRustCrateWithFeatures { + packageId = "numtower"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; + "zset" = rec { + packageId = "zset"; + build = internal.buildRustCrateWithFeatures { + packageId = "zset"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; + "zset-ffi" = rec { + packageId = "zset-ffi"; + build = internal.buildRustCrateWithFeatures { + packageId = "zset-ffi"; + }; + # Debug support which might change between releases. # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; @@ -403,6 +433,22 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "arrayvec" = rec { + crateName = "arrayvec"; + version = "0.7.8"; + edition = "2018"; + sha256 = "0mmd8lrijbvg1qp4c5zis5dq41a3mjv2rb6bxkyj9kwaw2k6gyyk"; + authors = [ + "bluss" + ]; + features = { + "borsh" = [ "dep:borsh" ]; + "default" = [ "std" ]; + "serde" = [ "dep:serde" ]; + "zeroize" = [ "dep:zeroize" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "atomic" = rec { crateName = "atomic"; version = "0.6.1"; @@ -603,6 +649,93 @@ rec { "zeroize" = [ "dep:zeroize" ]; }; }; + "borsh" = rec { + crateName = "borsh"; + version = "1.8.1"; + edition = "2018"; + crateBin = []; + sha256 = "02p2lhz49944wc6j5dzr42y1mkrv0znbicg3cl61b9bbda25sg2m"; + authors = [ + "Near Inc " + ]; + dependencies = [ + { + name = "borsh-derive"; + packageId = "borsh-derive"; + optional = true; + } + { + name = "bytes"; + packageId = "bytes"; + optional = true; + usesDefaultFeatures = false; + } + ]; + buildDependencies = [ + { + name = "cfg_aliases"; + packageId = "cfg_aliases"; + } + ]; + features = { + "ascii" = [ "dep:ascii" ]; + "borsh-derive" = [ "dep:borsh-derive" ]; + "bson" = [ "dep:bson" ]; + "bytes" = [ "dep:bytes" ]; + "default" = [ "std" ]; + "derive" = [ "borsh-derive" ]; + "hashbrown" = [ "dep:hashbrown" ]; + "indexmap" = [ "dep:indexmap" ]; + "std" = [ "bytes?/std" ]; + "unstable__schema" = [ "derive" "borsh-derive/schema" ]; + "uuid" = [ "dep:uuid" ]; + }; + resolvedDefaultFeatures = [ "borsh-derive" "derive" "std" "unstable__schema" ]; + }; + "borsh-derive" = rec { + crateName = "borsh-derive"; + version = "1.8.1"; + edition = "2018"; + sha256 = "0mlzrbwzcr7qaky1iap09m0nrzp6crjd79sinj4iz808cxjzxk8j"; + procMacro = true; + libName = "borsh_derive"; + authors = [ + "Near Inc " + ]; + dependencies = [ + { + name = "once_cell"; + packageId = "once_cell"; + } + { + name = "proc-macro-crate"; + packageId = "proc-macro-crate"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 3.0.6"; + features = [ "full" "fold" ]; + } + ]; + devDependencies = [ + { + name = "syn"; + packageId = "syn 3.0.6"; + features = [ "full" "fold" "parsing" ]; + } + ]; + features = { + }; + resolvedDefaultFeatures = [ "default" "schema" ]; + }; "bumpalo" = rec { crateName = "bumpalo"; version = "3.20.3"; @@ -2147,6 +2280,42 @@ rec { }; resolvedDefaultFeatures = [ "more_lengths" ]; }; + "getrandom 0.2.17" = rec { + crateName = "getrandom"; + version = "0.2.17"; + edition = "2018"; + sha256 = "1l2ac6jfj9xhpjjgmcx6s1x89bbnw9x6j9258yy6xjkzpq0bqapz"; + authors = [ + "The Rand Project Developers" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (target."unix" or false); + } + { + name = "wasi"; + packageId = "wasi"; + usesDefaultFeatures = false; + target = { target, features }: ("wasi" == target."os" or null); + } + ]; + features = { + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "js" = [ "wasm-bindgen" "js-sys" ]; + "js-sys" = [ "dep:js-sys" ]; + "rustc-dep-of-std" = [ "compiler_builtins" "core" "libc/rustc-dep-of-std" "wasi/rustc-dep-of-std" ]; + "wasm-bindgen" = [ "dep:wasm-bindgen" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "getrandom 0.3.4" = rec { crateName = "getrandom"; version = "0.3.4"; @@ -3412,6 +3581,28 @@ rec { }; resolvedDefaultFeatures = [ "default" "extra_traits" "std" ]; }; + "libloading" = rec { + crateName = "libloading"; + version = "0.8.9"; + edition = "2015"; + sha256 = "0mfwxwjwi2cf0plxcd685yxzavlslz7xirss3b9cbrzyk4hv1i6p"; + authors = [ + "Simonas Kazlauskas " + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + target = { target, features }: (target."unix" or false); + } + { + name = "windows-link"; + packageId = "windows-link"; + target = { target, features }: (target."windows" or false); + } + ]; + + }; "libm" = rec { crateName = "libm"; version = "0.2.16"; @@ -3785,6 +3976,39 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "std" ]; }; + "num-bigint" = rec { + crateName = "num-bigint"; + version = "0.4.8"; + edition = "2021"; + sha256 = "0ry3xjal8f5xhdinani268ci13h14mf7j4w0y1gflfzhw3knk7n8"; + libName = "num_bigint"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "default" = [ "std" ]; + "quickcheck" = [ "dep:quickcheck" ]; + "rand" = [ "dep:rand" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "num-integer/std" "num-traits/std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "num-conv" = rec { crateName = "num-conv"; version = "0.2.2"; @@ -3822,6 +4046,67 @@ rec { ]; }; + "num-integer" = rec { + crateName = "num-integer"; + version = "0.1.47"; + edition = "2018"; + sha256 = "02z1p3azy6p10n99skrab4a6hhfd4amf2i9gm8sxqd1p9dfxkqkw"; + libName = "num_integer"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "num-traits/std" ]; + }; + resolvedDefaultFeatures = [ "i128" "std" ]; + }; + "num-rational" = rec { + crateName = "num-rational"; + version = "0.4.2"; + edition = "2021"; + sha256 = "093qndy02817vpgcqjnj139im3jl7vkq4h68kykdqqh577d18ggq"; + libName = "num_rational"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "num-bigint"; + packageId = "num-bigint"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + features = { + "default" = [ "num-bigint" "std" ]; + "num-bigint" = [ "dep:num-bigint" ]; + "num-bigint-std" = [ "num-bigint/std" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "num-bigint?/std" "num-integer/std" "num-traits/std" ]; + }; + resolvedDefaultFeatures = [ "default" "num-bigint" "num-bigint-std" "std" ]; + }; "num-traits" = rec { crateName = "num-traits"; version = "0.2.19"; @@ -3841,7 +4126,7 @@ rec { "default" = [ "std" ]; "libm" = [ "dep:libm" ]; }; - resolvedDefaultFeatures = [ "default" "std" ]; + resolvedDefaultFeatures = [ "default" "i128" "std" ]; }; "num_threads" = rec { crateName = "num_threads"; @@ -3859,6 +4144,54 @@ rec { } ]; + }; + "numtower" = rec { + crateName = "numtower"; + version = "0.1.0"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./crates/numtower; }; + dependencies = [ + { + name = "num-bigint"; + packageId = "num-bigint"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + } + { + name = "num-rational"; + packageId = "num-rational"; + usesDefaultFeatures = false; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "rust_decimal"; + packageId = "rust_decimal"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + ]; + devDependencies = [ + { + name = "num-rational"; + packageId = "num-rational"; + features = [ "num-bigint-std" ]; + } + { + name = "proptest"; + packageId = "proptest"; + } + ]; + }; "once_cell" = rec { crateName = "once_cell"; @@ -4406,6 +4739,25 @@ rec { }; resolvedDefaultFeatures = [ "simd" "std" ]; }; + "proc-macro-crate" = rec { + crateName = "proc-macro-crate"; + version = "3.5.0"; + edition = "2021"; + sha256 = "0kv1g1d1zjwxlgcaba2qlshzyy32j03xic8rskqlcr5mnblsfyz6"; + libName = "proc_macro_crate"; + authors = [ + "Bastian Köcher " + ]; + dependencies = [ + { + name = "toml_edit"; + packageId = "toml_edit"; + usesDefaultFeatures = false; + features = [ "parse" ]; + } + ]; + + }; "proc-macro2" = rec { crateName = "proc-macro2"; version = "1.0.107"; @@ -4463,7 +4815,7 @@ rec { } { name = "rand_chacha"; - packageId = "rand_chacha"; + packageId = "rand_chacha 0.9.0"; usesDefaultFeatures = false; } { @@ -4616,6 +4968,19 @@ rec { "The Rust Project Developers" ]; dependencies = [ + { + name = "libc"; + packageId = "libc"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (target."unix" or false); + } + { + name = "rand_chacha"; + packageId = "rand_chacha 0.3.1"; + optional = true; + usesDefaultFeatures = false; + } { name = "rand_core"; packageId = "rand_core 0.6.4"; @@ -4632,7 +4997,7 @@ rec { "std" = [ "rand_core/std" "rand_chacha/std" "alloc" "getrandom" "libc" ]; "std_rng" = [ "rand_chacha" ]; }; - resolvedDefaultFeatures = [ "small_rng" ]; + resolvedDefaultFeatures = [ "alloc" "getrandom" "libc" "rand_chacha" "small_rng" "std" ]; }; "rand 0.9.5" = rec { crateName = "rand"; @@ -4646,7 +5011,7 @@ rec { dependencies = [ { name = "rand_chacha"; - packageId = "rand_chacha"; + packageId = "rand_chacha 0.9.0"; optional = true; usesDefaultFeatures = false; } @@ -4666,7 +5031,37 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "os_rng" "std" ]; }; - "rand_chacha" = rec { + "rand_chacha 0.3.1" = rec { + crateName = "rand_chacha"; + version = "0.3.1"; + edition = "2018"; + sha256 = "123x2adin558xbhvqb8w4f6syjsdkmqff8cxwhmjacpsl1ihmhg6"; + authors = [ + "The Rand Project Developers" + "The Rust Project Developers" + "The CryptoCorrosion Contributors" + ]; + dependencies = [ + { + name = "ppv-lite86"; + packageId = "ppv-lite86"; + usesDefaultFeatures = false; + features = [ "simd" ]; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + } + ]; + features = { + "default" = [ "std" ]; + "serde" = [ "dep:serde" ]; + "serde1" = [ "serde" ]; + "std" = [ "ppv-lite86/std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; + "rand_chacha 0.9.0" = rec { crateName = "rand_chacha"; version = "0.9.0"; edition = "2021"; @@ -4722,12 +5117,20 @@ rec { "The Rand Project Developers" "The Rust Project Developers" ]; + dependencies = [ + { + name = "getrandom"; + packageId = "getrandom 0.2.17"; + optional = true; + } + ]; features = { "getrandom" = [ "dep:getrandom" ]; "serde" = [ "dep:serde" ]; "serde1" = [ "serde" ]; "std" = [ "alloc" "getrandom" "getrandom/std" ]; }; + resolvedDefaultFeatures = [ "alloc" "getrandom" "std" ]; }; "rand_core 0.9.5" = rec { crateName = "rand_core"; @@ -5311,6 +5714,147 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ]; }; + "rust_decimal" = rec { + crateName = "rust_decimal"; + version = "1.43.0"; + edition = "2021"; + sha256 = "0fnd56zidby90qy98ffkmx7rjcvng7ssdglz37f43b6wflp2flvn"; + authors = [ + "Paul Mason " + ]; + dependencies = [ + { + name = "arrayvec"; + packageId = "arrayvec"; + usesDefaultFeatures = false; + } + { + name = "borsh"; + packageId = "borsh"; + optional = true; + usesDefaultFeatures = false; + features = [ "derive" "unstable__schema" ]; + } + { + name = "bytes"; + packageId = "bytes"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "rand"; + packageId = "rand 0.8.8"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "rand"; + packageId = "rand 0.9.5"; + rename = "rand-0_9"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde_json"; + packageId = "serde_json"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: ("wasm32" == target."arch" or null); + } + ]; + devDependencies = [ + { + name = "bytes"; + packageId = "bytes"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch" or null)); + } + { + name = "rand"; + packageId = "rand 0.8.8"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch" or null)); + features = [ "getrandom" ]; + } + { + name = "rand"; + packageId = "rand 0.9.5"; + rename = "rand-0_9"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch" or null)); + features = [ "thread_rng" ]; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + usesDefaultFeatures = false; + target = { target, features }: ("wasm32" == target."arch" or null); + } + ]; + features = { + "borsh" = [ "dep:borsh" "std" ]; + "bytemuck" = [ "c-repr" "dep:bytemuck" "dep:bytemuck_derive" ]; + "db-diesel-mysql" = [ "diesel/mysql_backend" "std" ]; + "db-diesel-postgres" = [ "diesel/postgres_backend" "std" ]; + "db-diesel2-mysql" = [ "db-diesel-mysql" ]; + "db-diesel2-postgres" = [ "db-diesel-postgres" ]; + "db-postgres" = [ "dep:bytes" "dep:postgres-types" "std" ]; + "db-tokio-postgres" = [ "dep:bytes" "dep:postgres-types" "std" ]; + "default" = [ "serde" "std" ]; + "diesel" = [ "dep:diesel" ]; + "macros" = [ "dep:rust_decimal_macros" ]; + "maths-nopanic" = [ "maths" ]; + "ndarray" = [ "dep:ndarray" ]; + "proptest" = [ "dep:proptest" ]; + "rand" = [ "dep:rand" ]; + "rand-0_9" = [ "dep:rand-0_9" ]; + "rocket-traits" = [ "dep:rocket" "std" ]; + "rust-fuzz" = [ "dep:arbitrary" ]; + "serde" = [ "dep:serde" "wasm-bindgen?/serde" ]; + "serde-arbitrary-precision" = [ "serde-with-arbitrary-precision" ]; + "serde-bincode" = [ "serde-str" ]; + "serde-float" = [ "serde-with-float" ]; + "serde-str" = [ "serde-with-str" ]; + "serde-with-arbitrary-precision" = [ "serde" "serde_json/arbitrary_precision" "serde_json/std" "zmij" ]; + "serde-with-float" = [ "serde" ]; + "serde-with-str" = [ "serde" ]; + "serde_json" = [ "dep:serde_json" ]; + "std" = [ "arrayvec/std" "wasm-bindgen?/std" "borsh?/std" "bytes?/std" "rand?/std" "rand-0_9?/std" "serde?/std" "serde_json?/std" ]; + "tokio-pg" = [ "db-tokio-postgres" ]; + "tokio-postgres" = [ "dep:tokio-postgres" ]; + "wasm" = [ "dep:wasm-bindgen" ]; + "zmij" = [ "dep:zmij" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "rustc_version" = rec { crateName = "rustc_version"; version = "0.4.1"; @@ -6110,7 +6654,7 @@ rec { "proc-macro" = [ "proc-macro2/proc-macro" "quote?/proc-macro" ]; "test" = [ "syn-test-suite/all-features" ]; }; - resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; + resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "fold" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; "tempfile" = rec { crateName = "tempfile"; @@ -6785,6 +7329,82 @@ rec { }; resolvedDefaultFeatures = [ "connect" "default" "handshake" "stream" ]; }; + "toml_datetime" = rec { + crateName = "toml_datetime"; + version = "1.1.2+spec-1.1.0"; + edition = "2024"; + sha256 = "0lrhcmqvhjr259w4f2kijya7fgi0kpmhg9fb3m144v3cj1kxg1ib"; + dependencies = [ + { + name = "serde_core"; + packageId = "serde_core"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "serde_core?/alloc" ]; + "default" = [ "std" ]; + "serde" = [ "dep:serde_core" ]; + "std" = [ "alloc" "serde_core?/std" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; + "toml_edit" = rec { + crateName = "toml_edit"; + version = "0.25.17+spec-1.1.0"; + edition = "2024"; + sha256 = "174w1b2fjg1jb0wi7384v1m59g7ga79458i022gafjakpddisr73"; + dependencies = [ + { + name = "indexmap"; + packageId = "indexmap"; + features = [ "std" ]; + } + { + name = "toml_datetime"; + packageId = "toml_datetime"; + } + { + name = "toml_parser"; + packageId = "toml_parser"; + optional = true; + } + { + name = "winnow"; + packageId = "winnow"; + optional = true; + } + ]; + features = { + "debug" = [ "toml_parser?/debug" "dep:anstream" "dep:anstyle" "display" ]; + "default" = [ "parse" "display" ]; + "display" = [ "dep:toml_writer" ]; + "parse" = [ "dep:toml_parser" "dep:winnow" ]; + "serde" = [ "dep:serde_core" "toml_datetime/serde" "dep:serde_spanned" ]; + }; + resolvedDefaultFeatures = [ "parse" ]; + }; + "toml_parser" = rec { + crateName = "toml_parser"; + version = "1.1.5+spec-1.1.0"; + edition = "2024"; + sha256 = "0k3lljyi4zxchdklaqghkwbl7wkd2ab1w16hlyniqzid0fl979ms"; + dependencies = [ + { + name = "winnow"; + packageId = "winnow"; + usesDefaultFeatures = false; + } + ]; + features = { + "debug" = [ "std" "dep:anstream" "dep:anstyle" ]; + "default" = [ "std" ]; + "simd" = [ "winnow/simd" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; "tungstenite" = rec { crateName = "tungstenite"; version = "0.30.0"; @@ -8447,6 +9067,31 @@ rec { }; resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; + "winnow" = rec { + crateName = "winnow"; + version = "1.0.4"; + edition = "2021"; + sha256 = "10fzxipa7lx16172p3aca9j60hzbqgjki2f95kqksd5qywcp7f93"; + dependencies = [ + { + name = "memchr"; + packageId = "memchr"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "ascii" = [ "parser" ]; + "binary" = [ "parser" ]; + "debug" = [ "std" "dep:anstream" "dep:anstyle" "dep:is_terminal_polyfill" "dep:terminal_size" ]; + "default" = [ "std" "ascii" "binary" ]; + "simd" = [ "dep:memchr" ]; + "std" = [ "alloc" "memchr?/std" ]; + "unstable-doc" = [ "alloc" "std" "ascii" "binary" "simd" "unstable-recover" ]; + "unstable-recover" = [ "parser" ]; + }; + resolvedDefaultFeatures = [ "alloc" "ascii" "binary" "default" "parser" "std" ]; + }; "wit-bindgen" = rec { crateName = "wit-bindgen"; version = "0.57.1"; @@ -8559,6 +9204,39 @@ rec { "no-panic" = [ "dep:no-panic" ]; }; }; + "zset" = rec { + crateName = "zset"; + version = "0.1.0"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./crates/zset; }; + devDependencies = [ + { + name = "proptest"; + packageId = "proptest"; + } + ]; + + }; + "zset-ffi" = rec { + crateName = "zset-ffi"; + version = "0.1.0"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./crates/zset-ffi; }; + libName = "zset_ffi";type = [ "cdylib" "staticlib" "rlib" ]; + dependencies = [ + { + name = "zset"; + packageId = "zset"; + } + ]; + devDependencies = [ + { + name = "libloading"; + packageId = "libloading"; + } + ]; + + }; }; # @@ -8887,15 +9565,7 @@ rec { ) crateConfigs; target = makeTarget pkgs.stdenv.hostPlatform; - # Build-time dependency graph (for proc-macros and build - # dependencies). When not cross-compiling it equals the host - # graph, so reuse `self`; otherwise build it for - # `pkgs.buildPackages`. - build = - if pkgs.stdenv.buildPlatform.config == pkgs.stdenv.hostPlatform.config then - self - else - mkBuiltByPackageIdByPkgs pkgs.buildPackages; + build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; in self; @@ -8911,35 +9581,38 @@ rec { devDependencies = lib.optionals (runTests && packageId == rootPackageId) ( crateConfig'.devDependencies or [ ] ); - # Enabled (platform- and feature-filtered) dependency lists, reused - # for both derivation wiring and the crate renames below. - enabledDependencies = filterEnabledDependencies { + dependencies = dependencyDerivations { inherit features; inherit (self) target; + buildByPackageId = + depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false then + self.build.crates.${depPackageId} + else + self.crates.${depPackageId}; dependencies = (crateConfig.dependencies or [ ]) ++ devDependencies; }; - enabledBuildDependencies = filterEnabledDependencies { + buildDependencies = dependencyDerivations { inherit features; inherit (self.build) target; + buildByPackageId = depPackageId: self.build.crates.${depPackageId}; dependencies = crateConfig.buildDependencies or [ ]; }; - dependencies = map - ( - dependency: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${dependency.packageId}.procMacro or false then - self.build.crates.${dependency.packageId} - else - self.crates.${dependency.packageId} - ) - enabledDependencies; - buildDependencies = map - (dependency: self.build.crates.${dependency.packageId}) - enabledBuildDependencies; - # Order (build dependencies, then normal dependencies) feeds the - # crateRenames grouping below. dependenciesWithRenames = - lib.filter (d: d ? "rename") (enabledBuildDependencies ++ enabledDependencies); + let + buildDeps = filterEnabledDependencies { + inherit features; + inherit (self) target; + dependencies = crateConfig.dependencies or [ ] ++ devDependencies; + }; + hostDeps = filterEnabledDependencies { + inherit features; + inherit (self.build) target; + dependencies = crateConfig.buildDependencies or [ ]; + }; + in + lib.filter (d: d ? "rename") (hostDeps ++ buildDeps); # Crate renames have the form: # # { @@ -9111,17 +9784,6 @@ rec { corresponding feature sets are merged. Features in rust are additive. */ mergePackageFeatures = - args: builtins.mapAttrs (_packageId: builtins.attrNames) (mergePackageFeaturesImpl args); - - /* - Core of the feature-resolution fixpoint. The cache (`featuresByPackageId`) - maps each packageId to a feature *set* (an attrset `feature -> 1`) rather - than a sorted list, so the fold merges with `//` and detects convergence - with attrset equality instead of re-concatenating and re-sorting the - accumulated feature list on every step. `mergePackageFeatures` projects the - result back to canonical sorted lists. - */ - mergePackageFeaturesImpl = { crateConfigs ? crates , packageId , rootPackageId ? packageId @@ -9170,15 +9832,14 @@ rec { cache: { packageId, features }: let - cacheFeatures = cache.${packageId} or { }; - # `features` is the (small) incoming list; merge it into the set. - combinedFeatures = cacheFeatures // listToSet features; + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); in - if cache ? ${packageId} && cacheFeatures == combinedFeatures then + if cache ? ${packageId} && cache.${packageId} == combinedFeatures then cache else - mergePackageFeaturesImpl { - features = builtins.attrNames combinedFeatures; + mergePackageFeatures { + features = combinedFeatures; featuresByPackageId = cache; inherit crateConfigs @@ -9191,8 +9852,8 @@ rec { ); cacheWithSelf = let - cacheFeatures = featuresByPackageId.${packageId} or { }; - combinedFeatures = cacheFeatures // listToSet enabledFeatures; + cacheFeatures = featuresByPackageId.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); in featuresByPackageId // { @@ -9219,31 +9880,27 @@ rec { assert (builtins.isList features); assert (builtins.isAttrs target); - let - # Identical for every dep in this call; build the predicate arg once. - targetArgs = { inherit features target; }; - in lib.filter ( dep: - (dep.target or (features: true)) targetArgs + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } && (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep) features) ) dependencies; # Returns whether the given feature should enable the given dependency. doesFeatureEnableDependency = - dependency: - # Callers partially apply this once per dependency, then test every feature, - # so hoist the dep-invariant strings out of the per-feature comparison. + dependency: feature: let name = dependency.rename or dependency.name; - depName = "dep:" + name; prefix = "${name}/"; len = builtins.stringLength prefix; + startsWithPrefix = builtins.substring 0 len feature == prefix; in - feature: - feature == name || feature == depName || builtins.substring 0 len feature == prefix; + feature == name || feature == "dep:" + name || startsWithPrefix; /* Returns the expanded features for the given inputFeatures by applying the @@ -9257,24 +9914,29 @@ rec { assert (builtins.isAttrs featureMap); assert (builtins.isList inputFeatures); let - # Transitive closure of `inputFeatures` under `featureMap`. `seen` - # tracks already-expanded features so each is visited at most once; - # this also breaks feature cycles (issue #209). expandFeaturesNoCycle = - seen: features: - builtins.foldl' - ( - acc: feature: - if acc ? ${feature} then - acc - else - expandFeaturesNoCycle (acc // { ${feature} = 1; }) (featureMap.${feature} or [ ]) - ) - seen - features; - seen = expandFeaturesNoCycle { } inputFeatures; + oldSeen: inputFeatures: + if inputFeatures != [ ] then + let + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { + ${feature} = 1; + }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + in + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else + [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; in - sortedUnique (builtins.attrNames seen); + sortedUnique outFeatures; /* This function adds optional dependencies as features if they are enabled @@ -9333,19 +9995,16 @@ rec { in defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - # Builds a feature set (attrset `feature -> 1`) from a list of feature names. - listToSet = - features: - builtins.listToAttrs (map (feature: { name = feature; value = 1; }) features); - # Sorts and removes duplicates from a list of strings. sortedUnique = features: assert (builtins.isList features); - # attrNames returns keys in ascending string order, so the result is - # sorted and deduplicated. The feature-merge fixpoint relies on this - # canonical order to detect convergence. - builtins.attrNames (listToSet features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; deprecationWarning = message: value: From 42d808732b20f39e39890ed06d813e177b85c8e5 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-circuit)" Date: Sat, 10 Oct 2026 12:35:58 -0700 Subject: [PATCH 06/10] fix(zset-ffi): first fault wins, no free or re-entry under a running step, circuit ids, interrupted rebuilds, typed callbacks (improvise-bw4p.1) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/code/zset-ffi.md | 23 +++-- context/test-evidence/zset-ffi.md | 34 ++++++++ crates/zset-ffi/include/zset_ffi.h | 16 ++-- crates/zset-ffi/src/engine.rs | 41 ++++++--- crates/zset-ffi/src/lib.rs | 39 +++++++-- crates/zset-ffi/tests/abi.rs | 131 +++++++++++++++++++++++++++++ crates/zset-ffi/tests/header.rs | 84 ++++++++++++++++-- 7 files changed, 333 insertions(+), 35 deletions(-) diff --git a/context/code/zset-ffi.md b/context/code/zset-ffi.md index 649bb2af..b7e36581 100644 --- a/context/code/zset-ffi.md +++ b/context/code/zset-ffi.md @@ -62,7 +62,12 @@ link, so they call the `extern "C"` functions as plain functions. duration of `step` and `rebuild_step`, and cleared only when they return. Every entry point first reads it, so a circuit whose frames were skipped answers `ZS_ERR_POISONED` forever, and a callback that re-enters its own circuit gets `ZS_ERR_POISONED` without the - library touching the engine. The CL rule is to put `handler-case` *inside* the callback and + library touching the engine. `zs_rebuild_step` reads `circuit(c)` before it borrows the job, + so re-entry from an `expired` callback is refused first. `zs_circuit_free` on a mid-step + circuit and `zs_rebuild_free` on a job being stepped do nothing: the object is leaked, not + freed under the frames that are running. The first fault recorded in a step is the one + reported (`note`), so a failing `index_by` callback is `ZS_ERR_CALLBACK`, not the + consequent `BAD_VALUE`. The CL rule is to put `handler-case` *inside* the callback and return the failure value (`tests/smoke.lisp` does). `index_by` needs exactly one emitted key and a keyed stream valid pairs, else `ZS_ERR_BAD_VALUE`; these faults are recorded in a shared cell rather than panicking. @@ -87,8 +92,9 @@ last chunk ran, whatever the deadline callback says. All exported functions are `pub unsafe extern "C" fn`: the rlib is public API, and a safe Rust caller could otherwise pass any pointer. Output pointers are checked before any work (`with_out`): a refused call changes nothing and leaks nothing. Stream ids and sink ids are -different spaces (sink ids carry the top bit). A `ZsRebuild` records its circuit and is -refused on another. An id from another circuit is not detected. `zs_zset_add(a, a)` is +different spaces (sink ids carry the top bit). A `ZsRebuild` records its circuit's id (a process-wide counter, not an address, which can be +reused) and is refused on another. A `Circuit::step` between two `zs_rebuild_step` calls is +`ZS_ERR_INTERRUPTED`, without poisoning: zset's `Rebuild::interrupted`, checked first. An id from another circuit is not detected. `zs_zset_add(a, a)` is defined: the sum is computed through shared references and then stored. Keys of 4 GiB or longer are `ZS_ERR_BAD_VALUE`, not truncated. A non-empty aggregate group always has a row, even when its count or sum is zero. @@ -96,14 +102,17 @@ even when its count or sum is zero. ## Miri Miri checks the aliasing, leak and null rules the C compiler cannot. It runs `tests/abi.rs` -on a nightly built through the flake's `rust-overlay` input (passes at this commit, 12s): +on a nightly built through the flake's `rust-overlay` input. The recipe below is **not +pinned**: `builtins.getFlake "github:oxalica/rust-overlay"` and `` (hence `--impure`) +resolve to whatever is current, and only the nightly date is fixed. It passed at the date of +this commit (about 12s): ``` -nix shell --impure --expr 'let p = import { overlays = [ (builtins.getFlake "github:oxalica/rust-overlay").overlays.default ]; }; in p.rust-bin.nightly."2026-09-24".default.override { extensions = [ "miri" "rust-src" ]; }' --command cargo miri test -p zset-ffi --test abi +nix shell --impure --expr 'let p = import { overlays = [ (builtins.getFlake "github:oxalica/rust-overlay").overlays.default ]; }; in p.rust-bin.nightly."2026-09-24".default.override { extensions = [ "miri" "rust-src" ]; }' --command cargo miri test -p zset-ffi --lib --test abi ``` -It is not a CI job. A proposed one: a `miri` job in `.github/workflows/ci.yml` running that -command in `nix develop .#ci`. +It is not a CI job. A proposed one: a `miri` job in `.github/workflows/ci.yml` that takes the +nightly from the flake's own pinned `rust-overlay` input in `nix develop .#ci`. ## Calling it from Common Lisp diff --git a/context/test-evidence/zset-ffi.md b/context/test-evidence/zset-ffi.md index 3f9a7031..b1a83910 100644 --- a/context/test-evidence/zset-ffi.md +++ b/context/test-evidence/zset-ffi.md @@ -132,3 +132,37 @@ Codex's finding: `Some(0)` was filtered away for a non-empty group. ### a_length_of_4_gib_or_more_is_refused_not_truncated `len32` refuses `1 << 32` and accepts `u32::MAX`. + +## Delta review of #347 (9bcfa9d1) + +**Injected**, one at a time: a fault that overwrites the first (`note` replaced by a plain +`set`) fails `an_index_by_callback_that_fails_reports_the_callback_code_not_a_later_fault`; +`zs_rebuild_step` borrowing the job before checking the circuit (Miri only), and a `free` that ignores +`in_step` or `busy`, fail `a_callback_cannot_step_or_free_the_job_or_circuit_it_runs_under` +(and under Miri, as use-after-free); an owner compared by address instead of id fails +`a_circuit_freed_and_another_made_does_not_inherit_its_jobs`; the interrupted check removed +fails `a_host_step_between_rebuild_calls_is_interrupted_not_a_poison`; a retyped +`ZsKeepFn` fails `the_header_callback_typedefs_match_the_rust_callback_types`. + +### an_index_by_callback_that_fails_reports_the_callback_code_not_a_later_fault + +The callback fails (returns 3) and emits nothing, so `index_by` also sees zero keys. The step +must report `ZS_ERR_CALLBACK`, the first fault, not the consequent `BAD_VALUE`. + +### a_callback_cannot_step_or_free_the_job_or_circuit_it_runs_under + +An `expired` callback steps its own job, frees the job and frees the circuit. The step is +refused with `ZS_ERR_POISONED`, both frees do nothing, and the job then finishes normally. + +### a_circuit_freed_and_another_made_does_not_inherit_its_jobs + +A job outlives its circuit; sixty-four new circuits (some at the old address) all refuse it. + +### a_host_step_between_rebuild_calls_is_interrupted_not_a_poison + +`ZS_ERR_INTERRUPTED`, and the circuit still steps afterwards. + +### the_header_callback_typedefs_match_the_rust_callback_types + +Retyping `ZsKeepFn`'s return in the header fails. The `unary_op!` signatures are now read +from the macro body rather than written into the test. diff --git a/crates/zset-ffi/include/zset_ffi.h b/crates/zset-ffi/include/zset_ffi.h index bcb61b20..ae38877f 100644 --- a/crates/zset-ffi/include/zset_ffi.h +++ b/crates/zset-ffi/include/zset_ffi.h @@ -21,8 +21,8 @@ * a reference count): keep the whole group on one thread at a time. Z-sets are separate * handles. Streams, inputs and sinks are uint32_t ids local to their circuit; a sink id is * never a stream id (they are refused as ZS_ERR_BAD_HANDLE), but an id from another circuit - * is not detected. A ZsRebuild is stepped only on its own circuit (else ZS_ERR_BAD_HANDLE); - * free the jobs before the circuit. + * is not detected. A ZsRebuild is stepped only on the circuit that made it (else + * ZS_ERR_BAD_HANDLE), even if another circuit now lives at the old address. * - Keys of 4 GiB or longer are refused with ZS_ERR_BAD_VALUE: the wire form has u32 lengths. */ #ifndef ZSET_FFI_H @@ -46,6 +46,7 @@ extern "C" { #define ZS_ERR_BAD_VALUE 6 #define ZS_ERR_POISONED 7 #define ZS_ERR_CALLBACK 8 +#define ZS_ERR_INTERRUPTED 9 #define ZS_AGG_COUNT 0 #define ZS_AGG_SUM 1 @@ -66,8 +67,11 @@ typedef struct ZsRebuild ZsRebuild; * - A callback MUST return normally: no longjmp, no C++ exception, no Lisp non-local exit * through it (wrap the body in handler-case and return the failure value). If one does, * the circuit is left marked mid-step and every later call on it is ZS_ERR_POISONED. - * - A callback must not call back into the library for its own circuit: that call returns - * ZS_ERR_POISONED. */ + * - A callback that calls the library with its own circuit (or the rebuild job being + * stepped) gets ZS_ERR_POISONED and nothing changes. zs_circuit_free on a circuit that is + * mid-step, and zs_rebuild_free on a job being stepped, do nothing (the object is leaked + * rather than freed under the running frames); free after the call returns. + * A circuit whose step was left by a non-local exit stays mid-step, so its free leaks too. */ typedef void (*ZsEmit)(void *emit_ctx, const uint8_t *bytes, size_t len); typedef int32_t (*ZsFlatMapFn)(void *ctx, const uint8_t *key, size_t len, ZsEmit emit, void *emit_ctx); /* 1 keep, 0 drop, negative fail. */ @@ -132,7 +136,9 @@ int32_t zs_circuit_step(ZsCircuit *c); int32_t zs_circuit_take(ZsCircuit *c, uint32_t sink, ZsZSet **out); /* Resumable snapshot load: chunk keys per piece (0 reads as 1). The whole load is one - * logical tick. done is 1 as soon as the last chunk ran. */ + * logical tick, and done is 1 as soon as the last chunk ran. While a job is pending it + * owns the clock: a zs_circuit_step in between makes the next zs_rebuild_step return + * ZS_ERR_INTERRUPTED (the circuit stays usable; drop the job). */ int32_t zs_rebuild_new(ZsCircuit *c, uint32_t input, const ZsZSet *snapshot, size_t chunk, ZsRebuild **out); int32_t zs_rebuild_step(ZsRebuild *job, ZsCircuit *c, ZsExpiredFn expired, void *ctx, int32_t *done); void zs_rebuild_free(ZsRebuild *job); diff --git a/crates/zset-ffi/src/engine.rs b/crates/zset-ffi/src/engine.rs index 5655f8c5..68f90e50 100644 --- a/crates/zset-ffi/src/engine.rs +++ b/crates/zset-ffi/src/engine.rs @@ -39,6 +39,7 @@ codes! { BadValue = 6 => "ZS_ERR_BAD_VALUE", Poisoned = 7 => "ZS_ERR_POISONED", Callback = 8 => "ZS_ERR_CALLBACK", + Interrupted = 9 => "ZS_ERR_INTERRUPTED", } #[derive(Clone, Debug, PartialEq, Eq)] @@ -134,6 +135,15 @@ struct Ctx(*mut c_void); type Fault = Rc>>; +/// Records a fault unless one is already recorded: the first one is the cause. +fn note(fault: &Fault, code: Code, msg: &'static str) { + if let Some(first) = fault.take() { + fault.set(Some(first)); + } else { + fault.set(Some((code, msg))); + } +} + extern "C" fn collect(ctx: *mut c_void, p: *const u8, n: usize) { // SAFETY: `ctx` is the `Vec` `images` passes below, alive for the callback's duration. let out = unsafe { &mut *(ctx as *mut Vec) }; @@ -157,7 +167,7 @@ fn images(f: FlatMapFn, ctx: Ctx, key: &Bytes, fault: &Fault) -> Vec { &mut out as *mut Vec as *mut c_void, ); if rc != 0 { - fault.set(Some((Code::Callback, "a callback reported failure"))); + note(fault, Code::Callback, "a callback reported failure"); out.clear(); } out @@ -299,7 +309,7 @@ impl Engine { 0 => false, n if n > 0 => true, _ => { - fault.set(Some((Code::Callback, "a callback reported failure"))); + note(&fault, Code::Callback, "a callback reported failure"); false } } @@ -325,16 +335,17 @@ impl Engine { let out = self.circuit.flat_map(&s, move |k: &Bytes| { let mut gs = images(f, ctx, k, &fault); if gs.len() != 1 { - fault.set(Some(( + note( + &fault, Code::BadValue, "index_by: the key callback must emit exactly one key", - ))); + ); return Vec::new(); } match pair(&gs.remove(0), k) { Ok(p) => vec![p], Err(_) => { - fault.set(Some((Code::BadValue, "a key is 4 GiB or longer"))); + note(&fault, Code::BadValue, "a key is 4 GiB or longer"); Vec::new() } } @@ -348,10 +359,11 @@ impl Engine { self.circuit.flat_map(s, move |b: &Bytes| match unpair(b) { Some((k, v)) => vec![(k.to_vec(), v.to_vec())], None => { - fault.set(Some(( + note( + &fault, Code::BadValue, "a keyed stream carried an element that is not a pair", - ))); + ); Vec::new() } }) @@ -367,7 +379,7 @@ impl Engine { .circuit .join_indexed(&ka, &kb, move |k: &Bytes, x: &Bytes, y: &Bytes| { pair(x, y).and_then(|xy| pair(k, &xy)).unwrap_or_else(|_| { - fault.set(Some((Code::BadValue, "a key is 4 GiB or longer"))); + note(&fault, Code::BadValue, "a key is 4 GiB or longer"); Vec::new() }) }); @@ -393,7 +405,7 @@ impl Engine { match <[u8; 8]>::try_from(v.as_slice()) { Ok(b) => nums.push((i64::from_le_bytes(b), *w)), Err(_) => { - fault.set(Some((Code::BadValue, "an aggregated value is not 8 bytes"))); + note(&fault, Code::BadValue, "an aggregated value is not 8 bytes"); return None; } } @@ -418,7 +430,7 @@ impl Engine { match pair(k, &r.to_le_bytes()) { Ok(p) => vec![p], Err(_) => { - fault.set(Some((Code::BadValue, "a key is 4 GiB or longer"))); + note(&fault, Code::BadValue, "a key is 4 GiB or longer"); Vec::new() } } @@ -482,6 +494,13 @@ impl Engine { job: &mut Rebuild, expired: Option<(ExpiredFn, *mut c_void)>, ) -> Res { + self.alive()?; + if job.interrupted(&self.circuit) { + return Err(Fail::new( + Code::Interrupted, + "the rebuild was interrupted by a circuit step before it finished", + )); + } let fault = Rc::clone(&self.fault); let progress = self.guarded(|c| { job.step(c, || match expired { @@ -490,7 +509,7 @@ impl Engine { 0 => false, n if n > 0 => true, _ => { - fault.set(Some((Code::Callback, "a callback reported failure"))); + note(&fault, Code::Callback, "a callback reported failure"); true } }, diff --git a/crates/zset-ffi/src/lib.rs b/crates/zset-ffi/src/lib.rs index 38c29ab5..835e5e45 100644 --- a/crates/zset-ffi/src/lib.rs +++ b/crates/zset-ffi/src/lib.rs @@ -9,6 +9,7 @@ use engine::{ use std::cell::{Cell, RefCell, UnsafeCell}; use std::ffi::{CString, c_char, c_void}; use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicU64, Ordering}; use zset::{Rebuild, ZSet}; pub const ZS_ABI_VERSION: u32 = 1; @@ -21,12 +22,16 @@ pub struct ZsZSet(BZ); pub struct ZsCircuit { engine: UnsafeCell, in_step: Cell, + id: u64, } +static NEXT_CIRCUIT_ID: AtomicU64 = AtomicU64::new(1); + /// An opaque resumable snapshot load, bound to the circuit that made it. pub struct ZsRebuild { job: Rebuild, - owner: *const ZsCircuit, + owner: u64, + busy: Cell, } thread_local! { @@ -318,6 +323,7 @@ pub unsafe extern "C" fn zs_circuit_new(out: *mut *mut ZsCircuit) -> i32 { boxed(ZsCircuit { engine: UnsafeCell::new(Engine::new()), in_step: Cell::new(false), + id: NEXT_CIRCUIT_ID.fetch_add(1, Ordering::Relaxed), }) }) } @@ -328,7 +334,10 @@ pub unsafe extern "C" fn zs_circuit_new(out: *mut *mut ZsCircuit) -> i32 { /// `c` is null, or a live circuit freed once and not mid-step. #[unsafe(no_mangle)] pub unsafe extern "C" fn zs_circuit_free(c: *mut ZsCircuit) { - // SAFETY: the caller's contract. + // SAFETY: the caller's contract; a mid-step circuit is leaked, never freed under its frames. + if !c.is_null() && unsafe { &*c }.in_step.get() { + return; + } unsafe { release(c) } } @@ -520,7 +529,11 @@ pub unsafe extern "C" fn zs_rebuild_new( with_out(out, || { let (cr, snap) = (circuit(c)?, as_ref(snapshot)?); let job = cr.engine().start_rebuild(input, snap.0.clone(), chunk)?; - boxed(ZsRebuild { job, owner: c }) + boxed(ZsRebuild { + job, + owner: cr.id, + busy: Cell::new(false), + }) }) } }) @@ -540,15 +553,23 @@ pub unsafe extern "C" fn zs_rebuild_step( // SAFETY: the caller's contract. unsafe { with_out(done, || { - let (job, cr) = (as_mut(job)?, circuit(c)?); - if !std::ptr::eq(job.owner, c) { + let cr = circuit(c)?; + if job.is_null() { + return Err(null()); + } + // Field places only, never a `&mut ZsRebuild`: a callback reads `busy` through + // `zs_rebuild_free` while `job.job` is mutably borrowed here. + if (*job).owner != cr.id { return Err(Fail::new( Code::BadHandle, "the rebuild belongs to another circuit", )); } + (*job).busy.set(true); let finished = - cr.stepping(|e| e.rebuild_step(&mut job.job, expired.map(|f| (f, ctx))))?; + cr.stepping(|e| e.rebuild_step(&mut (*job).job, expired.map(|f| (f, ctx)))); + (*job).busy.set(false); + let finished = finished?; Ok(i32::from(finished)) }) } @@ -559,7 +580,10 @@ pub unsafe extern "C" fn zs_rebuild_step( /// `job` is null, or a live job freed once. #[unsafe(no_mangle)] pub unsafe extern "C" fn zs_rebuild_free(job: *mut ZsRebuild) { - // SAFETY: the caller's contract. + // SAFETY: the caller's contract; a job being stepped is leaked, not freed under its frame. + if !job.is_null() && unsafe { (*job).busy.get() } { + return; + } unsafe { release(job) } } @@ -582,6 +606,7 @@ mod tests { Code::Poisoned as i32 ); assert_eq!(id, 77, "nothing was written"); + unsafe { &*c }.in_step.set(false); unsafe { zs_circuit_free(c) }; } } diff --git a/crates/zset-ffi/tests/abi.rs b/crates/zset-ffi/tests/abi.rs index c5d4e66a..770efa4a 100644 --- a/crates/zset-ffi/tests/abi.rs +++ b/crates/zset-ffi/tests/abi.rs @@ -834,3 +834,134 @@ fn a_length_of_4_gib_or_more_is_refused_not_truncated() { Code::BadValue ); } + +#[test] +fn an_index_by_callback_that_fails_reports_the_callback_code_not_a_later_fault() { + extern "C" fn fails_after_emitting( + _: *mut c_void, + _: *const u8, + _: usize, + _: engine::Emit, + _: *mut c_void, + ) -> i32 { + 3 + } + let c = C::new(); + let i = c.input(); + let mut o = 0; + assert_eq!( + unsafe { zs_circuit_index_by(c.0, i, Some(fails_after_emitting), null_mut(), &mut o) }, + OK + ); + c.push(i, &[(b"x", 1)]); + assert_eq!(unsafe { zs_circuit_step(c.0) }, Code::Callback as i32); +} + +struct Reenter { + c: *mut ZsCircuit, + job: *mut ZsRebuild, + seen: Vec, +} + +extern "C" fn expired_reenters(ctx: *mut c_void) -> i32 { + let r = unsafe { &mut *(ctx as *mut Reenter) }; + let mut done = 0; + r.seen + .push(unsafe { zs_rebuild_step(r.job, r.c, None, null_mut(), &mut done) }); + unsafe { zs_rebuild_free(r.job) }; + unsafe { zs_circuit_free(r.c) }; + 1 +} + +#[test] +fn a_callback_cannot_step_or_free_the_job_or_circuit_it_runs_under() { + let snapshot = zset(&[(b"a", 1), (b"b", 1), (b"c", 1)]); + let c = C::new(); + let i = c.input(); + let s = c.sink(i); + let mut job = null_mut(); + assert_eq!(unsafe { zs_rebuild_new(c.0, i, snapshot, 1, &mut job) }, OK); + let mut r = Reenter { + c: c.0, + job, + seen: Vec::new(), + }; + let mut done = 0; + let ctx = (&mut r as *mut Reenter).cast(); + assert_eq!( + unsafe { zs_rebuild_step(job, c.0, Some(expired_reenters), ctx, &mut done) }, + OK + ); + assert_eq!( + r.seen, + vec![Code::Poisoned as i32], + "stepping from inside is refused" + ); + assert_eq!(done, 0); + assert_eq!( + unsafe { zs_rebuild_step(job, c.0, None, null_mut(), &mut done) }, + OK + ); + assert_eq!( + (done, c.take(s).len()), + (1, 3), + "the job and circuit survived the frees" + ); + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; +} + +#[test] +fn a_circuit_freed_and_another_made_does_not_inherit_its_jobs() { + let snapshot = zset(&[(b"a", 1), (b"b", 1)]); + let c1 = C::new(); + let i1 = c1.input(); + let mut job = null_mut(); + assert_eq!( + unsafe { zs_rebuild_new(c1.0, i1, snapshot, 1, &mut job) }, + OK + ); + drop(c1); + for _ in 0..64 { + let c2 = C::new(); + let _ = c2.input(); + let mut done = 0; + assert_eq!( + unsafe { zs_rebuild_step(job, c2.0, None, null_mut(), &mut done) }, + Code::BadHandle as i32, + "a new circuit, even at the old address, is not the job's" + ); + } + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; +} + +#[test] +fn a_host_step_between_rebuild_calls_is_interrupted_not_a_poison() { + let snapshot = zset(&[(b"a", 1), (b"b", 1)]); + let c = C::new(); + let i = c.input(); + let s = c.sink(i); + let mut job = null_mut(); + assert_eq!(unsafe { zs_rebuild_new(c.0, i, snapshot, 1, &mut job) }, OK); + let (mut calls, mut done) = (0u32, 0); + let ctx = (&mut calls as *mut u32).cast(); + assert_eq!( + unsafe { zs_rebuild_step(job, c.0, Some(out_of_time), ctx, &mut done) }, + OK + ); + assert_eq!(unsafe { zs_circuit_step(c.0) }, OK); + assert_eq!( + unsafe { zs_rebuild_step(job, c.0, None, null_mut(), &mut done) }, + Code::Interrupted as i32 + ); + assert!(last_error().contains("interrupted"), "{}", last_error()); + assert_eq!( + unsafe { zs_circuit_step(c.0) }, + OK, + "the circuit is still usable" + ); + let _ = c.take(s); + unsafe { zs_rebuild_free(job) }; + unsafe { zs_zset_free(snapshot) }; +} diff --git a/crates/zset-ffi/tests/header.rs b/crates/zset-ffi/tests/header.rs index 6e58076e..514a640a 100644 --- a/crates/zset-ffi/tests/header.rs +++ b/crates/zset-ffi/tests/header.rs @@ -60,6 +60,7 @@ fn c_type(rust: &str) -> String { "Option" => "ZsKeepFn", "Option" => "ZsFlatMapFn", "Option" => "ZsExpiredFn", + "Emit" => "ZsEmit", other => panic!("no C type is declared for the Rust type `{other}`"), }; squash(c) @@ -95,13 +96,24 @@ fn exported() -> BTreeMap)> { }; out.insert(name, (ret, params)); } - // The `unary_op!` expansions all take (circuit, stream, out). + // The `unary_op!` expansions share the macro body's signature. + let body = &SRC[SRC.find("macro_rules! unary_op").unwrap()..]; + let sig = body.split("extern \"C\" fn $name(").nth(1).unwrap(); + let (params, tail) = sig.split_once(')').unwrap(); + let unary_params: Vec = split_params(params) + .iter() + .map(|p| c_type(p.split_once(':').unwrap().1)) + .collect(); + let unary_ret = c_type( + tail.split('{') + .next() + .unwrap() + .trim() + .trim_start_matches("->"), + ); for line in SRC.lines().filter(|l| l.starts_with("unary_op!(")) { let name = line["unary_op!(".len()..].split(',').next().unwrap(); - let params = ["ZsCircuit*", "uint32_t", "uint32_t*"] - .map(String::from) - .to_vec(); - out.insert(name.to_string(), ("int32_t".to_string(), params)); + out.insert(name.to_string(), (unary_ret.clone(), unary_params.clone())); } out } @@ -202,3 +214,65 @@ fn the_header_constants_are_the_rust_constants() { [0, 1, 2, 3] ); } + +/// `typedef R (*Name)(params)` from the header, and `pub type X = extern "C" fn(..) -> R` from +/// `engine.rs`, as `name -> (return, params)`. +type Sigs = BTreeMap)>; + +fn callback_types() -> (Sigs, Sigs) { + let mut hdr = BTreeMap::new(); + for decl in strip_comments(HEADER).split(';') { + let Some(at) = decl.find("(*") else { continue }; + if !decl.contains("typedef") { + continue; + } + let ret = squash( + decl.split("typedef") + .nth(1) + .unwrap() + .split('(') + .next() + .unwrap(), + ); + let rest = &decl[at + 2..]; + let (name, after) = rest.split_once(')').unwrap(); + let params = after + .trim() + .trim_start_matches('(') + .rsplit_once(')') + .unwrap() + .0; + let params = split_params(params).iter().map(|p| param_type(p)).collect(); + hdr.insert(name.trim().to_string(), (ret, params)); + } + let engine = include_str!("../src/engine.rs"); + let mut rust = BTreeMap::new(); + for line in engine.lines().filter(|l| l.contains("= extern \"C\" fn(")) { + let name = line + .trim_start_matches("pub type ") + .split(' ') + .next() + .unwrap(); + let sig = line.split("fn(").nth(1).unwrap(); + let (params, tail) = sig.rsplit_once(')').unwrap(); + let ret = match tail.trim().trim_end_matches(';').trim().strip_prefix("->") { + Some(r) => c_type(r), + None => "void".to_string(), + }; + let params = split_params(params).iter().map(|p| c_type(p)).collect(); + rust.insert(format!("Zs{name}"), (ret, params)); + } + (hdr, rust) +} + +#[test] +fn the_header_callback_typedefs_match_the_rust_callback_types() { + let (hdr, rust) = callback_types(); + assert_eq!( + rust.len(), + 4, + "the scanner saw {} callback types", + rust.len() + ); + assert_eq!(rust, hdr); +} From 5f2755dc8a02b66d85361ec9ab18f90a70ab979b Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-circuit)" Date: Sat, 10 Oct 2026 14:30:19 -0700 Subject: [PATCH 07/10] docs(doc-site): publish the zset-ffi pages (improvise-bw4p.1) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- scripts/build-doc-site.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build-doc-site.mjs b/scripts/build-doc-site.mjs index 2fdb30ca..03e9f122 100644 --- a/scripts/build-doc-site.mjs +++ b/scripts/build-doc-site.mjs @@ -69,6 +69,7 @@ const SECTIONS = [ ["context/code/gp.md", "code/gp", "gp-tree and improvise-gp"], ["context/code/numtower.md", "code/numtower", "numtower, the numeric tower"], ["context/code/zset.md", "code/zset", "zset, the Z-set group"], + ["context/code/zset-ffi.md", "code/zset-ffi", "zset-ffi, the C ABI"], ["context/code/improvise-incremental.md", "code/improvise-incremental", "improvise-incremental, the plug"], ["context/code/kb-tool.md", "code/kb-tool", "The kb tool"], ["context/code/tooling.md", "code/tooling", "The repository's own checkers"], @@ -90,6 +91,7 @@ const SECTIONS = [ ["context/test-evidence/gp.md", "test-evidence/gp", "gp-tree and improvise-gp"], ["context/test-evidence/numtower.md", "test-evidence/numtower", "numtower"], ["context/test-evidence/zset.md", "test-evidence/zset", "zset"], + ["context/test-evidence/zset-ffi.md", "test-evidence/zset-ffi", "zset-ffi"], ["context/test-evidence/improvise-incremental.md", "test-evidence/improvise-incremental", "improvise-incremental"], ["context/test-evidence/tooling.md", "test-evidence/tooling", "The repository's own gates"], ["context/test-evidence/layout-cost.md", "test-evidence/layout-cost", "Layout and keypress cost"], From 1bd665a7549e7300457fc63b3fbf7c2dffcf73a1 Mon Sep 17 00:00:00 2001 From: Edward Langley Date: Sat, 10 Oct 2026 15:28:08 -0700 Subject: [PATCH 08/10] docs(context): bring the repo map, gotchas and kb up to ADR 005/009 crates Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/file-inventory.md | 15 ++++++++------- context/gotchas.md | 6 ++++-- context/repo-map.md | 28 ++++++++++++++++++++-------- prolog-sessions/kb.pl | 22 ++++++++++++++++++++++ scripts/build-doc-site.mjs | 2 +- 5 files changed, 55 insertions(+), 18 deletions(-) diff --git a/context/file-inventory.md b/context/file-inventory.md index aa56a4f2..3da13bdf 100644 --- a/context/file-inventory.md +++ b/context/file-inventory.md @@ -359,14 +359,15 @@ The three repo-map sections of these names, as they stood before the map was con ## Wasm The browser bundle is `improvise-core`, `improvise-formula`, `improvise-io`, -`improvise-protocol`, `improvise-command`, `improvise-wasm-client` and -`improvise-worker-server`. All seven +`improvise-protocol`, `improvise-command`, `improvise-wasm-client`, +`improvise-worker-server`, and (ADR 005, ADR 009) `numtower`, `zset` and +`improvise-incremental`, the last three because `improvise-protocol` and +`improvise-command` depend on them. Each `cargo check --target wasm32-unknown-unknown` clean with no feature flags -(improvise-ywd), gated by the `wasm` job in `.github/workflows/ci.yml`. That -job gates **eight** crates, not seven: `improvise-gp` was added (improvise-z5eg) -explicitly *not* as a bundle member but as a candidate, because staying -wasm-clean is free now and expensive to retrofit — so the bundle is seven and -the gate is eight, and neither number is the other one's typo. +(improvise-ywd); `wasmCrates` in `nix/ci.nix` lists the set the CI gate checks, and +it also holds `improvise-gp` (improvise-z5eg), a candidate rather than a member, +because staying wasm-clean is free now and expensive to retrofit. `zset-ffi` is +native only and not in that list. **A feature that adds a dependency to a bundle crate must be checked with `cargo tree --edges normal`** — the default output includes **dev** edges, so a plain `cargo tree -p improvise-protocol --target wasm32-unknown-unknown` reports a dev-only `serde_json` as *present* and reads exactly like a leak. Check per-crate too, because the `wasm` CI job invokes each crate separately and feature unification cannot reach across that (improvise-26dm added a `test-support` feature on `improvise-protocol` on those terms; it is off in every shipped build). `improvise-ws-server` is the host and is native only — tokio, tungstenite and diff --git a/context/gotchas.md b/context/gotchas.md index d98edc42..b78e02fe 100644 --- a/context/gotchas.md +++ b/context/gotchas.md @@ -71,7 +71,7 @@ Dated claims and counts inside an entry are snapshots: recount rather than cite. 39. **`Applied` is an enum because a document swap is not a diff.** `Sessions::apply_checked` returns `Applied::Diff { dirty, broadcast }`, `Applied::Rebased { snapshots }` or — since ADR 001 layer 1 — `Applied::Rejected { reason }`, and the enum *is* the mechanism: a host that wants the `Dirty` must match, and matching names the case where a `Snapshot` is owed. The server's mirrors self-heal — `ClientSession::project` reseeds on the `document_generation` bump — but `ViewState::cursors` lives on the **client**, keyed by the outgoing document's view names (gotcha 24), and only a fresh `Snapshot` replaces it. `Command::Model` put the three `WholeWorkbook` mutations on the wire (gotcha 38), so the host that must send the snapshots is no longer the host that asked. `apply_model` therefore calls `rebase` itself rather than instructing the host to, exactly as `apply_switch` adopts the accepted view itself. **The trigger is the generation moving, not the command variant**: a `LoadModel` that cannot read its path leaves the document alone, and reseeding on the variant would cost a client its cursors for a *failed* load — `a_load_that_fails_is_not_a_rebase` guards that arm. A `rebase: bool` beside `dirty` was rejected, because the host that forgets to call `rebase` is the host that forgets to read the flag. The **third** case is a *normal outcome*, not an error, and it is in the enum for the same reason the other two are: a host that wants the `Dirty` must match, and matching names it. `ServerMsg::Rejected` is the matching frame, addressed to **the asker alone** — nothing happened, so nobody else is owed anything — and the client keeps it as a `Rejection` rather than a sentence, because force needs to know *which* record and *which* kind (gotcha 43). **Fixture warning.** When testing a document swap, the incoming document's active view must have the **same name** as the outgoing one. A rename makes `apply_model` broadcast a `SwitchView`; the client folds it, and the stale cursor is then read under a key nothing has written — right *by accident*, with the bug invisible. `other_document()` renames, and a first version of this test passed because of it; `a_document_shaped_differently_under_the_same_view_name()` keeps the name and changes only the shape, so there is nothing to hide behind. -40. **A batch's basis is three tiers, and only two of them can be read off the effect.** `App` carries `document_generation` (bumped by `install_workbook`), `layout_version` (per *view shape*) and `record_versions` (per *record*) — ADR 001's stale-basis check, and shared document state, so never on `ViewState` (gotcha 30). **The unit is the record**: `record_of(key)` is the coordinate tuple minus `_Measure` (`record_row_identity`), because renaming a category coordinate in wide records mode moves every measure cell in the row (gotcha 1) — a cell-level unit is violated by one legal edit, so the unit must be at least as coarse as the coarsest legal write. **Only the `Dirty` is read off the `EffectKind`; both lower tiers are *observed* around the loop** — the source says so at `apply_effects` and this paragraph said the opposite until improvise-ilvl's audit. The `EffectKind` decides only *whether to look*: `touches_model` arms the shape snapshot, `writes_cells` the record fingerprints. `record_versions` **must** be observed, because an `EffectKind` says *that* a batch writes cells, never *which*; `layout_version` is observed for a better reason than necessity — `observe_reshape` diffs `view_shapes` before and after and bumps only where a view's shape actually moved (improvise-pdvm/nynk), so it is a **fact about the document** rather than a claim about the batch. Two consequences the declared form did not have. An **idempotent** `set-axis` bumps nothing, and batching two real changes into one line or two gives the same `2`. And the gotcha 21 argument is subsumed rather than restated: an aborted batch moved no shape, so there is nothing to not-claim — which is why observing once **after** the loop is correct where accumulating `Dirty` inside it is. **`view_shape` hashes what is drawn, not occupancy** (its doc comment: "Drawn, not configured") — axes, the drawn item names minus hidden ones, folds, page selections and the `prune_empty` *flag* — so an occupancy write does **not** bump `layout_version`: a `CellData`-only batch that prunes a row out of a view moves the cursor epoch (the frame observation) and not `layout_version`, as `context/code/command.md` records (the question that raised was improvise-6nsd, closed). The arming is wider than the tier (`touches_model` takes the shape snapshot), but a bump needs a shape hash to have moved. Three further consequences. A write landing the value already there **bumps nothing**, which is correct — no basis was invalidated. A `LayoutGeometry` effect that drops cells **will** bump their records when it shares a batch with a `CellData` one: an over-bump costing at most an unnecessary rejection, the same direction `Dirty::of` is deliberately sloppy in. And only *raw* data is fingerprinted — formula results live in `Model::formula_cache`, and a record whose derived value moved because someone edited elsewhere has not itself been written. The diff collects into a **set**: chaining `before.keys()` with `after.keys()` counts any rewritten record twice, a bug that survived one test run. +40. **A batch's basis is three tiers, and only two of them can be read off the effect.** `App` carries `document_generation` (bumped by `install_workbook`), `layout_version` (per *view shape*) and `record_versions` (per *record*) — ADR 001's stale-basis check, and shared document state, so never on `ViewState` (gotcha 30). **The unit is the record**: `record_of(key)` is the coordinate tuple minus `_Measure` (`record_row_identity`), because renaming a category coordinate in wide records mode moves every measure cell in the row (gotcha 1) — a cell-level unit is violated by one legal edit, so the unit must be at least as coarse as the coarsest legal write. **Only the `Dirty` is read off the `EffectKind`; both lower tiers are *observed* around the loop** — the source says so at `apply_effects` and this paragraph said the opposite until improvise-ilvl's audit. The `EffectKind` decides only *whether to look*: `touches_model` arms the shape snapshot, `writes_cells` the record fingerprints. `record_versions` **must** be observed, because an `EffectKind` says *that* a batch writes cells, never *which*; `layout_version` is observed for a better reason than necessity — `observe_reshape` diffs `view_shapes` before and after and bumps only where a view's shape actually moved (improvise-pdvm/nynk), so it is a **fact about the document** rather than a claim about the batch. Two consequences the declared form did not have. An **idempotent** `set-axis` bumps nothing, and batching two real changes into one line or two gives the same `2`. And the gotcha 21 argument is subsumed rather than restated: an aborted batch moved no shape, so there is nothing to not-claim — which is why observing once **after** the loop is correct where accumulating `Dirty` inside it is. **`view_shape` hashes what is drawn, not occupancy** (its doc comment: "Drawn, not configured") — axes, the drawn item names minus hidden ones, folds, page selections and the `prune_empty` *flag* — so an occupancy write does **not** bump `layout_version`: a `CellData`-only batch that prunes a row out of a view moves the cursor epoch (the frame observation) and not `layout_version`, as `context/code/command.md` records (the question that raised was improvise-6nsd, closed). The arming is wider than the tier (`touches_model` takes the shape snapshot), but a bump needs a shape hash to have moved. Three further consequences. A write landing the value already there **bumps nothing**, which is correct — no basis was invalidated. A `LayoutGeometry` effect that drops cells **will** bump their records when it shares a batch with a `CellData` one: an over-bump costing at most an unnecessary rejection, the same direction `Dirty::of` is deliberately sloppy in. And only *raw* data is fingerprinted — formula results live in `Model::formula_cache`, and a record whose derived value moved because someone edited elsewhere has not itself been written. The diff collects into a **set**: chaining `before.keys()` with `after.keys()` counts any rewritten record twice, a bug that survived one test run. **Superseded (ADR 009 step 7):** the record tier is no longer fingerprinted. `apply_effects` turns the store's change log (`DataStore::record_changes`/`drain_changes`) into a `CellDelta` (`improvise_incremental::cell_delta`) and `bump_moved_records` bumps every record in `moved_records(delta)`; `record_fingerprints` and the text above naming it are historical. 41. **`Storage` is not a VFS, and that closes the SQLite question.** The trait is whole-document `read`/`write` bytes (gotcha 2, `invariant(storage_is_two_ops)`) — no offsets, seeking, handles or locking — so an `sqlite3_vfs`, which needs `xRead(offset,len)`/`xWrite`/`xTruncate`/`xLock`, has nothing to bind to. SQLite *can* still sit behind it, as an in-memory database serialized to a blob (`rusqlite` `serialize`/`deserialize_read_exact`, proven with a probe before declining — declining on a capability that actually exists would have been the wrong decline). It is worth nothing: measured on 18,000 cells the SQLite container is **22% larger** than `.improv`, **4.7× larger gzipped**, ~**160×** larger at the floor (24 KB against ~150 bytes), and still rewritten whole on every save, so the one dimension SQLite normally wins — incremental page writes — is unavailable. improvise-9ix was declined on that evidence. Two things generalise past SQLite. **A second serialization format is a second implementation of the grammar**, obliged to track `improv.pest` forever; and **its own round-trip test cannot detect drift from `.improv`**, because it never compares the two — that missing cross-format equality test is what made the ticket look cheap. If a structured-query story is ever wanted it is a one-way `export_sqlite` beside `export_csv`, native-only and feature-gated: `libsqlite3-sys`' C build fails under this repo's nix cc-wrapper for `wasm32-unknown-unknown`, which would break the `wasm` CI gate the day it landed. @@ -91,7 +91,7 @@ Dated claims and counts inside an entry are snapshots: recount rather than cite. 45. **One session loop, two transports — and the browser saves itself.** `Host` is `App` + `Sessions` + `handle(id, ClientFrame) -> Vec` and lives in `improvise_command::ui::host`, **not** in `improvise-ws-server`, which re-exports it. It moved for improvise-9x6: the worker hosts the same authoritative `App` over a `MessagePort` and cannot depend on tokio, and a second copy would have been exactly the mirror improvise-xks6 had just deleted for the frames — a drifting `Applied`/`Broadcast` match is invisible until a client sees it. `registry` stayed on the server, because a worker hosts **one** document. What the worker adds is only what a `MessagePort` differs in: **one client by construction** (the `Addressed` filter is honoured anyway, against a future `SharedWorker`), and **a handshake with no close** — `ClientFrame::Viewport` must still come first, but a pre-handshake frame is *dropped* rather than closing a connection that does not exist. - **Saving is a frame, and dirtiness is read off the classification the effect already carries.** `effect::Save` is `ViewOnly` (gotcha 3) — correctly, since it writes *this host's* `file_path` through *this host's* `Storage` — so the derivation that builds `ModelCommand` from the model effects cannot name it, and for a while that meant no client on any transport could ask its host to save. `ClientFrame::Save` closes it (improvise-368h): a **request** carrying no path, because a client cannot see the host's storage, answered `ServerFrame::Rejected(NoDocumentPath | SaveFailed)` to the asker alone and **silently on success** — nothing moved, so no widget is dirty. It is a frame and not a `Command` for one concrete reason: as a `Command` it put a runtime-fallible operation inside a batch `ClientFrame::Commands` promises is all-or-nothing, so `[Save, SetCell]` reported a failed write and then applied the edit. It also fails the test every `Command` passes — it mutates no shared state and no client's slice, so `reduce_view` has nothing to fold and `Checked` has no basis to answer for. **And `MarkDirty` is gone.** It was a separate effect the *command* layer composed beside each mutation — reasonable on the key path, unreachable from anywhere else, because improvise-kmgs derived `ModelCommand` from the effects **one at a time** and so put `SetCell` on the wire without its companion (improvise-gv9r): a derivation correct per element and wrong for a set, whose symptom was that "save if dirty" never fired. `App::apply_effects` now raises the flag from the effect's `EffectKind`, which makes omitting it unrepresentable rather than validated — **declared** for `WholeWorkbook` (a replaced document differs from the file its path names, so a `LoadModel` that cannot read its path still raises it, costing one redundant write) and **observed** for `CellData` and `LayoutGeometry`, where `document_content`/`document_config` fingerprint the document before and after, so a write that leaves it as it was raises nothing (improvise-748f). The observation is read inside the loop, and `Save` clears the flag in loop order, so a batch that edits and then saves ends clean. The worker's persist trigger is therefore the **flag**, not a table of command variants: an observation rather than a second description of which commands change the document, for the reason `cursor_epoch` beats matching on `CycleAxis` (gotcha 24) and `document_generation` beats matching on `LoadModel` (gotcha 39). It is also strictly tighter — a refused batch raises nothing, and neither does a write that changes nothing, so the "was anything rejected" guard went with it. A browser still never *sends* `ClientFrame::Save`, and that is the answer rather than a gap: there is no file the user owns and no Ctrl-S semantics, the document *is* the OPFS entry, so the worker persists on its own behalf through the one existing `Save` effect. The cost that remains: `Storage` is whole-document (gotcha 41), so OPFS rewrites the file per accepted model command. + **Saving is a frame, and dirtiness is read off the classification the effect already carries.** `effect::Save` is `ViewOnly` (gotcha 3) — correctly, since it writes *this host's* `file_path` through *this host's* `Storage` — so the derivation that builds `ModelCommand` from the model effects cannot name it, and for a while that meant no client on any transport could ask its host to save. `ClientFrame::Save` closes it (improvise-368h): a **request** carrying no path, because a client cannot see the host's storage, answered `ServerFrame::Rejected(NoDocumentPath | SaveFailed)` to the asker alone and **silently on success** — nothing moved, so no widget is dirty. It is a frame and not a `Command` for one concrete reason: as a `Command` it put a runtime-fallible operation inside a batch `ClientFrame::Commands` promises is all-or-nothing, so `[Save, SetCell]` reported a failed write and then applied the edit. It also fails the test every `Command` passes — it mutates no shared state and no client's slice, so `reduce_view` has nothing to fold and `Checked` has no basis to answer for. **And `MarkDirty` is gone.** It was a separate effect the *command* layer composed beside each mutation — reasonable on the key path, unreachable from anywhere else, because improvise-kmgs derived `ModelCommand` from the effects **one at a time** and so put `SetCell` on the wire without its companion (improvise-gv9r): a derivation correct per element and wrong for a set, whose symptom was that "save if dirty" never fired. `App::apply_effects` now raises the flag from the effect's `EffectKind`, which makes omitting it unrepresentable rather than validated — **declared** for `WholeWorkbook` (a replaced document differs from the file its path names, so a `LoadModel` that cannot read its path still raises it, costing one redundant write) and **observed** for `CellData` and `LayoutGeometry`, where `document_content`/`document_config` fingerprint the document before and after, so a write that leaves it as it was raises nothing (improvise-748f). The observation is read inside the loop, and `Save` clears the flag in loop order, so a batch that edits and then saves ends clean. The worker's persist trigger is therefore the **flag**, not a table of command variants: an observation rather than a second description of which commands change the document, for the reason `cursor_epoch` beats matching on `CycleAxis` (gotcha 24) and `document_generation` beats matching on `LoadModel` (gotcha 39). It is also strictly tighter — a refused batch raises nothing, and neither does a write that changes nothing, so the "was anything rejected" guard went with it. A browser still never *sends* `ClientFrame::Save`, and that is the answer rather than a gap: there is no file the user owns and no Ctrl-S semantics, the document *is* the OPFS entry, so the worker persists on its own behalf through the one existing `Save` effect. The cost that remains: `Storage` is whole-document (gotcha 41), so OPFS rewrites the file per accepted model command. **Superseded (ADR 009 step 7):** `document_content` is gone. For `CellData` the flag is `reordered || !delta.is_zero() || document_shape moved` over the batch cell delta; `LayoutGeometry` still compares `document_config`. The text above naming `document_content` is historical. **The async seam does not survive being boxed.** `OpfsFs::open`/`attach`/`refresh`/`commit_deletes` are on the struct, not on `Storage`. `App::storage` is a `Box` and a trait object can name none of the four, so **handing the backend to the `App` puts the async seam permanently out of reach**; the host must hold the concrete backend *and* give the `App` a `Storage`, which means sharing one. That is `impl Storage for Rc`, and it lives in **`improvise-io`** rather than in each worker (improvise-hmws): `impl Storage for Rc` is two foreign types in any other crate, so improvise-9x6 wrote a forwarding newtype that every future worker would have written again. `Rc` and not `Arc` because the one caller is a Web Worker and wasm32 has no other thread. **The flat root is no longer load-bearing**: `OpfsFs::list` joins `dir` like the other two backends, so `storage_contract` takes `dir` as a parameter and the three agree at *every* directory rather than only at `Path::new("")` where joining is the identity (improvise-vhcz). What the index holds is **documents, not entries**, and the split between its three writers is the rule: `attach` indexes nothing (what it creates is zero bytes), `Storage::write` indexes the name and un-queues any pending delete of it, and `refresh` rebuilds from OPFS skipping zero-byte entries — counting an entry it cannot measure as a document, since the one cause of that is another tab holding the exclusive handle (improvise-vvl3). @@ -136,4 +136,6 @@ Dated claims and counts inside an entry are snapshots: recount rather than cite. 52. **A date column imports as one hierarchical category, and the import refuses what it cannot keep (improvise-8jk6.3, 8jk6.6, d0n6).** Reasoning: `context/code/formula-and-io.md` ("Import: …") and `cli.md` §13. The default `DateShape` nests Year > Quarter > Month (the chosen levels) in ONE category named after the column and opens folded to its years; `DateShape::Split` is the old flat split, chosen by `--split-date ` headless or `s` on the wizard's date step. `Plan::spelling_clash` refuses a raw, unparsed date spelled like a generated item or group (`2024`, `2024-01`) because the two would silently merge — asked at the wizard's ConfigureDates step **and** again at build time, so a headless import cannot reach it either. Headless, `--strict` turns a partial skip into a refusal before the build, and a folder whose CSVs are all header-only (no records) is refused as an import that kept no record. +53. **Delivery: a watermark is a `Stamp`; a correction is `target - held`, applied only at the watermark it starts from.** `through` is not `at` (the watermark advances only to `through`), a refused frame restores `Delivery` with the app snapshot, and a lost last notice is found only by `poll`. Axis changes refetch the window until ADR 008's render model. The argument is `context/code/delivery.md` (sections 3, 4 and 6); a refused frame restoring `Delivery` is §6, Rollback. + --- diff --git a/context/repo-map.md b/context/repo-map.md index 7a64ddf9..a3d2f0ed 100644 --- a/context/repo-map.md +++ b/context/repo-map.md @@ -6,8 +6,12 @@ Apache-2.0. Root binary+lib `improvise`; sub-crates under `crates/`: - `improvise-core` — `Model`, `View`, `Workbook`, number formatting. No UI/IO. - `improvise-formula` — formula parser, AST, `parse_formula`. - `improvise-io` — `.improv` save/load, CSV/JSON import, `Storage` backends. No UI/commands. -- `improvise-protocol` — the wire vocabulary and the portable session slice: `Command`, `Projection`, `ViewState`, the per-widget caches, `reduce_view`, key types. Depends only on `improvise-core` + serde. No IO, no command layer, no terminal. -- `improvise-command` — command/effect layer + semantic `App` state (app.rs, effect.rs, cat_tree.rs, viewport.rs). Depends on `improvise-protocol`. No ratatui/crossterm. +- `improvise-protocol` — the wire vocabulary and the portable session slice: `Command`, `Projection`, `ViewState`, the per-widget caches, `reduce_view`, key types, and `delivery/{wire,notice,exchange,client}.rs` (notices, watermarks, corrections). Depends on `improvise-core`, `zset`, `numtower` + serde. No IO, no command layer, no terminal. +- `improvise-command` — command/effect layer + semantic `App` state (app.rs, effect.rs, cat_tree.rs, viewport.rs, delivery.rs). Depends on `improvise-protocol`, `zset`, `improvise-incremental`. No ratatui/crossterm. +- `numtower` — exact decimal arithmetic tower (`Num`, canonical text), ADR 005. Pure Rust, no workspace deps; `improvise-protocol` uses it for the wire form. Long form `context/code/numtower.md`. +- `zset` — std-only Z-sets (non-zero checked `i64` weights, panic on overflow) plus `Circuit`, operators, `Trace`, `Rebuild` (ADR 009 clause 1). No dependencies. `context/code/zset.md`. +- `zset-ffi` — C ABI over `zset` (`cdylib`/`staticlib`/`rlib`) for CFFI; `include/zset_ffi.h`. Native only, **not** in `wasmCrates`. `context/code/zset-ffi.md`. +- `improvise-incremental` — the `Derived` trait, `Reading`, `Eager`, `differential_in`/`drawn_probes`, and `cell_delta`/`DeltaValue`/`moved_records` (the store's `CellChange` log to a `ZSet`). Depends on `improvise-core` + `zset`; `core` never depends on `zset`. `context/code/improvise-incremental.md`. - `gp-tree` — problem-agnostic genetic-programming regression trees (`Tree`, `Spec`, `breed`, `simplify`, `render`). Knows nothing about improvise. - `improvise-gp` — the one seam between the two: emits a fitted `gp_tree::Tree` as improvise formula text (`formula_text` / `fitted_formula`). Depends on `gp-tree`, `improvise-formula` **and `improvise-core`** — `dataset.rs` builds a feature matrix from a `View`'s records (improvise-4s8) and `fitted_formula` takes the `&Model` it must resolve against (improvise-8xhy). Nothing depends on it yet. - `improvise-ws-server` — the host: tokio + tungstenite, a `Registry` of `Host`s keyed by `DocumentId` (multi-document, ADR 001), `Sessions` fan-out, and the socket framing (`wire.rs`). Native only. @@ -44,6 +48,12 @@ Architectural intent lives in `context/design-principles.md` — read that for t | Change how a name is quoted anywhere | `improvise-formula/src/quote.rs` — the one home, both halves | | Change import-wizard keys/steps | `command/cmd/wizard.rs` (keys) + `import/wizard.rs` (steps) | | Change frame layout | `draw.rs` → `draw()` | +| Change an incremental operator or the circuit | `crates/zset/src/{ops,circuit}.rs` | +| Change the C ABI | `crates/zset-ffi/src/lib.rs` + `crates/zset-ffi/include/zset_ffi.h` | +| Change the wire form of a value or delta | `improvise-protocol/src/delivery/wire.rs` | +| Change what the host publishes/answers | `improvise-command/src/ui/delivery.rs` | +| Change what counts as a document or record write | `improvise-command/src/ui/app.rs::apply_effects` (reads the batch cell delta) | +| Ask "does this grid draw column totals" | `GridLayout::draws_totals` (`view/layout.rs`) — the one condition | | Change how wide the TUI grid is beside a side panel | `improvise-command/src/ui/viewport.rs` → `grid_width` (reached through `App::grid_width`); `draw.rs::draw_content` splits on it and `App::cmd_context` counts `visible_cols` on it (improvise-gtn7.1) | | Change what the browser page draws | `improvise-wasm-client/src/render.rs` → `RenderModel` | | Change a browser↔server frame | `improvise-protocol/src/frame.rs` → `ClientFrame`/`ServerFrame`. The **one** pair; both ends `pub use` it and neither defines one (improvise-xks6) | @@ -202,12 +212,12 @@ the incidents and the ticket ids.** Read the long form of every gotcha your diff 37. **`gp-tree` returns data and never prints; a fit needs seed *and* `FitConfig`.** A scorer takes `Row` slices. `fitted_formula` refuses features the document cannot name; categories are forced `Categorical` (zv8t). 38. **The wire vocabulary is derived twice** (`Command` view half, `ModelCommand` model half). Completeness is checked (`vocabulary_is_complete.rs`). `ApplyAndClearDrill` and `AcceptWizardModel` are deliberately off the wire (session-local operands). Recount, don't cite. 39. **`Applied` is an enum (`Diff | Rebased | Rejected`)** because a document swap is not a diff; the trigger is the generation moving, not the variant. A swap fixture must keep the view *name*. -40. **A batch's basis is three tiers** (`document_generation`, `layout_version`, `record_versions`); the unit is the record; lower tiers are *observed*, not read off the effect. +40. **A batch's basis is three tiers** (`document_generation`, `layout_version`, `record_versions`); the unit is the record; lower tiers are *observed*, not read off the effect. The record tier is the batch's cell delta (`cell_delta` over the store's change log, `moved_records`); `record_fingerprints` is gone. 41. **`Storage` is not a VFS;** SQLite was declined on measurement (a second serialization is a second grammar). 42. **Browser render decisions are made in Rust** (`RenderModel`: `editable` from the `CellId` variant, selection reconciled with offsets, `flatten_cat_tree`). `improvise_protocol::frame` is the one envelope pair for both deployments. 43. **A batch carries its basis, and the server publishes it** (`Projection::Basis`). `Basis` has no `Default` or public constructor; `Checked`/`CheckedBatch` make "ignoring the basis" unrepresentable; `Rejection` keeps `RecordGone` / `RecordMoved` / `RecordUnstated` apart. Anti-staleness, not security. The check is a property of the session door (`apply_command` does not check), and only two of the three tiers hoist per frame — the record tier is asked per command (`records_named`) against the frame's basis. Read gotcha 43 before touching it. 44. **The TUI grid draws from `GridCache`**, which frames it too (`AxisFrame`; width rule `header_level_widths`). Use the one-pass accessors, not per-position scans. `RecordHash` is document-scoped, `AggregateHash` layout-scoped. -45. **One session loop (`Host`), two transports.** Saving is a request frame (`ClientFrame::Save`); dirtiness is *declared* for `WholeWorkbook` and *observed* for `CellData`/`LayoutGeometry` (`MarkDirty` is gone). `impl Storage for Rc` lives in `improvise-io`. `harness/` runs `OpfsFs` for real; Safari is unseen. +45. **One session loop (`Host`), two transports.** Saving is a request frame (`ClientFrame::Save`); dirtiness is *declared* for `WholeWorkbook` and *observed* for `CellData`/`LayoutGeometry` (`MarkDirty` is gone); for `CellData` the flag is the batch delta plus reorders, formulas and config, not a document fingerprint. `impl Storage for Rc` lives in `improvise-io`. `harness/` runs `OpfsFs` for real; Safari is unseen. 46. **The page picks a peer:** `?ws=` → socket, else a module `Worker`. `#status` composes two facts; the first-frame watchdog is per-peer-agnostic. The demo model is built so every cell is a `Record`. 47. **Keys become commands in the browser;** the server must not know the keymap. `GridExtent`, `set_viewport`, `Commands(Vec)`, `resolve::NEEDS_THE_HOST` and the five client rules (`projected_at`, `opened`, `refused_here`, `Step::Declined`, `notify`). 48. **The browser JS is partly executed** (`harness/tests/shell.spec.js`); changes to `web/*.js` belong there. Serialize frames on one tail promise; disarm, don't pre-empt, a watchdog. Evidence headings may be module-qualified (and must be when the leaf collides). @@ -215,6 +225,7 @@ the incidents and the ticket ids.** Read the long form of every gotcha your diff 50. **A composed TUI row measures in cells and names which side yields** (`fill_line` + `Keeps`). Keep whichever key table is *derived from what it describes*. Scrolls are measured where drawn (`grid_width`). 51. **The command language reads pipes, refuses surplus, and checks names when applied** (`only_when_omitted`; `RequireKnown`/`unless_unknown`). 52. **A date column imports as one hierarchical category;** `--split-date` restores flat. `Plan::spelling_clash` refuses what would silently merge; `--strict` refuses partial skips. +53. **Delivery: a watermark is a `Stamp`; a correction is `target - held`, applied only at the watermark it starts from.** `through` is not `at` (the watermark advances only to `through`), a refused frame restores `Delivery` with the app snapshot, and a lost last notice is found only by `poll`. Axis changes refetch the window until ADR 008's render model. Long form `context/code/delivery.md`. --- @@ -239,15 +250,16 @@ Per-crate entry points. Per-file notes, test counts and a dated line-count snaps | Browser harness | `harness/` | Playwright (`tests/shell.spec.js`, `tests/opfs.spec.js`), `opfs-contract/`, `build.sh` — not a workspace member; its own CI job | | Examples | `examples/`, `crates/*/examples/` | `gen-grammar.rs`, `pretty-print.rs`, `category_width_bench.rs`, `fit_csv.rs` | | Scripts | `scripts/` | `checkers.sh` (every `*-test.sh` suite), `check-comment-runs.py`, `check-ci-graph.py`, `bd-audit-ids`, `kb`, `pr-watch`, `pr-reviews`, `build-doc-site.mjs` | -| Context docs | `context/` | `design-principles.md`, `gotchas.md`, `file-inventory.md`, `code/.md` (arguments), `test-evidence/` (bug evidence by test name), `adr/` (001–004, 007), `cloud-agents.md`; `prolog-sessions/kb.pl` is the KB; `docs/design-notes.md` is product vision | +| Context docs | `context/` | `design-principles.md`, `gotchas.md`, `file-inventory.md`, `code/.md` (arguments), `test-evidence/` (bug evidence by test name), `adr/` (001–005, 007, 009; no 006/008 file yet), `cloud-agents.md`; `prolog-sessions/kb.pl` is the KB; `docs/design-notes.md` is product vision | --- ## Wasm -The browser bundle is `improvise-{core,formula,io,protocol,command,wasm-client,worker-server}`; all seven -`cargo check --target wasm32-unknown-unknown` clean with no feature flags, and the `wasm` CI job also gates -`improvise-gp` (a candidate, not a member). `improvise-ws-server` and the root crate are native-only. +The browser bundle is `improvise-{core,formula,io,protocol,command,wasm-client,worker-server,incremental}` +plus `zset` and `numtower`; each `cargo check --target wasm32-unknown-unknown` clean with no feature flags +(`wasmCrates` in `nix/ci.nix`, which also gates `improvise-gp`, a candidate). `improvise-ws-server`, +`zset-ffi` and the root crate are native-only. Check new bundle dependencies with `cargo tree --edges normal` (the default output includes dev edges), per crate. `cargo check` cannot see runtime hazards: `Instant::now()` panics on wasm (`web-time`), and `std::fs` compiles but fails in a browser. Long form: `context/file-inventory.md`, appendix. diff --git a/prolog-sessions/kb.pl b/prolog-sessions/kb.pl index 62d656f5..662f1d25 100644 --- a/prolog-sessions/kb.pl +++ b/prolog-sessions/kb.pl @@ -9,6 +9,10 @@ crate(improvise_gp,'crates/improvise-gp','bridge: emits a fitted gp-tree Tree as crate(improvise_ws_server,'crates/improvise-ws-server','the socket host: tokio + tungstenite transport and a Registry of Hosts keyed by DocumentId. Re-exports Host and the frames; defines neither. Native only.'). crate(improvise_wasm_client,'crates/improvise-wasm-client','the browser client: the fold, the key classifier, and the render model the DOM paints. Non-dev deps are improvise-protocol + serde + wasm-bindgen and nothing else.'). crate(improvise_worker_server,'crates/improvise-worker-server','the other host: the same App, Sessions and Host as the socket server, behind self.onmessage/self.postMessage with OPFS underneath. worker.rs and vfs.rs are ordinary Rust and hold every decision and every test; bindings.rs is the wasm32 shim, run by harness/tests/shell.spec.js under Chromium and by no Rust test.'). +crate(numtower,'crates/numtower','exact decimal arithmetic tower (ADR 005): Num, canonical text; no workspace deps. Used by improvise-protocol for the wire form.'). +crate(zset,'crates/zset','std-only Z-sets (checked i64 weights) plus Circuit, operators, Trace and Rebuild (ADR 009 clause 1); no dependencies.'). +crate(zset_ffi,'crates/zset-ffi','C ABI over zset (cdylib, staticlib, rlib; include/zset_ffi.h) for CFFI. native_only: not in nix/ci.nix wasmCrates.'). +crate(improvise_incremental,'crates/improvise-incremental','the Derived trait, Reading, Eager, differential_in/drawn_probes and cell_delta/DeltaValue; plugs zset in behind a trait (ADR 009).'). layer(model,improvise_core). layer(view,improvise_core). layer(formula,improvise_formula). @@ -581,6 +585,13 @@ decision('improvise-cgqt',import_collisions_are_asked_not_defaulted,'PR #304, me decision('improvise-cgqt',occurrence_category_goes_on_axis_none,'Under DuplicatePolicy::Separate the build adds the occurrence category with items 1..=deepest collision and sets it to Axis::None in the active view, so the default grid still shows each tuple total and nothing new is pivoted. A record that collides with nothing is occurrence 1 (improvise-cgqt, improvise-9bit).'). decision('improvise-felo.1',category_cap_has_one_home,'The regular-category cap is improvise_core::model::MAX_CATEGORIES and nothing else. Until felo.1 the import wizard kept a private const of its own, so raising the model number alone would have left the wizard refusing a 13th category the model accepts -- a derivation stated twice that agreed only by coincidence. The only place the literal 24 is spelled is the test add_category_max_limit; every other test that fills to the cap derives it. No recorded reason for 12 was found; the user raised it on 2026-09-25 (12 categories is feeling small, we should increase to 24).'). decision('improvise-felo.13',skip_rebuild_keyed_on_declared_model_effects_plus_observed_view_and_drill,'Renamed from skip_rebuild_keyed_on_observed_inputs_not_effect_kind, whose name overstated. App::refresh_layout skips the rebuild unless layout_stale (DECLARED: set from EffectKind::Model(_) in apply_effects, and by install_workbook), or the drawn view name or the drill snapshot pointer (OBSERVED, compared as LayoutBasis) moved. It is not a hand-kept list of effects, the shape cursor_epoch and document_generation avoid, but it inherits gotcha 3 classification hole: a ViewOnly effect that changes what the layout reads leaves app.layout stale, where before felo.13 every batch rebuilt. A writer outside effects must call rebuild_layout itself (gotcha 23).'). +decision('improvise-felo.4',zset_weight_overflow,'zset weights are checked i64 and panic on overflow, in debug and release (user ruling 2026-10-10). The group is partial past i64.'). +decision('improvise-felo.4',derived_lives_in_incremental,'Derived lives in improvise-incremental, not improvise-core. core never depends on zset: the store logs CellChange and improvise-incremental builds the ZSet (cell_delta).'). +decision('improvise-felo.4',dirty_flag_is_delta_plus,'The document dirty flag is the batch cell delta plus reorders, formulas and config. record_fingerprints and document_content are gone; the record tier is moved_records(delta).'). +decision('improvise-felo.4',ffi_circuit_poisoned,'A failed circuit step (a panic in step or in a rebuild step) poisons the zset-ffi circuit: every later call returns ZS_ERR_POISONED and the caller builds another.'). +decision('improvise-felo.4',resumable_deadline_predicate,'The resumable step takes a predicate, not an Instant, because Instant::now() panics on wasm32 and the host closes over its own clock (context/code/zset.md, point 8).'). +decision('improvise-felo.4',eager_occupancy,'Eager::occupancy counts the visible stored members of a collapsed group, and formula-target keys are not asked.'). +decision('improvise-z2vr',delivery_axis_refetch,'A delivery axis change makes the correction a window refetch until ADR 008 supplies the render model (context/code/delivery.md section 3).'). wire_carries(command,both_directions,'user intent up, projection commands down'). wire_carries(snapshot,on_connect_only,'improvise-q08: full ViewState + render cache filled from current viewport'). wire_carries(command,both_directions,'user intent client->server, projections server->client'). @@ -640,6 +651,13 @@ crate_depends_on(improvise_protocol,improvise_core). crate_depends_on(improvise_command,improvise_protocol). crate_depends_on(improvise_gp,gp_tree). crate_depends_on(improvise_gp,improvise_formula). +crate_depends_on(improvise_incremental,improvise_core). +crate_depends_on(improvise_incremental,zset). +crate_depends_on(zset_ffi,zset). +crate_depends_on(improvise_protocol,zset). +crate_depends_on(improvise_protocol,numtower). +crate_depends_on(improvise_command,zset). +crate_depends_on(improvise_command,improvise_incremental). depends_transitively(A,B):-depends_via(A,B,[A]). depends_via(A,B,C):-crate_depends_on(A,B). depends_via(A,B,C):-crate_depends_on(A,D), \+memberchk(D,C),depends_via(D,B,[D|C]). @@ -690,7 +708,11 @@ wasm_bundle_crate(improvise_command). wasm_bundle_crate(improvise_protocol). wasm_bundle_crate(improvise_wasm_client). wasm_bundle_crate(improvise_worker_server). +wasm_bundle_crate(numtower). +wasm_bundle_crate(zset). +wasm_bundle_crate(improvise_incremental). not_in_wasm_bundle(improvise,'terminal binary; depends on crossterm'). +not_in_wasm_bundle(zset_ffi,'C ABI cdylib/staticlib for CFFI; native only, absent from nix/ci.nix wasmCrates'). wasm_checks_clean(all_bundle_crates,no_feature_flags_needed). wasm_runtime_hazard(instant_now,'panics on wasm32-unknown-unknown',fixed_with(web_time)). wasm_runtime_hazard(std_fs,'compiles on wasm but fails in a browser',tracked_as('improvise-fuj')). diff --git a/scripts/build-doc-site.mjs b/scripts/build-doc-site.mjs index 4d3bae9a..325ce8fd 100644 --- a/scripts/build-doc-site.mjs +++ b/scripts/build-doc-site.mjs @@ -50,7 +50,7 @@ const SECTIONS = [ ["context/adr/005-exact-decimal-arithmetic.md", "adr/005-exact-decimal-arithmetic", "ADR 005 — Exact decimal arithmetic (proposed)"], ["context/adr/007-stable-identities.md", "adr/007-stable-identities", "ADR 007 — Stable identities"], ["context/adr/007-stable-identities-amendment-1.md", "adr/007-stable-identities-amendment-1", "ADR 007, amendment 1"], - ["context/adr/009-incremental-evaluation.md", "adr/009-incremental-evaluation", "ADR 009 — Incremental evaluation (proposed)"], + ["context/adr/009-incremental-evaluation.md", "adr/009-incremental-evaluation", "ADR 009 — Incremental evaluation"], ], }, { From 0b261165d129f0f996d9cb9d563e009ce76b4217 Mon Sep 17 00:00:00 2001 From: Edward Langley Date: Sat, 10 Oct 2026 15:49:04 -0700 Subject: [PATCH 09/10] docs(context): the dirty flag is per-effect; only the record tier uses the batch delta Also record the improvise-command to improvise-gp and improvise-gp to improvise-core edges. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/gotchas.md | 4 ++-- context/repo-map.md | 6 +++--- prolog-sessions/kb.pl | 4 +++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/context/gotchas.md b/context/gotchas.md index b78e02fe..f6692978 100644 --- a/context/gotchas.md +++ b/context/gotchas.md @@ -71,7 +71,7 @@ Dated claims and counts inside an entry are snapshots: recount rather than cite. 39. **`Applied` is an enum because a document swap is not a diff.** `Sessions::apply_checked` returns `Applied::Diff { dirty, broadcast }`, `Applied::Rebased { snapshots }` or — since ADR 001 layer 1 — `Applied::Rejected { reason }`, and the enum *is* the mechanism: a host that wants the `Dirty` must match, and matching names the case where a `Snapshot` is owed. The server's mirrors self-heal — `ClientSession::project` reseeds on the `document_generation` bump — but `ViewState::cursors` lives on the **client**, keyed by the outgoing document's view names (gotcha 24), and only a fresh `Snapshot` replaces it. `Command::Model` put the three `WholeWorkbook` mutations on the wire (gotcha 38), so the host that must send the snapshots is no longer the host that asked. `apply_model` therefore calls `rebase` itself rather than instructing the host to, exactly as `apply_switch` adopts the accepted view itself. **The trigger is the generation moving, not the command variant**: a `LoadModel` that cannot read its path leaves the document alone, and reseeding on the variant would cost a client its cursors for a *failed* load — `a_load_that_fails_is_not_a_rebase` guards that arm. A `rebase: bool` beside `dirty` was rejected, because the host that forgets to call `rebase` is the host that forgets to read the flag. The **third** case is a *normal outcome*, not an error, and it is in the enum for the same reason the other two are: a host that wants the `Dirty` must match, and matching names it. `ServerMsg::Rejected` is the matching frame, addressed to **the asker alone** — nothing happened, so nobody else is owed anything — and the client keeps it as a `Rejection` rather than a sentence, because force needs to know *which* record and *which* kind (gotcha 43). **Fixture warning.** When testing a document swap, the incoming document's active view must have the **same name** as the outgoing one. A rename makes `apply_model` broadcast a `SwitchView`; the client folds it, and the stale cursor is then read under a key nothing has written — right *by accident*, with the bug invisible. `other_document()` renames, and a first version of this test passed because of it; `a_document_shaped_differently_under_the_same_view_name()` keeps the name and changes only the shape, so there is nothing to hide behind. -40. **A batch's basis is three tiers, and only two of them can be read off the effect.** `App` carries `document_generation` (bumped by `install_workbook`), `layout_version` (per *view shape*) and `record_versions` (per *record*) — ADR 001's stale-basis check, and shared document state, so never on `ViewState` (gotcha 30). **The unit is the record**: `record_of(key)` is the coordinate tuple minus `_Measure` (`record_row_identity`), because renaming a category coordinate in wide records mode moves every measure cell in the row (gotcha 1) — a cell-level unit is violated by one legal edit, so the unit must be at least as coarse as the coarsest legal write. **Only the `Dirty` is read off the `EffectKind`; both lower tiers are *observed* around the loop** — the source says so at `apply_effects` and this paragraph said the opposite until improvise-ilvl's audit. The `EffectKind` decides only *whether to look*: `touches_model` arms the shape snapshot, `writes_cells` the record fingerprints. `record_versions` **must** be observed, because an `EffectKind` says *that* a batch writes cells, never *which*; `layout_version` is observed for a better reason than necessity — `observe_reshape` diffs `view_shapes` before and after and bumps only where a view's shape actually moved (improvise-pdvm/nynk), so it is a **fact about the document** rather than a claim about the batch. Two consequences the declared form did not have. An **idempotent** `set-axis` bumps nothing, and batching two real changes into one line or two gives the same `2`. And the gotcha 21 argument is subsumed rather than restated: an aborted batch moved no shape, so there is nothing to not-claim — which is why observing once **after** the loop is correct where accumulating `Dirty` inside it is. **`view_shape` hashes what is drawn, not occupancy** (its doc comment: "Drawn, not configured") — axes, the drawn item names minus hidden ones, folds, page selections and the `prune_empty` *flag* — so an occupancy write does **not** bump `layout_version`: a `CellData`-only batch that prunes a row out of a view moves the cursor epoch (the frame observation) and not `layout_version`, as `context/code/command.md` records (the question that raised was improvise-6nsd, closed). The arming is wider than the tier (`touches_model` takes the shape snapshot), but a bump needs a shape hash to have moved. Three further consequences. A write landing the value already there **bumps nothing**, which is correct — no basis was invalidated. A `LayoutGeometry` effect that drops cells **will** bump their records when it shares a batch with a `CellData` one: an over-bump costing at most an unnecessary rejection, the same direction `Dirty::of` is deliberately sloppy in. And only *raw* data is fingerprinted — formula results live in `Model::formula_cache`, and a record whose derived value moved because someone edited elsewhere has not itself been written. The diff collects into a **set**: chaining `before.keys()` with `after.keys()` counts any rewritten record twice, a bug that survived one test run. **Superseded (ADR 009 step 7):** the record tier is no longer fingerprinted. `apply_effects` turns the store's change log (`DataStore::record_changes`/`drain_changes`) into a `CellDelta` (`improvise_incremental::cell_delta`) and `bump_moved_records` bumps every record in `moved_records(delta)`; `record_fingerprints` and the text above naming it are historical. +40. **A batch's basis is three tiers, and only two of them can be read off the effect.** `App` carries `document_generation` (bumped by `install_workbook`), `layout_version` (per *view shape*) and `record_versions` (per *record*) — ADR 001's stale-basis check, and shared document state, so never on `ViewState` (gotcha 30). **The unit is the record**: `record_of(key)` is the coordinate tuple minus `_Measure` (`record_row_identity`), because renaming a category coordinate in wide records mode moves every measure cell in the row (gotcha 1) — a cell-level unit is violated by one legal edit, so the unit must be at least as coarse as the coarsest legal write. **Only the `Dirty` is read off the `EffectKind`; both lower tiers are *observed* around the loop** — the source says so at `apply_effects` and this paragraph said the opposite until improvise-ilvl's audit. The `EffectKind` decides only *whether to look*: `touches_model` arms the shape snapshot, `writes_cells` the record fingerprints. `record_versions` **must** be observed, because an `EffectKind` says *that* a batch writes cells, never *which*; `layout_version` is observed for a better reason than necessity — `observe_reshape` diffs `view_shapes` before and after and bumps only where a view's shape actually moved (improvise-pdvm/nynk), so it is a **fact about the document** rather than a claim about the batch. Two consequences the declared form did not have. An **idempotent** `set-axis` bumps nothing, and batching two real changes into one line or two gives the same `2`. And the gotcha 21 argument is subsumed rather than restated: an aborted batch moved no shape, so there is nothing to not-claim — which is why observing once **after** the loop is correct where accumulating `Dirty` inside it is. **`view_shape` hashes what is drawn, not occupancy** (its doc comment: "Drawn, not configured") — axes, the drawn item names minus hidden ones, folds, page selections and the `prune_empty` *flag* — so an occupancy write does **not** bump `layout_version`: a `CellData`-only batch that prunes a row out of a view moves the cursor epoch (the frame observation) and not `layout_version`, as `context/code/command.md` records (the question that raised was improvise-6nsd, closed). The arming is wider than the tier (`touches_model` takes the shape snapshot), but a bump needs a shape hash to have moved. Three further consequences. A write landing the value already there **bumps nothing**, which is correct — no basis was invalidated. A `LayoutGeometry` effect that drops cells **will** bump their records when it shares a batch with a `CellData` one: an over-bump costing at most an unnecessary rejection, the same direction `Dirty::of` is deliberately sloppy in. And only *raw* data is fingerprinted — formula results live in `Model::formula_cache`, and a record whose derived value moved because someone edited elsewhere has not itself been written. The diff collects into a **set**: chaining `before.keys()` with `after.keys()` counts any rewritten record twice, a bug that survived one test run. **Superseded (ADR 009 step 7):** the record tier is no longer fingerprinted. `apply_effects` accumulates the store's change log (`DataStore::record_changes`/`drain_changes`) into one consolidated batch `CellDelta` (`improvise_incremental::cell_delta`) and `bump_moved_records` bumps every record in `moved_records(delta)`; `record_fingerprints` and the text above naming it are historical. 41. **`Storage` is not a VFS, and that closes the SQLite question.** The trait is whole-document `read`/`write` bytes (gotcha 2, `invariant(storage_is_two_ops)`) — no offsets, seeking, handles or locking — so an `sqlite3_vfs`, which needs `xRead(offset,len)`/`xWrite`/`xTruncate`/`xLock`, has nothing to bind to. SQLite *can* still sit behind it, as an in-memory database serialized to a blob (`rusqlite` `serialize`/`deserialize_read_exact`, proven with a probe before declining — declining on a capability that actually exists would have been the wrong decline). It is worth nothing: measured on 18,000 cells the SQLite container is **22% larger** than `.improv`, **4.7× larger gzipped**, ~**160×** larger at the floor (24 KB against ~150 bytes), and still rewritten whole on every save, so the one dimension SQLite normally wins — incremental page writes — is unavailable. improvise-9ix was declined on that evidence. Two things generalise past SQLite. **A second serialization format is a second implementation of the grammar**, obliged to track `improv.pest` forever; and **its own round-trip test cannot detect drift from `.improv`**, because it never compares the two — that missing cross-format equality test is what made the ticket look cheap. If a structured-query story is ever wanted it is a one-way `export_sqlite` beside `export_csv`, native-only and feature-gated: `libsqlite3-sys`' C build fails under this repo's nix cc-wrapper for `wasm32-unknown-unknown`, which would break the `wasm` CI gate the day it landed. @@ -91,7 +91,7 @@ Dated claims and counts inside an entry are snapshots: recount rather than cite. 45. **One session loop, two transports — and the browser saves itself.** `Host` is `App` + `Sessions` + `handle(id, ClientFrame) -> Vec` and lives in `improvise_command::ui::host`, **not** in `improvise-ws-server`, which re-exports it. It moved for improvise-9x6: the worker hosts the same authoritative `App` over a `MessagePort` and cannot depend on tokio, and a second copy would have been exactly the mirror improvise-xks6 had just deleted for the frames — a drifting `Applied`/`Broadcast` match is invisible until a client sees it. `registry` stayed on the server, because a worker hosts **one** document. What the worker adds is only what a `MessagePort` differs in: **one client by construction** (the `Addressed` filter is honoured anyway, against a future `SharedWorker`), and **a handshake with no close** — `ClientFrame::Viewport` must still come first, but a pre-handshake frame is *dropped* rather than closing a connection that does not exist. - **Saving is a frame, and dirtiness is read off the classification the effect already carries.** `effect::Save` is `ViewOnly` (gotcha 3) — correctly, since it writes *this host's* `file_path` through *this host's* `Storage` — so the derivation that builds `ModelCommand` from the model effects cannot name it, and for a while that meant no client on any transport could ask its host to save. `ClientFrame::Save` closes it (improvise-368h): a **request** carrying no path, because a client cannot see the host's storage, answered `ServerFrame::Rejected(NoDocumentPath | SaveFailed)` to the asker alone and **silently on success** — nothing moved, so no widget is dirty. It is a frame and not a `Command` for one concrete reason: as a `Command` it put a runtime-fallible operation inside a batch `ClientFrame::Commands` promises is all-or-nothing, so `[Save, SetCell]` reported a failed write and then applied the edit. It also fails the test every `Command` passes — it mutates no shared state and no client's slice, so `reduce_view` has nothing to fold and `Checked` has no basis to answer for. **And `MarkDirty` is gone.** It was a separate effect the *command* layer composed beside each mutation — reasonable on the key path, unreachable from anywhere else, because improvise-kmgs derived `ModelCommand` from the effects **one at a time** and so put `SetCell` on the wire without its companion (improvise-gv9r): a derivation correct per element and wrong for a set, whose symptom was that "save if dirty" never fired. `App::apply_effects` now raises the flag from the effect's `EffectKind`, which makes omitting it unrepresentable rather than validated — **declared** for `WholeWorkbook` (a replaced document differs from the file its path names, so a `LoadModel` that cannot read its path still raises it, costing one redundant write) and **observed** for `CellData` and `LayoutGeometry`, where `document_content`/`document_config` fingerprint the document before and after, so a write that leaves it as it was raises nothing (improvise-748f). The observation is read inside the loop, and `Save` clears the flag in loop order, so a batch that edits and then saves ends clean. The worker's persist trigger is therefore the **flag**, not a table of command variants: an observation rather than a second description of which commands change the document, for the reason `cursor_epoch` beats matching on `CycleAxis` (gotcha 24) and `document_generation` beats matching on `LoadModel` (gotcha 39). It is also strictly tighter — a refused batch raises nothing, and neither does a write that changes nothing, so the "was anything rejected" guard went with it. A browser still never *sends* `ClientFrame::Save`, and that is the answer rather than a gap: there is no file the user owns and no Ctrl-S semantics, the document *is* the OPFS entry, so the worker persists on its own behalf through the one existing `Save` effect. The cost that remains: `Storage` is whole-document (gotcha 41), so OPFS rewrites the file per accepted model command. **Superseded (ADR 009 step 7):** `document_content` is gone. For `CellData` the flag is `reordered || !delta.is_zero() || document_shape moved` over the batch cell delta; `LayoutGeometry` still compares `document_config`. The text above naming `document_content` is historical. + **Saving is a frame, and dirtiness is read off the classification the effect already carries.** `effect::Save` is `ViewOnly` (gotcha 3) — correctly, since it writes *this host's* `file_path` through *this host's* `Storage` — so the derivation that builds `ModelCommand` from the model effects cannot name it, and for a while that meant no client on any transport could ask its host to save. `ClientFrame::Save` closes it (improvise-368h): a **request** carrying no path, because a client cannot see the host's storage, answered `ServerFrame::Rejected(NoDocumentPath | SaveFailed)` to the asker alone and **silently on success** — nothing moved, so no widget is dirty. It is a frame and not a `Command` for one concrete reason: as a `Command` it put a runtime-fallible operation inside a batch `ClientFrame::Commands` promises is all-or-nothing, so `[Save, SetCell]` reported a failed write and then applied the edit. It also fails the test every `Command` passes — it mutates no shared state and no client's slice, so `reduce_view` has nothing to fold and `Checked` has no basis to answer for. **And `MarkDirty` is gone.** It was a separate effect the *command* layer composed beside each mutation — reasonable on the key path, unreachable from anywhere else, because improvise-kmgs derived `ModelCommand` from the effects **one at a time** and so put `SetCell` on the wire without its companion (improvise-gv9r): a derivation correct per element and wrong for a set, whose symptom was that "save if dirty" never fired. `App::apply_effects` now raises the flag from the effect's `EffectKind`, which makes omitting it unrepresentable rather than validated — **declared** for `WholeWorkbook` (a replaced document differs from the file its path names, so a `LoadModel` that cannot read its path still raises it, costing one redundant write) and **observed** for `CellData` and `LayoutGeometry`, where `document_content`/`document_config` fingerprint the document before and after, so a write that leaves it as it was raises nothing (improvise-748f). The observation is read inside the loop, and `Save` clears the flag in loop order, so a batch that edits and then saves ends clean. The worker's persist trigger is therefore the **flag**, not a table of command variants: an observation rather than a second description of which commands change the document, for the reason `cursor_epoch` beats matching on `CycleAxis` (gotcha 24) and `document_generation` beats matching on `LoadModel` (gotcha 39). It is also strictly tighter — a refused batch raises nothing, and neither does a write that changes nothing, so the "was anything rejected" guard went with it. A browser still never *sends* `ClientFrame::Save`, and that is the answer rather than a gap: there is no file the user owns and no Ctrl-S semantics, the document *is* the OPFS entry, so the worker persists on its own behalf through the one existing `Save` effect. The cost that remains: `Storage` is whole-document (gotcha 41), so OPFS rewrites the file per accepted model command. **Superseded (ADR 009 step 7):** `document_content` is gone. For `CellData` the flag is `reordered || !delta.is_zero() || document_shape moved` computed from **each `CellData` effect's own delta** (drained after every such effect, so ordering with `Save` holds; writes that cancel across effects still leave it set); only the record tier (`bump_moved_records`) uses the consolidated batch delta; `LayoutGeometry` still compares `document_config`. The text above naming `document_content` is historical. **The async seam does not survive being boxed.** `OpfsFs::open`/`attach`/`refresh`/`commit_deletes` are on the struct, not on `Storage`. `App::storage` is a `Box` and a trait object can name none of the four, so **handing the backend to the `App` puts the async seam permanently out of reach**; the host must hold the concrete backend *and* give the `App` a `Storage`, which means sharing one. That is `impl Storage for Rc`, and it lives in **`improvise-io`** rather than in each worker (improvise-hmws): `impl Storage for Rc` is two foreign types in any other crate, so improvise-9x6 wrote a forwarding newtype that every future worker would have written again. `Rc` and not `Arc` because the one caller is a Web Worker and wasm32 has no other thread. **The flat root is no longer load-bearing**: `OpfsFs::list` joins `dir` like the other two backends, so `storage_contract` takes `dir` as a parameter and the three agree at *every* directory rather than only at `Path::new("")` where joining is the identity (improvise-vhcz). What the index holds is **documents, not entries**, and the split between its three writers is the rule: `attach` indexes nothing (what it creates is zero bytes), `Storage::write` indexes the name and un-queues any pending delete of it, and `refresh` rebuilds from OPFS skipping zero-byte entries — counting an entry it cannot measure as a document, since the one cause of that is another tab holding the exclusive handle (improvise-vvl3). diff --git a/context/repo-map.md b/context/repo-map.md index a3d2f0ed..57efb2a9 100644 --- a/context/repo-map.md +++ b/context/repo-map.md @@ -7,7 +7,7 @@ Apache-2.0. Root binary+lib `improvise`; sub-crates under `crates/`: - `improvise-formula` — formula parser, AST, `parse_formula`. - `improvise-io` — `.improv` save/load, CSV/JSON import, `Storage` backends. No UI/commands. - `improvise-protocol` — the wire vocabulary and the portable session slice: `Command`, `Projection`, `ViewState`, the per-widget caches, `reduce_view`, key types, and `delivery/{wire,notice,exchange,client}.rs` (notices, watermarks, corrections). Depends on `improvise-core`, `zset`, `numtower` + serde. No IO, no command layer, no terminal. -- `improvise-command` — command/effect layer + semantic `App` state (app.rs, effect.rs, cat_tree.rs, viewport.rs, delivery.rs). Depends on `improvise-protocol`, `zset`, `improvise-incremental`. No ratatui/crossterm. +- `improvise-command` — command/effect layer + semantic `App` state (app.rs, effect.rs, cat_tree.rs, viewport.rs, delivery.rs). Depends on `improvise-protocol`, `improvise-gp`, `zset`, `improvise-incremental`. No ratatui/crossterm. - `numtower` — exact decimal arithmetic tower (`Num`, canonical text), ADR 005. Pure Rust, no workspace deps; `improvise-protocol` uses it for the wire form. Long form `context/code/numtower.md`. - `zset` — std-only Z-sets (non-zero checked `i64` weights, panic on overflow) plus `Circuit`, operators, `Trace`, `Rebuild` (ADR 009 clause 1). No dependencies. `context/code/zset.md`. - `zset-ffi` — C ABI over `zset` (`cdylib`/`staticlib`/`rlib`) for CFFI; `include/zset_ffi.h`. Native only, **not** in `wasmCrates`. `context/code/zset-ffi.md`. @@ -212,12 +212,12 @@ the incidents and the ticket ids.** Read the long form of every gotcha your diff 37. **`gp-tree` returns data and never prints; a fit needs seed *and* `FitConfig`.** A scorer takes `Row` slices. `fitted_formula` refuses features the document cannot name; categories are forced `Categorical` (zv8t). 38. **The wire vocabulary is derived twice** (`Command` view half, `ModelCommand` model half). Completeness is checked (`vocabulary_is_complete.rs`). `ApplyAndClearDrill` and `AcceptWizardModel` are deliberately off the wire (session-local operands). Recount, don't cite. 39. **`Applied` is an enum (`Diff | Rebased | Rejected`)** because a document swap is not a diff; the trigger is the generation moving, not the variant. A swap fixture must keep the view *name*. -40. **A batch's basis is three tiers** (`document_generation`, `layout_version`, `record_versions`); the unit is the record; lower tiers are *observed*, not read off the effect. The record tier is the batch's cell delta (`cell_delta` over the store's change log, `moved_records`); `record_fingerprints` is gone. +40. **A batch's basis is three tiers** (`document_generation`, `layout_version`, `record_versions`); the unit is the record; lower tiers are *observed*, not read off the effect. The record tier is the consolidated batch cell delta (`cell_delta` over the store's change log, `moved_records`); `record_fingerprints` is gone. 41. **`Storage` is not a VFS;** SQLite was declined on measurement (a second serialization is a second grammar). 42. **Browser render decisions are made in Rust** (`RenderModel`: `editable` from the `CellId` variant, selection reconciled with offsets, `flatten_cat_tree`). `improvise_protocol::frame` is the one envelope pair for both deployments. 43. **A batch carries its basis, and the server publishes it** (`Projection::Basis`). `Basis` has no `Default` or public constructor; `Checked`/`CheckedBatch` make "ignoring the basis" unrepresentable; `Rejection` keeps `RecordGone` / `RecordMoved` / `RecordUnstated` apart. Anti-staleness, not security. The check is a property of the session door (`apply_command` does not check), and only two of the three tiers hoist per frame — the record tier is asked per command (`records_named`) against the frame's basis. Read gotcha 43 before touching it. 44. **The TUI grid draws from `GridCache`**, which frames it too (`AxisFrame`; width rule `header_level_widths`). Use the one-pass accessors, not per-position scans. `RecordHash` is document-scoped, `AggregateHash` layout-scoped. -45. **One session loop (`Host`), two transports.** Saving is a request frame (`ClientFrame::Save`); dirtiness is *declared* for `WholeWorkbook` and *observed* for `CellData`/`LayoutGeometry` (`MarkDirty` is gone); for `CellData` the flag is the batch delta plus reorders, formulas and config, not a document fingerprint. `impl Storage for Rc` lives in `improvise-io`. `harness/` runs `OpfsFs` for real; Safari is unseen. +45. **One session loop (`Host`), two transports.** Saving is a request frame (`ClientFrame::Save`); dirtiness is *declared* for `WholeWorkbook` and *observed* for `CellData`/`LayoutGeometry` (`MarkDirty` is gone); for `CellData` the flag is each effect's own delta (drained after every `CellData` effect, so it keeps its order with `Save`) plus reorders, formulas and config, not a document fingerprint; writes that cancel across effects still leave it set. `impl Storage for Rc` lives in `improvise-io`. `harness/` runs `OpfsFs` for real; Safari is unseen. 46. **The page picks a peer:** `?ws=` → socket, else a module `Worker`. `#status` composes two facts; the first-frame watchdog is per-peer-agnostic. The demo model is built so every cell is a `Record`. 47. **Keys become commands in the browser;** the server must not know the keymap. `GridExtent`, `set_viewport`, `Commands(Vec)`, `resolve::NEEDS_THE_HOST` and the five client rules (`projected_at`, `opened`, `refused_here`, `Step::Declined`, `notify`). 48. **The browser JS is partly executed** (`harness/tests/shell.spec.js`); changes to `web/*.js` belong there. Serialize frames on one tail promise; disarm, don't pre-empt, a watchdog. Evidence headings may be module-qualified (and must be when the leaf collides). diff --git a/prolog-sessions/kb.pl b/prolog-sessions/kb.pl index 662f1d25..da838702 100644 --- a/prolog-sessions/kb.pl +++ b/prolog-sessions/kb.pl @@ -587,11 +587,11 @@ decision('improvise-felo.1',category_cap_has_one_home,'The regular-category cap decision('improvise-felo.13',skip_rebuild_keyed_on_declared_model_effects_plus_observed_view_and_drill,'Renamed from skip_rebuild_keyed_on_observed_inputs_not_effect_kind, whose name overstated. App::refresh_layout skips the rebuild unless layout_stale (DECLARED: set from EffectKind::Model(_) in apply_effects, and by install_workbook), or the drawn view name or the drill snapshot pointer (OBSERVED, compared as LayoutBasis) moved. It is not a hand-kept list of effects, the shape cursor_epoch and document_generation avoid, but it inherits gotcha 3 classification hole: a ViewOnly effect that changes what the layout reads leaves app.layout stale, where before felo.13 every batch rebuilt. A writer outside effects must call rebuild_layout itself (gotcha 23).'). decision('improvise-felo.4',zset_weight_overflow,'zset weights are checked i64 and panic on overflow, in debug and release (user ruling 2026-10-10). The group is partial past i64.'). decision('improvise-felo.4',derived_lives_in_incremental,'Derived lives in improvise-incremental, not improvise-core. core never depends on zset: the store logs CellChange and improvise-incremental builds the ZSet (cell_delta).'). -decision('improvise-felo.4',dirty_flag_is_delta_plus,'The document dirty flag is the batch cell delta plus reorders, formulas and config. record_fingerprints and document_content are gone; the record tier is moved_records(delta).'). decision('improvise-felo.4',ffi_circuit_poisoned,'A failed circuit step (a panic in step or in a rebuild step) poisons the zset-ffi circuit: every later call returns ZS_ERR_POISONED and the caller builds another.'). decision('improvise-felo.4',resumable_deadline_predicate,'The resumable step takes a predicate, not an Instant, because Instant::now() panics on wasm32 and the host closes over its own clock (context/code/zset.md, point 8).'). decision('improvise-felo.4',eager_occupancy,'Eager::occupancy counts the visible stored members of a collapsed group, and formula-target keys are not asked.'). decision('improvise-z2vr',delivery_axis_refetch,'A delivery axis change makes the correction a window refetch until ADR 008 supplies the render model (context/code/delivery.md section 3).'). +decision('improvise-felo.4',dirty_flag_is_delta_plus,'The document dirty flag is each CellData effect own cell delta (drained after every such effect, so ordering with Save holds) plus reorders, formulas and config; writes cancelling across effects still leave it set. The record tier alone uses the consolidated batch delta (moved_records). record_fingerprints and document_content are gone.'). wire_carries(command,both_directions,'user intent up, projection commands down'). wire_carries(snapshot,on_connect_only,'improvise-q08: full ViewState + render cache filled from current viewport'). wire_carries(command,both_directions,'user intent client->server, projections server->client'). @@ -658,6 +658,8 @@ crate_depends_on(improvise_protocol,zset). crate_depends_on(improvise_protocol,numtower). crate_depends_on(improvise_command,zset). crate_depends_on(improvise_command,improvise_incremental). +crate_depends_on(improvise_gp,improvise_core). +crate_depends_on(improvise_command,improvise_gp). depends_transitively(A,B):-depends_via(A,B,[A]). depends_via(A,B,C):-crate_depends_on(A,B). depends_via(A,B,C):-crate_depends_on(A,D), \+memberchk(D,C),depends_via(D,B,[D|C]). From 28e9836759d72075632a9273abcb95ae49a4d8e9 Mon Sep 17 00:00:00 2001 From: "Claude (dbsp-backend)" Date: Sat, 10 Oct 2026 16:06:22 -0700 Subject: [PATCH 10/10] fix(incremental): a new aggregation reaches the stems that carry its measure; ordered evaluation re-runs the passes over a zero (improvise-shbc) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- context/code/improvise-incremental.md | 21 ++-- .../test-evidence/improvise-incremental.md | 27 ++++- crates/improvise-incremental/src/formulas.rs | 10 ++ .../improvise-incremental/src/incremental.rs | 25 ++++- .../improvise-incremental/tests/formulas.rs | 106 ++++++++++++++++++ .../tests/support/mod.rs | 28 +++-- 6 files changed, 191 insertions(+), 26 deletions(-) diff --git a/context/code/improvise-incremental.md b/context/code/improvise-incremental.md index f7a11ffd..af5dc684 100644 --- a/context/code/improvise-incremental.md +++ b/context/code/improvise-incremental.md @@ -269,17 +269,24 @@ is followed in outline, and the places it is not are the reason for this section formulas' dependency graph by name. When it has no cycle and there are fewer formulas than passes, one evaluation per cell in dependency order reaches the fixed point eager's passes reach (a chain of fifteen links needs fifteen passes and a sixteenth to see no change). It - differs from the passes in one place: a cell holding NaN is never equal to itself, so eager's passes - never see it settle and call it `circular`, and the cells that read it keep what they computed - from the NaN. `mark_nan_circular` does the same afterwards. `Evaluation::Plain` always runs - the passes; `tests/ordering.rs` compares the two and `tests/formulas.rs` compares both with - eager. + differs from the passes in two places. A cell holding NaN is never equal to itself, so eager's + passes never see it settle and call it `circular`, and the cells that read it keep what they + computed from the NaN; `mark_nan_circular` does the same afterwards. And eager stops on `==`, + which calls `-0.0` equal to the `0.0` an early pass read from a stored cell, so a stem with a + zero cell can settle on a sign dependency order never sees (a stored `0.0` at `F1`, `F3 = F1`, + `F4 = F3 ^ -1`, then `F1 = -0`: eager's `F4` is `inf`). A stem whose ordered evaluation holds any + zero is therefore evaluated again by the passes (`holds_a_zero`). `Evaluation::Plain` always + runs the passes; `tests/ordering.rs` compares the two after every edit, zero signs included, and + `tests/formulas.rs` compares both with eager. 6. **A formula edit rebuilds the stems.** Adding, removing or replacing a formula (the fingerprint of the formula list moves) rebuilds the stems from the store by one pass, clears the cache and evaluates every stem, because a formula has a cell at each. Clause 5 wants a formula delta; the index is not maintained across a formula edit. `Meter::rebuilds` counts it. -7. **A new aggregation dirties every stem if a formula reads that measure.** `measure_agg` is not - a cell, so no change reaches the readers; `reads_name` asks the formulas. +7. **A new aggregation dirties the stems it can reach.** `measure_agg` is not a cell, so no change + reaches the readers. A fold under a None category takes its aggregation from the `_Measure` + coordinate of the key, which for a formula targeting another category comes from the stem and + from no name the formula reads, so every live stem carrying that measure is dirtied + (`Stems::carrying`). Where a formula names the measure (it targets `_Measure`) `reads_name` dirties every stem. 8. **A write of the same bits changes nothing; a write of `-0.0` over `0.0` does.** The cell delta treats the two as one value (clause 3), but a formula may not (`Units ^ -1` is `inf` and `-inf`), so the formula side reads the raw change log, not the canonical delta. diff --git a/context/test-evidence/improvise-incremental.md b/context/test-evidence/improvise-incremental.md index 31c87228..e143ab2b 100644 --- a/context/test-evidence/improvise-incremental.md +++ b/context/test-evidence/improvise-incremental.md @@ -460,7 +460,6 @@ What each fault was and the test that names it: `a_cell_edit_evaluates_the_one_stem_that_reads_it_whatever_the_table_size`, `every_formula_in_the_pool_is_added_edited_around_and_removed_like_eager` and six others; - a stem that no cell gives keeps its cells: `a_new_record_adds_its_stem_and_a_cleared_one_takes_it_away`; -- reads skip the formulas a name targets: `formulas::tests::what_a_formula_reads_includes_what_the_formulas_it_names_read`; - reads skip the filters' categories: `a_filter_on_a_category_the_stem_has_reads_the_cells_of_other_stems`; - reads skip the formula's own target name: `a_new_aggregation_for_the_measure_a_filtered_formula_falls_back_to_changes_it`; @@ -488,7 +487,10 @@ What each fault was and the test that names it: the cache: `every_formula_in_the_pool_is_added_edited_around_and_removed_like_eager`, `incremental_matches_eager_on_a_pooled_total_of_a_ratio`. -**Faults that cannot be seen, and why.** A born stem is found again by the lookup of the cell that +**Faults that cannot be seen, and why.** Reads that skip the formulas a name targets change no answer +either (only the unit test `formulas::tests::what_a_formula_reads_includes_what_the_formulas_it_names_read` +fails): a stem is evaluated whole, so whatever the skipped formula would have added is +re-evaluated because the formula itself dirties the stem. A born stem is found again by the lookup of the cell that gave it, since the cell agrees with the stem outside the target category, so skipping the extra "born means dirty" step changes nothing. The target category being in a formula's reads cannot matter either: stems are derived by removing it, so no stem has it to drop. @@ -632,3 +634,24 @@ Both zeros in both orders. Fails on `Max` negative if any zero is. **Speed.** The corpus self-comparison and the maintained-against-fresh property moved to their own binaries (`tests/corpus_eager.rs`, `tests/maintained.rs`): `oracle.rs` ran 2.45 to 3.04 s in loaded full runs because its two heaviest tests shared it. The tests are unchanged. + +### Review of PR #351: aggregation edits, zero signs (improvise-shbc) + +### a_new_aggregation_reaches_the_stems_of_a_formula_that_targets_another_category + +`R0 = East + West` on Region, Time folded, `SetMeasureAgg Revenue Max`: eager reads 13 (the maximum of +East plus the maximum of West), the first version kept 14. Fails when a new aggregation dirties only +the stems of formulas that name the measure, and when `Stems::carrying` ignores the measure. + +### a_new_aggregation_reaches_a_product_targeted_filtered_sum + +The reviewer's second repro, a Product-targeted `SUM(F0 WHERE Region = East)`. It already agreed on this +fixture and stays as the second shape of the same hazard. + +### ordered_evaluation_keeps_the_zero_eager_settles_on + +A stored `0.0` at `F1`, then `F3 = F1`, `F4 = F3 ^ -1`, then `F1 = -0`: eager settles on `F3 = 0.0` and `F4 = inf` +because it calls `-0.0` equal to the `0.0` it first read; the first ordered evaluation gave `-inf`. Fails +(only `Ordered`) when a stem with a zero is not re-evaluated by the passes. The ordering property's pool +now holds `-0`, `Units * -1` and `-Cost`, and compares after every edit, but did not find this fault +alone in 1,500 cases: this test is the pin. diff --git a/crates/improvise-incremental/src/formulas.rs b/crates/improvise-incremental/src/formulas.rs index 78f100fd..f23d4147 100644 --- a/crates/improvise-incremental/src/formulas.rs +++ b/crates/improvise-incremental/src/formulas.rs @@ -3,6 +3,7 @@ use improvise_core::formula::ast::Expr; use improvise_core::model::Model; +use improvise_core::model::category::MEASURE_CATEGORY; use improvise_core::model::cell::{CellChange, CellKey}; use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -255,6 +256,15 @@ impl Stems { self.live.keys() } + /// The live stems whose own `_Measure` coordinate is `measure`: a fold under a None category + /// takes its aggregation from that coordinate, whether or not a formula names the measure. + pub fn carrying(&self, measure: &str) -> impl Iterator + '_ { + let measure = measure.to_string(); + self.live() + .filter(move |s| s.get(MEASURE_CATEGORY) == Some(measure.as_str())) + .cloned() + } + /// Whether any formula reads a cell under `name`, directly or through another formula. pub fn reads_name(&self, name: &str) -> bool { self.reads.iter().any(|r| r.names.contains(name)) diff --git a/crates/improvise-incremental/src/incremental.rs b/crates/improvise-incremental/src/incremental.rs index ede2e1f0..2ba5fff7 100644 --- a/crates/improvise-incremental/src/incremental.rs +++ b/crates/improvise-incremental/src/incremental.rs @@ -204,10 +204,11 @@ impl Incremental { self.clear_stem(stem); } let mut dirty = moved.dirty; - if let Edit::SetMeasureAgg { measure, .. } = edit - && self.stems.reads_name(measure) - { - dirty.extend(self.stems.live().cloned()); + if let Edit::SetMeasureAgg { measure, .. } = edit { + dirty.extend(self.stems.carrying(measure)); + if self.stems.reads_name(measure) { + dirty.extend(self.stems.live().cloned()); + } } self.evaluate(dirty); } @@ -253,7 +254,14 @@ impl Incremental { let value = self.evaluate_cell(&keys[i]); self.model.set_formula_cell(keys[i].clone(), value); } - self.mark_nan_circular(&keys); + if self.holds_a_zero(&keys) { + for key in &keys { + self.model.set_formula_cell(key.clone(), None); + } + self.passes(&keys); + } else { + self.mark_nan_circular(&keys); + } } _ => self.passes(&keys), } @@ -286,6 +294,13 @@ impl Incremental { } } + /// Eager stops on `==`, which calls `-0.0` equal to the `0.0` an early pass read, so a stem + /// with a zero cell can settle on a sign the dependency order never sees: it runs the passes. + fn holds_a_zero(&self, keys: &[CellKey]) -> bool { + let zero = |k: &CellKey| matches!(self.model.formula_cell(k), Some(CellValue::Number(n)) if *n == 0.0); + keys.iter().any(zero) + } + /// A NaN is not equal to itself, so eager's passes never see it settle and call it circular /// once they run out; the cells that read it keep what they computed from the NaN. fn mark_nan_circular(&mut self, keys: &[CellKey]) { diff --git a/crates/improvise-incremental/tests/formulas.rs b/crates/improvise-incremental/tests/formulas.rs index 2bc4883f..03cabdd2 100644 --- a/crates/improvise-incremental/tests/formulas.rs +++ b/crates/improvise-incremental/tests/formulas.rs @@ -292,3 +292,109 @@ fn a_new_record_adds_its_stem_and_a_cleared_one_takes_it_away() { ]; same_model(&wb.model, &none, &edits, Evaluation::Ordered).unwrap(); } + +/// Region {East, West}, a Time category folded away, Revenue by quarter: `R0 = East + West` +/// targets Region, so its stems carry the measure but no formula names it. +fn region_target_model() -> Model { + let mut model = Model::new("regions"); + for (cat, items) in [("Region", vec!["East", "West"]), ("Time", vec!["Q1", "Q2"])] { + model.add_category(cat).unwrap(); + for item in items { + model.category_mut(cat).unwrap().add_item(item); + } + } + model.category_mut("_Measure").unwrap().add_item("Revenue"); + for (region, quarter, v) in [ + ("East", "Q1", 1.0), + ("East", "Q2", 3.0), + ("West", "Q1", 10.0), + ] { + let key = CellKey::new(vec![ + ("Region".into(), region.into()), + ("Time".into(), quarter.into()), + ("_Measure".into(), "Revenue".into()), + ]); + model.set_cell(key, CellValue::Number(v)); + } + model.add_formula(parse_formula("R0 = East + West", "Region").unwrap()); + model +} + +#[test] +fn a_new_aggregation_reaches_the_stems_of_a_formula_that_targets_another_category() { + let model = region_target_model(); + let none = vec!["Time".to_string()]; + let edits = [Edit::SetMeasureAgg { + measure: "Revenue".into(), + func: improvise_core::formula::ast::AggFunc::Max, + }]; + for by in [Evaluation::Ordered, Evaluation::Plain] { + same_model(&model, &none, &edits, by).unwrap(); + } + let mut backend = Incremental::new(model, none); + backend.apply(&edits[0]); + let at = CellKey::new(vec![ + ("Region".into(), "R0".into()), + ("_Measure".into(), "Revenue".into()), + ]); + let want = Some(CellValue::Number(13.0)); + assert_eq!( + backend.value(&at, Aggregation::OverHidden), + want, + "Max 3 + Max 10" + ); +} + +#[test] +fn a_new_aggregation_reaches_a_product_targeted_filtered_sum() { + let mut model = Model::new("products"); + for (cat, items) in [("Region", vec!["East", "West"]), ("Product", vec!["F0"])] { + model.add_category(cat).unwrap(); + for item in items { + model.category_mut(cat).unwrap().add_item(item); + } + } + model.category_mut("_Measure").unwrap().add_item("Cost"); + for (region, v) in [("East", 4.0), ("West", 9.0)] { + let key = CellKey::new(vec![ + ("Region".into(), region.into()), + ("Product".into(), "F0".into()), + ("_Measure".into(), "Cost".into()), + ]); + model.set_cell(key, CellValue::Number(v)); + } + model.add_formula(parse_formula("P0 = SUM(F0 WHERE Region = East)", "Product").unwrap()); + let none = vec![]; + let edits = [Edit::SetMeasureAgg { + measure: "Cost".into(), + func: improvise_core::formula::ast::AggFunc::Min, + }]; + for by in [Evaluation::Ordered, Evaluation::Plain] { + same_model(&model, &none, &edits, by).unwrap(); + } +} + +/// A stored `0.0` at `F1`, then `F3 = F1`, `F4 = F3 ^ -1` and last `F1 = -0`: eager's first +/// pass reads the stored zero, and its later passes call `-0.0` equal to it and stop. +#[test] +fn ordered_evaluation_keeps_the_zero_eager_settles_on() { + let mut model = Model::new("zeros"); + model.add_category("Region").unwrap(); + model.category_mut("Region").unwrap().add_item("East"); + let key = CellKey::new(vec![ + ("Region".into(), "East".into()), + ("_Measure".into(), "F1".into()), + ]); + model.set_cell(key, CellValue::Number(0.0)); + let edits: Vec = [ + ("F3 = F1", "_Measure"), + ("F4 = F3 ^ -1", "_Measure"), + ("F1 = -0", "_Measure"), + ] + .iter() + .map(|(raw, cat)| Edit::AddFormula(parse_formula(raw, cat).unwrap())) + .collect(); + for by in [Evaluation::Ordered, Evaluation::Plain] { + same_model(&model, &[], &edits, by).unwrap_or_else(|e| panic!("{by:?}: {e}")); + } +} diff --git a/crates/improvise-incremental/tests/support/mod.rs b/crates/improvise-incremental/tests/support/mod.rs index d4f6f45c..edc5a71c 100644 --- a/crates/improvise-incremental/tests/support/mod.rs +++ b/crates/improvise-incremental/tests/support/mod.rs @@ -238,7 +238,7 @@ pub fn hidden_counted_workbook() -> improvise_core::workbook::Workbook { /// Formulas the random tests add and remove, as `(text, target category)`: chains, a cycle, /// aggregates over expressions, filters, a ratio, and two that target Region. -pub const FORMULA_POOL: [(&str, &str); 18] = [ +pub const FORMULA_POOL: [(&str, &str); 21] = [ ("Profit = Revenue - Cost", "_Measure"), ("Margin = Profit / Revenue", "_Measure"), ("Weighted = SUM(Revenue * Units)", "_Measure"), @@ -257,6 +257,9 @@ pub const FORMULA_POOL: [(&str, &str); 18] = [ ("EastRev = Revenue", "Region"), ("Recip = Units ^ -1", "_Measure"), ("Pa = A + Revenue", "_Measure"), + ("NegZero = -0", "_Measure"), + ("Flip = Units * -1", "_Measure"), + ("Negate = -Cost", "_Measure"), ]; pub fn formula_edit(index: usize) -> Edit { @@ -371,23 +374,24 @@ pub fn any_formula_edit() -> impl Strategy { ] } -/// The same edits told to a backend that evaluates in dependency order and to one that always -/// runs eager's passes: their models, formula cache included, must print alike. +/// The same edits told to a backend that evaluates in dependency order and to one that runs +/// eager's passes: models, cache and zero signs must print alike after every edit. pub fn same_orders(model: &Model, none: &[String], edits: &[Edit]) -> Result<(), String> { let side = |by| Incremental::new(model.clone(), none.to_vec()).with_evaluation(by); let (mut ordered, mut plain) = (side(Evaluation::Ordered), side(Evaluation::Plain)); - for edit in edits { + for (n, edit) in edits.iter().enumerate() { ordered.apply(edit); plain.apply(edit); + let (want, got) = ( + format!("{:?}", plain.model()), + format!("{:?}", ordered.model()), + ); + if want != got { + let told = &edits[..=n]; + return Err(format!("after {told:?}:\n plain {want}\n ordered {got}")); + } } - let (want, got) = ( - format!("{:?}", plain.model()), - format!("{:?}", ordered.model()), - ); - match want == got { - true => Ok(()), - false => Err(format!("after {edits:?}:\n plain {want}\n ordered {got}")), - } + Ok(()) } /// An `Eager` rebuilt from scratch on every edit: maintaining one in place