diff --git a/.agents/skills/code-review/SKILL.md b/.agents/skills/code-review/SKILL.md index 1594097a..b4687084 100644 --- a/.agents/skills/code-review/SKILL.md +++ b/.agents/skills/code-review/SKILL.md @@ -25,7 +25,7 @@ them here would let the copy drift: | Source | What it holds | |---|---| | `context/design-principles.md` | The "why" — functional-first, polymorphism over conditionals, correctness by construction, layering, §7 testing doctrine | -| `context/repo-map.md` | The "where", plus **numbered gotchas** — read the ones your diff touches | +| `context/repo-map.md` | The "where", plus a **numbered gotcha digest** (the full text is `context/gotchas.md`) — read the long form of the ones your diff touches | | `prolog-sessions/kb.pl` | Executable knowledge: `invariant/2`, `doctrine/2`, `forbidden/2`, `decision/3`, and prior `work_done/3` | | `AGENTS.md` | Working rules: bug-first tests, `Option`/`Result`/`Box` at the edges, history reconciled only by merge, a new conventional commit by default (amend only an unpushed HEAD, for a small fix), and nothing that needs a force push | | `.agents/skills/wave/agent-rules.md`, "GitHub" | The reviewer's rule: what to read before your first comment, how to answer each existing review, and what every review you post must state. Read it before reviewing a PR | diff --git a/.agents/skills/wave/agent-rules.md b/.agents/skills/wave/agent-rules.md index 444059f8..16840d3b 100644 --- a/.agents/skills/wave/agent-rules.md +++ b/.agents/skills/wave/agent-rules.md @@ -341,7 +341,8 @@ out on purpose: ## Files you must not write `.beads/**` (never commit `issues.jsonl`, never `git checkout` it), -`prolog-sessions/kb.pl`, `context/repo-map.md`: `AGENTS.md`, "Who owns what". +`prolog-sessions/kb.pl`, `context/repo-map.md`, `context/gotchas.md`, +`context/file-inventory.md`: `AGENTS.md`, "Who owns what". Also `AGENTS.md`, unless your brief grants you a named section. Report proposed text for any of these in your hand-back. diff --git a/AGENTS.md b/AGENTS.md index b5c5e18a..8f0431f9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -151,7 +151,7 @@ that costs to get right. Everything in it was learned by getting it wrong first. ### Who owns what -Three files are **orchestrator-only**, and the rule exists because they are +These files are **orchestrator-only**, and the rule exists because they are guaranteed conflicts — every agent has a reason to touch them and they merge badly: @@ -160,8 +160,9 @@ badly: | `.beads/` (incl. `issues.jsonl`) | bd owns it. See "Never hand-edit" below. | | `prolog-sessions/kb.pl` | Written by `scripts/kb` only. See `context/code/kb-tool.md`. | | `context/repo-map.md` | Every agent wants to add a gotcha; every wave would conflict. | +| `context/gotchas.md`, `context/file-inventory.md` | The long forms the map digests; same conflict, same owner. | -Wave agents **read** all three freely (`bd show`, `bd ready`, grep `kb.pl`) and +Wave agents **read** all of them freely (`bd show`, `bd ready`, grep `kb.pl`) and **write** none of them. They report proposed repo-map lines and `kb.pl` facts as *text* in their hand-back, and the orchestrator applies them. Repo-map lines are batched at the end of the wave, after the review, because they conflict. diff --git a/Cargo.lock b/Cargo.lock index 77a3c5b8..51a21dda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "atomic" version = "0.6.1" @@ -178,6 +184,30 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -705,6 +735,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -1216,6 +1257,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1233,6 +1284,26 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1251,6 +1322,18 @@ dependencies = [ "libc", ] +[[package]] +name = "numtower" +version = "0.1.0" +dependencies = [ + "num-bigint", + "num-integer", + "num-rational", + "num-traits", + "proptest", + "rust_decimal", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1449,6 +1532,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1469,7 +1561,7 @@ dependencies = [ "bitflags 2.13.2", "num-traits", "rand 0.9.5", - "rand_chacha", + "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -1510,6 +1602,8 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ + "libc", + "rand_chacha 0.3.1", "rand_core 0.6.4", ] @@ -1519,7 +1613,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -1534,6 +1628,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -1549,6 +1653,9 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] [[package]] name = "rand_core" @@ -1713,6 +1820,23 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rust_decimal" +version = "1.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.8", + "rand 0.9.5", + "serde", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "rustc_version" version = "0.4.1" @@ -2161,6 +2285,36 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "tungstenite" version = "0.30.0" @@ -2525,6 +2679,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/Cargo.toml b/Cargo.toml index 1a5467a7..a07fe043 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/improvise-formula", "crates/improvise-core", "crates/improvise-protocol", "crates/improvise-io", "crates/improvise-command", "crates/improvise-wasm-client", "crates/improvise-ws-server", "crates/improvise-worker-server", "crates/gp-tree", "crates/improvise-gp"] +members = [".", "crates/improvise-formula", "crates/improvise-core", "crates/improvise-protocol", "crates/improvise-io", "crates/improvise-command", "crates/improvise-wasm-client", "crates/improvise-ws-server", "crates/improvise-worker-server", "crates/gp-tree", "crates/improvise-gp", "crates/numtower"] [package] name = "improvise" diff --git a/context/adr/009-incremental-evaluation.md b/context/adr/009-incremental-evaluation.md new file mode 100644 index 00000000..68135713 --- /dev/null +++ b/context/adr/009-incremental-evaluation.md @@ -0,0 +1,447 @@ +# ADR 009 — Incremental evaluation: an in-house DBSP core, plugged in behind a trait + +**Status:** proposed. · +**Ticket:** improvise-felo.4 (the record of this ADR; implementation goes to its own +epic) · +**Relates to:** ADR 001 (the basis a write claims), ADR 002 (surrogates on the wire), +ADR 003, ADR 005, ADR 006, ADR 007, ADR 008 + +## Problem + +Every change recomputes every formula cell (`Model::recompute_formulas`, called by +`rebuild_layout` after each batch), and raw aggregates are re-folded on every read. That +does not scale to a live source (ADR 003) streaming changes into a large table, nor to +ADR 008's virtualized grid, which needs occupancy, totals and widths maintained without +building the grid. A change should cost in proportion to what it touches. + +## Decision + +Formulas are evaluated incrementally by an in-house implementation of the DBSP core. A +cell write becomes a delta, aggregates are maintained, and only the formula cells whose +inputs moved are re-evaluated by the existing evaluator. The incremental results must +equal today's eager results exactly. + +**Principle: everything is as incremental as possible**, so the work done per model +mutation is minimised. Where this ADR does work that grows with more than the +mutation, it says so and names the incremental path that normally applies. + +### 1. The `zset` crate + +A problem-agnostic workspace crate with no dependencies beyond `std` (so it is +wasm32-clean and serde-free). It provides: +- `ZSet`, a map from `K: Ord` to a non-zero `i64` weight, which is an abelian group; +- a synchronous, single-threaded `Circuit` per process, with `delay`, `integrate`, + `differentiate`, `map`, `filter`, `flat_map`, `index_by`, the bilinear delta join + `join_indexed`, `distinct` and `aggregate`; +- resumable rebuilds: building an operator from a snapshot is a `step(deadline)`, so it + can run as a sliced job; +- operator state behind one trace interface (look up, read a range, apply a delta), so + later store-memory work can page or compress it. + +**Operators are self-contained.** An operator's state and behaviour depend only on its +inputs and its own state, never on a shared in-process global or on another operator's +internals. That keeps the graph distributable: in principle the model could run across a +cluster of machines. Nothing more is designed for that here. + +There are no nested streams, no recursion operator and no timestamps beyond one clock. +**The formula language is the query language.** What a request asks of the model beyond +its axes and window (filters, aggregation, derived values) is written in formula text +over ids (ADR 007). It goes through the same evaluator and circuit as a document +formula, so there is no second query language. + +### 2. The plug + +improvise-core gains a trait, `Derived`, for the two things the evaluator reads: the +formula result at a key, and the raw fold of the cells that match a partial key. The +incremental backend implements it in a seam crate, `improvise-incremental`, which +depends on `zset` and improvise-core. improvise-core never depends on `zset`. + +A live source (ADR 003) that reports its changes feeds cell deltas like `DataStore`. A +source that answers aggregate queries but reports no changes implements the derived-value +operator directly, by query pushdown, with no change feed. + +The circuit decides *which* formula cells to re-evaluate. The one existing evaluator +computes each value, so formulas are never compiled into operators. + +**`Eager` is temporary.** Today's code moves behind the trait as `Eager`, unchanged. It +exists only to show discrepancies between the new evaluator and the old one: a +differential property test applies one random edit sequence to both backends and +compares every drawn cell, total and occupancy count. Eventually it is deleted, and +nothing depends on it surviving. Long-term tests compare against exact references, such +as `ExactSum` against rational sums, not against `Eager`. + +### 3. The cell delta + +`DataStore::set` and `DataStore::remove` emit a `ZSet<(CellKey, CellValue)>`: +- `+(k, v)` for a new cell; +- `−(k, v₀) + (k, v₁)` for an overwrite; +- `−(k, v₀)` for a removal. + +Values compare by normal float rules, except that every NaN is one canonical value: +`-0.0` equals `+0.0`, and an equal-value overwrite consolidates to nothing. NaN payloads +are not preserved, so nothing can be smuggled through them. Today's record tier compares +bits, so overwriting `0.0` with `-0.0` stops counting as a change. Multisets order values +by `f64::total_cmp` after mapping every NaN to the canonical one and `-0.0` to `+0.0`. `CellKey` gains `Ord`, which is its canonical order: the +natural order of ids (ADR 007 amendment 1, clause 4), the order eager folds in. The +circuit steps once per model effect. + +The record tier and the dirty flag read this delta instead of scanning the store. The +dirty flag fires on a non-empty `(key, value)` delta, or when a fingerprint of the +formula set moves, because formula edits move no cell. + +Under ADR 007 and its amendment, `CellKey` pairs are `(CategoryId, ItemHandle)`. So a +rename is a display write and emits no delta. The formula-set fingerprint must use the id +form too, or a rename reads as a formula change. + +### 4. Aggregates + +A group is the cell key minus the None categories. One aggregation circuit serves every +shape key with the same None list, which removes the per-session model clone. + +| aggregate | state per group | per delta element | +|---|---|---| +| `Count` | count of values, count of numbers | O(1) | +| `Sum`, `Avg` | an exact accumulator (clause 8), and the count | O(1) | +| `Min`, `Max`, text unanimity | a multiset of values | O(log d) | + +A group is read with eager's rules, including NaN and signed zero. `LAST`/`FIRST` and +semi-additive measures (improvise-felo.7, felo.8) reuse the multiset shape, keyed by the +ordering category's order. Those tickets choose the key. It must keep a reorder or an +insert incremental: only the entries under the moved items change, and nothing is +rebuilt. + +### 5. Formulas + +**Stems** are `distinct` over the delta, keyed by "cell key minus the target and None +categories". A formula counts at a stem where its top-level `WHERE` matches or raw data +exists at the target. + +**Inputs.** A formula cell is re-evaluated when one of its inputs moved: +- a `Ref(R)`: the group at the stem with `R`'s category set to `R`; +- an `Agg`: the group at the stem; +- an `Agg` whose inner `WHERE` names `Cat = Item`: the group at the stem with `Cat` set + to `Item`. This is the one join: stems indexed by "stem minus `Cat`". + +The affected pairs are evaluated in the formula graph's topological order. + +**An aggregate over an expression** (`SUM(Revenue * Price)`) is a pipeline: +1. A per-record stage retracts and reinserts each touched record. It reads the record's + previous cells as the store minus the delta, so it keeps no copy of the values. The + store is the `integrate` operator's trace over the cell delta, read as an input of + the stage, so the stage stays self-contained. +2. A map produces `(context, order key, expr)`. The order key is (context rank, record + key in `CellKey` order), which is eager's concatenation order. +3. The group aggregate, with an error channel that reads the first `Err` in order. + +No cross product is built. A formula whose expression reads other records is evaluated +over the **occupied contexts** only, and `Eager` uses the same enumeration. The answer is +unchanged, because a context with no records contributes nothing. One exception: an +inner bare ref that names no item errors at every context, so for it the ref is checked +before enumerating. This replaces `Expansion::members`. + +### 6. Circular references + +DBSP's least fixed point is not improvise's circularity: `X = X + 1` has none, and the +answer must be `Error("circular")`. The formula graph's strongly connected components +are maintained incrementally as formulas change. Acyclic formulas are evaluated once +each, in topological order. A cyclic component is maintained incrementally too: a change +re-iterates only the cells it reaches. A component that does not converge within the +existing bound of sixteen passes (`MAX_EVAL_DEPTH`) propagates `Error("circular")`, as +today. An optimisation that speeds convergence is allowed only where it is exact: it +gives the same result as the plain iteration, and a test compares the two. + +### 7. Operator state follows the request's shape key + +A projection request's **shape key** selects operator state, shared by every request +with the same key, from any client. The key is: +- the categories on each axis, by level; +- the None categories; +- the filter keys: each filter category's item, and the hidden items of categories on + None, Page or Filter; +- the page categories, so each page is a maintained slice and paging rebuilds nothing; +- `measure_agg`. + +The key uses the canonical parsed id form of the request's query text, so two spellings +of one query share their operator state. Everything else in a request is applied when output is read: hidden and collapsed items +on axis categories, item order, the page selection and prune. Occupancy and column +aggregates are keyed by axis **items**, so a collapse, hide, show, reorder or added item +changes no operator. The visible axis is a remap. + +- **A new key** builds its operators as a sliced job under ADR 008's job runner. Deltas + arriving during the build are buffered and added at install. The finished result is the + response to the request that asked for it, not a notice. +- **A formula edit** is a formula delta fed to the circuit. +- **A document swap** rebuilds everything, and a new key's first build is a full pass. + These are the only rebuilds; every other change is a delta. +- **Saving a view definition** touches no operator. +- **Live keys are a cache.** A key is evicted after a period in which no request names + it. The period is a local duration on the host, set by measurement, and never compared + with another machine's clock. Every eviction is announced (clause 9). + +**Column totals and hidden rows.** Totals are kept over all rows, because the shape key +is shared and a hide is one client's. Whether hidden items travel in a request is ADR +008's open question. If they do not, a total is read as it is kept. If they do, a total +must leave the hidden rows out, so a read subtracts the request's H hidden cells as a +temporary overlay: +- `Count` subtracts; +- `Min` and `Max` skip keys whose whole count is hidden; +- text counts the distinct keys left; +- `Sum` subtracts exactly through the accumulator. + +That is O(H log d) per read for every aggregate, with no view-store state in the model +store. +Aggregate-over-expression totals index their entries by row tuple for the same +subtraction. + +### 8. Exact sums + +An f64 sum maintained by `+v` and `−v` drifts. Measured by +`crates/improvise-core/examples/retraction_drift.rs`, at 18,000 cells after a million +cycles, it is 1.7e-9 from eager, past `FLOAT_EQ_EPSILON`, so `=` could answer +differently on the two paths. So every `Sum` and `Avg` uses **`ExactSum`**, in both +backends: +- a fixed-point integer covering every finite f64, with counts of NaN and ±∞, which + makes it an abelian group that rounds once, on read; +- measured bit-equal to a fresh sum after a million cycles, and exact against rational + sums on 20,000 cases; +- 568 bytes per group, and about 1.3× the cost of an f64 add per update. + +The sign of a zero sum follows IEEE rules for the same addends: it is `-0.0` only if +every addend is `-0.0`, and an exact cancellation gives `+0.0`. It lives in ADR 005's +`numtower` crate. **User-visible change:** displayed sums change in their last bits, +because they become correctly rounded. + +### 9. Delivery + +The **model store** (the document model and the DBSP graph) and the **view store** (the +cursor, the window and unsaved view definitions) are conceptual boxes, not a network +boundary. Where a piece of state physically lives (a client, the host or a worker) is +an implementation detail. The model store's decisions depend only on the model. A +server-side twin of a client's state, such as its filter, watermark or window, may be +held at the serving end of an edge where that makes the graph genuinely simpler. It is +then part of the edge, not of the model store, and it is disposable: losing it never +gives wrong data. A client that keeps its own watermark recovers with a correction from +it. A client that relied on the twin for its watermark recovers with a full refetch. + +**Every edge may cross a network.** An edge, meaning the stream from one operator to the +next, may be in-process or remote. No edge assumes a function call or shared +memory. An operator that sits at a wire boundary is just one implementation of an edge, +an **edge adapter**. So every edge has the same semantics: +- **A wire form.** Every element type that flows on an edge has a serializable form, + defined in `improvise-protocol`. The adapter converts to and from it, and `zset` stays + free of serde. `CellKey` is not `Serialize`, so on the wire a cell is named by its + tuple of ids, its position or an ADR 002 surrogate. +- **The wire form is lossless.** Nothing is lost in transport, numeric precision above + all: + - a value never crosses as a JSON or JS double. It crosses in ADR 005's canonical text: + an inexact finite value as `#i` with its shortest round-trip spelling, which keeps + the sign of zero, infinities as `inf` and `-inf`, and every NaN as the one token + `nan`; + - positions and counts below 2⁵³ are exact as JSON numbers. Wider integers cross as + strings; + - text crosses byte-exact, with no normalisation; + - the delta is the unit that crosses, never `ExactSum`'s limbs. A receiver that needs + a sum gets the host's exact read or rebuilds it from the exact deltas. + + The stamp rule detects lost messages, but nothing would detect a lossy encoding. So + every element type has a round-trip property test, where encoding then decoding is + the identity for every value, including `-0.0`, infinities, 28-digit decimals and + `i128`-wide fractions. A NaN round-trips as the canonical NaN. +- **The clock is causal, never a wall clock.** Ordering, staleness and loss detection + use only the stamp sequence (below), a logical clock the host advances once per + committed command. Each process runs its own synchronous sub-circuit, and across an + edge the stamped commands are the clock. No decision compares wall-clock times across + machines, so clock skew cannot affect it. A delta crosses a wire only after its command + commits, so a remote operator never sees a delta that is later rolled back. +- **Ordering and backpressure.** An edge delivers in order. A slow edge coalesces + exactly: deltas are summed, and notices merged. A merged notice carries the range of + publications it covers, `(from, to]`, so the client that merges notices sees no gap + where nothing was lost. +- **An edge is a session, not a connection.** An edge may be one long-lived connection, + or a sequence of requests carrying the same session token or cookie, for example + several HTTP requests. The host decides nothing from the token: it is transport, and + any later authentication is out of scope. What continues across connections is the + client's own watermark and shape keys, which it sends with each request. Every request + can be retried, and a resync from a watermark is idempotent. A client asks for + "notices after my watermark", and the host answers from **one shared, bounded log + of published notices**: a single log, not one per edge or client, holding the notices + themselves (keys and flags, not values). A request older than the log's start is + answered with a correction (below). +- **Failure.** Timeouts are local monotonic durations, never compared across machines. + On a long-lived connection, both ends keep alive, and a host-side timeout is tied to + that open connection. On a sequence of requests, only the + client detects failure: a request that fails or times out. In either case the client + then asks for a correction from its watermark. + +- **Correction is Z-set arithmetic.** A correction is `target − held`, where `target` + is the current state for the client's window and `held` is what the client holds. + Applying it with ordinary Z-set addition gives `held + (target − held) = target`, so + the group structure of `ZSet` provides it, with no separate machinery. The client's + position is a **watermark**: the whole stamp `(host instance, document_generation, + publication)` up to which it has applied the stream, so everything at or before it is + applied. A watermark from another host instance or generation is behind every current + stamp. The client holds one per shape key it reads. A watermark is view-store state: + the model store keeps none, so a client sends its own with each request unless a twin + at the serving end holds it. + - where the shared log covers the watermark and the covered range holds only moved + keys, the host sends the current values of those keys, restricted to the client's + window, and the client applies `target − held` over them. The log holds keys and + flags, not values, so the model store keeps no view-store state; + - where the covered range also holds axis changes, relabels or flags, which positions + and labels depend on, the correction carries them too, in order, as a merged notice + does, so the client re-derives positions incrementally; + - only where the log no longer covers the watermark, or the covered range holds an + oversized notice that lists no keys, does the host fall back to `target` + for the whole window, and the client applies `target − held`; + - a full refetch is `target − ∅`. + + **Every read sees only the last committed state**, whether it is a correction, a + fetch or `project`, and it is labelled with that state's stamp. A value a failed + command wrote and then rolled back is never read. A correction is labelled with the + publication range it covers, so it is applied only at the watermark it starts from, + which makes it idempotent and resumable. Its elements are `(key, value)` pairs with + weights, in the lossless wire form, so a client and a worker apply it with the same + code. + +Aggregation stays on the model side (ADR 008), so the operators that aggregate run on +the model side. Which other edges cross a wire is a deployment choice. The engine's invalidation signal, which ADR 006's replay trigger +relies on, is a correction trigger below, followed from a displayed cell's key back to +the model. + +**A request** is a shape key plus a window, given as an absolute cross-product top-left +and dimensions. The window is view-store state, and a request costs O(visible). +- **Positions** are bigints where they exceed 2⁵³, sent as JSON strings. +- **Output** is keyed by tuple identity, with the position attached when sent. +- **An item insert, removal or reorder** is sent as an axis change, and each client + re-derives its positions. + +**Pruning support.** Pruning is window-relative: a row or column empty within the +displayed values is hidden. This replaces today's whole-grid `prune_empty`. The model +answers one restricted successor query: "the next k rows after position p with a value +in a column of S", and the same for columns. A per-column sorted index of occupied rows +answers it in O(|S| log n + k|S|), holding O(occupied cells). Whether pruning runs on +the client and which parameters travel are ADR 008's questions, and this design supports +either answer. If the client prunes, it prunes its window, fills forward with that +query, and, when scrolling up or left, finds a new top-left by galloping backwards in +doubling jumps (about 14 to 28 probes at density 10⁻⁴). + +**The change notice.** Each command commits on its own (ADR 006), and after each +committed command the host emits one notice, addressed to no client. A frame +whose later command fails keeps its earlier commands, and their notices, published. A +notice carries: +- the stamp `(host instance, document_generation, publication)`. The publication number + increments once per committed command. The host instance changes on restart; it is a + random id, never a wall-clock time; +- axis changes, display relabels as `(id, display)`, and a flag when the formula set, + `measure_agg`, the view definitions or the default view moved. A shared display's + ` #2` suffix is never sent: the client computes it from the displays in scope; +- the shape keys evicted since the last notice; +- for each live shape key, the moved output keys (cells, formula cells and totals), as + the circuit computes them. Moved records alone would miss formulas with cross-stem + inputs. + +**A client receives only what it needs.** Filtering is part of the DBSP graph, not a +separate mechanism. A client either pulls just the state it needs, or pushes a filter to +the host's side of its connection: a filter operator on its shape's query, written in the +query language, through which its notices pass. Whether the filter also carries the +client's window is an optimisation, since the model store's decisions depend only on the +model. A pushed filter is held by the open connection, or by a twin. Otherwise the +client sends it with each request. While a connection's filter is open, it counts as a request +naming its shape, so its shape is not evicted. + +A notice whose key lists exceed a fixed size carries "correct" instead. That is a +fallback, and it costs every reader a correction. + +**What a client does.** It intersects the moved keys with its window. A fold row has no +maintained key, so it maps member keys through its own remap first. It then fetches what +moved. It asks for a correction when it sees a publication gap, a new +`document_generation` or host instance, its shape in the eviction list, a notice it +cannot read, or a dropped connection or failed request. A slow client may merge pending notices: the union of +keys, the covered range of publications, and axis changes kept in order. Because the +wire is lossless, outputs keep the circuit's value identity end to end. + +**What this replaces.** Today's per-session mirrors and pushed diffs (`Sessions`, +`ClientSession`, `grid_updates`, `Projection`, `Dirty`). The model store keeps no +view-store state, only the shared notice log. `GridExtent` is derived by the client, and in records +mode the record count is fetched. `Projection` also published the ADR 001 basis a write +claims. Whether `project` returns it instead, and what replaces the layout tier, are +ADR 008's questions. + +**Edge adapters.** An edge that crosses a wire has one adapter at each end. Toward the +circuit, the client's adapter is a sink for results and notices and a source of requests. +Toward the client it is the mirror image. The client's end keeps the watermark. In +process, an edge is a function call. + +### 10. Column width hints + +Every column incrementally maintains a width hint, which optimises grid sizing and +reduces jitter. The hint is an integrated aggregate: a histogram from display width to +count, so a retraction is O(log d). Where in the graph the hint is computed is a +scheduling choice, made as an optimisation later. + +### 11. Rollback and memory + +- **Rollback.** The circuit's state joins ADR 006's per-command snapshot, which is taken + before a command's effects and restored if the command fails. An inverse delta is not + a rewind, because `delay` and `differentiate` keep history. The requirement is that + this snapshot does not cost O(cells) per command against ADR 008's 8 ms budget, and + that writes do not block reads. How is for the implementation epic. +- **Memory.** Resident state beside the store is the multisets, `ExactSum` per summed + group, occupancy per live shape, the successor-query index, the width histograms and + the bounded notice log. The store itself is the integrated base and is never copied. It + is measured at ADR 008's fixture before the store-memory work begins. + +## Alternatives rejected + +- **Feldera's `dbsp` crate:** 67 required dependencies, including tokio and zstd, and it + fails to build for wasm32. +- **Compiling formulas into operators:** a second evaluator. +- **An ad-hoc dirty-tracking cache:** one invalidation rule per derived fact, each a + second description of evaluation. +- **A topic or subscription mechanism beside the graph:** filtering is an operator in + the graph (clause 9). + +## Consequences + +**Good:** +- A write's evaluation costs O(delta) plus the formula cells it reaches. +- The store scans and the per-view model clones go. +- ADR 008 gets occupancy, totals and widths from one evaluator. +- Live sources that report changes plug in. +- The model store holds no view-store state. + +**Bad:** +- Two backends exist until `Eager` goes. +- Integrated state is O(cells) beside the store. +- `ExactSum` costs 568 bytes per summed group. +- Each command snapshots the circuit's state for rollback. + +**Order** (each step is a ticket in the implementation epic, behind the differential +test): +1. `zset`; +2. the `Derived` trait with `Eager`; +3. the cell delta, the record tier and the dirty flag; +4. order-free aggregates and occupancy, measured at ADR 008's fixture; +5. formulas and the aggregate-over-expression pipeline; +6. `Sum`, `Avg` and totals through `ExactSum`; +7. live sources; +8. delivery. + +Step 8's required tests: +- a frame `[AddItem, SetCell]` publishes one notice per committed command, numbered + consecutively, and no client asks for a correction; +- a dropped notice is detected; +- a reconnect after a host restart asks for a correction; +- a corrected state equals the target, `held + (target − held) = target`, and + correcting a state that is already correct is the zero Z-set (property tests over + random edit sequences and losses); +- a refused command consumes no publication number, and the commands before it in its + frame stay published; +- a dropped connection or a failed request triggers a correction, and a later request + with the same session resumes from the watermark. +- a zero sum has the sign IEEE gives the same addends: `-0.0` only if every addend is + `-0.0`, and `+0.0` for an exact cancellation (a property test against IEEE). + +## Open questions + +None. diff --git a/context/code/core.md b/context/code/core.md index 24f123bb..6544057f 100644 --- a/context/code/core.md +++ b/context/code/core.md @@ -1792,9 +1792,51 @@ command name a key that does not exist once the two differ. That is the gap improvise-cq1q.11 names, and cq1q.7 later turns these key fields into ids. The active-view marker in both view panels still compares keys for the same reason. -**What this first slice covers:** the category panel's headers, the tile bar and -the view list, in both front ends. Item rows, the grid frame, the page bar, -records-mode labels, CSV export headers, status messages and the view title -still show keys. They follow in later slices of this step, because the grid's -label tuples are also cell coordinates (`cell_key`, `innermost_group`), so their -display has to be added at projection time rather than in place. +**What the first slice covers:** the category panel's headers, the tile bar and +the view list, in both front ends. + +**What the second slice covers (improvise-cq1q.12):** the grid frame, the page +bar, records-mode headers and coordinate cells, the category panel's item rows, +the view panel's axis line and CSV export. The frame's label tuples are also cell +coordinates (`cell_key`, `innermost_group`, `run_start`), so the display is added +beside the key and never in its place: + +- `AxisEntry::DataItem` carries `display` parallel to `tuple`, and `GroupHeader` + a `group_display`. They are built where the layout is, because a folded row's + tuple holds a *group* name where an item name would be and only `expand_category` + knows which. `Group` gained a private `display` for it. +- `AxisFrame` carries `cat_displays`, `FrameEntry::Item` a `display` tuple and + `FrameGroup` a `display`, beside `cats`, `tuple` and `name`. Deserialising + refuses a display list that is not the length of its keys. `header_labels` + decides *whether* a repeat blanks from the keys and draws the display, and + `header_widths` measures what is drawn. +- `PageAxis` carries `cat_display`, `selected_display` and `item_displays` + beside `cat`, `selected` and `items`. A page command is composed from `items` + at an index, so `item_displays` can never be read back as a key. +- `CategoryRow.items` is `Vec`, and + `ViewSummary.axes` is `Vec`. +- Records mode: a column header shows the category's or measure's display, and a + coordinate cell's *text* is the item's display (`GridLayout::coordinate_display`). + Its `edit_text` and `value` stay the key, because the editor seeds the rename + from them. +- CSV export writes displays in its corner cell, column headers, row labels and + page note. +- The browser's render model keeps its field names and carries the display in them + (`RenderPageAxis.cat`, `RenderCatRow.label`), so `web/` is unchanged. + +Still keys: `GridLayout::row_label`/`col_label` (commands and the status line +match on them), `edit_text`, the active-view marker, and everything status +messages and the view title say (improvise-cq1q.13). A display change alone does +not move `layout_fingerprint`, so it resets no cursor; no production path changes +one yet. + +`AxisFrame::cat_displays` is projected and validated, but nothing paints it yet +(the corner label has no consumer); that is a gap to close when one does, not a +bug. The new wire fields carry no `serde(default)`, so a payload from before this +change is refused: deliberate, since zero is a claim and not an absence (the same +reason as `AxisFrame::first`), and bundles are stamped so a client and host +cannot differ. + +**`.improv` neither writes nor reads a display in this step.** A saved file still +carries keys only, so a display set in memory is lost on save; display persists +only with the new version literal (improvise-cq1q.5). diff --git a/context/code/doc-site.md b/context/code/doc-site.md index cfd4b734..5a364b4c 100644 --- a/context/code/doc-site.md +++ b/context/code/doc-site.md @@ -18,7 +18,7 @@ Published, in four sections: | Section | Contents | Why | |---|---|---| -| Start here | `docs/design-notes.md`, `context/design-principles.md`, `context/repo-map.md` | What the project is for, the doctrine, and the single best explanation of how it is built. | +| Start here | `docs/design-notes.md`, `context/design-principles.md`, `context/repo-map.md`, `context/gotchas.md`, `context/file-inventory.md` | What the project is for, the doctrine, and the single best explanation of how it is built. The map is a digest; the last two are its long forms, published so a `gotcha N` citation and the per-file listing have somewhere to land. | | Decision records | `context/adr/*.md` | An ADR is written for an outside reader by construction — it states the options that were rejected, which is the part that is worth reading and the part that is nowhere else. | | The arguments, by area | `context/code/*.md` | This is the reasoning that used to live in code comments and was moved out under the two-line cap (design-principles §6). It is the most valuable prose in the repository and it is addressed to exactly the reader a documentation site has. | | Test evidence | `context/test-evidence/*.md` | One entry per test whose narrative is longer than its code. | @@ -43,7 +43,7 @@ Not published: will meet, and nobody visiting a website is about to read a PR thread. None of the three is referenced by any published document except as a row in -`repo-map.md`'s file inventory, which is inside a code fence, so leaving them +the file inventory (`file-inventory.md`), which is inside a code fence, so leaving them out creates no dangling link. That was checked rather than assumed. **And "not published" is now derived rather than assumed** (improvise-res9). @@ -233,11 +233,13 @@ recorded as a decision rather than left to be rediscovered as a bug. ## 6. Navigation -`repo-map.md` is one long document whose gotcha list is numbered 1 to 48, and -splitting it per gotcha was rejected: the gotchas cross-reference each other -constantly ("see gotcha 35", "for the reason gotcha 39 watches…"), and a reader -following those wants a scroll, not 48 page loads. It stays one page, with an -id on every list item and a compact strip of numbered links under the heading. +`repo-map.md` carries a digest of the numbered gotchas and `gotchas.md` the long +form; the map was condensed because it was too long to put in every new agent's +context. Splitting per gotcha was rejected: the gotchas cross-reference each +other constantly ("see gotcha 35", "for the reason gotcha 39 watches…"), and a +reader following those wants a scroll, not one page load per number. Each of the +two stays one page, with an id on every list item and a compact strip of +numbered links under the heading. The ids come from the list item's **own number**, not from a counter, so an inserted gotcha renumbers the anchors exactly as it renumbers the prose that cites them. diff --git a/context/code/formula-and-io.md b/context/code/formula-and-io.md index 9a8b88ad..7c7da5d8 100644 --- a/context/code/formula-and-io.md +++ b/context/code/formula-and-io.md @@ -24,7 +24,7 @@ over `unescape_pipe_body`. Before improvise-i2m5 there were **three** copies and they had drifted, over the keyword rule below. **`improvise-io` no longer has one** — its -`persistence/quoting.rs` is deleted. `context/repo-map.md`'s file inventory +`persistence/quoting.rs` is deleted. `context/file-inventory.md` kept claiming io "still has its own unguarded copy" long after the deletion — it was corrected in wave 1, but any *other* doc or comment still saying so is wrong, and that is the shape this particular fact goes stale in. diff --git a/context/code/numtower.md b/context/code/numtower.md new file mode 100644 index 00000000..ea27d290 --- /dev/null +++ b/context/code/numtower.md @@ -0,0 +1,328 @@ +# `numtower`: the reasoning + +`crates/numtower` is ADR 005 migration step 1 (improvise-59nw.1): the numeric tower, +first and alone. Nothing in improvise depends on it yet, so it changes no behaviour. +The decisions are the ADR's (`context/adr/005-exact-decimal-arithmetic.md`). This file +records how the crate reads them, the places the ADR left a choice open or said two +things, and the measurements. The evidence for each test is in +`context/test-evidence/numtower.md`. + +## The shape + +| type | what it is | ADR | +|---|---|---| +| `Num` | the stored value: an exact `Decimal`, an exact `Ratio`, or an `f64` | clauses 1, 2 | +| `NumRef` | a borrowed view of a `Num`'s arm, for exhaustive matching | — | +| `Interim` | a value inside one evaluation; exact intermediates widen to a bigint rational | question 13 (b) | +| `NumSum` | a retractable, order-free `SUM` over any `Num` | clause 8, question 16 | +| `ExactSum` | ADR 009's f64 superaccumulator | question 15 (a) | +| `parse_number`, `is_number`, `Typing` | clause 3's number language, gate and parse | clause 3, question 12 | +| `Display` / `Num::from_canonical` | the canonical text: `12.50`, `10/3`, `#i0.1`, `inf`, `-inf`, `nan` | clauses 10, 11, 14 | +| `Num::round_to_places` | display rounding, the mode a parameter | clause 12 "Format" | + +**`Num` is a struct with private arms, not the ADR's `pub enum`.** The ADR's sketch +(clause 2) shows `pub enum Num { Exact(Decimal), Fraction(Box>), +Inexact(f64) }`. A public enum would let any caller build values the tower never +produces: `Fraction(1/2)` (which must be the decimal `0.5`), an unreduced fraction, or an +exact `-0`. Each would break exact equality or the fingerprint of clause 9. So the arms +are private, every constructor normalizes, and callers match on `NumRef`. That is +`context/design-principles.md` §3, smart constructors. It is also what clause 14's table +asks for, since improvise calls "`Num`'s accessors for exactness, mantissa, scale and +fraction parts". The `Box` lives in one constructor, as `AGENTS.md` asks. `Num` is 24 +bytes, as the ADR measured, and `a_num_is_24_bytes` pins it. + +**The normal form.** An exact zero has no sign. A fraction is reduced and has a positive +denominator. A value whose reduced denominator is `2^p·5^q`, with `k = max(p, q) <= 28` +and a mantissa that fits 96 bits, is always the `Decimal`, never a `Fraction`. So +`Fraction(n/1)` exists only when `n` does not fit 96 bits, as with `10^35/1`. Every `i128` +is a valid numerator, `i128::MIN` included, because the arm is `Ratio`. That one +value cannot be negated in `i128`, so `Operand::parts` keeps it out of the `i128` fast +path and its arithmetic takes the bigint path (Copilot on PR #337). + +## Arithmetic + +### Why there are two types: `Num` and `Interim` + +Question 13 (b) bounds only the **stored** result: "every intermediate inside one formula +evaluation is unbounded up to the ceiling". The bounded `Num` cannot be such an +intermediate, so `Interim` is the widened type. It is either a stored `Num` or an exact +`BigQ` (a reduced `BigInt` fraction) that does not fit `i128`. `Interim::store` applies +the stored bound. The operators on `&Num` are one-operation evaluations: each builds the +`Interim`, applies the operation and stores the result. So `(a * a) / a` returns +`a` exactly when computed through `Interim`, and is inexact when computed through `Num` +(`an_intermediate_beyond_i128_comes_back_exact`). That difference is (b)'s whole point. +Step 3's evaluators should evaluate through `Interim` and store once per cell. + +**So `Interim` compares, and knows zero, without storing.** `IF(a*a/a = a, …)` and the +division guard both run before the stored bound. If `=` had to store its operands first, +its answer would depend on how the formula is spelled, which is what question 13 (b) +rules out. `Interim::cmp_exact` is clause 6's exact order over every pairing: two stored +values go through `Num::cmp_exact`, and anything wide compares as a bigint rational, +against an f64 by its exact binary value. `Interim::is_zero` is the one zero predicate. A +wide value is never zero, because zero always fits a stored arm. The comparison through +storing is the rival (`an_interim_compares_exactly_without_storing_catches_storing_first`): +`a` and `a ± 1/(3·10^30)` store to one f64 and compare equal. The review of `302594bf` +asked for this or for a note naming it as a step-3 item; the crate does the first. + +### Contagion, and how many roundings a demotion has + +An inexact operand makes the result inexact. Each exact operand is converted by the +correctly rounded conversion, and then the f64 operation runs. That is two roundings, +which is clause 4's "computed in f64". + +**An exact ∘ exact result that does not fit is rounded once.** It is the correctly rounded +f64 of the exact value. Clause 4 still says a demoted sum or product is "rounded twice, +once per operand and once in the operation". That text predates option (d). Option (d)'s +"Operations" and question 13 (b) both say that a result failing the stored bound is "the +correctly rounded f64 of the exact fraction", and under (b) the exact value is always in +hand. The crate follows (d), because (d) is decided and later. Both texts cannot hold for +one operation, so this is a reading, and it is flagged. + +### Clause 4: the hint, the recompute, the stored scale + +`Decimal + Decimal` and `Decimal × Decimal` first try `rust_decimal`'s checked operation. +The result is trusted only when its scale equals the nominal one (`max(sa, sb)` for `+` +and `−`, `sa + sb` for `×`). A lower scale is the **hint**, and so is a `None`. On the +hint the exact integer result is recomputed in a 256-bit integer, and its trailing zeros +are stripped. If the stripped mantissa fits 96 bits at a scale of at most 28, the result +is a decimal at the **stored scale**, `min(nominal, 28)`, padded back with zeros while the +mantissa fits. Otherwise the result goes to the fraction arm, then to a wide `Interim`. +The ADR's own example holds: `1.2500000000000000 × 4.0000000000000000` is `5` at scale 28 +(`a_recomputed_result_keeps_the_stored_scale_clause_4_names`). A `None` counts as the +hint, so that an overflow `rust_decimal` reports is never mistaken for an exact result. + +### Division: the direct test, one implementation + +Clause 5 is normative: write the quotient as a reduced `n / (2^p·5^q)`, then it is a +decimal iff `k <= 28` and the padded mantissa fits. The crate does this the same way for +every exact pairing, not only for decimals. It reduces the quotient (in `i128` when that +cannot overflow, and in a bigint otherwise), then normalizes. `small_ratio` and +`decimal_of` are the one normalization, so a decimal reached through any path has the +same representation. Multiplying back is never used. + +**A quotient carries its minimal scale** (`10.00 / 2` is `5`, `1 / 4` is `0.25`). The ADR +does not say what scale a quotient has. A quotient's scale is not a function of its +operands' scales (clause 5), and a result that passed through the fraction arm carries +its minimal scale (option (d)), so the minimal scale is the consistent choice. It is a +reading, and it is flagged. + +**The ADR's argument against multiplying back is weaker than it says.** Clause 5 rejects +it as incomplete, on `1e-28 / 0.5`: `q × 0.5` "needs scale 29, so the check flags it". +That holds under revision 6's rule, which demotes on the hint +(`division_decides_fit_directly_catches_multiplying_back` keeps that rival). But under +revision 7's recompute the product `1e-28` is found exact after its zeros are stripped. +So multiplying back by `Num`'s own `*` accepts the ADR's counterexample. In 20 runs of the +division property (5,120 random cases plus the three seeds), it agreed with the direct +test every time. That was an experiment and is not committed. The direct test stays the +rule, because the ADR decided it and because it needs no second division. + +### `POW` + +- **The exponent.** It is exact when it is an exact **integer value**, so `2.0` counts. +- **The size, decided before any multiplication.** The result's numerator has at least + `|n|·(bits(num) − 1)` bits, and likewise its denominator. If either lower bound passes + the ceiling, the result is `powf`, inexact, with nothing allocated. So + `POW(1/3, 10^9)` costs nothing. Inside the bound the result is computed exactly, and it + can still exceed the ceiling (`3^6000` is 9,510 bits). It then demotes, which is the + ceiling's stated exception. +- **The scale.** A positive power of a decimal takes clause 4's stored scale with nominal + `s·n`, the scale repeated multiplication would give, so `1.50^2` is `2.2500`. A + negative power is a division and has its minimal scale, **unit bases included**: + `1.0^-2` is `1` and `(-1.0)^-3` is `-1`, as `2.0^-2` is `0.25`. Until the review of + `302594bf` the unit-base arm padded every power to `s·|n|`, so `1.0^-2` read `1.00` + (Copilot on PR #337). The power property now checks the scale of every decimal result + against this rule, with unit bases among its seeds. +- **The edge values.** `0^0` is `1`. `0^-n` is `div/0`. `±1^n` is `±1` for any `n`, + including one too large to compute. +- **A demoted power takes its sign from the exact exponent.** `powf` gets the base's + magnitude, and the sign is set from `base < 0 && n odd`. An exponent above 2^53 rounds + to an even f64, so handing `powf` the signed base made `(-2)^(2^53 + 1)` read `+inf` + (Copilot on PR #337). + +### The ceiling + +`CEILING_BITS = 8,192`. It applies to the bit length of an intermediate's numerator *or* +denominator. The ADR states the ceiling as a width, not as which of the two it measures. +Past it, the intermediate becomes the correctly rounded f64 of its exact value. + +## Comparison + +- **Exact with exact.** Cross-multiplication in 256 bits. A decimal is `m / 10^s` with + `10^28 < 2^94`, so every product fits. +- **Exact with inexact.** The ADR's method: convert the exact operand, and the rounded + value decides unless it lands on the f64 itself. A tie is resolved against the f64's + exact binary value, in a bigint. +- **That fast path is sound only because the conversion is correctly rounded.** Injecting + `rust_decimal`'s `to_f64` failed `exact_mixed_comparison_is_transitive` as well as the + conversion test. A future integer conversion (clause 4 allows one) must keep that. + +`MIN` and `MAX` skip NaN. Equal values break their tie as follows. Only the first rule is +the ADR's (question 11); the other two are this crate's, and they make the result +independent of argument order: +- the exact value is preferred to the inexact one (question 11); +- of two equal decimals, the larger scale is kept, for the same reason: it keeps more of + what was typed; +- of `0.0` and `-0.0`, `MIN` takes `-0.0` and `MAX` takes `0.0`, as IEEE 754's + `minimum` and `maximum` do. + +## The number language + +**The gate is literally `f64::from_str` reading a finite value.** The parse is the +crate's own lexer, and it never goes through an f64 on an exact path. The property test +checks that the two agree, and that the exact value read rounds to the f64 +`f64::from_str` reads. + +- **A lexeme's scale.** It is the scale the lexeme implies (`1.50` → 2, `1.5e3` → 0). When + the implied scale is above 28 but the value fits a decimal (`0.1` written with 29 + places), clause 4's stored-scale rule applies: strip the zeros, then pad toward 28. + The ADR only says "with the scale the lexeme implies", which cannot exceed 28. +- **Zero.** It is exact, unsigned, at the implied scale clamped to 0–28, so + `0e99999999999999999999` is `0`. +- **An exponent too long to read** saturates, because no exact arm can reach it. A + significand of more than 180 digits, or a leading digit outside 10^-130 to 10^40, goes + straight to f64. Those bounds are wider than the fraction arm can reach (a reduced + denominator `10^k / 2^a` is at least `2^k`, so `k < 127`). +- **The forced `n/d` arm.** Its value is a decimal, then a fraction, then **the nearest + f64**. The ADR says `n/d` "is a fraction or, once normalized, a decimal", and does not + say what a forced `n/d` too large for `i128` becomes. The crate applies clause 3's order + to it, as it does to every other spelled number. This is a reading, and it is flagged. + **Except that the nearest f64 must be finite.** A forced `n/d` whose value rounds to + infinity (`n` of 400 digits, `/1`) is text, exactly as the gate makes `1e999` text. + Clause 3: "a finite lexeme must never read as infinity". One that underflows is the + inexact zero, as `1e-999` is. `is_number` and `parse_number` share the one helper, + `forced_fraction`, so the gate and the parse cannot disagree on it. + +**The canonical text is `Display`.** An inexact finite value is `#i` followed by Rust's +`{:?}` spelling, which is the shortest that reads back (`#i1e300`, `#i0.1`). +`from_canonical` reads all six spellings back, and reads an exact spelling only as an +exact value. Clause 14's list puts "`#i…`" in the crate, while its table says "the `#i` +prefix and the grammar are improvise's". Those two contradict each other. The crate +holds the prefix, because the wire and the file both need the one spelling, and step 7 +may move the prefix out if the grammar wants it. + +## Rounding to places + +Every arm goes through one exact path: the value as a bigint fraction, +`n·10^places / d`, then rounded by the caller's mode (`HalfAwayFromZero` or `HalfEven`). +An inexact value is first read exactly from its shortest round-trip spelling, which is +improvise-felo.9's fix, so a typed-then-computed `1.005` shows `1.01`. A result that +rounds to zero has no sign (`-0.001` at two places is `0.00`). The ADR does not decide +that, and today's f64 path would print `-0.00`. + +## Sums + +`NumSum` keeps five things, and every one is an abelian group, so the state is a function +of the multiset of terms and not of their order. + +| part | holds | width | +|---|---|---| +| the decimal partition | every decimal, and every fraction whose denominator is `2^p·5^q` with `k <= 28`, as an integer at scale 28 | 320 bits | +| the groups | per other reduced denominator, the summed numerator | 256 bits each, in a `BTreeMap` | +| `group_floats` | an `ExactSum` of each group's correctly rounded value | ADR 009's | +| `inexact` | an `ExactSum` of the f64 terms | ADR 009's | +| `inexact_terms` | the net weight per f64 bit pattern, entries at zero dropped | a `BTreeMap` | + +**The read.** It is exact when no inexact term is present and at most K = 8 groups +remain. The groups and the decimal partition are then combined **over the lcm of their +denominators, and reduced once**, and the stored bound applies to the result. Past K the +read is the f64 accumulators: `group_floats`, the decimal partition converted once, and +the inexact terms. That reading rounds each group, then the total. With inexact terms and +at most K groups, the exact total is converted once and added to the inexact terms. A +group whose numerator returns to zero is dropped, so "how many groups" is a function of +state. + +**"Is an inexact term present" is asked per value, never of the net weight.** The first +version kept one net count. Inserting `1.0` and retracting `2.0` leaves the count at zero +while the accumulator holds −1.0, so the read took the exact branch and dropped it +(ChatGPT Codex, P1 on PR #337). Presence is now a map from f64 bit pattern to net weight, +dropped at zero, the same rule the groups follow. It costs O(distinct inexact values), +the same order as the groups. The K cutoff overrides representability: nine denominators whose exact total fits +still read inexact (`more_than_k_denominators_read_inexact_even_when_the_total_fits`). +That is question 13's second exception, and the ADR's flagged reading. + +**`ExactSum` is ADR 009's prototype**, from `crates/improvise-core/examples/retraction_drift.rs` +on PR #309. It has three changes: +- it splits a weight into base-2^30 digits, as ADR 009 says the `numtower` version must, + instead of looping per chunk; +- it holds 70 limbs, not 67, because a digit's shift adds up to 60 bits; +- it reads through the crate's one rational-to-f64 rounding. + +It counts NaN and the infinities, so they can be retracted. A count that is nonzero in +either direction is treated as present. + +**`ExactSum`'s `==` compares sums, not representations.** A derived equality compared the +raw limbs and the `pending` carry counter, so `0.5 + 0.5 != 1.0`, an insert then retract +was not equal to an empty sum, and the same weights added in another order compared +unequal (Copilot on PR #337). For a type whose whole point is order-freedom, and which +ADR 009's group newtype will wrap, that equality is a trap. It now compares the +propagated limbs, which are canonical (every limb but the top in `[0, 2^32)`), plus the +three special-value counts, and ignores `pending`. `NumSum` derives no equality: compare +its `read()`. + +**For ADR 009's seam crate.** `NumSum` and `ExactSum` both have the shape a Z-set group +needs: `add(&Num, i64)` / `add(f64, i64)`, `insert`, `retract`, `read`/`value`, `Clone`. +`ExactSum` also has `absorb`, to merge two accumulators. Neither implements a group trait +yet, because `zset` does not exist yet. The newtype of question 15 is where that goes. + +## The wide integers + +`Fixed` is a two's-complement integer of `N` 64-bit limbs that never allocates. It is +used for the clause 4 recompute (256 bits), exact comparison (256 bits), the decimal +partition (320 bits) and the group numerators (256 bits). Clause 14 also names "256-bit +helpers for the division reduction". Division reduces in `i128` when the products cannot +overflow, and otherwise in a bigint. It does not have a hand-rolled 256-bit gcd. That +path is the slow path, and it is measured below. + +## Dependencies + +The normal edges are `rust_decimal` 1.43.0 (std only), `num-rational` 0.4.2 (no default +features), `num-bigint` 0.4.8, `num-integer` 0.1.47 and `num-traits` 0.2.19. The last two +are direct dependencies, used for `gcd`/`lcm` and the numeric traits. They were already +in the tree through `num-rational` and `num-bigint`, so they add no crate. The ADR names +only the first three. There is no serde, and `cargo tree --edges normal` shows no +`borsh`. `BigRational` is a **dev**-dependency feature only, for the oracle, so the +oracle and the implementation do not share a rational type. + +## Measured + +On an Apple M4 Max (64 GB), rustc 1.98.1, `cargo run --release -p numtower --example +bench`. Each range is two runs. + +| what | figure | +|---|---| +| SUM, 18k and 90k typed two-place values, ns per element | f64 0.53; `rust_decimal` fold 2.7–3.0; `Num` `+` fold 7.8–8.2; `NumSum` insert and read 7.9–8.1 | +| per operation over 18,000 typed pairs, ns | `Num` add 3.1–3.3, mul 3.0–3.2, div 146–153; `rust_decimal` checked add 1.4–1.5, mul 1.2, div 24.7–25.3 | +| `(Rev − Cost) / Rev`, ns per record | 151–153 | +| `to_f64` of a decimal (the text path), ns | 36.7–37.1 | +| `round_to_places(2)`, ns | 370–382 | +| `NumSum` read, K distinct 126-bit denominators, numerators of about 190 bits | K = 2: 5.2–5.4 µs; K = 8: 41.4–42.0 µs | +| `ExactSum`, 1M values | 1.78–1.81 ns per insert, 0.29 µs per read | + +Here is how these sit against the ADR: +- **The grouped read.** At K = 8 it is inside the ADR's 52–127 µs. The ADR measured + 250-bit numerators, and these are about 190 bits, because `NumSum` can only reach a + group's numerator through its public `add`. +- **`ExactSum`.** It agrees with ADR 009's 2.0 ns per update and 0.58 µs per read. +- **Division.** At about 150 ns it is above the ADR's *estimate* of 60–110 ns. Most + quotients of typed values do not terminate, so each one pays two `i128` gcds and the + `Box` of a `Fraction`. +- **The rest.** The ADR's whole-recompute measurement at 18k and 90k cells waits for + step 3, as the ADR says. + +**The wasm32 cost** comes from a probe outside the repository, built exactly as the +ADR's appendix builds one: a `cdylib`, `opt-level = "s"`, LTO, `wasm-opt -Oz +--enable-bulk-memory --enable-nontrapping-float-to-int`, `gzip -9`. The baseline already +parses and prints an f64. + +| probe | after `wasm-opt -Oz` | gzipped | +|---|---|---| +| f64 only | 48,331 B | 29,162 B | +| the ADR's five operations (parse, `+`, `−`, `/`, print) through `numtower` | +89,375 B | **+36,714 B** | +| plus `NumSum`, `cmp_exact` and `round_to_places` | +119,277 B | **+48,221 B** | + +The narrow figure confirms the ADR's estimate of "up to about +36 KB together". The full +surface is about +47 KB gzipped. That is what the bundle pays once step 4 sums through +the crate and step 6 rounds through it. It is more than the +15.4 KB the user answered +question 8 against, and the ADR says question 8 "reopens" if the figure changes the +answer. Neither figure is the real bundle, which shares allocator and formatting code +with everything else. diff --git a/context/code/protocol.md b/context/code/protocol.md index 30f4ec7d..e9b8bd2c 100644 --- a/context/code/protocol.md +++ b/context/code/protocol.md @@ -902,7 +902,7 @@ The feature is on for **every workspace crate with a normal dependency on serde_json** (improvise-dz8c). Each crate's own manifest names it, because each build is made from one crate alone: the `wasm` CI job and `wasm-pack` invoke per crate, and feature unification cannot reach across separate invocations -(`context/repo-map.md`, `## Wasm`). Enabling it in one crate and relying on the +(`context/file-inventory.md`, appendix "Wasm"). Enabling it in one crate and relying on the others to inherit it works in `cargo test` and nowhere else. **Why "can parse" and not "parses a frame".** The first version of the rule diff --git a/context/file-inventory.md b/context/file-inventory.md new file mode 100644 index 00000000..aa56a4f2 --- /dev/null +++ b/context/file-inventory.md @@ -0,0 +1,430 @@ +# File Inventory (the long form) + +The per-file listing that `context/repo-map.md` summarises. Moved here unchanged. + +Line counts are a **dated snapshot**, not a maintained fact: measured 2026-09-24 at `c1772ca9`, and **70 of 105 had drifted past 10%** in the three days since the 2026-09-21 refresh — eleven had doubled or more (`client.rs` 255 → 943, `lockstep.rs` 1126 → 4064). Treat them as "is this file big", never as a figure to cite; re-derive with `wc -l`. The `ws-server/` and `wasm-client/` entries use a shorthand prefix, so a refresh must map it to `crates/improvise-/{src,}/` by hand or they drift silently. Test counts are informational — run `cargo test --workspace` for live numbers. Files under 100 lines and render-only widgets omitted. + +### `improvise-core` (`crates/improvise-core/src/`) +``` +model/types.rs 3560 / 112t Model, formula eval (fixed-point), CRUD +model/cell.rs 1358 / 47t CellKey (sorted), CellValue, DataStore (interned + index), + live_records (insertion order — gotcha 18) +model/record.rs 226 / 8t RecordHash / CellId surrogates: the opaque identities a + thin client names a cell by (ADR 002) +model/category.rs 310 / 9t Category, Item, Group, CategoryKind +model/symbol.rs 89 / 3t Symbol interning +view/layout.rs 2604 / 57t GridLayout, drill, records mode, records_columns; a private + per-axis position index (`row_pos`/`col_pos`, rebuilt by `reindex`) + so a data position resolves in O(1) (improvise-felo.13) +view/types.rs 817 / 43t View config (axes, pages, hidden, collapsed, format) +model/id.rs 290 slugify + IdAllocator: how a generated id is made (ADR 007 + clause 4, core.md §20). Nothing outside the file calls it yet +view/axis.rs 190 / 5t Axis enum + label/next methods (no colour: that is a Face) +workbook.rs 413 / 22t Workbook: Model + cross-view ops, item facade +format.rs 519 / 40t format_f64, parse_number_format (display only) +``` + +### `improvise-formula` (`crates/improvise-formula/src/`) +``` +parser.rs 1394 / 79t pest grammar + tokenizer → Formula AST +quote.rs 160 / 7t quote_name / is_bare_name / pipe_quote / unescape_pipe_body: + the one home, both halves (improvise-i2m5). improvise-gp + re-exports it; improvise-io calls it from + persistence/{format,parse}.rs +formula.pest 108 PEG grammar for formula text +ast.rs 130 Expr, BinOp, AggFunc, Formula, Filter +``` + +### `improvise-io` (`crates/improvise-io/src/`) +``` +storage.rs 334 / 13t Storage trait (read/write/list/delete), PhysicalFs, MemoryFs, + `impl Storage for Rc` (improvise-hmws), and + contract::storage_contract — public, not cfg(test), because + a browser must run that sequence (harness/) +opfs.rs 289 / 0t OpfsFs: Storage over the browser's Origin Private File + System. wasm32 only. Run by harness/, not by cargo test +persistence/improv.pest 179 PEG grammar — single source of truth +persistence/mod.rs 441 / 11t document lifecycle: save/load/gzip/legacy-JSON over Storage; + facade re-exporting format_md/parse_md/export_csv* +persistence/format.rs 401 / 11t format_md: Workbook → .improv text +persistence/parse.rs 1684 / 46t parse_md: pest tree-walk → Workbook (+ grammar generator) +persistence/export.rs 243 / 5t export_csv / render_csv. Builds its layout with + with_frozen_records, NOT new — `new` skips prune_empty + (improvise-iot6) +persistence/roundtrip_tests.rs + 1236 / 62t format_md ↔ parse_md characterization (tests only) +import/wizard.rs 1967 / 62t ImportPipeline + ImportWizard. `Plan::place` is the one + placement pass, shared by `collisions()` and the build so + the two cannot disagree; `DuplicatePolicy` / `Collision` + (improvise-cgqt) +import/analyzer.rs 434 / 17t Field kind detection (Category/Measure/Time/Skip); + analyze_records is first-seen order, not HashSet (gotcha 18) +import/csv_parser.rs 590 / 26t CSV parsing, multi-file merge +``` + +### `improvise-protocol` (`crates/improvise-protocol/src/`) +``` +lib.rs 619 / 2t ViewState, ViewCursor, ViewFrame, AppMode (17), Panel, + MinibufferConfig, MinibufColor, HELP_PAGE_COUNT +reduce.rs 749 / 10t Command + ModelCommand (the wire format), HelpPage, reduce_view +render.rs 2157 / 46t CellDisplay, WidgetCache trait + one cache per widget + (GridCache/CategoryTreeCache/FormulaCache/ViewListCache/ + TileBarCache), AxisFrame/FrameEntry/FrameGroup and + header_level_widths (the grid frame, gotcha 44), + GridCache::seed (the bootstrap compaction), Projection, + bootstrap-only Snapshot +keymap.rs 975 / 0t KeyPattern, Binding, Keymap, ModeKey (17), TextEntry, + PanelKeymap, Keymaps::default_keymaps/prefix_keymap — the + shared table both front ends look up. Tests live with + dispatch in improvise-command/src/command/keymap.rs +viewmodel.rs 170 / 5t flatten_cat_tree — derivations between a cache and a + renderer, called by both front ends (improvise-edp) +face.rs 682 Face (a role: Axis, WizardField, Mode*, MinibufferOf…), Theme, + AnsiSlot — appearance as data a front end applies (ADR 004, + protocol.md §10); draw.rs::face_style is the TUI's one translation +key.rs 96 / 3t KeyCode, KeyModifiers, KeyEvent (moved from command/) +frame.rs 156 / 3t ClientFrame/ServerFrame — the ONE envelope pair, both + deployments (improvise-xks6). Nothing else defines one. + ClientFrame::Save is a request, not a Command (gotcha 45) +basis.rs 257 / 6t Basis (the three tiers), Rejection, BasisCache — ADR 001 + layer 1. Basis has no Default and no public constructor +help.rs 1241 / 14t HelpPage content and Hint { keys: Vec<(KeyCode, KeyModifiers)>, + command, caption }. `compact` drops a chord by its MODIFIERS + (anything beyond shift), never by parsing a label it just + formatted (improvise-thpe) — the guard is the signature +resolve.rs 4883 / 88t the browser's key resolvers: a BrowserKey to the Commands it + composes, mirroring each Cmd line for line INCLUDING where + siblings disagree (gotcha 47). The largest file in the crate +client.rs 943 / 15t ClientState: ViewState + one cache per widget, and the fold; + in_flight, `opened` (OpenedCell), notify, settle_status + (gotcha 47). EditorSession is in lib.rs beside ViewState +``` + +### `gp-tree` / `improvise-gp` (`crates/gp-tree/`, `crates/improvise-gp/`) +``` +gp-tree/src/lib.rs 575 / 8t Tree, Spec, eval, simplify, render, breed, Population, + SplitMix64 (vendored seeded Rng; seed() keeps the seed so + a fit can be shown and recorded, pinned to the published + reference vectors) +gp-tree/src/data.rs 1000 / 21t raw string rows -> f32 feature vectors: per-column type + inference, one-hot, mean-imputation + col.missing flag, + ColumnSpec/encode_row, Dataset (incl. warnings), + Model = trees + plan + seed + FitConfig +gp-tree/src/fit.rs 961 / 32t the search: FitConfig (Default = the CLI's numbers), fit() + -> simplified champion, cv_regress/cv_classify, + permutation_importance, folds/standardize/mse. + Returns data; never prints +gp-tree/examples/fit_csv.rs 583 CLI over the above: flags, demo dataset, all printing +gp-tree/tests/scoring_known_answers.rs + 250 / 6t f32 bit patterns + a next_u64 witness: the evidence any + change to the scoring paths needs (gotcha 37c) +improvise-gp/src/lib.rs 978 / 25t formula_text / fitted_formula: a Tree as IF(...) formula + text, validated back through parse_formula. A threshold is + the SHORTEST decimal that reads back as the same f64 (not + the exact expansion), so a row on a split boundary routes + the same in f64 as in f32 eval; non-finite values panic +improvise-gp/src/dataset.rs 506 / 15t a View's records -> the feature matrix a fit consumes — + the improvise-side counterpart of preprocess. Records mode + is already a feature matrix (gotcha 1) +improvise-gp/src/fit.rs 1778 / 35t one fit end to end: a View's records in, an improvise + Formula and the evidence for believing it out. Not in a + command, because none of it is terminal-specific +``` + +### Command layer (`crates/improvise-command/src/command/`) +``` +cmd/core.rs 475 / 2t Cmd trait, CmdContext, CmdRegistry, parse helpers +cmd/registry.rs 903 / 13t default_registry() — all registrations +cmd/navigation.rs 507 / 12t Move, EnterAdvance, Page* +cmd/cell.rs 724 / 20t ClearCellCommand, YankCell, PasteCell, TransposeAxes, SaveCmd +cmd/commit.rs 1304 / 27t CommitAndAdvance, CommitBuffer(CommitKind), CommitExport, + CommitSave +cmd/effect_cmds.rs 519 / 6t effect_cmd! macro, 25+ simple wrappers, `unless_unknown` (gotcha 51) +cmd/fit.rs 293 / 8t fit-measure: evolve a tree over the active view and store it + as an ordinary `## Formulas` line — nothing GP-specific is + left in the document afterwards +cmd/grid.rs 988 / 22t ToggleGroupAtCursor, ViewNavigate, DrillIntoCell, TogglePruneEmpty +cmd/mode.rs 469 / 13t EnterMode, Quit, EditOrDrill, EnterTileSelect +cmd/panel.rs 936 / 24t Panel toggle/cycle/cursor, formula/category/view panels +cmd/search.rs 202 / 4t SearchNavigate, SearchOrCategoryAdd, ExitSearchMode +cmd/text_buffer.rs 399 / 14t AppendChar, PopChar, CommandModeBackspace, ExecuteCommand +cmd/tile.rs 170 / 6t MoveTileCursor, TileAxisOp +cmd/wizard.rs 244 / 6t HandleWizardKey — per-step import-wizard key tables +keymap.rs 1353 / 44t KeymapSet, Keymap::dispatch/dispatch_transient, effects_of, + SetTransientPrefix — what a thin client cannot have. The + table itself is improvise-protocol/src/keymap.rs +parse.rs 350 / 23t Script/command-line parser (prefix syntax) +``` + +### Semantic UI state (`crates/improvise-command/src/ui/`) +``` +ui/effect.rs 4020 / 99t Effect trait. 63 effects: 53 `impl Effect for` blocks, 9 + simple_effect! expansions, and SetTransientPrefix in + command/keymap.rs. The 10 outside the brace-match are all + ViewOnly; 28 of the 53 are Model(_) (gotcha 38, h3y7) +ui/app.rs 5558 / 134t App, ModelState, ViewState + dispatch methods, Panel, handle_key, dispatch_key, document_generation, autosave; apply_effects declares `dirty` for WholeWorkbook and observes it for CellData/LayoutGeometry (gotcha 45) +ui/project.rs 6383 / 118t Dirty, Viewport, grid_of/category_rows/formula_texts/ + view_summaries/tile_entries, grid_updates (the diff), + ClientSession + Sessions (connect/apply/rebase/project_all) +ui/reduce.rs 1217 / 24t Checked / CheckedBatch / apply_command / model_effects / + reduce_key — one door from a Command or a keypress to the + authoritative App. `reduce_full` is GONE (improvise-h3y7) +ui/host.rs 3410 / 67t Host: one App + one Sessions; handle, commands, project_all, + rebase. Lives here, not in ws-server (gotcha 45) +ui/cat_tree.rs 174 / 8t Category tree flattener for panel +ui/viewport.rs 259 compute_col_widths / compute_row_header_width / visible_cols + (pure geometry; the width rule itself is the protocol's); + grid_width, SIDE_PANEL_WIDTH, MIN_GRID_WIDTH (improvise-gtn7.1); + GridSummary / grid_summary / grid_window (improvise-felo.13) +``` + +### TUI rendering, draw, main (`src/ui/`, `src/draw.rs`, `src/main.rs`) +``` +ui/grid.rs 905 / 14t GridWidget (ratatui): no &Model, no &View — the frame comes + off GridCache (gotcha 44). Its module doc is the best-written + statement of that rule in the repo +ui/help.rs 277 5-page help overlay (render only) +ui/import_wizard_ui.rs 829 / 9t Import wizard rendering. `Panel` is the one writer, and + every write is clipped to the panel's rows and width +ui/text.rs 169 truncate and wrap, both measured in terminal cells, never bytes +draw.rs 1122 / 20t TUI event loop, frame composition; `content_tests` draws a + whole frame to check the selection beside a side panel +main.rs 428 CLI entry (clap): open, import, cmd, script +# other ui/*.rs are small panel renderers — skip unless changing layout/style +tests/grid_frame_agreement.rs + 479 / 6t the two front ends draw the same grid frame (improvise-anr9) +tests/records_wide_smoke.rs 67 / 1t +tests/shipped_documents_load.rs 333 / 8t every TRACKED .improv/.improv.gz loads, and every record + it projects is writable. "Shipped" is read off `git ls-files`, + not a .gitignore reading (improvise-c0aj), and the walk's git + is scrubbed of every hostile GIT_* setting the caller may carry +tests/web_demo_model.rs 83 / 2t every visible cell in web/demo.improv is a Record +``` + +### Browser host and client (`crates/improvise-ws-server/src/`, `crates/improvise-wasm-client/src/`, `web/`) +``` +ws-server/registry.rs 616 / 14t Registry of Hosts keyed by DocumentId; ClientId {document, + session} — a SessionId is unique only within its document. + A document is never evicted while the server runs (ADR 001) +ws-server/host.rs 4 / 0t a re-export shim only. The session loop is + improvise-command/src/ui/host.rs (gotcha 45) +ws-server/origin.rs 132 / 6t OriginPolicy: which origins may open a socket. Loopback by + default, --allow-origin widens, absent allowed, two refused + (improvise-kc7n). The only decision transport::admitted makes +ws-server/transport.rs 330 / 4t tokio + tungstenite plumbing, plus the Origin check at the + handshake — a refused upgrade, not an ignored socket +ws-server/main.rs 190 / 5t CLI entry: serve a file on a port; --bind, --allow-origin +ws-server/tests/socket.rs 389 / 6t the one test that uses a real socket: handshake, JSON + framing, SessionId routing, which document a URL asked for +wasm-client/lib.rs 455 / 8t Client: the wasm-bindgen surface (init, on_message, on_frame, + viewport/set_viewport, render_model, set_cell_at, typed_cell) +wasm-client/keys.rs 187 / 5t BrowserKey ONLY — a KeyboardEvent flattened to an improvise + key. It does NOT resolve: `resolve` and KeyOutcome live in + improvise-protocol/src/resolve.rs, shared by both front ends +wasm-client/render.rs 1249 / 21t RenderModel: editable / selected / cat tree, decided here +wasm-client/socket.rs 547 / 13t the fold (fold_frame/handle_frame), the two compositions + and typed() text→CellValue. Defines no enums +# state.rs is GONE — ClientState moved to improvise-protocol/src/client.rs +wasm-client/tests/browser_round_trip.rs 2136 / 34t improvise-1ey's scope against a real Host +wasm-client/tests/lockstep.rs 4064 / 78t host and client agree on a keypress +web/app.js 816 string calls only, no protocol knowledge, no framework. + Run by harness/tests/shell.spec.js (Chromium; the standalone half on Firefox too) — see + gotcha 48 for what that does and does not reach +web/worker.js 86 three string calls, no protocol knowledge. RUN by + shell.spec.js's no-?ws path, with bindings.rs (gotcha 45) +web/shell.js 399 everything app.js DECIDES that is not painting: no DOM, + no browser API, one `node --test` away. Owns socketUrl, + gridPlan, frameCostOf, documentTitle — so a chunk of the + browser's render decisions is NOT in the file gotcha 42 + and gotcha 46 name. Reasoning: context/code/clients.md +web/serial.js 19 the one rule worker.js exists to keep: one frame at a + time, in arrival order (gotcha 48's async-listener race), + extracted so it is testable without worker globals +web/index.html, style.css the page; legibility only +web/build.sh 38 wasm-pack build → web/pkg (gitignored). Needs + wasm-opt = ["-O", "--enable-bulk-memory"] or the bundled + wasm-opt rejects LLVM's memory.copy +web/assemble.sh 112 assembles the deployable site (copies web/*.improv) +web/demo.improv 65 the document the site ships and seeds from. Every cell a + Record — asserted (gotcha 46) +``` + +### Browser worker host (`crates/improvise-worker-server/`) +``` +improvise-worker-server/src/worker.rs 765 / 17t Worker: one port, one client; handshake, Addressed + filter, persist trigger (model_state.dirty). Every decision, every test +improvise-worker-server/src/vfs.rs 348 / 8t documents/open: seeding is a Storage::write +improvise-worker-server/src/bindings.rs 185 / 0t wasm32 shim: OpfsFs awaits + postMessage. No Rust test + reaches it, but harness/tests/shell.spec.js DOES: the + no-?ws path runs start()/OpfsFs::open under Chromium + and Firefox. + Called NEVER RUN here for months: see gotcha 45 +improvise-worker-server/src/lib.rs 51 / 1t the module doc that states exactly what is and is not + covered — the one correct copy of that fact +improvise-worker-server/tests/post_message.rs 407 / 8t main-thread traffic, strings only +``` + +### Browser harness (`harness/`) — not a workspace member + +``` +harness/playwright.config.js 50 projects: chromium (every spec), firefox (opfs, plus + shell.spec's standalone half: grepInvert @mocked-peer), + webkit (opfs only, which skips on a runtime probe: no + OPFS). CI runs `--project=chromium --project=firefox` only: + nixpkgs' WebKit aborts inside the runner VMs (ci.yml) +harness/tests/shell.spec.js 345 the first execution of web/app.js. A change to web/*.js + belongs here (gotcha 48). Playwright serves harness/serve/, + which harness/build.sh ASSEMBLES: edit web/*.js, rebuild, + or the specs run the old page +harness/tests/opfs.spec.js 103 drives the OPFS contract page and reads back its result +harness/opfs-contract/src/lib.rs 470 runs improvise_io::storage::contract::storage_contract + against a real OpfsFs inside a real Web Worker +harness/www/opfs/{index.html,worker.js} the page and worker that host it +harness/build.sh 50 assembles harness/serve/ (web/assemble.sh, the fixtures, + and the contract crate's wasm-pack build) — gitignored +# gated by the `browser` job in .github/workflows/ci.yml, which can fail the build +``` + +### Examples +``` +examples/gen-grammar.rs Grammar-walking random .improv generator (pest_meta) +examples/pretty-print.rs Parse stdin, print formatted .improv +crates/improvise-core/examples/category_width_bench.rs + The same records at 12 and at 24 regular categories + (improvise-felo.1; context/code/core.md) +``` + +### Context docs +``` +.claude/skills/code-review/ Review skill: checks a change against the doctrine in these docs + kb.pl +scripts/check-comment-runs.py The two-line comment cap AND --check-evidence. The flag is + ADDITIVE since improvise-ijql — every invocation that can + exit 0 answers the cap — guarded by a subprocess self-test, + the only level a dispatch bug is visible at. Reasoning: + context/code/tooling.md §2 +scripts/check-ci-graph.py pages-deploy must `needs` every job in ci.yml, and every + suite `checkers.sh --list` knows must run in some CI job + (improvise-dak5). It REFUSES what it cannot read rather than + parsing YAML: the values it accepts are an allowlist + (tooling.md). --self-test; run in the `test` job. README + states the rule and lists no jobs (improvise-x7cq) +scripts/checkers.sh The one entry point for every *-test.sh suite: scrubs the + env to an allowlist, runs under the bash that runs it (no + version floor since #297). `ci_suites` is the table of which + suites each CI job runs (`--ci `); ci.yml names none +scripts/pr-watch Waits on a PR's checks (the pr-watch skill). Exit 5: the PR + is already MERGED or CLOSED, so there is nothing to wait for +scripts/assemble-stamp.sh The cache-bust stamp gate web/assemble.sh calls, extracted + so its refusals can be exercised without a web/pkg +scripts/assemble-stamp-test.sh Its check_the_checker, run first in the `build` CI job +context/code/tooling.md The repository's own checkers: why an evidence heading's + names come from cargo rather than from the source, and + why a leaf name is not an identity +context/test-evidence/tooling.md Evidence for the gates in scripts/ and for the tests that + hold CI's shape +scripts/nix-seed-flake-inputs.sh Seeds locked flake inputs over git so `nix develop` works in cloud sessions +prolog-sessions/kb.pl Executable knowledge base: invariants, doctrine, decisions (CI-checked) +context/adr/007-stable-identities.md Stable identities: ids unique across the document are the + keys and names are display-only; gensym suffixes apply to + generated ids only, and hand-written ids are canonical. + Accepted 2026-09-25. (There is no ADR 005 or 006 file: + improvise-l2b8 and improvise-ilvl claim those numbers.) +context/adr/004-faces-and-themes.md Faces: appearance as UI-agnostic data a front end + applies, emacs-style. Accepted 2026-09-22. +context/adr/003-live-backed-models.md A model may be backed by a live query source — + improvise as a DB client. Accepted, NOT a priority. +context/adr/002-naming-a-cell.md How a client names a cell: an opaque surrogate on + CellDisplay, whose absence means the cell is an + aggregate. Records why _Index cannot serve. +context/adr/001-concurrent-edits.md Concurrent edits between clients: record-level atomicity, + optimistic basis check per batch, advisory reservations. + Records why a causal clock was rejected. +context/cloud-agents.md Cloud-session environment: tracker sync, bd pitfalls, what is blocked +context/design-principles.md Architectural principles & testing doctrine +context/plan.md Show HN launch plan +context/repo-map.md The map: crates, how-to-find-things, the gotcha digest, a file map +context/gotchas.md The long form of every numbered gotcha (the digest is in repo-map.md) +context/file-inventory.md This file: the per-file listing the repo map summarises +docs/design-notes.md Product vision & non-goals +``` + +--- + +--- + +# Wasm, CLI, testing and dependencies (long form) + +The three repo-map sections of these names, as they stood before the map was condensed. + +## Wasm + +The browser bundle is `improvise-core`, `improvise-formula`, `improvise-io`, +`improvise-protocol`, `improvise-command`, `improvise-wasm-client` and +`improvise-worker-server`. All seven +`cargo check --target wasm32-unknown-unknown` clean with no feature flags +(improvise-ywd), gated by the `wasm` job in `.github/workflows/ci.yml`. That +job gates **eight** crates, not seven: `improvise-gp` was added (improvise-z5eg) +explicitly *not* as a bundle member but as a candidate, because staying +wasm-clean is free now and expensive to retrofit — so the bundle is seven and +the gate is eight, and neither number is the other one's typo. +**A feature that adds a dependency to a bundle crate must be checked with `cargo tree --edges normal`** — the default output includes **dev** edges, so a plain `cargo tree -p improvise-protocol --target wasm32-unknown-unknown` reports a dev-only `serde_json` as *present* and reads exactly like a leak. Check per-crate too, because the `wasm` CI job invokes each crate separately and feature unification cannot reach across that (improvise-26dm added a `test-support` feature on `improvise-protocol` on those terms; it is off in every shipped build). + +`improvise-ws-server` is the host and is native only — tokio, tungstenite and +`Box` — so `improvise-wasm-client` cannot depend on it in anger. It +no longer needs to: both ends `pub use` `improvise_protocol::frame` and neither +defines a pair (gotcha 42). The **dev**-dependency remains, for +`tests/browser_round_trip.rs`, which hands the client's own serialized frame to +a real `Host::handle`. The root `improvise` crate is +the terminal binary and does **not** build for wasm — it depends on +crossterm; that is correct, not a gap. + +`cargo check` passing is not the same as running. Two known runtime hazards +it cannot see: `Instant::now()` panics on wasm (hence `web-time` in +`improvise-command`), and any `std::fs` call compiles but fails in a browser +— improvise-fuj closed the effects that bypassed `Storage`; autosave and the headless final write still do (improvise-31oe.11). + +## CLI + +``` +improvise [model.improv] # open TUI (default) +improvise import data.csv [--no-wizard] [-o out] # import CSV/JSON +improvise cmd 'add-cat Region' -f model.improv # headless command(s) +improvise script setup.txt -f model.improv # run script file +``` + +Import flags: `--category`, `--measure`, `--time`, `--skip`, `--extract`, `--axis`, `--formula`, `--name`, `--split-date ` (a date column imports as ONE hierarchical category by default; this restores the flat split, and `--extract` still picks the components — improvise-8jk6.3), `--strict` (with `--no-wizard`, any skipped record is a refusal and nothing is written, where the default writes and warns with a count — improvise-8jk6.6), and `--on-duplicate refuse|sum|separate` (default `refuse`; applies with `--no-wizard`, where the wizard would otherwise ask — improvise-cgqt). A headless import also refuses a `--split-date` that names a non-date field and an `--axis` that names a category the import did not build (exit 1). + +--- + +## Testing — the short version + +Full guidance lives in `context/design-principles.md` §7. Quick reminders: + +- **Two numbers, not one** — the ambiguity between them is what let the old + "suite runs in <2s" drift: **no single test binary over 2 seconds**, and the + workspace wall clock is the sum. Measured at `63a5c13`: **1464 tests over 35 + binaries** (1 ignored), longest single binary **0.6–1.0s across two runs on one machine** — + the spread being its own argument for reading your own run. Excludes compilation, + and excludes `harness/` (Playwright, minutes, its own CI job). *Re-measure* rather + than re-assert — `cargo test --workspace` prints a `finished in` per binary, and + **the portable signal is the ratio, not the seconds**. **Not the ratio to the + median, though: it is degenerate.** Most binaries print `0.00s` at two decimals, + so the median is zero and every ratio to it is undefined (improvise-i5z6, 39 + binaries). Compare the slowest binary to the next few slowest instead: an outlier + is one that sits well clear of them *in every run*. That is how i5z6 was read — + `improvise-formula`'s lib longest of 39 in all three runs, at 0.91–1.95s against a + runner-up's 0.50–0.92s. Both outliers so far had one cause: a generator re-parsing + its pest grammar per case. `improvise-io` (improvise-8shu, 1.42s → 0.21s) and + `improvise-formula` (improvise-i5z6, 0.84–0.98s → 0.06–0.07s) each read it once through a + `OnceLock` now. A new grammar-walking strategy should do the same from the start. + The full argument is `context/design-principles.md` §7. +- Commands: build a `CmdContext`, call `execute`, assert on returned effects. No terminal needed. +- Property tests (`proptest`, default 256 cases) cover invariants: CellKey sort, axis consistency, save/load roundtrip, `parse(format(parse(generate())))` stability. +- **Bug-fix workflow**: write a failing test *before* the fix (regression guard). Record the bug in `context/test-evidence/`, **keyed by test name** — *not* in the test's doc comment, which the two-line comment cap forbids and `scripts/check-comment-runs.py` enforces. A heading may be module-qualified and must be when its leaf is not unique. +- Coverage target ~80% line/branch on logic code; skip ratatui render paths. + +--- + +## Key dependencies + +ratatui 0.30, crossterm 0.29, clap 4.6 (derive), serde/serde_json, indexmap 2, anyhow, chrono 0.4, pest + pest_derive, flate2 (gzip), csv, enum_dispatch. Dev: proptest, tempfile, pest_meta. diff --git a/context/gotchas.md b/context/gotchas.md new file mode 100644 index 00000000..d98edc42 --- /dev/null +++ b/context/gotchas.md @@ -0,0 +1,139 @@ +# Gotchas (the long form) + +The numbered gotchas that `context/repo-map.md` summarises in a line or two each. +**The numbers are stable and are cited across the repo (`gotcha 24`, `repo-map gotcha 35`) +— a citation means the entry below.** Read the ones your diff touches; the repo map's +digest says which those are. Every entry is the original text, moved here unchanged. + +Dated claims and counts inside an entry are snapshots: recount rather than cite. + +1. **Records mode is wide format.** A row is one *coordinate tuple* — the cells sharing everything but `_Measure` — and the columns are `[coordinates…, **`+cat`**, raw measures…, `+`]`: a column per regular category, then the column that creates a **new category** (improvise-2zq, sitting where the column it creates will appear), then a column per raw measure, then the column that creates a new measure (improvise-c5v). Consequences: two measures on one coordinate are ONE row; editing a category column moves **every** measure cell in the row, not just the one under the cursor; `_Dim` in the synthetic key carries the column **index**, not a label, because a category and a measure may share a name — resolve it with `GridLayout::record_col` into a `RecordCol`. Formula targets are not columns (nothing to type into them). Drill pending edits are keyed by `(row, RecordCol)`. + **`prune_axis_items` takes a predicate, not a flag vector.** `prune_empty` hands it `impl Fn(usize) -> bool` over the *data* index (group headers consume no index). The domain is derived from `items` itself, so there is no parallel length a caller could get wrong — the earlier `&[bool]` + `debug_assert` pair could disagree between debug and release if a third caller ever sized `keep` from something other than `row_count()`/`col_count()` (improvise-n3wt). **It re-marks the survivors rather than filtering the markers** (improvise-e265): runs nest, so dropping an item can leave a header with nothing under it or a `GroupEnd` closing a run that was never reopened, and no per-marker filter can tell. `remark` instead emits, before each kept item, exactly the headers and `GroupEnd`s that make the output's open runs equal the input's at every level — each survivor keeps exactly the runs it was in. Reasoning: `context/code/core.md` §6. **`prune_empty` shares one `GridEval`** across the whole row×col emptiness test, and **a grid evaluates only through `GridLayout::evaluator`**, which builds the fold memo from that layout's own `none_cats` and hidden items — so no other grid's memo can reach its cells or totals (improvise-metf, improvise-opbo; the memo itself is improvise-h6z1, `core.md` §14). + **`GridLayout::new` does not prune; `with_frozen_records` does** — and that split has now produced **four** bugs: `view_frame`, `project::grid_in` (PR #78), and `persistence/export.rs::render_csv` (improvise-iot6), which wrote rows *and columns* the grid does not draw for as long as CSV export has existed. Every remaining `GridLayout::new` call site in the workspace is test code — verified — so a new production caller almost certainly wants `with_frozen_records(m, v, None)`. + +2. **Effects never touch `std::fs`.** Every document read or write goes through `App::storage: Box` (improvise-fuj, closed). Two writes still go around it: `App`'s autosave and the headless final write call the path-taking `persistence::save` (improvise-31oe.11, open). The host picks the backend: `App::new` supplies `PhysicalFs`, `App::with_storage` takes any other — the browser worker supplies OPFS (improvise-i34, closed), and tests use `MemoryFs`. `improvise-io` keeps path-taking wrappers (`save`, `load`, `export_csv`, `parse_csv_with_warnings`) for the native CLI, each delegating to a `_to`/`_from`/`_bytes` variant that takes the backend or the bytes. +3. **Every effect declares an `EffectKind`, and there is no default.** `ViewOnly` never reaches other clients; `Model(CellData | LayoutGeometry | WholeWorkbook)` does (improvise-mae). A new effect that forgets to classify itself fails to compile — that is deliberate, since the failure mode is silent: a mis-declared `ViewOnly` edit would simply never reach connected clients. Watch two traps: the *View* lives inside `model_state.workbook`, so "touches model_state" does NOT mean "is a model change" (axis/page/format are `LayoutGeometry`, cursor and scroll are `ViewOnly` — see improvise-8uz; and `SwitchView`, `ViewBack` and `ViewForward` moved from `LayoutGeometry` to `ViewOnly` with improvise-jytm, writing `view_state` and nothing else, which is what took the latter two off the wire — but a sound derivation is not a guard: only `SwitchView` had one, and review showed reclassifying either sibling back was silent across the whole suite); and `Save`/`SaveAs` touch `model_state` for bookkeeping only and are `ViewOnly`. `AddCategory`/`RemoveCategory` are the third trap and go the other way: they look like data edits, but both go through `Workbook`, whose `on_category_added`/`on_category_removed` rewrite **every** view's axis map, so they are `LayoutGeometry`. The two "an effect declaring X actually changes X" tests cannot separate them — a category change moves model data *and* view config, so it clears either bar; `structural_effects_declare_their_kind` pins the declaration itself. A third trap, now guarded: **`EffectKind` is too coarse to check a *mapping*.** `ui/reduce.rs::model_effects` sends each of the 26 `ModelCommand` variants (counted on `c1772ca9`; gotcha 38) to one effect, and every inverse pair — `HideItem`/`ShowItem`, `AddItem`/`RemoveItem`, `CreateView`/`DeleteView`, `SetCell`/`ClearCell` — declares the *same* kind, so swapping three of them at once left the suite at 918 passing while three wire messages did the exact inverse of what they said. The mapping is name-for-name by construction, so `every_model_command_names_the_effect_of_the_same_name` asserts that, reading the type name off the effect's derived `Debug` (`Effect` is a trait object, so `type_name` is out of reach). A variant whose effect must be named something else is a variant that stopped being derived — it should fail there and be argued about. Their reach is wider than the name suggests: `Model::remove_category` also drops every formula targeting the category, which is why `Dirty::of(LayoutGeometry)` reports `formulas: true` — see gotcha 35. +4. **Commands never mutate.** `&CmdContext` is read-only; return `Vec>`. If you want to touch `&mut App`, add or extend an Effect. +5. **`CellKey` is always sorted.** Use the constructors; equality and hashing rely on canonical form. +6. **No `Model::add_item`.** With a `Workbook` in hand use the facade: `wb.add_item("Region", "East")?` / `add_item_in_group` / `remove_item` (improvise-tvt). With only a `Model`, go through the category: `m.category_mut("Region").unwrap().add_item("East")`. +7. **Virtual categories.** `_Measure` items = explicit items ∪ formula targets. Use `measure_item_names()` / `effective_item_names("_Measure")`. `add_formula` does *not* add items to `_Measure`. Formula-target lookup also falls back into `_Measure` via `Model::find_item_category`. Cell aggregation goes through `apply_agg`, formula `SUM(…)`/`AVG(…)` through `eval_agg` — they differ on empty input (absent vs. A third rule joins them for **non-numeric** values: `aggregate_cell_values` reduces text by **unanimity** — one matching value, or several copies of the same value, is that value; differing values, or text among numbers, is `CellValue::Error("mixed")` (improvise-l4qx). The **all-numeric branch is checked first**, so `Count` of a lone number is still `1` and not the number — that ordering is the whole difference between this rule and "a single value passes through". What the rule forbids is *dropping*: `filter_map(as_f64)` used to, and since `_Index`/`_Dim` are on `Axis::None` in every default view that made **every text cell in every document render blank**, including the pipe-quoted text `.improv` carries as a first-class value. `evaluate_aggregated` and `aggregate_raw` share the helper, so a formula ref over mixed values now errors `"mixed"` rather than silently summing the numeric ones. `0` for Sum/Count, error for Avg/Min/Max). `Model::evaluate_aggregated` takes `Aggregation::{Exact, OverHidden}` rather than `none_cats: &[String]` (improvise-5jm5) — the list was only ever tested for emptiness, and the guard against that shape returning is the **signature**. `recompute_formulas`/`aggregate_raw` keep the list because they walk it. +8. **Display rounding is view-only.** `format_f64` (half-away-from-zero) is only called in rendering. Formula eval uses full `f64`. Never feed a rounded value back into eval. +9. **Formula eval is fixed-point.** `recompute_formulas(none_cats)` iterates until stable, bounded by `MAX_EVAL_DEPTH`; circular refs converge to `CellValue::Error("circular")`. `App::new` runs it before the first frame so formula cells render on startup. +10. **Drill into formula cells** strips the `_Measure` coordinate when it names a formula target, so `matching_cells` returns the raw records that feed the formula instead of returning empty. +11. **Minibuffer keymaps come from `TextEntry`.** The eight text-entry modes are built by the `TextEntry` builder in `improvise-protocol/src/keymap.rs` — seven by `TextEntry::new`, and `RecordsEditing` by `TextEntry::with_parent` off `Editing`'s map, which is why a count taken off the `::new` call sites reads seven. It binds the shared spine (AnyChar→`append-char `, Backspace→`pop-char `, Esc→`[clear-buffer , enter-mode ]`) and welds every commit key to its `clear-buffer` via `.commit(key, cmd, args)` (improvise-w3q). `TextEntry::new` takes the `AppMode` and reads `` off its `MinibufferConfig`, so the buffer the prompt names and the buffer the keys edit are the same string by construction (improvise-k8h) — the keymap never spells a buffer name. `.rebind` exists only for CommandMode's Backspace. `SearchMode` is hand-written on purpose — no commit key, and `exit-search-mode` owns the query. `EnterMode` clears the incoming mode's minibuffer through `write_buffer` too — building `SetBuffer` directly there was unreachable only because no shipped `AppMode` is keyed `"search"`. `MinibufferConfig::buffer_key` is a `String`, not `&'static str`: a borrowed-'static field makes `Deserialize` unimplementable, and every text-entry `AppMode` variant carries a `MinibufferConfig` (improvise-a2x). A buffer named at runtime is read with `cmd::core::read_buffer` and written with its mirror `write_buffer` — "search" is not in the buffers map but its own `ViewState` field, and those two functions are the only code that knows it (improvise-gdl). +12. **Panel keymaps come from `PanelKeymap`.** The three panel modes share a spine (Esc/Tab→Normal, arrows and j/k→`move-panel-cursor `, every panel's toggle key, `:`→command mode) built by `PanelKeymap::new(panel)`; the keys that act on the panel's contents arrive through `.bind`/`.bind_args`/`.delete` because they vary in shape, not just command name (improvise-4pg). `Panel::ALL`/`name()`/`key()` in `ui/app.rs` drive the toggle keys for Normal and all three panels, so a fourth panel reaches every keymap from one `ALL` entry. `name()` must spell what `parse_panel` accepts. +13. **Prefix keys are stored as an address, not a keymap.** `ViewState::transient_prefix` holds the `(KeyCode, KeyModifiers)` that opened a prefix; `KeymapSet::prefix_keymap` resolves it back to the sub-keymap, walking parents so `RecordsNormal` reaches Normal's `g`/`y`/`Z` (improvise-jc8) — that is the command-layer wrapper; the resolution itself is `Keymaps::prefix_keymap` in `improvise-protocol/src/keymap.rs`. `ViewState` must stay free of `improvise-command` types — it is destined for the protocol crate, which `improvise-command` will depend on. Resolution assumes prefixes are **flat**; `prefix_keymaps_are_flat` fails the build if a nested one is added. +14. **Keybindings are per-mode.** `ModeKey::from_app_mode()` resolves the mode; Normal + `search_mode=true` maps to `SearchMode`. Minibuffer modes bind Enter/Esc as `Binding::Sequence` so `clear-buffer` fires alongside commit/cancel. Hints are checked against the keymaps: `every_hint_key_resolves_in_its_mode` (`command/keymap.rs`) splits each `AppMode::hint_text()` and requires every key it names to resolve in that mode's map, because `only_quit_has_no_hint` asserts only that a hint is *present*. It checks which keys are named, not what they do. Its tokenizer knows the hint dialect explicitly (named keys, `Ctrl-`, `F/C/V` alternatives, character runs like `hjkl`/`[:]`, the em-dash prose tail, CommandMode's `:q` ex-commands) and panics on a token it cannot classify rather than skipping it. +15. **Registry names come from prototypes, not strings.** Every `register*` call reads the name off a prototype `Cmd` instance, so a typo is impossible (improvise-9cn) — so no `register*` call spells a name. Name literals do remain in `registry.rs`, and only tests guard them — the keymap test `every_bound_command_name_is_registered` for the `Move` literals (all bound; `registered_names_are_unique` covers them too), and for the four aliases `registry.rs::every_alias_resolves_to_a_registered_command` (a hand-copied list, not the table) plus the `parse.rs` tests `alias_add_cat_resolves_to_add_category`, `alias_formula_resolves_to_add_formula`, `alias_add_view_resolves_to_create_view` and `alias_q_bang_resolves_to_force_quit` — the `Move` prototypes carry `cmd_name: "move-selection"` / `"scroll-rows"` / `"page-scroll"` (the four `jump-*` names go through `reg_jump!`), and four `r.alias` calls spell an alias and its target (`add-cat`, `formula`, `add-view`, `q!`). What types can't reach: five commands name themselves off a field (`Move` off `cmd_name`, `ViewNavigate`, `TileAxisOp`, `ToggleGroupAtCursor`, `CommitAndAdvance`), so a prototype can name the wrong *variant* and still compile. That shows up as a **duplicate** registration, not a mismatch — the wrong-variant entry collides with its sibling and the sibling's name vanishes. `registered_names_are_unique` catches it; `every_bound_command_name_is_registered` catches the keymap side, and checks **both** halves of `dispatch`'s `registry.interactive(name, args, ctx).ok()?` — the name resolves *and* the args parse — because improvise-s13 moved `enter-mode ["export-prompt"]`/`["formula-edit"]` out of the nullary class, where a typo'd mode name compiles into a dead key. It builds the `CmdContext` in each keymap's own `AppMode`, since `commit-buffer item` reads `ctx.mode`. +16. **`effect_cmd!` macro** wraps a pre-built effect as a parseable command. Use it for simple wrappers; reach for a hand-written `Cmd` once there's real decision logic. Its counterpart on the effect side is `simple_effect!` in `ui/effect.rs`, which declares a one-field newtype that writes one place on the `App`. Its **kind argument is mandatory** — `simple_effect!(Name(T) => |app| place, EffectKind::ViewOnly)` — even though every call site so far answers `ViewOnly`: hard-coding it reintroduced exactly the default gotcha 3 does without, since `$place` is an unconstrained expression and a one-line `active_view_mut().number_format` setter would have reported `ViewOnly` with the suite green (improvise-8fy). Exactly six wrappers return `unless_unknown(needs, effect)` rather than the bare effect (gotcha 51): `set-axis`, `set-page`, `toggle-group`, `hide-item`, `show-item` and `switch-view`. The rest return bare effects — `delete-view` (whose `DeleteView` effect refuses an unknown view itself), `add-item`, `add-items`, `set-cell`, `create-view` — and `remove-formula` checks in `execute`. Likewise, mode entry has no per-mode shim commands — bind `enter-mode ` (improvise-s13); the name is decoded by `parse_mode_name` in `registry.rs`, and `EnterMode` clears the mode's minibuffer on the way in. +17. **Float equality inside formula eval has one epsilon, and the div/0 guard shares it.** `FLOAT_EQ_EPSILON = 1e-10` in `model/types.rs` is used by `=`/`!=` **and** by the division-by-zero guard (`rv.abs() < FLOAT_EQ_EPSILON`), in both `eval_expr` and its aggregated twin — six uses across the two evaluators (a div/0 guard plus the `Eq` and `Ne` arms in each), one constant. This gotcha previously recorded the opposite (`=`/`!=` epsilon, div/0 exact) and was stale; the mix is gone. Why an epsilon at all and why this one: `context/code/core.md` §4. +18. **`IndexMap`** preserves insertion order for categories, views, and items. Persistence relies on this — and so, since improvise-shyt, do `DataStore::matching_cells`/`matching_values`: the secondary index **narrows** the candidates, then `matching_positions` resolves each to its position in the primary `IndexMap`, sorts, and reads back. **An index whose values are `HashSet`s is for O(1) lookup, never for iteration.** Walking it gave a different order in every process, which was not tidiness. Records-mode rows reordered between sessions; `SUM`/`AVG` drifted in the last bits because `f64` addition is not associative (`Min`/`Max`/`Count` and the text-unanimity rule of gotcha 7 are order-free, so they were never affected); and worst, the **drill snapshot** froze its record list in hash order while `drill_state.pending_edits` are keyed by *row index* (gotcha 1), so the same drill opened in a second process addressed **different records** — silent misdirected writes, and nothing in the tracker knew. Measured at 18,000 cells: +20% worst case on `matching_values`, and `matching_cells` 7% *faster*, sorted positions resolving `CellKey`s with better locality. `DataStore::live_records` was the open choice and is now decided (improvise-2yc2): it walks `cells` and projects insertion order onto records, deliberately paying O(cells) rather than O(records), because `cells`' order is already the answer `matching_positions` gives and a second answer to "what order is this store in" is the disease. It is derived per call rather than memoised on `RecordEntry`, since a stored position is a copy and a record deleted and re-created would hold its old one. Note what is *not* on offer: an unordered return — every iterator has an order, and a type that merely disclaims one still hands the caller hash order, which is the guard living one level away from the hazard and the shape this whole class of bug returns through. **The second real instance was `import::analyzer::analyze_records`** (improvise-7yx), which built its distinct values through a `HashSet`: `build_model` adds items in exactly that order and `IndexMap` preserves it, so the *same CSV imported twice* produced two different `.improv` files and two different row orders. Now first-seen order, pinned by a five-iteration test because each `HashSet` gets its own `RandomState`. Also answered here since: an **empty `partial`** matches every cell vacuously (improvise-hkt6) — it used to index `sets[0]` and panic, unreachable only because both callers guarded. **A group's fold reads through the same rule.** `Model::fold_stored` folds a collapsed group's stored values once, over every record the group covers, through `DataStore::spanned_values` → `spanned_positions` (improvise-yb4b) — one scan, not a lookup per member. `spanned_positions` chooses **by index-set size** between narrowing once by the base key and narrowing once per covered item of the smallest span, whichever visits fewer (improvise-jq0m), and **both arms return store order** — sorted positions, as `matching_positions` does. That is load-bearing, not tidy: a fold that returned hash or per-item order would reintroduce the `SUM` drift above, and the choice between the arms would then be visible in the last bits of a total. **A partial as wide as the widest stored cell needs no narrowing** (improvise-felo.13): `DataStore` keeps `arities` (how many stored cells have each width), and `exact_positions` answers a strictly-sorted partial at least that wide with one probe of `cells`, since only an equal cell can match; anything else falls through to the index. The answer is the same sorted positions. +19. **Commit paths for cell edits** split across `commit_cell_value` → `commit_regular_cell_value` / `commit_plain_records_edit` in `command/cmd/commit.rs`. Keep the synthetic-records branch in sync with the plain branch when changing behavior. **Which cells refuse an editor is one function, asked at both ends of an edit**: `cmd::core::uneditable(model, layout, row, col)` is the shared decline for **opening** an editor (`EnterEditAtCursor`) and for **committing** one (`CmdContext::committable_cell_key`) — a fold or other aggregate, a formula's result, or a `CellId::NonItem` (improvise-enz6, improvise-gk51). It reads the cell's `CellId` through `project::cell_id`, the same derivation the projection uses, and the browser's `resolve::declined_at` matches on the projected `CellId`, so the two front ends refuse the same cells. Asking only at open is the bug enz6 was: Enter-to-advance moved onto a collapsed group's row and committed a write to the group coordinate. **`CellId::NonItem { category, item }` is a stored value whose coordinate names no item of its category** — records mode can draw one, and no write can address it, so it is projected as its own variant rather than as a `Record` that the host then refuses (gk51 was exactly that mismatch). Its one wording is `improvise_protocol::non_item_refusal`. +20. **Minibuffer commits go through one command.** `commit-buffer ` (`CommitBuffer { kind: CommitKind }`) owns buffer→trim→bail-on-empty→dirty→status for all three; the varying parts are methods on `CommitKind`. The add effect is emitted **last** so its failure message wins over the command's optimistic status (improvise-6t3). `CommitKind::Item` carries its target category, resolved from `AppMode::ItemAdd` by `parse_commit_kind` at registry time — so `commit-buffer item` is not constructible from a script line. +21. **An abort stops the chain, never the bookkeeping — and ordering is what separates them.** Three effects set `abort_effects`, and they are not all saves: `Save` and `SaveAs` on a failed write or a missing path (improvise-8tk — `SaveAs` was the arm left out of the original fix, so `:w /bad/path . q` quit on a failed write), and `AddCategory` when `Model::add_category` refuses, because what follows assumes the category exists — from the `+cat` column (improvise-2zq) the rest of the batch would advance the cursor and re-open an editor on a column that was never created. A `Binding::Sequence` flattens into one effect vec, so the abort carries across sequence steps; that is a property of the *flattening*, and it is why improvise-0hb's proposal to apply effects between steps was rejected — `apply_effects` resets the flag per batch, so step 2 in a fresh batch would land the `Quit` after a failed `Save`, silently reopening a P1. `a_sequence_whose_first_step_aborts_does_not_run_the_second` is the guard. **The rule the aborts taught, which is the part to carry:** an abort at the *tail* of a batch also ate the mode restoration and the buffer clearing, stranding the user in `CommandMode`/`Editing`/`SavePrompt` — where `draw_bottom_bar` renders the minibuffer *instead of* the status line, so the error the effect had just written was never drawn and the operation appeared to do nothing at all. So **tidy-up goes at the head, where an abort cannot reach it**: `ExecuteCommand` fronts its batch with `ReturnToNormalUnlessMoved`, `CommitAndAdvance` with the mode change, `CommitSave` with the mode change (found by Copilot on PR #27). Ordering answers it without giving any effect a "runs anyway" flag, and it does not weaken gotcha 31's observed-not-predicted mode settling: whatever a later effect sets, it sets *after*, so the mode a `:` line ends in is unchanged in every case — the only difference is what happens when the batch never reaches the end. **An abort is not a rollback**; that is the hole clause 4 leaves (gotcha 47). +22. **The import wizard's back path is a history stack, not a reverse table.** `advance` pushes the step it leaves onto `ImportWizard::history`; `previous` pops it, so back always retraces the path actually taken through the conditional transitions (improvise-907). Preconditions live in `validation_error()`, checked on the step where the user can still fix them — never deferred to `build_model`. `NameModel` is terminal for `advance`: Enter there calls `build()`, which parks the workbook in `built` on `Done`, and `AcceptWizardModel` takes it from there. Going back from `Done` discards `built`. **`ResolveDuplicates` appears only when `ImportPipeline::collisions()` is non-empty**: `after_fields` routes there from `ReviewProposals` or `ConfigureDates`, so it comes after the field and date steps and before `DefineFormulas`, with `NameDuplicateCategory` between them when the answer is Separate. Enter there is `WizardOp::ChooseDuplicate` (sum, keep separate, or go back). `ImportPipeline::on_duplicate` defaults to `DuplicatePolicy::Refuse`, so a path that never asked — the headless `--no-wizard` import included — errors listing the collisions rather than summing them. **A collision is a cell written by two records** (a tuple *and* a measure), not a shared tuple: records with disjoint measures on one tuple write different cells and collide with nothing (improvise-9bit). Reasoning: `context/code/formula-and-io.md`, "Colliding records are a question, not a default". +23. **`app.layout` goes stale mid-batch.** `apply_effects` rebuilds only *after* the whole batch, so an effect that reads the layout after an earlier effect changed the grid needs a `RebuildLayout` effect in front of it — the command decides the order (see `EnterEditAtCursorCmd`, `CommitAndAdvance`). Read the cursor's cell text via `App::display_value_at_cursor()`; don't build a `CmdContext` inside `apply` (it reads `layout`, the memoised `grid_geometry()` and the cursor, and a stale layout there is the bug). **Since improvise-felo.13 the rebuild is skipped when nothing it reads has moved**: `App::refresh_layout` rebuilds only when `layout_stale` (any `EffectKind::Model(_)` in the batch, or `install_workbook`), the drawn view's name, or the drill snapshot's pointer (`LayoutBasis`) changed, so a cursor key rebuilds nothing, and the column widths and row-header width are measured once per layout (`App::grid_geometry`, dropped with the layout). Two of the three inputs are **observed** (the view name and the drill pointer); `layout_stale` is **declared** from `EffectKind::Model(_)` (`touches_model`), so gotcha 3's classification hole — a `ViewOnly` effect that changes what the layout reads — now also leaves `app.layout` stale, where before felo.13 every batch rebuilt. It is still not a hand-kept list of effects. A writer that bypasses effects and changes what the layout reads must call `rebuild_layout()` itself. Measurements: `context/test-evidence/layout-cost.md`. + +24. **The cursor is per-client and keyed by view.** `ViewState::cursors: HashMap` holds `selected`/`row_offset`/`col_offset`, keyed by view name; an absent key means the origin. Reach it through `App::cursor()` (a `Copy` snapshot), `App::cursor_mut()`, and `App::reset_cursor()` — never off `View`, which no longer has the fields (improvise-8uz). Two consequences. **Changing the grid's shape must reset the cursor explicitly**: `View::transpose_axes`/`cycle_axis` used to do it themselves, and now `TransposeAxes`/`CycleAxis` call `app.reset_cursor()` — a new geometry effect that renumbers rows or columns has to do the same. **Replacing the document must clear the map**: `LoadModel` and `AcceptWizardModel` call `cursors.clear()`, because the incoming workbook's view names have nothing to do with the outgoing one's. That clearing replaces `Workbook::normalize_view_state`, which is gone — a `View` can no longer carry a stale offset. The same clearing owes the **session mirrors**: `install_workbook` bumps `App::document_generation`, and `ClientSession::project` reseeds any mirror seeded at an earlier one (improvise-cqq). An effect cannot reach `Sessions`, so the generation is the mechanism and `Sessions::rebase` is only the client-facing half. The same is true of a **geometry** change, and it owes the mirrors *both halves*: `App::cursor_epoch` is bumped by `reset_cursor` and watched by `Sessions::adopt_cursor_epoch`, which resets every mirror **and** returns `Broadcast::ToAll(Command::ResetCursor)` (improvise-gvnb). Doing only the first is **strictly worse than doing nothing** — the diff then believes every client already holds origin-positioned content and sends nothing, so they stay stale *permanently* rather than until their next scroll. The trigger is the **epoch moving**, not the command variant, for the reason gotcha 39 watches `document_generation` instead of matching on `LoadModel`: which effects renumber the grid is a per-effect decision, so a table of variants would be a second description of it, free to disagree. What is broadcast is an **invalidation, never a position** — another client's transpose must not move where I am looking (that is UI and it is mine, improvise-g0f), only declare that where I was looking has stopped meaning anything. **Since improvise-ni6k no effect calls `reset_cursor` at all — the cursor is invalidated by OBSERVING the grid's frame.** `apply_effects` diffs the frame after every model effect, **inside the loop**, because a batch that reframes and then places the cursor loses the placement otherwise (four records-mode tests pin that position). There is no list of effects anywhere, so the mechanism is total over `EffectKind::Model(_)` by construction — a genuinely stronger shape than a hand-kept table, and the reason the old "a geometry effect that forgets `reset_cursor` is invisible in both directions" hazard is closed. + + **Two arms, and the asymmetry is the whole design (improvise-mcq4).** `LayoutGeometry` hashes only the **active** view — it is the held-key path, and other views cannot reframe there since `view_mut()` writes only the active view, with the category and item effects already covered by `view_shapes`. `CellData` hashes **every** view, because a cell write can prune a row out of a view the host is not on; narrowing that arm to one view measured as *free* and silently lost a multi-client invalidation, which is gotcha 24's own strictly-worse-than-nothing case. Cost of the wide arm: 0% on the held-key path, +5.7% on an 18k two-view `SetCell`, linear in view count, accepted. State the absolute as well as the ratio: the observation roughly doubles held-key latency on large pruned documents (3.04 → 5.69 ms at 18k) and 56 ms at 90,000 cells is 3.4 frames. + + **Totality is conditional on classification**, and nothing makes classification total: a `ViewOnly` effect that renumbers the grid is invisible to all of this (gotcha 3's hole, unchanged). And two known gaps are filed rather than fixed — a geometry effect immediately after a mid-batch `SwitchView` skips the check entirely, because there is no basis for the new view yet (improvise-9c9o; under-reporting, which gotcha 35 says is the expensive direction), and `Model(WholeWorkbook)` matches neither arm, which is benign only because all three such effects go through `install_workbook` → `rebase` → a fresh `Snapshot`. + +25. **`ViewState` and its field types are `Serialize`/`Deserialize`.** It is the wire payload of the client Snapshot (improvise-q08), so `AppMode`, `MinibufferConfig`, `MinibufColor`, `ViewFrame`, `KeyCode` and `KeyModifiers` all derive both (improvise-a2x), and `improvise-command` depends on `serde` for it. The constraint this puts on new state: **no borrowed-`'static` fields anywhere ViewState can reach** — `Deserialize` cannot produce one, and the error surfaces on every enclosing variant rather than at the offending field. `view_state_survives_a_json_round_trip` guards it. `ViewState` and `Snapshot` also derive `Clone` and `PartialEq`, so the bootstrap message can be compared in a round-trip test. + +26. **Axis presentation lives on `Axis`; its colour is a `Face`.** Four vocabularies hang off the enum in `view/axis.rs` — `plain_label()` ("Row"/"Col"/"Page"/"None"), `glyph()` (which `Display` appends, giving "Row ↕"), `short_label()` (three columns for the tile bar; `None` is "·", *not* a truncation), and `initial()` (one letter for the view panel; `None` is empty). Plus `next()` for the Row→Column→Page→None rotation. **`Axis` carries no colour**: `AxisColor` and `Axis::color()` are gone. A colour is a `Face::Axis(axis)` in `improvise-protocol/src/face.rs` (ADR 004, `context/code/protocol.md` §10), resolved by the `Theme`, and `draw.rs::face_style` is the one place a face becomes a ratatui `Style` — the tile bar and the category panel both call it, so neither keeps a table. `improvise-core` still names no ratatui type. Not covered here: `persistence/format.rs` maps `Axis` to the on-disk keywords ("row"/"column"/"page"), which is a format concern and stays in the io layer. + +27. **`Axis::Filter` constrains without being navigable.** Drilling fixes its coordinates on `Filter`, not `Page` (improvise-dz8): both restrict which records are visible, but only `Page` is something the user may move or page through. The split shows up as two different questions, and picking the wrong one is the trap: **"what data is visible" is `View::filtering_cats()`** (Page ∪ Filter — used by `GridLayout` and by `add-record-row` when it builds a key from the current constraints), while **"what can the user act on" is `categories_on(Axis::Page)`** (used by `[`/`]` page cycling) or `View::tileable_cats()` (used by all three tile-select sites). `Axis::next()` maps `Filter` to itself and no navigable axis rotates into it, so tile select cannot hand one out. The `.improv` grammar accepts `filter` and round-trips a selection with it — a Filter axis only ever occurs in the never-persisted `_Drill` view, but the alternative was writing it out as some other axis. **What a virtual view must *be* is model-side and has one home** (improvise-gd28): `view::{records_spine, records_steps, drill_steps}` return `Vec` — pure data naming no `Effect`, no `App` and no front end — and `cmd/grid.rs::steps_into_active_view` is the whole of *how the TUI gets there*, twenty lines a browser answers its own way. Before the split, the drill's filter placement and the records spine were spelled out twice in `cmd/grid.rs` with nothing keeping them in agreement. `DrillState` deliberately did **not** move: it holds an `Rc` and its `pending_edits` are uncommitted writes, so gotcha 30's session-local tag already places it. Reasoning: `context/code/core.md` §11. + +28. **The browser backend is sync on purpose, with an async seam.** `OpfsFs` (`improvise-io/src/opfs.rs`, wasm32 only) implements the same synchronous `Storage` trait as `PhysicalFs`, because `FileSystemFileHandle.createSyncAccessHandle()` yields a handle whose read/write/truncate/flush **are** blocking — available only inside a dedicated Web Worker, which is where improvise-9x6 puts the app. So `Storage` never became async. What is still async splits out into four explicit methods the worker awaits from its message loop: `open()`, `attach(path)` (must precede any read/write of that document — acquiring a handle is async even though using it is not), `refresh()` (repopulates the index `list` answers from), and `commit_deletes()`. Consequence: **`delete` is visible immediately but lands in OPFS at the end of the batch** — it drops the name from the index and queues a `removeEntry`. Handles are exclusive per file, so `attach` closes any handle it replaces and `detach` releases the lock. `storage_contract` in `storage.rs` is the behaviour every backend owes; it runs against both host backends and is the written-down spec `OpfsFs` must satisfy in a browser, where the harness cannot reach. **And the premise is dedicated-worker-only, which is narrower than this paragraph used to read.** `createSyncAccessHandle` is spec'd `[Exposed=DedicatedWorker]`, and measured in a module `SharedWorker` on Chromium and Firefox it is **absent** — not rejected, not flagged, not transient: `getDirectory()` and `getFileHandle()` both succeed and the method is simply not on the object (improvise-12ha). So `OpfsFs` cannot be hosted by a `SharedWorker` as written, and "a browser worker" in this paragraph means a *dedicated* one everywhere it appears. `OpfsFs::open`'s own guard does not enforce that — it `dyn_into`s `WorkerGlobalScope`, which a `SharedWorkerGlobalScope` **satisfies**, so a shared worker sails past a check whose message names the right requirement and fails later as a raw `TypeError` deep in the glue (improvise-7gip). The consequence for multi-tab is that the machinery everyone reaches for first is the storage seam, not `Sessions`: `Sessions`, the `Addressed` filter and the one `frame` pair are all ready, and none of that helps, because the no is entirely on the handle. + +29. **"Which view is active" is stored twice, and the copies are now allowed to differ.** `Workbook::active_view` is the document's **default** — what a freshly connected client opens, and what `.improv` writes as `Initial View`. `ViewState::active_view` is **authoritative for that client**, and it is what keys `ViewState::cursors`. The second copy exists because `reduce_view` runs on a client that holds no `Workbook` (improvise-cqi); the *split* is improvise-jytm answering improvise-g0f, on the user's ruling that **the model is shared and the UI is not**. Two clients may sit on different views of one document. Three things follow. **`App::resolve_view` is the one answer** to what a view name names in this document, falling back to the default so a slice naming a deleted view cannot panic in `rebuild_layout`; `view()`, `view_mut()`, `view_name()`, `project::grid_in` and `ClientSession::view_in` all go through it, and a second copy of that fallback was the first thing review found. **The guard was replaced, not deleted** — `the_two_active_view_names_never_diverge` became `the_documents_default_view_outlives_this_clients_switches` plus `a_fresh_session_opens_the_documents_default_view`, because a deliberate split with no guard is how one silently regresses into the other. And **a deleted view is the case the split creates**: `DeleteView` can take away the view another client is on, so `Sessions::adopt_deleted_views` moves that mirror to the default and tells it, and `effect::DeleteView` does the same for the host — observed ("does this mirror still name a view the document has"), never predicted from the command variant. **Testing any of it needs a bystander that can be moved**: both audience tests first sat theirs on the default, where a wrongly-addressed relay folds to a no-op, and review demonstrated the vacuity by injecting `ToAll` twice. One gap remains: `Initial View` no longer follows the TUI user (improvise-azhb). The other — a view-scoped `ModelCommand` naming no view — is **closed** (improvise-lltt), and this paragraph described it as an open gap for a day after: every view-scoped variant carries `view: String` and `model_effects` maps it to `ViewTarget::Named`, so a connected client's `SetAxis` lands on the view it names. `ViewTarget::Active` resolves through `App::view_mut()` and is therefore the *host's* view — but **the wire never constructs one**, which is what closed it. Follow mode is deliberately not foreclosed. + +30. **Two tests gate `ViewState` membership.** `ViewState` is the *portable* session slice — the one thing serialized into the Snapshot a newly-connected client gets (improvise-q08). A field may live there only if it is **portable** (plain data from the allowlisted crates, no `Rc`/`Arc`) and **snapshot-safe** (a fresh client inheriting the value is correct). `wizard` fails both — `ImportWizard` is an `improvise-io` type, and a half-open dialog is not something to inherit — and `drill_state` fails both — it holds an `Rc`, and `pending_edits` are uncommitted writes belonging to whoever staged them. Both live on `App` under the **session-local** residue tag (improvise-h0b). The `ViewState: Send` static assertion in `app.rs`'s tests is the compile-time proxy for portability: `Rc` is not `Send`, so a re-added shared pointer fails the build (the `Deserialize` derive from gotcha 25 now rejects it too, serde's `rc` feature being off). `Send` does *not* catch the origin-crate or snapshot-safety halves — check those by hand. + +31. **`:` commands settle their mode by observation, not prediction.** `ExecuteCommand` ends every batch with `ReturnToNormalUnlessMoved(ctx.mode)`, which drops to `Normal` only if the mode is still what the batch started in. There is no `Effect::changes_mode()` any more: it was a per-type constant, and `ViewBack`/`ViewForward`/`StartImportWizard` swap the mode only on some runs, so no up-front answer was right — `:import` opened the wizard and was snapped straight back out (improvise-toc). A new effect that changes the mode needs no annotation; just don't return the batch's starting mode from a success path. + +32. **There is no aggregate render cache — each widget folds its own stream.** `improvise-protocol::render` holds one cache type per widget (`GridCache`, `CategoryTreeCache`, `FormulaCache`, `ViewListCache`, `TileBarCache`), each implementing `WidgetCache` with its own `Update` type, and a `Projection` is one such update addressed to one widget (improvise-rkeq). `reduce_view` folds only `ViewState` now — it no longer takes a cache, and `Command::Project` passes through it untouched. Two consequences. **The overlap between caches is deliberate**: the tile bar and the category panel both carry `(category, axis)` and are updated by separate projections, so neither is woken by the other's changes; do not "fix" it by sharing a variant. **Bootstrap is a *compaction*, not a replay**: `Snapshot { view, seed: Vec }` seeds a new client with a batch of the same projections the steady stream carries, so there is no whole-state type that could disagree with the incremental one. The seed is the server's *current* state, not the head of the history — `GridCache::seed()` builds it, and the property that earns it is `GridCache::folded(&cache.seed()) == *cache`. Nothing is defined for the help overlay (its content is static and `help_page` is already in `ViewState`) or for the import wizard, which now has a shape and **deliberately no projection**: `WizardCache` is built by `project::wizard_cache` and drawn locally, because a half-open dialogue is addressed to precisely the session that opened it, never to every client (improvise-764 — the rendering-side twin of the argument that removed `AcceptWizardModel` from `ModelCommand`, gotcha 38). **A cache without a `Projection` is the shape session-local residue takes when it still has to be drawn by two different front ends.** It lives in `improvise-protocol` rather than `improvise-command` for the `flatten_cat_tree` reason: when the wasm client grows a wizard it must draw *these* rows, not a second derivation. + +33. **The grid's labels frame its cells, and `apply` enforces it — the fold is not commutative.** Three consequences in `improvise-protocol/src/render.rs`. (a) A `GridUpdate::Cells` that overtakes the `RowFrame`/`ColFrame` naming its positions is **dropped**, because `fit_to_labels` truncates to the labels it has; a client must not reorder the stream, and `a_cell_update_that_overtakes_its_labels_is_lost` pins it. (b) A label change that *renames* a position blanks the cell under it: `fit_to_labels` preserves by POSITION, which is only right where the label did not change, so `apply` calls `invalidate_relabelled` itself rather than trusting a producer to send `GridUpdate::Invalidated` first. `Invalidated` therefore now means only "contents stale, frame unchanged". (c) **every arm that touches labels *or* widths re-fits** — `col_widths` is re-fitted with the columns — a new column is drawn at its label's own character count until `ColWidths` catches up — because a renderer zips widths with labels and a length mismatch indexes off the end. Fitting on the label arms alone is what let a late `ColWidths` put the mismatch straight back — a short update keeps each unheard-of column at its label's width, and a long one is dropped as having overtaken the frame that names it, exactly as (a) holds `Cells`. `GridCache` also carries `totals` and `page_coords` (improvise-n10). `totals` is framed by `col_frame` on the same terms as the cells and widths and re-fits with them, with one asymmetry: an **empty** strip means "this grid has no totals" (records mode, or an empty grid) and is therefore *not* padded, where a non-empty one is — padding it would draw a blank total row under a records grid that has none. `Invalidated` blanks the totals too: they are contents, and a stale sum under fresh cells is the one number a user takes at face value. `page_coords` is framed by nothing (it names categories, not positions) and survives a reshape. Note the fold tests: asserting `fold(prefix)+fold(rest) == fold(whole)` is vacuous (it is the definition of a left fold), so the tests that earn their place are the order-sensitivity one and the compaction-equivalence one — see `doctrine(fold_tests_must_test_order_not_the_axiom)`. + +34. **A formula target is unquoted, and pipe quoting has one home.** `parse_formula` strips the pipes off a target exactly as it does off an `Expr::Ref` — the target names a `_Measure` item, so a target that kept its delimiters made `add_formula` create an item literally named `|Total Revenue|`: a ghost column the user's real item never matched, and `Expr::Ref("Total Revenue")` never resolved to. Both the TUI formula minibuffer (`AddFormula`) and `.improv` load go through that one call, so both were affected. The writer emits `Formula::raw`, not `target`, so file text is unaffected either way — which is exactly why the round-trip tests never saw it, and why the fix repairs existing files on next load with no migration. Pipe quoting has exactly **one home**: `improvise_formula::quote`, owning both halves — `quote_name`/`is_bare_name`, `pipe_quote` (unconditional, for a `.improv` text cell that must not read back as a number) and `unescape_pipe_body`. `improvise-gp` re-exports it, `improvise-io` calls it from `persistence/{format,parse}.rs`, `parser.rs::unquote_pipe` is a shim over it, and io's own copy is **deleted** (improvise-i2m5). **The keyword guard is canonical — user-confirmed, so do not revert it as over-caution**: an item literally named `SUM` is written `|SUM|`. Bare keywords do parse, but only because `agg_call` needs a `(` and PEG backtracking falls through to `ref_expr` — rule ordering, not a stated rule — and a `## Formulas` line carries formula text verbatim, so a name `format_md` writes is read back by `parse_formula`: one namespace, one spelling rule. One thing the guard does not cover, and one that used to be an exception and no longer is. **A section header takes its name as the whole line, totally** (improvise-pczv): `header_name = { pipe_quoted ~ &NEWLINE | line_name }` — the pipe-quoted arm needs the lookahead, or `|A|B` would parse as `A` and silently drop `B`, and `line_name` catches everything else, so a header name can no longer fail to parse. Four rules share it (`model_name`, `initial_view`, `category_section`, `view_section`), which is why it is a rule and not four copies. Note a header is still *written* raw where a reference is quoted — the grammar accepts both arms, so the quoting asymmetry between a header and a reference is a writer convention, not a format one. And the two former *unquoting* copies differed only on a trailing lone backslash, unreachable from the grammar, where io returned it and `parser.rs` panicked — the surviving `unescape_pipe_body` is total. Two `## Category:` sections naming one category are **one** category, concatenated by `ParsedFile::push_category` in **pass 1** (improvise-349m) — merging in pass 2 was not idempotent, because `build_category` creates its own section's item-groups before its own `>` lines while `format_md` writes every item before every `>` line. No grammar change was needed: this is the tree-walker, not the format, and duplicate headers stay legal. And **`WHERE` now carries `IS`'s word boundary** — both are compound-atomic with `sp+` (improvise-r08u), so `SUM(Rev WHEREAS x = y)` is a syntax error rather than a `WHERE` on a category `AS`. The guard is `sp+` and **not** the `!bare_char` lookahead the ticket proposed, because the grammar-walking generator reads the grammar through `pest_meta` and **skips predicates**, emitting `WHEREa=""` — a rule whose only guard is a lookahead is a rule the generator can violate. It costs one spelling: a category may no longer abut the keyword. + +35. **`Dirty` over-reports on purpose, and the asymmetry is the reason.** `Dirty::of` maps an `EffectKind` to the widgets that kind can reach, and the two directions do not cost the same: over-reporting pays one redundant `Projection`, under-reporting leaves a client's cache wrong **forever**, with nothing in the stream to correct it. `LayoutGeometry` therefore includes `formulas`, because `RemoveCategory` wears it and `Model::remove_category` ends by dropping every formula targeting the category — a client's formula panel otherwise kept showing a formula the document no longer had. It now coincides with `Dirty::EVERYTHING` and is still spelled out, because the two are derived from different effects and need not stay equal. The derivation now lives in `context/code/command.md`, and the doc comment on `Dirty::of` points there rather than restating it; the reason `LayoutGeometry` is spelled out rather than written as `EVERYTHING` is a two-line comment inside the match body. It went stale once already — claiming `AddCategory` was `CellData` after it was reclassified — so reclassifying an effect means re-reading the prose, not just the comment. + +36. **A client's command has one door, and the door takes the `App`.** `Sessions::apply_checked(app, id, Checked<'_>)` is the only way a connected client's `Command` reaches the server — `Host` calls *that* (improvise-ahdf). `Sessions::apply` is not `pub`, delegates to `apply_checked` after `Checked::against`, and has no production caller: its call sites are tests, which is why it takes a raw `&Command`. `ClientSession::fold` and the mirror's `ViewState` are private, so a transport cannot reach the mirror at all. `apply_switch` now **takes a `SessionId`** — it is addressed to exactly one session, which is the opposite of what this gotcha used to say — moves only that session's mirror, and dirties nothing. The one thing it still asks the document is whether the view exists, because a mirror keys `ViewState::cursors` and is read by `grid_in`. The audience is conditional: an accepted switch relays **nothing** (the asker folded it, and no other client's view moves), a refused one is `Broadcast::To(asker, …)` — improvise-6c6q, where `ToOthers` told every client but the one that was wrong. `adopt_active_view` and `Broadcast::ToOthers` are **deleted**; the variants are `ToAll` and `To(SessionId, _)`, the second because the deleted-view case needs a one-session audience that is *not* the asker. The paragraph's warning about the "what do the others hear" helper survives and is why the 6c6q test drives the asker's own `ViewState` — that helper is itself deleted. What survives unchanged: the model is shared and the UI is not, and `Broadcast` is a `Vec` because one command can owe two relays with different audiences. **The trap this paragraph used to describe is gone with `ToOthers`, and the lesson it taught is not.** It was copying `ToOthers` from `apply_switch` into `apply_model`, invisible from the bystander's side because `adopt_active_view` moved the asker's mirror either way — so the drift sat between the server's picture of the asker and the asker's own slice, where a test written through a "what do the others hear" helper looks away by construction. improvise-6c6q closed the mirror-image hole on the refused path. Two rules survive the mechanism: **assert the `Broadcast` itself**, since the audience is the message rather than something to infer from folded state, and **seat a bystander somewhere a wrong relay can move it from** — a bystander already on the relayed value cannot see a wrongly *widened* audience either, because the wrong relay folds to a no-op. Review demonstrated that vacuity twice on PR #48. A `Vec` rather than an `Option` because one command can owe two relays with different audiences, and an `Option` would have had to drop one silently. Everything else is that client's alone and never touches the `App`'s own view slice: **the model is shared, the UI is not.** Folding `SwitchView` into the mirror alone was the bug — `viewport()` then read the cursor under the new view's key and the next `project()` built that client a grid out of the *other* view. + +37. **`gp-tree` returns data and never prints, and a fit takes two things to reproduce.** Two rules with one root — the crate is a library a TUI, a wasm worker or a CLI may each host. (a) A decision the user should hear about comes back as **data**: `Dataset::warnings` carries the columns `preprocess` dropped for being entirely missing (improvise-d44s), and `fit.rs` hands back pooled `(prediction, actual)` pairs and a score per feature rather than tables. `examples/fit_csv.rs` owns every `println!`, and that is the only reason its output stayed byte-identical across these moves — adding an `eprintln!` to either module is the regression. (b) **The seed is necessary and not sufficient**: `data::Model` records `seed: Option` *and* `config: Option`, because the same seed under a different population/generations/elite/penalty lands on a different tree (improvise-ksst). Both are `Option` for one reason — every `FitConfig` value and the seed `0` are legal, so nothing can double as "unknown". Note `#[serde(default)]` on an `Option` field is **documentation, not the mechanism**: serde defaults it anyway, so deleting the attribute fails no test, and a backward-compatibility test that only deletes it is not discriminating. Aim such a test at an absent field loading as the *default* instead of as unknown. The GA loop itself is `fit::fit` (improvise-0ai); `breed` remains one fitness-agnostic generation for callers whose fitness is not "predict this column". (c) **A scoring function takes one slice of rows.** `Row` pairs a feature vector with what it predicts, so the parallel `x`/`y` lengths that `zip` used to truncate silently cannot be built (improvise-dkqn, `doctrine(an_invariant_is_made_unrepresentable_not_validated)` — the same answer `prune_axis_items` got, and the reason no `Result` entered five signatures). The pairing is made at the boundary that *builds* the data: `preprocess` walks one row index and the rows live inside `Target`, because a `Dataset` carrying `rows` and `target` separately just moves the zip up one level into every caller. The empty row set is one shape, answered once in `mean_over`. Changing any of this needs **bit-level** evidence: `tests/scoring_known_answers.rs` pins every path as f32 bit patterns plus a `next_u64` witness that catches a changed RNG call order, because the CLI prints three or four decimals and byte-identical output alone cannot see a moved accumulation. (d) **A fit is storable only when the document can *name* its features.** `preprocess` invents `Region=North`, `Cost.missing`, `when.year`; those are **encoding artefacts, not items of the document**, and `Expr::Ref` resolves through `Model::find_item_category` — so a tree splitting on one emitted a well-formed formula whose every cell was `Error("ref:…")`, silently (improvise-8xhy). `fitted_formula` therefore takes the `&Model` and refuses, listing what it could not resolve; `formula_text` still spells it, being the tree's own text. **The check asks the document, never the name's shape**, because a measure literally named `Region=North` is a legal document and the two are indistinguishable as strings — which also means `.missing` and the date parts are covered with no second copy of `preprocess`'s expansion rules to drift from. **This is an interim** (improvise-4zl7): a `WHERE` predicate needs a *scalar coordinate predicate* the formula language does not have — `Filter` gates a formula or an aggregate, it is not an `Expr` — would still leave `.missing` and the date parts, and must first answer what such a predicate means where `context.get(cat)` is `None`, since `Formula::filter` falls through to `aggregate_raw` and a categorical fit would silently degrade in every view that aggregates over the category. **improvise-zv8t closed the categorical half of the accident, and the close cost a document.** `fit_measure` handed `preprocess` an `Overrides::default()`, so it never said which header columns are document categories *although the very next line told `Features::of`* — and `preprocess` infers `Numeric → Date → Categorical` from the values, so a `Year` category with items `2024`/`2025` was a *numeric column* and a category of ISO dates a *date column*, both refused with `Year IS 2024` spellable all along. It now forces the same `header[..category_columns]` slice to `Forced::Categorical`. Two things follow. The **date decline is narrower than it read** — it covers a genuine date *measure*, never a category misread as one. And the forcing **widens the encoding**: a hundred-item numeric category one-hots past `MAX_FEATURES` and refuses a fit that previously succeeded by never splitting on it (improvise-e8no — and **not** to be closed by making the forcing conditional on cardinality, which is inference by another name and makes the *encoding* depend on a budget rather than on what the document says). The `Artefact` arms stay: `crate::dataset` is `pub` and takes the caller's `Overrides`, so the roles are still reachable. **Forcing by name while budgeting by index defeats the budget** — a category and a measure may legally share a name, and `no_ambiguous_column_names` ran *after* `preprocess`, so the expansion the preflight exists to prevent was allocated and only then refused; `as_categories` now returns `Result`, so the ordering is a compile error rather than a convention. The test that missed all this is the shape to remember: `improvise_eval` **fabricates** a `_Measure` item per feature name, so it asserted the emitter and `gp_tree::eval` agree against a model that could not exist. It is kept, with its scope now stated, and the property it could not state is asserted separately against a real document. + +38. **The wire vocabulary is derived twice, by the same recipe.** `Command`'s view half is the `EffectKind::ViewOnly` effects targeting the portable slice (`decision(improvise-cqi, command_set_derived_not_invented)`); its model half, `ModelCommand`, is the `EffectKind::Model(..)` effects — **28** of them, brace-matched on `c1772ca9` (2026-09-24) — minus `ApplyAndClearDrill` and `AcceptWizardModel`, leaving **26** variants (improvise-kmgs). The two added since this paragraph counted 26 and 24 are `ReplaceFormula` (improvise-km71) and `RenameRecordCoordinate` (improvise-fdk1); `SwitchView`, `ViewBack` and `ViewForward` left the model half with improvise-jytm. Every count in the rest of this paragraph is dated, so recount rather than cite it. **Completeness is now checked, not counted**: `ui/reduce/vocabulary_is_complete.rs` requires every effect to be on the wire under its own name or listed in `NOT_ON_THE_WIRE` with a verdict — `Command`, `Frame`, `Off` with its argument, or `Gap` naming the ticket that owns it (`SaveAs` and `ExportCsv` are gaps, improvise-2mqz). It is the forward twin of `vocabulary_is_composed.rs`, and it is what the enumeration below lacked. `SwitchView` no longer needs subtracting separately: it is not a model effect any more. `ViewBack`/`ViewForward` left for a reason this paragraph **stated wrongly until 2026-09-21**: it said they went for the *same* reason as the other two exclusions, session-local residue. That is false, and the falseness mattered. `ViewState::view_back_stack` **is in `ViewState`**, which is the portable slice — so by the view half's own rule (`decision(improvise-cqi)`: the `ViewOnly` effects targeting the portable slice) they were always `Command`s and were simply **missing**. The cost of the mis-statement was a live divergence nobody could see: `effect::SwitchView` pushes the leaving frame and clears the forward stack, while `reduce_view`'s arm set `active_view` **and nothing else** — so **no client ever had a back history at all**, and that, not the drill commit the `declined()` comment blames, is why `view-back` was dead in a browser (improvise-gd28, PR #115). **The lesson is about derivations, not about these two variants**: a derivation stated as a rule is only as good as the last time somebody enumerated it *against* the rule. This one had been checked for correctness — every listed variant belongs — and never for **completeness**. Note the obvious `awk` extraction reports **30**: it picks up `HelpPageSet`, which is `ViewOnly`, because that is the last `pub struct` before `#[cfg(test)]` and the test module mentions `ModelChange::` — brace-match the `impl Effect` blocks instead (`doctrine(derive_then_verify)`). Three things follow. **A `ModelCommand` does not mutate**: `ui/reduce.rs::model_effects` maps it back to the effect it came from and `apply_command` runs that through `App::apply_effects`, so effects stay the only mutation site and the `Dirty` is the effect's own `EffectKind`, never a second classification beside gotcha 35's table. **Two model effects are deliberately off the wire.** `ApplyAndClearDrill` and `AcceptWizardModel` were once kept on the grounds that the *command* is portable even where the operand is not; **that was the wrong criterion** (improvise-3x3u). A `Command::Model` is routed with no `SessionId`, which is right exactly when the change is addressed to *no one* session — and these are addressed to precisely one, never the sender: their operands are `drill_state.pending_edits` and the built wizard workbook, session-local residue (gotcha 30) private to whoever staged them (improvise-tom1). Unaddressed, B's `ApplyAndClearDrill` commits A's uncommitted typing and B's `AcceptWizardModel` swaps the document out of a dialogue B never opened — both demonstrated against `Sessions::apply` before removal. So **28** − these two = **26** on `c1772ca9` (it was 26 − 2 = 24 before km71 and fdk1), and the older spelling of this sum (`29 − SwitchView − these two = 26`) was itself a survival of the same staleness this paragraph documents: it subtracted `SwitchView` and forgot its two siblings, so it contradicted the count at the head of the paragraph and the enum’s own 24 variants (found by ChatGPT Codex on PR #116). The effects are untouched and still reached by the key path. Not permanent: multi-client drill wants the *addressed* form, which is where ADR 001 layer 2 goes anyway. Guarded by `no_model_command_commits_another_clients_session_local_state`, plus a companion asserting both effects still exist and are still model changes so the first cannot pass by naming nothing. **`Sessions::apply_checked` routes the model half to `apply_model`**, which relays `adopt_cursor_epoch`'s `ResetCursor` and `adopt_deleted_views`'s per-client switch; a document-replacing `LoadModel` bumps the generation and lands in the rebase arm. The former claim here — that `ViewBack`, `ViewForward` and `LoadModel` all end in `App::switch_to_view` so every mirror takes the accepted active view — was wrong on all three clauses after improvise-jytm and is deleted. A keymap lookup still cannot return a `Command`, and not for want of vocabulary: a binding names an *intent* resolved against `CmdContext` (`move-cursor down`) while a `Command` is the *resolved mutation* (`SetSelected { row, col }`) — improvise-gxi's AC 4 asks for something the split forbids. + +39. **`Applied` is an enum because a document swap is not a diff.** `Sessions::apply_checked` returns `Applied::Diff { dirty, broadcast }`, `Applied::Rebased { snapshots }` or — since ADR 001 layer 1 — `Applied::Rejected { reason }`, and the enum *is* the mechanism: a host that wants the `Dirty` must match, and matching names the case where a `Snapshot` is owed. The server's mirrors self-heal — `ClientSession::project` reseeds on the `document_generation` bump — but `ViewState::cursors` lives on the **client**, keyed by the outgoing document's view names (gotcha 24), and only a fresh `Snapshot` replaces it. `Command::Model` put the three `WholeWorkbook` mutations on the wire (gotcha 38), so the host that must send the snapshots is no longer the host that asked. `apply_model` therefore calls `rebase` itself rather than instructing the host to, exactly as `apply_switch` adopts the accepted view itself. **The trigger is the generation moving, not the command variant**: a `LoadModel` that cannot read its path leaves the document alone, and reseeding on the variant would cost a client its cursors for a *failed* load — `a_load_that_fails_is_not_a_rebase` guards that arm. A `rebase: bool` beside `dirty` was rejected, because the host that forgets to call `rebase` is the host that forgets to read the flag. The **third** case is a *normal outcome*, not an error, and it is in the enum for the same reason the other two are: a host that wants the `Dirty` must match, and matching names it. `ServerMsg::Rejected` is the matching frame, addressed to **the asker alone** — nothing happened, so nobody else is owed anything — and the client keeps it as a `Rejection` rather than a sentence, because force needs to know *which* record and *which* kind (gotcha 43). + **Fixture warning.** When testing a document swap, the incoming document's active view must have the **same name** as the outgoing one. A rename makes `apply_model` broadcast a `SwitchView`; the client folds it, and the stale cursor is then read under a key nothing has written — right *by accident*, with the bug invisible. `other_document()` renames, and a first version of this test passed because of it; `a_document_shaped_differently_under_the_same_view_name()` keeps the name and changes only the shape, so there is nothing to hide behind. + +40. **A batch's basis is three tiers, and only two of them can be read off the effect.** `App` carries `document_generation` (bumped by `install_workbook`), `layout_version` (per *view shape*) and `record_versions` (per *record*) — ADR 001's stale-basis check, and shared document state, so never on `ViewState` (gotcha 30). **The unit is the record**: `record_of(key)` is the coordinate tuple minus `_Measure` (`record_row_identity`), because renaming a category coordinate in wide records mode moves every measure cell in the row (gotcha 1) — a cell-level unit is violated by one legal edit, so the unit must be at least as coarse as the coarsest legal write. **Only the `Dirty` is read off the `EffectKind`; both lower tiers are *observed* around the loop** — the source says so at `apply_effects` and this paragraph said the opposite until improvise-ilvl's audit. The `EffectKind` decides only *whether to look*: `touches_model` arms the shape snapshot, `writes_cells` the record fingerprints. `record_versions` **must** be observed, because an `EffectKind` says *that* a batch writes cells, never *which*; `layout_version` is observed for a better reason than necessity — `observe_reshape` diffs `view_shapes` before and after and bumps only where a view's shape actually moved (improvise-pdvm/nynk), so it is a **fact about the document** rather than a claim about the batch. Two consequences the declared form did not have. An **idempotent** `set-axis` bumps nothing, and batching two real changes into one line or two gives the same `2`. And the gotcha 21 argument is subsumed rather than restated: an aborted batch moved no shape, so there is nothing to not-claim — which is why observing once **after** the loop is correct where accumulating `Dirty` inside it is. **`view_shape` hashes what is drawn, not occupancy** (its doc comment: "Drawn, not configured") — axes, the drawn item names minus hidden ones, folds, page selections and the `prune_empty` *flag* — so an occupancy write does **not** bump `layout_version`: a `CellData`-only batch that prunes a row out of a view moves the cursor epoch (the frame observation) and not `layout_version`, as `context/code/command.md` records (the question that raised was improvise-6nsd, closed). The arming is wider than the tier (`touches_model` takes the shape snapshot), but a bump needs a shape hash to have moved. Three further consequences. A write landing the value already there **bumps nothing**, which is correct — no basis was invalidated. A `LayoutGeometry` effect that drops cells **will** bump their records when it shares a batch with a `CellData` one: an over-bump costing at most an unnecessary rejection, the same direction `Dirty::of` is deliberately sloppy in. And only *raw* data is fingerprinted — formula results live in `Model::formula_cache`, and a record whose derived value moved because someone edited elsewhere has not itself been written. The diff collects into a **set**: chaining `before.keys()` with `after.keys()` counts any rewritten record twice, a bug that survived one test run. + +41. **`Storage` is not a VFS, and that closes the SQLite question.** The trait is whole-document `read`/`write` bytes (gotcha 2, `invariant(storage_is_two_ops)`) — no offsets, seeking, handles or locking — so an `sqlite3_vfs`, which needs `xRead(offset,len)`/`xWrite`/`xTruncate`/`xLock`, has nothing to bind to. SQLite *can* still sit behind it, as an in-memory database serialized to a blob (`rusqlite` `serialize`/`deserialize_read_exact`, proven with a probe before declining — declining on a capability that actually exists would have been the wrong decline). It is worth nothing: measured on 18,000 cells the SQLite container is **22% larger** than `.improv`, **4.7× larger gzipped**, ~**160×** larger at the floor (24 KB against ~150 bytes), and still rewritten whole on every save, so the one dimension SQLite normally wins — incremental page writes — is unavailable. improvise-9ix was declined on that evidence. Two things generalise past SQLite. **A second serialization format is a second implementation of the grammar**, obliged to track `improv.pest` forever; and **its own round-trip test cannot detect drift from `.improv`**, because it never compares the two — that missing cross-format equality test is what made the ticket look cheap. If a structured-query story is ever wanted it is a one-way `export_sqlite` beside `export_csv`, native-only and feature-gated: `libsqlite3-sys`' C build fails under this repo's nix cc-wrapper for `wasm32-unknown-unknown`, which would break the `wasm` CI gate the day it landed. + +42. **The browser client's render decisions are made in Rust, and the frames it speaks are nobody's but the protocol's.** `improvise-wasm-client/src/render.rs` builds a `RenderModel` in which every question with a *silent* wrong answer is already answered, because the DOM half (`web/app.js`) is untested JavaScript and a wrong answer there shows up as a wrong pixel rather than a failing test. Three such answers. **`editable` is a fact about the `CellId` variant** — `Record` only, per ADR 002 — and the trap is the third case: `None` means "no projected cell", **not** "aggregate", so conflating them either hides a cell or offers a write the type system exists to make unconstructible. **The selection is placed by reconciling the scroll offset**, because `GridCache` is viewport-indexed while `ViewState::selected` is absolute. `render.rs` does it by *adding* — `r + cursor.row_offset == sel_row` — rather than subtracting, which is why there is no underflow path and why a selection above the window simply never matches. Omitting the offset entirely looks right until the first scroll, so the test that earns its place uses **nonzero and unequal** offsets: at `selected=(5,1)`, `row_offset=4`, `col_offset=1` it separates "reconciles" from "ignores" *and* from "swaps the two offsets". **The category panel is `viewmodel::flatten_cat_tree`**, never a second walk of `CategoryTreeCache` — that second walk was improvise-edp. The page is four string calls: `Client::viewport` (the handshake — the `Snapshot` seeds the *visible* cells, so one built at extent zero is an empty grid and a resize is a fan-out with nothing dirty), `on_frame` (one frame, one fold, arrival order — gotcha 33 holds *inside* a batch, so never coalesce), `render_model`, `typed_cell`. + + **One envelope pair, two deployments (improvise-xks6).** `improvise_protocol::frame` owns `ClientFrame { Viewport, Commands(Vec<_>), Save, Key }` and `ServerFrame { Snapshot(Box<_>), Commands(Vec<_>), Rejected, Accepted }` (as of `c1772ca9`; this line named a singleton `Command` and no `Save`/`Accepted` for some time after both changed); `improvise-ws-server` and `improvise-wasm-client` each `pub use` them and **neither defines one**. There used to be **three** pairs for two deployments — the socket's server side, a hand-kept *mirror* of it in the browser bundle, and a third written for the worker (improvise-tm6) before the projection architecture landed. The question that had to be answered before any of it moved was whether the **worker** and the **socket** want the same vocabulary, and the answer is **yes**: every difference was a *date* rather than a decision, and the test applied to each was whether the **argument** for it names the thing they differ in. None did. `Viewport` is deployment-independent — the extent is not in `ViewState` and no effect produces it, so no derivation reaches it, and a worker hosting the same `Sessions` owes `set_extent` the same answer; it is the handshake on both because a `Snapshot` seeds the *visible* cells. `Key` is deployment-independent too — a binding names an *intent* resolved against a `CmdContext` a thin client does not hold (gotcha 38), which says nothing about what the bytes travel over; the socket lacked it because key forwarding was never built on it (improvise-bpq2), and `Host::handle` now answers `ClientFrame::Key` with an explicit empty vec that **says so**, where the frame used to just fail to parse in `next_client_msg`. And `Commands(Vec<_>)` batches where a singleton cannot, because a fan-out must arrive whole and in sequence (gotcha 33) — a property of the **fold**, not of the transport; an in-process `postMessage` interleaves exactly as a socket does. The worker pair was not merely different but **incoherent by omission**: written before ADR 001, it could carry `Command::Model { basis, .. }` and had no `Rejected`, so a client speaking it composed a claim that could never be answered. That incoherence — not the difference — is the tell for staleness. + + **The mirror is gone, and so are the two tests that held it honest**; that deletion was the ticket's measure of success. `the_socket_framing_matches_the_server` round-tripped hand-listed fixtures through the other crate's types to catch a **rename**; `every_server_variant_is_one_this_crate_mirrors` matched exhaustively to catch a **variant added on the server and never mirrored**. With one definition both are **type identity**: a rename is a compile error at every call site in both crates, and the un-mirrored variant is not a constructible state. That is stronger than either test, and it is the shape to insist on — **if a consolidation leaves one of its compatibility tests alive, the consolidation is not finished.** What survives is `the_retired_spellings_do_not_parse` in `frame.rs`, aimed at the one thing still possible (the retired `Bootstrap`/`Command` spellings returning as serde **aliases**, which would be two homes again wearing one type), and it keeps the injected-fault discipline this paragraph earned the hard way: **assert the mutated input is still well-formed** before feeding it in, and **assert the error kind** — serde's `unknown variant` — not merely that an error occurred, because a rewrite that also breaks the JSON makes the check pass against an enum that really does have the variant. End-to-end compatibility is now `tests/browser_round_trip.rs`, which hands the wasm client's own serialized frame to a real `Host::handle`; `improvise-ws-server` stays a **dev**-dependency of the browser bundle for exactly that. + + **What the harness does and does not reach is gotcha 48's subject** — `web/app.js` is no longer unrun (`harness/tests/shell.spec.js` executes it under Chromium, and its standalone half under Firefox), improvise-hwpi is closed, and the quiet-failure and never-reads-a-key claims that used to stand here went with it. What remains unexercised is named there: the grouped and multi-level DOM, the CSS and the `ResizeObserver` extent arithmetic. What a careful read *did* establish about the last of these: the observer cannot feed back into itself (`#grid` is a flex child with `overflow: auto`, so `replaceChildren` cannot resize it), and `ROW_PX`/`COL_PX` under-estimate, so the worst case is a row of over-fetch. What *is* verified end to end is the Rust half: `tests/browser_round_trip.rs` drives improvise-1ey's stated scope against a real `Host` over a real `App` touching the client only through the four string calls. One deployment note that is not a bug: `examples/demo.improv` leaves `Customer`/`Date` on the None axis, so **every** cell is an aggregate and nothing is editable — correct behaviour, bad demo. Pick a model with no regular category on None. + +43. **A batch carries its basis, and the server had to publish it first.** `Command::Model { basis, cmd }` is a **struct variant** so the basis cannot be forgotten, and `Sessions::apply_model` checks it *before* applying anything — a stale batch is never applied, not even partially (improvise-xwi6/fkfg, ADR 001 layer 1). Four things to know. **`Basis` has no `Default` and no public constructor**: a zero-valued basis is not "no claim", it is the claim that *the document is fresh*, which in a new `App` is true — so a default would be a forgeable bypass rather than an absence. `BasisCache::claim` is the only door a client has and it returns `None` until the server has published one. Note what that does and does not buy: `Basis` derives `Deserialize` because it must cross the wire, and its fields are public, so **a hand-rolled client can state any tiers it likes**. This is an *anti-staleness coordination check, not a security boundary* — `RecordUnstated` closes **omission**, not **invention**, and nothing here tries to close invention, because a client that forges a current basis is indistinguishable from one genuinely caught up. "The only constructor" is a statement about clients written against this crate, which is still worth having: it makes the honest path the easy one. **Which records are checked is the server's decision** — `reduce.rs::records_named`, an exhaustive match with one or-pattern for the variants that name no record, so a new record-writing variant fails the build. Reading the list off the *client's* claim would be a **force by omission**: the record tier skipped, with no flag to remove and nothing in the message saying so. A claim that omits a record the command writes gets `Rejection::RecordUnstated`, a case of its own because "you lost a race" and "you were never in one" are different things to tell a user. **`Rejection` keeps structural and data staleness apart**, which is the whole reason ADR 002 put a maintained `RecordHash` index in `DataStore`: `RecordGone` means nothing resolves and there is nothing to force (**force never creates structure**), `RecordMoved` means it resolves at another version and force is one keystroke — which is also why the wasm client stores the `Rejection` itself rather than collapsing it into `status_msg`, since a string cannot carry which record or which kind. **And the ADR omitted the publication.** It said a batch carries the basis it was composed against and never said how a client learns one; nothing in the projection stream carried any of the three tiers, so the check as written was satisfiable only by forging. `Projection::Basis` + `BasisCache` close that — the server publishes the two document tiers plus the counters for the records *this client's grid names*, diffed against what that client was last told and seeded in the bootstrap — and it is emitted **after** the grid updates in a batch, because the counters describe the cells those updates carry: a basis arriving first would let a client claim a version for a value it had not been shown, the mirror image of gotcha 33. That ordering was documented as load-bearing in three places and **held up only by the order of two statements** until `a_published_basis_follows_the_grid_updates_it_describes` pinned it (proved by injecting the swap) — the same job `a_cell_update_that_overtakes_its_labels_is_lost` does for the mirror-image rule. Two things are unchecked and deliberately so: `Command::SwitchView` carries no basis (it sits outside `Command::Model`; the workbook refuses a view it does not have, so it cannot corrupt, but it is a hole in the rule), and `apply_command` does not check at all — **the check is a property of the session door, not of the mutation path**, which is right because a host holding the `App` is the truth rather than a claimant. That second one **was** a door rather than an exemption, and needed a type to stop being one (improvise-ondj): both took a wire `Command` and their `Command::Model { cmd, .. }` arm **discarded the basis with `..`**, while `ui/reduce.rs`'s own module doc pointed every future host — the browser worker included — at exactly those two names. **The struct variant closed omission *from the message*; it did nothing to close ignoring *on arrival*, which is the half a struct variant is usually reached for.** So both now take `Checked<'_>`, a private-field newtype over a `&Command` whose only constructor is `Checked::against(&App, &Command) -> Result<_, Rejection>` — also the only caller of `stale_basis`. `Sessions::apply_checked` receives the proof and `Checked::against` is answered **once, before the dispatch**, so `apply_switch`/`apply_model` receive the proof rather than each remembering to ask: an invariant made unrepresentable rather than validated, the same answer `prune_axis_items` and `gp-tree`'s `Row` got. Visibility was the weaker option and was rejected for a concrete reason — `pub(crate)` closes only the out-of-crate path, and `reduce_full` had **no production caller anywhere in the workspace**, so it would have become dead code. It has since been **deleted** — which is the argument’s own conclusion arriving, not a contradiction of it; the surviving sibling is `reduce_key`. This paragraph named it in the present tense for some time after, so read every function name here as a claim to check. **The regression guard is therefore the signature**: the test that reproduced the bug no longer compiles, and what remains asserts the constructor refuses and that refusing leaves the document untouched. One residual hole, documented on the type and filed: a `Checked` does not name *which* `App` it was checked against, so a multi-document host could in principle check one and apply to another. That is also why `ui::reduce::caught_up(&App, cmd)` is not a bypass: it takes `&App`, so nothing reachable from a wire can call it. **Since improvise-955n the frame, not the command, is what is answered — and only two of the three tiers hoist.** `CheckedBatch::against` runs before anything in a `ClientFrame::Commands` is applied. `document_generation` and `layout_version` describe the *document*, so answering them once for the frame is exactly right, and a first version hoisted all three — which was a lost update: the record tier asks "is the client's picture of **this record** current", and which records are at stake is a property of the *command* (`records_named`). Skipping it for later commands let `[SetCell(North), SetCell(South)]` overwrite a South somebody else had moved, silently, with the frame reported accepted, and invisible to the other two tiers because a `SetCell` is `CellData` and moves neither. So every mutation answers for its own records **against the frame's basis** — the one the first basis-carrying command stated, never each command's own field, or a later command could claim a fresher picture than the frame was composed under. The one exemption is stated over a *property* rather than a list of commands: a record that **neither resolves in the document nor is claimed by the basis** is one the frame is about to create (`[AddItem(East), SetCell(East)]`, which `BasisCache::claim` cannot possibly claim for), and only that combination is exempt; a record that does not resolve but *is* claimed is still `RecordGone`. That the `Basis` inside a *later* `Command::Model` is ignored rather than compared is an honest interim: clause 1 says the **frame** states one basis, and moving the field from `Command::Model` to `ClientFrame::Commands` would make that unrepresentable-otherwise instead of a convention — a protocol change that belongs with clause 4. Two things worth keeping from the rejected design: **arity is not a property of a variant** (a future `Command::RunScript` is one variant performing unboundedly many mutations), so no count over the vocabulary can enforce "one mutation per frame"; and **the transaction boundary is not the frame**, because one command may already be a chain, which is why clause 4's mechanism belongs at `App::apply_effects`. + +44. **The TUI grid draws from its cache, and since improvise-anr9 the cache frames it too.** `GridWidget` holds no `&Model` and no `&View`: cell text, aggregate-ness (`CellId::Aggregate`, ADR 002), column widths, the page bar, the totals strip and now the **frame** all come off `GridCache`. The TUI still builds that cache over the **whole** grid (offsets zero), so a cache position is an absolute position — the browser's `Viewport` is a real window, the TUI's is not, and the two clients therefore use it differently; column totals **must** be projected rather than client-derived, because the sum runs over every row *including the ones a viewport scrolled past*. The frame is an `AxisFrame` per axis — `cats` (the categories, outermost first) plus `entries: Vec` interleaving `Group(FrameGroup)` headers and `GroupEnd { level }` markers with `Item { tuple, collapsed }` label *tuples*, exactly as `GridLayout::row_items` interleaves them. That replaced a `Vec` built by `GridLayout::row_label`, which slash-joined the tuple — the exact rendering `two_row_categories_produce_cross_product_labels` says must not appear — so the browser drew `Food/Alice` where the TUI drew two header columns, with no groups and no collapse indicators. It is **one** type and not a `labels` plus a `levels` beside it: two answers to "what are the labels" is the disease (improvise-edp), and this is the second cure, which cost a *type* change rather than an addition. Three things to know before touching it. **`leaf_labels()` is not "the labels"** — it is the innermost label, what a one-line summary calls a row by; a draw loop wants `header_labels(first_rendered)`, `groups()` and `shares_ancestry(sel)`, each of which is one pass, because `item`/`label`/`shows_level`/`group_of` scan `entries` from the start and asking them per position is quadratic in the frame *and grows with the TUI's absolute scroll offset* (found three times on anr9; the per-position forms survive for one-off lookups and say so). **The width rule has one home**, `improvise_protocol::header_level_widths` — widest label per level, plus a gap, clamped — reached by `AxisFrame::header_widths` and by `ui::viewport::compute_row_header_width`, which borrows level tuples off a `GridLayout` rather than building a frame on the keypress path. **A `cats` change re-identifies every position**: a cell's identity is `cats` zipped with its tuple, so `invalidate_relabelled` blanks the whole axis when the categories move, not merely the positions whose tuples differ, and blanks the **totals** on a recategorization of *either* axis — a column total is a sum over the rows, so re-identifying the rows leaves a number answering coordinates the grid no longer has (gotcha 33(b): the rule lives in `apply`, not in a producer's memory). What is still **not** on the cache is `record_col`, and that is correct: it keys pending drill edits and is session-local (gotcha 1). `body_from(idx)` takes an argument only because the two front ends scroll differently — a browser's frame is already windowed and passes `0`, the TUI's covers the whole grid and passes its row offset — not because the rule differs. `group_of` answers group **membership**, and since improvise-e265 membership is **the innermost open run**: `FrameGroup` and `FrameEntry::GroupEnd` each carry a `level` (an index into `cats`), runs **nest**, a header at level L replaces L and closes every deeper run, and a `GroupEnd { level }` closes L and deeper. Before e265 it was the nearest preceding header unless a level-less `GroupEnd` intervened (improvise-gd83, PR #77), which could not express one run inside another. `GroupEnd` draws nothing and exists only so membership is not inherited. `AxisFrame` deserialises through `UncheckedAxisFrame` and **refuses** a `level >= cats.len()` — a level naming no category is a malformed frame, not a clamp. And `FrameEntry::Item` is `{ tuple, collapsed }`: `collapsed` is the **layout's own fold flag**, read by `AxisFrame::folds`, so a client asks the frame whether a position folds rather than re-deriving it from the groups (improvise-4k1j). Both of the gaps this sentence used to list are **closed**, and it named them as open for a day after: `group_runs` coalesces by **header identity** (`std::ptr::eq`), not by name, so two runs under two headers equal as values are two runs (improvise-h0iv); and `AxisFrame::first` carries **no** `serde(default)` — its doc comment says why, since zero is a claim and not an absence (improvise-1wl0). `doctrine(a_stale_list_of_what_is_missing_is_worse_than_no_list)` is about exactly this paragraph, twice now. And `project::grid_of` used to evaluate every cell per frame where the widget evaluated only the visible ones; since improvise-felo.13 the host's own layout keeps a `GridSummary` (whole-grid widths, column formats and totals, `ui/viewport.rs::grid_summary`, held in `App::grid_geometry()`), and `grid_in` evaluates only a `grid_window` over it. The browser's window therefore costs its own cells. A session on another view, or a rederived model, still takes the whole-grid `grid_body` path. `grid_window` equals `grid_body` where the summary is current. Since improvise-jytm `grid_in` may additionally **derive a `GridLayout`** for a session on a view the host is not on; the host's layout is reused where the names agree, which is the common case and the only one that can carry a drill snapshot. + + **`RecordHash` is document-scoped; `AggregateHash` is layout-scoped and cannot be otherwise** (improvise-gd28). A `RecordHash` resolves through `DataStore`'s maintained index — but **only when the command names a view the document has** (improvise-wpxr; Codex P2 on #281). `SetCell`/`ClearCell` carry `view`, and the lookup is **strict**: a view the document lacks drew nothing, stored or not, so it is `Rejection::RecordGone` rather than a `resolve_view` fallback. An **empty** record — one the store holds no cell for, which the grid still drew — resolves against **that named view's layout**, the only place it exists. `RenameRecordCoordinate` is the exception and resolves in the **store alone**, since only a record holding cells can be renamed: `protocol::named_records` answers each command's records as `NamedRecord::Drawn { view, record }` or `NamedRecord::Stored(record)`, one exhaustive match. Every one of these questions goes through `App::grids()`, a `Grids` borrow that memoises **one derived `GridLayout` per view for as long as the borrow lives** (improvise-kimw) — hold one across a check, never across a mutation, since the layouts it holds describe the model it borrowed. An aggregate has no existence outside the layout that derives it, so there is no index to fall back to and none can be built — `App::resolve_aggregate` therefore answers only for the **host's** active view, which since improvise-jytm is not the client's. **That, and not ADR 001 layer 2, is what blocks browser drill.** The ticket's blocking argument was right about the *commit* (`ApplyAndClearDrill` is addressed to one session, gotcha 38) and wrong about the *entry*: entering a drill needs no commit, and `CreateView`/`SetAxis`/`SetPageSelection` are already unaddressed `ModelCommand`s that name their view. The gap is one level down from where it was looked for — **the mutation and its operand have different scopes**: the mutation configures the shared `_Drill` and is addressed to nobody, while the operand exists only inside one session's layout. So the fix is not a new mechanism but the `view: String` convention every other view-scoped variant already follows, resolved against a layout derived for that name (improvise-obtu). Two things travel with it: editing *inside* a drill grid from a browser does not follow from it, because `edit_or_drill` declines records-mode editing out loud before reaching a cell (improvise-fdk1, since closed, is the separate records-coordinate rename on the wire — `RenameRecordCoordinate`, gotcha 38 — and did not unblock drill editing); and closing it makes improvise-t219 live, `_Drill` being one shared view in `Workbook::views` that two clients drilling would contend for — the same masking relationship improvise-6uva has with improvise-up8g. Guarded by `an_aggregate_a_session_was_shown_does_not_resolve_against_the_hosts_view`, whose **control** is the load-bearing half — the same surrogate resolves once the host is moved onto that view, so a blanket `None` cannot pass. + +45. **One session loop, two transports — and the browser saves itself.** `Host` is `App` + `Sessions` + `handle(id, ClientFrame) -> Vec` and lives in `improvise_command::ui::host`, **not** in `improvise-ws-server`, which re-exports it. It moved for improvise-9x6: the worker hosts the same authoritative `App` over a `MessagePort` and cannot depend on tokio, and a second copy would have been exactly the mirror improvise-xks6 had just deleted for the frames — a drifting `Applied`/`Broadcast` match is invisible until a client sees it. `registry` stayed on the server, because a worker hosts **one** document. What the worker adds is only what a `MessagePort` differs in: **one client by construction** (the `Addressed` filter is honoured anyway, against a future `SharedWorker`), and **a handshake with no close** — `ClientFrame::Viewport` must still come first, but a pre-handshake frame is *dropped* rather than closing a connection that does not exist. + + **Saving is a frame, and dirtiness is read off the classification the effect already carries.** `effect::Save` is `ViewOnly` (gotcha 3) — correctly, since it writes *this host's* `file_path` through *this host's* `Storage` — so the derivation that builds `ModelCommand` from the model effects cannot name it, and for a while that meant no client on any transport could ask its host to save. `ClientFrame::Save` closes it (improvise-368h): a **request** carrying no path, because a client cannot see the host's storage, answered `ServerFrame::Rejected(NoDocumentPath | SaveFailed)` to the asker alone and **silently on success** — nothing moved, so no widget is dirty. It is a frame and not a `Command` for one concrete reason: as a `Command` it put a runtime-fallible operation inside a batch `ClientFrame::Commands` promises is all-or-nothing, so `[Save, SetCell]` reported a failed write and then applied the edit. It also fails the test every `Command` passes — it mutates no shared state and no client's slice, so `reduce_view` has nothing to fold and `Checked` has no basis to answer for. **And `MarkDirty` is gone.** It was a separate effect the *command* layer composed beside each mutation — reasonable on the key path, unreachable from anywhere else, because improvise-kmgs derived `ModelCommand` from the effects **one at a time** and so put `SetCell` on the wire without its companion (improvise-gv9r): a derivation correct per element and wrong for a set, whose symptom was that "save if dirty" never fired. `App::apply_effects` now raises the flag from the effect's `EffectKind`, which makes omitting it unrepresentable rather than validated — **declared** for `WholeWorkbook` (a replaced document differs from the file its path names, so a `LoadModel` that cannot read its path still raises it, costing one redundant write) and **observed** for `CellData` and `LayoutGeometry`, where `document_content`/`document_config` fingerprint the document before and after, so a write that leaves it as it was raises nothing (improvise-748f). The observation is read inside the loop, and `Save` clears the flag in loop order, so a batch that edits and then saves ends clean. The worker's persist trigger is therefore the **flag**, not a table of command variants: an observation rather than a second description of which commands change the document, for the reason `cursor_epoch` beats matching on `CycleAxis` (gotcha 24) and `document_generation` beats matching on `LoadModel` (gotcha 39). It is also strictly tighter — a refused batch raises nothing, and neither does a write that changes nothing, so the "was anything rejected" guard went with it. A browser still never *sends* `ClientFrame::Save`, and that is the answer rather than a gap: there is no file the user owns and no Ctrl-S semantics, the document *is* the OPFS entry, so the worker persists on its own behalf through the one existing `Save` effect. The cost that remains: `Storage` is whole-document (gotcha 41), so OPFS rewrites the file per accepted model command. + + **The async seam does not survive being boxed.** `OpfsFs::open`/`attach`/`refresh`/`commit_deletes` are on the struct, not on `Storage`. `App::storage` is a `Box` and a trait object can name none of the four, so **handing the backend to the `App` puts the async seam permanently out of reach**; the host must hold the concrete backend *and* give the `App` a `Storage`, which means sharing one. That is `impl Storage for Rc`, and it lives in **`improvise-io`** rather than in each worker (improvise-hmws): `impl Storage for Rc` is two foreign types in any other crate, so improvise-9x6 wrote a forwarding newtype that every future worker would have written again. `Rc` and not `Arc` because the one caller is a Web Worker and wasm32 has no other thread. **The flat root is no longer load-bearing**: `OpfsFs::list` joins `dir` like the other two backends, so `storage_contract` takes `dir` as a parameter and the three agree at *every* directory rather than only at `Path::new("")` where joining is the identity (improvise-vhcz). What the index holds is **documents, not entries**, and the split between its three writers is the rule: `attach` indexes nothing (what it creates is zero bytes), `Storage::write` indexes the name and un-queues any pending delete of it, and `refresh` rebuilds from OPFS skipping zero-byte entries — counting an entry it cannot measure as a document, since the one cause of that is another tab holding the exclusive handle (improvise-vvl3). + + **What no test here can see** has shrunk, and precisely. `harness/` (improvise-vvl3/xf1) runs `improvise_io::storage::contract::storage_contract` against a real `OpfsFs` inside a real dedicated Web Worker under Chromium and Firefox, and the `browser` job in CI can fail the build — so **`OpfsFs` is no longer compile-checked-and-never-run**, and two bugs fell out of its first execution. That is why `storage_contract` stopped being a `#[cfg(test)]` helper and became a public function returning `Result`: a browser must run *that* sequence, not a second transcription of it, and a failure has to travel out of a worker as a string. Still unseen: **Safari** (Playwright's WebKit has no OPFS, so xf1's criterion is met for two engines of three; `opfs.spec.js` skips on a runtime probe and an unexpected skip on any other engine is a failure). **`bindings.rs` is NOT in that list, and said it was for months.** `shell.spec.js`'s "the standalone deployment" loads the app with no `?ws=`, which spawns the worker, which imports the glue, which calls `bindings.rs::start` and `OpfsFs::open` — and the spec then asserts a grid draws, so the whole chain has run in CI since improvise-vvl3 (Chromium, and Firefox since 2026-09-20). The claim survived in **five** places (this line, the File Inventory, two `kb.pl` facts and improvise-bck's grooming note) because each was written before the harness landed and none was re-read after; the spec's own comment says the chain plainly. It is the reason to distrust a "never run" written in prose: nothing checks it, and `kb.pl`'s CI gate checks consistency rather than truth. What was already verified stays: `tests/post_message.rs` drives a real `Worker` over a real `App` on nothing but the JSON a `MessagePort` carries. Build note unchanged: the worker bundle needs `--enable-nontrapping-float-to-int` **as well as** `--enable-bulk-memory`, because it carries the whole `App` and so chrono's date arithmetic. + +46. **The page picks a peer, and the demo was built against an assertion.** `web/app.js` holds two transports and no protocol knowledge: `?ws=` selects `improvise-ws-server`, its absence spawns `new Worker('./worker.js?doc=&seed=', { type: "module" })` (improvise-djm). `type: "module"` is mandatory — `worker.js` `import`s the bundle — and there is **no `MessageChannel`**: a dedicated worker's own `postMessage`/`onmessage` *is* the port, so the ticket's channel wording was a date rather than a decision. The websocket path is **kept**, because the ws-server is how a TUI-hosted document is shared and nothing else in the browser can reach it; the cost is contained because the two differ only in *where strings come from* — handshake, fold, repaint and keyboard are all above the seam — and **the presence of the parameter is the whole selection mechanism**, a second knob being a second thing to get wrong. `#status` composes **two** facts, the transport's and the document's, because a single source let a repaint silently erase a connection error. **A worker that rejects in `start()` fires no `error` on the main thread** — `worker.js` catches its own — so the guard is a **first-frame watchdog**, deliberately *not* per-transport: "the peer never answered the handshake" is the one failure that means the same thing over a socket and over a port, and a check aimed at each peer's own failure modes would have missed exactly this one. + + **The demo is a model built to satisfy a check, not an axis flipped and hoped for.** `examples/demo.improv` leaves `Customer`/`Date` on None, so every cell is an `Aggregate` and nothing is editable — and the naive fix does not work either, since `Product × Region` by `Date_Month × Measure` still aggregates. `web/demo.improv` puts **exactly one category on each visible axis** (`Product` row, `_Measure` column, `Quarter` page, nothing regular on None) and carries `Profit = Revenue - Cost`, so the payoff is on one screen with no keystroke. **Settability is asked of `grid_of` as a `CellId` variant**, never re-derived from the axes — a second derivation could agree with itself while disagreeing with `render.rs::editable`, which is the whole point of ADR 002 — and the assertion was proved discriminating by pointing it at `examples/demo.improv`, where it names the aggregate at (0,0). **One category per axis is a property of the demo, not of the format**: when this was written `GridCache` slash-joined a multi-level tuple, so a second category on an axis put that known gap on the front page — improvise-anr9 has since closed it (gotcha 44), so what remains is the demo's own simplicity rather than a rendering limit. `web/*.improv` is un-ignored in `.gitignore` (`*.improv` is ignored repo-wide) and the CI assemble step copies it: leaving it out is an empty grid, not a 404 the page could report. One wart it now puts on the front page: a formula cell **is** a `Record`, so it is offered as editable and a write to it reverts on recompute — pre-existing and matching the TUI, newly visible. + +47. **Keys become commands in the browser, and the server must not know the keymap.** The user's ruling on improvise-bpq2, following from "the model is shared, the UI is not" — and it buys two clients different bindings over one document. What made it urgent is **not** latency. `a_key_resolved_on_the_host_moves_state_no_client_can_hear` shows nothing in the projection stream carries the host's own `ViewState`, so host-side resolution **diverges**; and `Host::handle` answers `ClientFrame::Key` with an explicit empty vec, so **an upstream key is a black hole, not a slow path** — every `KeyOutcome::Upstream` names a binding that does nothing at all in a browser. That test still holds and was not flipped: bpq2 closes the divergence by *avoiding* it. **The method is the ticket's** — enumerate the **bindings**, not the commands (`doctrine(binding_dump_diff)`) — and where a client cannot compose the `Command`, **enrich the projection** rather than moving resolution back. + + Three things that cost. **`GridExtent` on `GridCache`** — the whole grid, of which the labels are one window — because a viewport-indexed cache cannot tell "the last visible row" from "the last row". It is deliberately **not framed by the labels** (`fit_to_labels` leaves it alone) and its batch position is not load-bearing the way `RowFrame`-before-`Cells` is, since it addresses no position; the producer reads `GridLayout::row_count()`, and reading it off the projected ranges is **invisible in the TUI** (which builds at offsets zero, gotcha 44) and is exactly the browser bug. **The window is the client's own fact**, recorded by `Client::set_viewport` from what it announced — *not derivable*, since the row frame's own length is the window **clipped to the grid** and so differs precisely at the bottom edge where the scroll arithmetic matters. `Client::viewport` (associated, no recording) still exists and is the one that quietly turns cursor keys off; `web/app.js` does call `set_viewport`. **`ClientFrame::Commands(Vec)`** replaces the singleton, because one cursor key is `SetSelected` plus the offsets `viewport_effects` returns and a host that folds the selection but not the offset projects the wrong window. It is **answered whole, then applied whole** (improvise-955n): `CheckedBatch::against` checks the frame before anything runs, so there is no longer a moment at which part of a frame has landed and the rest has been refused, and `Applied::Rejected` is unreachable through `Host::commands` — still matched, because a host that wants the `Dirty` must name every case (gotcha 39). The contract is four clauses and **all four are now delivered** — clause 4 by improvise-yqpd on 2026-09-21, whose mechanism is `App::snapshot`/`restore` plus a `Sessions` clone taken in `Host::mutating_frame` and put back by `undo_frame`, deliberately *not* inside `apply_effects`, which runs on every keypress. **This paragraph said it was undelivered for a day after it landed**, and an orchestrator brief repeated that from here, so read the clause list as a claim to check. A delivered clause 4 is also what makes the CLIENT's rollback well-defined (improvise-nxah): a refusal means the host applied none of the frame and restored every mirror, so a client keeping its optimistic fold is the only thing left out of step. What rolls back there is `ViewState`, never a cache — a `Command::Model` is a no-op in `reduce_view`, so the optimistic fold never reached one. The old text, kept because the hazard it names is still the right shape to fear: one basis per frame, checked once, applied in order with each command seeing the last one's state, and all-or-nothing. was that the fourth was open because `App::apply_effects` aborts without undoing (gotcha 21 — an abort is not a rollback), so `[RemoveItem("North"), SetCell(the North record)]` would land the removal, resolve the write to nothing, and report the frame accepted having applied half of itself — partial *and* silent. That is what `yqpd` closed, and it closed it **outside** `apply_effects` for the reason the old text implies: the abort lives on the keypress path and must stay cheap. The old note that it was *unreachable* from any client here, the only multi-command producer being `keys::commit`, still reads correctly as a caution — **reachability is a reason not to rush a fix and not a reason not to notice one**, the lost update `CheckedBatch` fixed having had the same reachability argument and none of the documentation, which is how it shipped. + + Two corrections to the ticket's own gap list, both worth more than the code. **Panel cursor movement was never a gap**: improvise-edp had already made `viewmodel::flatten_cat_tree` the one home both ends call, so it is the *shared* derivation, not a second one — worth re-checking the "known gaps" in other old tickets against what has landed since. And the "unmodified characters only" rule was **mis-scoped, not wrong**: it was about `KeyPattern::AnyChar`, which is bound only in the text-entry modes, so `Ctrl-d` in Normal has an exact binding and mirroring it is a faithful subset. Each resolver mirrors its `Cmd` **line for line, including where siblings disagree** — `Move` and `MoveTileCursor` emit even when they clamp, `MovePanelCursor` returns nothing when the cursor does not move. Tidying that into one rule would make the client a second derivation, which is what `tests/lockstep.rs` exists to rule out. What still needs the host is **one list**, `resolve::NEEDS_THE_HOST` in `improvise-protocol/src/resolve.rs` (improvise-uln5) — this line named a list in `keys.rs` and a test called `the_normal_mode_keys_that_still_need_the_host_are_these` for some time after both had gone. Three tests hold it, one per direction: `the_command_names_that_still_need_the_host_are_these` (no **false** entry: each name is bound and really is silent), `a_help_row_the_browser_cannot_run_is_drawn_marked` in `wasm-client/render.rs` (the help overlay **marks** those rows, improvise-uln5), and `every_silent_binding_is_on_the_list` (every binding of **every mode a browser can reach** acts, declines out loud, or is on the list — improvise-oyaa, improvise-s2gb). The last replaced a check read off the help rows, which could not see a binding no help row names. Closing a gap therefore means deleting its entry, and the first test fails until you do. + + **What the browser client decides before a frame leaves it** (all in `improvise-protocol`, so both ends share it). Five rules, each bought by a bug: + - **Every client cache read goes through `projected_at`**, which subtracts the frame's own origin (`AxisFrame::first`) before indexing. A position outside the window the cache holds is **unknown**, never the cell at the same cache index (improvise-7mro) — once the client scrolled ahead of its cache, an index read answered a different cell's value. `unprojected` names that state, and a cell op on it waits rather than guessing. + - **The keyboard commit writes `ClientState::opened`**, not the cursor. `OpenedCell` captures the record, the `OpenedTarget` (`Value { measure }` or `Coordinate { category }`), the record's version, the `document_generation` and the view **at open**, so a commit lands on the cell the editor opened on however the cursor moved since (improvise-8xq0), and claims the basis it was **opened** against rather than one read at commit, which silently overwrote a value another client wrote while the editor was open (improvise-xz2z). A retry after a crossing `LayoutMoved` must match the **whole** `OpenedTarget`, not the record alone, or it wrote whatever cell was under the cursor by then (improvise-8phw for values, improvise-9ww8 for renames). + - **`refused_here` exempts a captured editor** (improvise-3h9w): with `opened` set, `commit-cell-edit` is answered by the capture, so the cursor's current cell cannot refuse a commit aimed elsewhere. + - **A decline that must stop a binding is `Step::Declined`**, not a `SetStatus` beside the binding's other steps: Enter's welded `clear-buffer` otherwise still ran and threw away the typed text (improvise-joed), and a touched editor over a declined cell went `Upstream` with no word at all (improvise-ik6l). + - **`ClientState::notify` queues a client-side notice at the tail of `in_flight`**, as if it were a sent frame, so a rebase that rebuilds the view from `settled` and replays `in_flight` replays the notice too instead of erasing it (improvise-n0pb). `settle_status` runs on **every** path that rebuilds the view (bootstrap, accept, refuse, relay), and assigns a message the editor open as it arrives, so an old message loses its owner once that editor closes (improvise-0dbk). `EditorSession`, the key it is owned by, lives in `improvise-protocol/src/lib.rs` beside `ViewState` so both front ends ask it (improvise-su4u, improvise-1x9h). + +48. **The browser JavaScript is argued *and now partly executed*, and the two rules it can break are both about *when*.** `web/app.js` and `web/worker.js` were syntax-checked, `.d.ts`-checked for call arity and read — never run — and every bug in them was found by reading. `harness/tests/shell.spec.js` is the first execution of either: it runs `app.js` under Chromium against a mocked peer, and it exists because PR #21 declined to add regression tests for its own close-listener fix on the ground that there was nowhere to put them. That decline is only honest because the assertions live there now, and **a change to `web/*.js` belongs in that spec directory** rather than in a second ad-hoc surface beside it. Playwright rather than a headless-DOM shim because what goes wrong in these files is a real Worker, a real WebSocket, real OPFS and real event ordering, none of which jsdom has. Two classes of bug still follow from *when*, both found by review. **An `async` `message` listener does not serialize.** `worker.js`'s `handle()` awaits `server.flush()`, so the event loop may start the next message while an earlier one is suspended, interleaving two `on_message`/`flush` pairs and letting a later frame's replies **overtake** an earlier frame's — which gotcha 33 answers by *dropping* the overtaken update, not by replaying it. Every frame is therefore chained onto **one tail promise** and the listener is deliberately not `async`; that also turns "frames that arrived before `start()` finished" into an ordinary case of the ordering rule instead of a drain loop, which had the same race twice over because `server` was assigned *before* the loop ran. **A watchdog must be disarmed, not pre-empted.** The handshake watchdog exists for one condition — the peer never answered — and every *loud* failure (worker `error`/`messageerror`, socket `error`) has already written something more precise; the timer then fired 8s later and replaced it, so `setShellFailure` clears it. Its neighbour, the socket's `close` listener overwriting a precise `error` with "disconnected", **is fixed** (improvise-taxx, PR #21) — and the ticket's own proposed mechanism was wrong: `readyState` cannot discriminate, an `opened` flag is the only thing that can. PR #21's *other* half is the one that keeps getting lost: it routes **both** close arms through `setShellFailure`, which disarms the watchdog, so a genuine mid-session disconnect is **not** overwritten eight seconds later. A bug report said it was, measured it honestly against a pre-#21 tree, and then "confirmed" it survived #21 by simulating the `opened` flag **and not the routing** — a simulated fix that is not the fix you named reproduces a bug the real code does not have, and reads exactly like a confirmation. The ticket was filed and closed invalid; the property is now asserted by `a socket that opened and then dropped reads as disconnected, and stays that way`, which runs past the 8s window, rather than argued in a comment that was wrong twice. + + **What the harness still cannot see**, and why the argued-not-tested discipline stays: the grouped and multi-level DOM, the CSS, and the `ResizeObserver` extent arithmetic. What a careful read established about the last of these — the observer cannot feed back into itself (`#grid` is a flex child with `overflow: auto`, so `replaceChildren` cannot resize it) and `ROW_PX`/`COL_PX` under-estimate, so the worst case is a row of over-fetch — is still a read and not a test. + + **A third thing no test here can see, which is why the docs must be read as code.** Three comments in one range said "not built yet" about work that had landed *in that same range* — `README.md`'s browser section, `app.js`'s keyboard note, and `Host::handle`'s `Key` arm — each because a follow-up commit updated **one copy of an argument and not its siblings**. When a fact is stated in more than one place, changing it is a grep, not an edit. + + **An evidence heading may be module-qualified, and must be when its leaf is not unique.** `scripts/check-comment-runs.py --check-evidence` takes its Rust names from `cargo test --workspace -- --list` minus the `--ignored` twin, as **full module paths**, and **refuses** an ambiguous leaf naming every candidate rather than warning. Six leaf names collide in this workspace. Note neither half suffices alone: rename one of a colliding pair and the leaf is unique again and resolves silently to the survivor, so qualification is the fix and the refusal is only what forces someone to write it (improvise-rm4s). The residual, filed: `--list` does not name the binary, so two crates sharing a module path are indistinguishable (improvise-sxsf) — **measured latent** (1444 names, zero collisions) and the proposed per-package fix **declined twice over**: 18× the cost (45.2s against 2.4s, and `--check-evidence` needs the `--ignored` twin, so 22 cargo invocations), and *a package is not a binary* — `improvise` alone has five integration test binaries beside its lib, and an integration test's top-level `fn` lists with **no module path at all** (60 of 1444 names are bare), so `-p improvise` prefixes all six identically. Per-**target** is the only shape that would close it. + +49. **A websocket handshake is exempt from the same-origin policy, and that is a server-side problem.** There is no CORS preflight, so any page a user visits can open a socket to `improvise-ws-server` *from that user's own allowed address* and read and rewrite every document it holds — cross-site websocket hijacking (improvise-kc7n). **No network control can see it**: an IP allow-list, a private network and loopback binding all answer "may this **client** reach the endpoint", and the honest answer is yes; only `Origin` says what is **driving** the client. **Loopback is the least protected case, not the safest** — no proxy in front of it, and loopback is not a boundary a browser enforces. `src/origin.rs` is the whole decision and `transport::admitted` its one caller; it refuses the **upgrade** with a 403 rather than accepting a socket and ignoring it, and `serve` **takes** the policy so a host cannot forget to apply it. Three arms are decided rather than defaulted: **loopback allowed** (the shipped local recipe serves its page from `http://localhost:8080`, so the one case worth optimising for must need no flag), **absent allowed** (no browser omits `Origin`, so absent is a non-browser client an address check already answers — and refusing it breaks `tests/socket.rs`, the repo's own instrument), and **two refused** (`HeaderMap::get` is first-wins, so a proxy that *appends* rather than sets leaves two, and answering from the first lets the second be anything). The deployed Pages origin is deliberately **not** hard-coded: it can only reach a plain-`ws://` server through a TLS proxy anyway, so one flag is an honest cost on an already multi-step path where a baked-in default would be a permanent hole for every other deployment. The residual hole is **stated rather than guarded** — a proxy that *strips* `Origin` turns the check off, which is why the proxy's own allow-list stays in `README.md` as defence in depth. **None of this is authentication**: it answers what page is driving a client, never who is connecting, and improvise-kpnr is still open. It correctly does **not** move to `improvise-command` beside `Host` (gotcha 45) — a `MessagePort` has no handshake and no headers, and a worker and its page are the same origin by construction, so this is genuinely transport-only and not a mirror waiting to drift. + + +50. **A composed TUI row measures in cells and names which side yields.** `draw.rs::fill_line` takes a `Keeps`, because the three rows that share a line genuinely disagree about it: the title bar keeps the **right** (the `?:help :q quit` escape hatch outranks an unbounded document name — improvise-t14d), while `minibuffer_line` and `draw_status` keep the **left**, which is what they already did, since right-priority there would let a long status message eat the text the user is editing. So the ticket's "give the right segment priority" was correct only for the title bar, and a global rule would have been the wrong shape. Widths go through `ui::text::truncate`, never `str::len` — and the ticket framed *that* half wrongly too: the title's own `·` separator is two bytes and one cell, so a byte count mis-padded **every** document, not only one with a non-ASCII name. + + **One key table on screen, and it is the one derived from the keymap** (improvise-rr8n). `TileBar` now advertises no keys at all: its literal read `Ctrl+↑↓←→ to move tiles` where those keys are bound to `enter-tile-select`, and the derived status line is a strict superset (`Space`, `h`/`l`, `Tab` appear only there). The ticket asked for the opposite — rewrite the literal, suppress the derived line — and was inverted deliberately, because suppressing a derived superset to preserve a literal that had already rotted keeps the stale table and deletes information. The **import wizard goes the other way**, and the asymmetry is the rule rather than an exception: its status line *is* derived, but from the **mode** keymap while the real keys are per-step inside `HandleWizardKey`, so there the derived table is the one that lies and `draw_status` suppresses the fallback for that one mode. **Keep whichever table is derived from the thing it describes.** Two blind spots make this class invisible: `every_hint_key_resolves_in_its_mode` **cannot see a string literal in a widget** (the same gap improvise-wk18 names for `help.rs`), and the widget's own characterization test renders at **80 columns**, where the hint does not fit and is never drawn — so the one test pinning that output was structurally incapable of seeing either hint. + + **A scroll is measured where it is drawn** (improvise-gtn7.1). The keypress path counted `visible_cols` from `term_width` while `draw_content` gave the grid `term_width` minus the 32-column side panel, so with any panel open the cursor scrolled to a column never drawn. Now `viewport::grid_width(term_width, side_panel_open)` is the one answer, reached through `App::grid_width`, and both `draw_content` and `App::cmd_context` read it; `App::side_panel_open` is the one answer to whether a panel took its column. The tile bar had the same shape: `first_tile` chose its scroll against a width that began one or two columns before where the tiles were drawn, and both now start at `tiles_left(start)`. The grid bug was latent from `b5418f2e`, when `compute_visible_cols` took `term_width`, and was exposed by `50e4a372`, which opened the category panel by default. Still open: columns of mixed width, where a `visible_cols` count taken at the old offset is applied at the new one (improvise-gtn7.2). + + +51. **The command language reads pipes, refuses surplus, and checks names when it applies (improvise-31oe.2, 31oe.3).** Reasoning: `context/code/cli.md` §14 and §15. (a) A `:` argument is read through `improvise_formula::quote`, so `create-view |By area|` names `By area`; a **surplus argument is refused** rather than ignored, and `cmd::core::only_when_omitted` gates the *cursor fallbacks* of `clear-cell`, `yank`, `paste` and `drill-into-cell` so a coordinate typed by the user is never overridden by the cursor. (b) `set-axis`, `set-page`, `toggle-group`, `hide-item`, `show-item` and `switch-view` return `RequireKnown(Needs)` (`effect.rs`) ahead of the real effect, built by `effect_cmds.rs::unless_unknown`. It is `ViewOnly`, writes nothing, is **off the wire** (listed in `vocabulary_is_complete.rs`) and checks the **document at apply time** — a `:` line builds every segment before any runs, so a check in `execute` refused `create-view Second . switch-view Second` — then refuses and aborts the batch (gotcha 21). (c) A refused name therefore stops the rest of a `:` line, while headless `improvise cmd` rebuilds the context per segment and runs each as its own batch (improvise-31oe.14, open). + +52. **A date column imports as one hierarchical category, and the import refuses what it cannot keep (improvise-8jk6.3, 8jk6.6, d0n6).** Reasoning: `context/code/formula-and-io.md` ("Import: …") and `cli.md` §13. The default `DateShape` nests Year > Quarter > Month (the chosen levels) in ONE category named after the column and opens folded to its years; `DateShape::Split` is the old flat split, chosen by `--split-date ` headless or `s` on the wizard's date step. `Plan::spelling_clash` refuses a raw, unparsed date spelled like a generated item or group (`2024`, `2024-01`) because the two would silently merge — asked at the wizard's ConfigureDates step **and** again at build time, so a headless import cannot reach it either. Headless, `--strict` turns a partial skip into a refusal before the build, and a folder whose CSVs are all header-only (no records) is refused as an import that kept no record. + +--- diff --git a/context/repo-map.md b/context/repo-map.md index b2888ace..7a64ddf9 100644 --- a/context/repo-map.md +++ b/context/repo-map.md @@ -156,513 +156,101 @@ Region=West, Measure=Revenue = |pending| # pipe-quoted text --- -## Gotchas (read before editing) +## Gotchas (digest — read before editing) -1. **Records mode is wide format.** A row is one *coordinate tuple* — the cells sharing everything but `_Measure` — and the columns are `[coordinates…, **`+cat`**, raw measures…, `+`]`: a column per regular category, then the column that creates a **new category** (improvise-2zq, sitting where the column it creates will appear), then a column per raw measure, then the column that creates a new measure (improvise-c5v). Consequences: two measures on one coordinate are ONE row; editing a category column moves **every** measure cell in the row, not just the one under the cursor; `_Dim` in the synthetic key carries the column **index**, not a label, because a category and a measure may share a name — resolve it with `GridLayout::record_col` into a `RecordCol`. Formula targets are not columns (nothing to type into them). Drill pending edits are keyed by `(row, RecordCol)`. - **`prune_axis_items` takes a predicate, not a flag vector.** `prune_empty` hands it `impl Fn(usize) -> bool` over the *data* index (group headers consume no index). The domain is derived from `items` itself, so there is no parallel length a caller could get wrong — the earlier `&[bool]` + `debug_assert` pair could disagree between debug and release if a third caller ever sized `keep` from something other than `row_count()`/`col_count()` (improvise-n3wt). **It re-marks the survivors rather than filtering the markers** (improvise-e265): runs nest, so dropping an item can leave a header with nothing under it or a `GroupEnd` closing a run that was never reopened, and no per-marker filter can tell. `remark` instead emits, before each kept item, exactly the headers and `GroupEnd`s that make the output's open runs equal the input's at every level — each survivor keeps exactly the runs it was in. Reasoning: `context/code/core.md` §6. **`prune_empty` shares one `GridEval`** across the whole row×col emptiness test, and **a grid evaluates only through `GridLayout::evaluator`**, which builds the fold memo from that layout's own `none_cats` and hidden items — so no other grid's memo can reach its cells or totals (improvise-metf, improvise-opbo; the memo itself is improvise-h6z1, `core.md` §14). - **`GridLayout::new` does not prune; `with_frozen_records` does** — and that split has now produced **four** bugs: `view_frame`, `project::grid_in` (PR #78), and `persistence/export.rs::render_csv` (improvise-iot6), which wrote rows *and columns* the grid does not draw for as long as CSV export has existed. Every remaining `GridLayout::new` call site in the workspace is test code — verified — so a new production caller almost certainly wants `with_frozen_records(m, v, None)`. +One or two lines each: the rule, and the trap. **The numbers are stable and cited across the repo; a +citation of `gotcha N` means entry N in `context/gotchas.md`, which holds the full argument, +the incidents and the ticket ids.** Read the long form of every gotcha your diff touches. Counts and +"closed/open" claims in the long form are dated snapshots. -2. **Effects never touch `std::fs`.** Every document read or write goes through `App::storage: Box` (improvise-fuj, closed). Two writes still go around it: `App`'s autosave and the headless final write call the path-taking `persistence::save` (improvise-31oe.11, open). The host picks the backend: `App::new` supplies `PhysicalFs`, `App::with_storage` takes any other — the browser worker supplies OPFS (improvise-i34, closed), and tests use `MemoryFs`. `improvise-io` keeps path-taking wrappers (`save`, `load`, `export_csv`, `parse_csv_with_warnings`) for the native CLI, each delegating to a `_to`/`_from`/`_bytes` variant that takes the backend or the bytes. -3. **Every effect declares an `EffectKind`, and there is no default.** `ViewOnly` never reaches other clients; `Model(CellData | LayoutGeometry | WholeWorkbook)` does (improvise-mae). A new effect that forgets to classify itself fails to compile — that is deliberate, since the failure mode is silent: a mis-declared `ViewOnly` edit would simply never reach connected clients. Watch two traps: the *View* lives inside `model_state.workbook`, so "touches model_state" does NOT mean "is a model change" (axis/page/format are `LayoutGeometry`, cursor and scroll are `ViewOnly` — see improvise-8uz; and `SwitchView`, `ViewBack` and `ViewForward` moved from `LayoutGeometry` to `ViewOnly` with improvise-jytm, writing `view_state` and nothing else, which is what took the latter two off the wire — but a sound derivation is not a guard: only `SwitchView` had one, and review showed reclassifying either sibling back was silent across the whole suite); and `Save`/`SaveAs` touch `model_state` for bookkeeping only and are `ViewOnly`. `AddCategory`/`RemoveCategory` are the third trap and go the other way: they look like data edits, but both go through `Workbook`, whose `on_category_added`/`on_category_removed` rewrite **every** view's axis map, so they are `LayoutGeometry`. The two "an effect declaring X actually changes X" tests cannot separate them — a category change moves model data *and* view config, so it clears either bar; `structural_effects_declare_their_kind` pins the declaration itself. A third trap, now guarded: **`EffectKind` is too coarse to check a *mapping*.** `ui/reduce.rs::model_effects` sends each of the 26 `ModelCommand` variants (counted on `c1772ca9`; gotcha 38) to one effect, and every inverse pair — `HideItem`/`ShowItem`, `AddItem`/`RemoveItem`, `CreateView`/`DeleteView`, `SetCell`/`ClearCell` — declares the *same* kind, so swapping three of them at once left the suite at 918 passing while three wire messages did the exact inverse of what they said. The mapping is name-for-name by construction, so `every_model_command_names_the_effect_of_the_same_name` asserts that, reading the type name off the effect's derived `Debug` (`Effect` is a trait object, so `type_name` is out of reach). A variant whose effect must be named something else is a variant that stopped being derived — it should fail there and be argued about. Their reach is wider than the name suggests: `Model::remove_category` also drops every formula targeting the category, which is why `Dirty::of(LayoutGeometry)` reports `formulas: true` — see gotcha 35. -4. **Commands never mutate.** `&CmdContext` is read-only; return `Vec>`. If you want to touch `&mut App`, add or extend an Effect. -5. **`CellKey` is always sorted.** Use the constructors; equality and hashing rely on canonical form. -6. **No `Model::add_item`.** With a `Workbook` in hand use the facade: `wb.add_item("Region", "East")?` / `add_item_in_group` / `remove_item` (improvise-tvt). With only a `Model`, go through the category: `m.category_mut("Region").unwrap().add_item("East")`. -7. **Virtual categories.** `_Measure` items = explicit items ∪ formula targets. Use `measure_item_names()` / `effective_item_names("_Measure")`. `add_formula` does *not* add items to `_Measure`. Formula-target lookup also falls back into `_Measure` via `Model::find_item_category`. Cell aggregation goes through `apply_agg`, formula `SUM(…)`/`AVG(…)` through `eval_agg` — they differ on empty input (absent vs. A third rule joins them for **non-numeric** values: `aggregate_cell_values` reduces text by **unanimity** — one matching value, or several copies of the same value, is that value; differing values, or text among numbers, is `CellValue::Error("mixed")` (improvise-l4qx). The **all-numeric branch is checked first**, so `Count` of a lone number is still `1` and not the number — that ordering is the whole difference between this rule and "a single value passes through". What the rule forbids is *dropping*: `filter_map(as_f64)` used to, and since `_Index`/`_Dim` are on `Axis::None` in every default view that made **every text cell in every document render blank**, including the pipe-quoted text `.improv` carries as a first-class value. `evaluate_aggregated` and `aggregate_raw` share the helper, so a formula ref over mixed values now errors `"mixed"` rather than silently summing the numeric ones. `0` for Sum/Count, error for Avg/Min/Max). `Model::evaluate_aggregated` takes `Aggregation::{Exact, OverHidden}` rather than `none_cats: &[String]` (improvise-5jm5) — the list was only ever tested for emptiness, and the guard against that shape returning is the **signature**. `recompute_formulas`/`aggregate_raw` keep the list because they walk it. -8. **Display rounding is view-only.** `format_f64` (half-away-from-zero) is only called in rendering. Formula eval uses full `f64`. Never feed a rounded value back into eval. -9. **Formula eval is fixed-point.** `recompute_formulas(none_cats)` iterates until stable, bounded by `MAX_EVAL_DEPTH`; circular refs converge to `CellValue::Error("circular")`. `App::new` runs it before the first frame so formula cells render on startup. -10. **Drill into formula cells** strips the `_Measure` coordinate when it names a formula target, so `matching_cells` returns the raw records that feed the formula instead of returning empty. -11. **Minibuffer keymaps come from `TextEntry`.** The eight text-entry modes are built by the `TextEntry` builder in `improvise-protocol/src/keymap.rs` — seven by `TextEntry::new`, and `RecordsEditing` by `TextEntry::with_parent` off `Editing`'s map, which is why a count taken off the `::new` call sites reads seven. It binds the shared spine (AnyChar→`append-char `, Backspace→`pop-char `, Esc→`[clear-buffer , enter-mode ]`) and welds every commit key to its `clear-buffer` via `.commit(key, cmd, args)` (improvise-w3q). `TextEntry::new` takes the `AppMode` and reads `` off its `MinibufferConfig`, so the buffer the prompt names and the buffer the keys edit are the same string by construction (improvise-k8h) — the keymap never spells a buffer name. `.rebind` exists only for CommandMode's Backspace. `SearchMode` is hand-written on purpose — no commit key, and `exit-search-mode` owns the query. `EnterMode` clears the incoming mode's minibuffer through `write_buffer` too — building `SetBuffer` directly there was unreachable only because no shipped `AppMode` is keyed `"search"`. `MinibufferConfig::buffer_key` is a `String`, not `&'static str`: a borrowed-'static field makes `Deserialize` unimplementable, and every text-entry `AppMode` variant carries a `MinibufferConfig` (improvise-a2x). A buffer named at runtime is read with `cmd::core::read_buffer` and written with its mirror `write_buffer` — "search" is not in the buffers map but its own `ViewState` field, and those two functions are the only code that knows it (improvise-gdl). -12. **Panel keymaps come from `PanelKeymap`.** The three panel modes share a spine (Esc/Tab→Normal, arrows and j/k→`move-panel-cursor `, every panel's toggle key, `:`→command mode) built by `PanelKeymap::new(panel)`; the keys that act on the panel's contents arrive through `.bind`/`.bind_args`/`.delete` because they vary in shape, not just command name (improvise-4pg). `Panel::ALL`/`name()`/`key()` in `ui/app.rs` drive the toggle keys for Normal and all three panels, so a fourth panel reaches every keymap from one `ALL` entry. `name()` must spell what `parse_panel` accepts. -13. **Prefix keys are stored as an address, not a keymap.** `ViewState::transient_prefix` holds the `(KeyCode, KeyModifiers)` that opened a prefix; `KeymapSet::prefix_keymap` resolves it back to the sub-keymap, walking parents so `RecordsNormal` reaches Normal's `g`/`y`/`Z` (improvise-jc8) — that is the command-layer wrapper; the resolution itself is `Keymaps::prefix_keymap` in `improvise-protocol/src/keymap.rs`. `ViewState` must stay free of `improvise-command` types — it is destined for the protocol crate, which `improvise-command` will depend on. Resolution assumes prefixes are **flat**; `prefix_keymaps_are_flat` fails the build if a nested one is added. -14. **Keybindings are per-mode.** `ModeKey::from_app_mode()` resolves the mode; Normal + `search_mode=true` maps to `SearchMode`. Minibuffer modes bind Enter/Esc as `Binding::Sequence` so `clear-buffer` fires alongside commit/cancel. Hints are checked against the keymaps: `every_hint_key_resolves_in_its_mode` (`command/keymap.rs`) splits each `AppMode::hint_text()` and requires every key it names to resolve in that mode's map, because `only_quit_has_no_hint` asserts only that a hint is *present*. It checks which keys are named, not what they do. Its tokenizer knows the hint dialect explicitly (named keys, `Ctrl-`, `F/C/V` alternatives, character runs like `hjkl`/`[:]`, the em-dash prose tail, CommandMode's `:q` ex-commands) and panics on a token it cannot classify rather than skipping it. -15. **Registry names come from prototypes, not strings.** Every `register*` call reads the name off a prototype `Cmd` instance, so a typo is impossible (improvise-9cn) — so no `register*` call spells a name. Name literals do remain in `registry.rs`, and only tests guard them — the keymap test `every_bound_command_name_is_registered` for the `Move` literals (all bound; `registered_names_are_unique` covers them too), and for the four aliases `registry.rs::every_alias_resolves_to_a_registered_command` (a hand-copied list, not the table) plus the `parse.rs` tests `alias_add_cat_resolves_to_add_category`, `alias_formula_resolves_to_add_formula`, `alias_add_view_resolves_to_create_view` and `alias_q_bang_resolves_to_force_quit` — the `Move` prototypes carry `cmd_name: "move-selection"` / `"scroll-rows"` / `"page-scroll"` (the four `jump-*` names go through `reg_jump!`), and four `r.alias` calls spell an alias and its target (`add-cat`, `formula`, `add-view`, `q!`). What types can't reach: five commands name themselves off a field (`Move` off `cmd_name`, `ViewNavigate`, `TileAxisOp`, `ToggleGroupAtCursor`, `CommitAndAdvance`), so a prototype can name the wrong *variant* and still compile. That shows up as a **duplicate** registration, not a mismatch — the wrong-variant entry collides with its sibling and the sibling's name vanishes. `registered_names_are_unique` catches it; `every_bound_command_name_is_registered` catches the keymap side, and checks **both** halves of `dispatch`'s `registry.interactive(name, args, ctx).ok()?` — the name resolves *and* the args parse — because improvise-s13 moved `enter-mode ["export-prompt"]`/`["formula-edit"]` out of the nullary class, where a typo'd mode name compiles into a dead key. It builds the `CmdContext` in each keymap's own `AppMode`, since `commit-buffer item` reads `ctx.mode`. -16. **`effect_cmd!` macro** wraps a pre-built effect as a parseable command. Use it for simple wrappers; reach for a hand-written `Cmd` once there's real decision logic. Its counterpart on the effect side is `simple_effect!` in `ui/effect.rs`, which declares a one-field newtype that writes one place on the `App`. Its **kind argument is mandatory** — `simple_effect!(Name(T) => |app| place, EffectKind::ViewOnly)` — even though every call site so far answers `ViewOnly`: hard-coding it reintroduced exactly the default gotcha 3 does without, since `$place` is an unconstrained expression and a one-line `active_view_mut().number_format` setter would have reported `ViewOnly` with the suite green (improvise-8fy). Exactly six wrappers return `unless_unknown(needs, effect)` rather than the bare effect (gotcha 51): `set-axis`, `set-page`, `toggle-group`, `hide-item`, `show-item` and `switch-view`. The rest return bare effects — `delete-view` (whose `DeleteView` effect refuses an unknown view itself), `add-item`, `add-items`, `set-cell`, `create-view` — and `remove-formula` checks in `execute`. Likewise, mode entry has no per-mode shim commands — bind `enter-mode ` (improvise-s13); the name is decoded by `parse_mode_name` in `registry.rs`, and `EnterMode` clears the mode's minibuffer on the way in. -17. **Float equality inside formula eval has one epsilon, and the div/0 guard shares it.** `FLOAT_EQ_EPSILON = 1e-10` in `model/types.rs` is used by `=`/`!=` **and** by the division-by-zero guard (`rv.abs() < FLOAT_EQ_EPSILON`), in both `eval_expr` and its aggregated twin — six uses across the two evaluators (a div/0 guard plus the `Eq` and `Ne` arms in each), one constant. This gotcha previously recorded the opposite (`=`/`!=` epsilon, div/0 exact) and was stale; the mix is gone. Why an epsilon at all and why this one: `context/code/core.md` §4. -18. **`IndexMap`** preserves insertion order for categories, views, and items. Persistence relies on this — and so, since improvise-shyt, do `DataStore::matching_cells`/`matching_values`: the secondary index **narrows** the candidates, then `matching_positions` resolves each to its position in the primary `IndexMap`, sorts, and reads back. **An index whose values are `HashSet`s is for O(1) lookup, never for iteration.** Walking it gave a different order in every process, which was not tidiness. Records-mode rows reordered between sessions; `SUM`/`AVG` drifted in the last bits because `f64` addition is not associative (`Min`/`Max`/`Count` and the text-unanimity rule of gotcha 7 are order-free, so they were never affected); and worst, the **drill snapshot** froze its record list in hash order while `drill_state.pending_edits` are keyed by *row index* (gotcha 1), so the same drill opened in a second process addressed **different records** — silent misdirected writes, and nothing in the tracker knew. Measured at 18,000 cells: +20% worst case on `matching_values`, and `matching_cells` 7% *faster*, sorted positions resolving `CellKey`s with better locality. `DataStore::live_records` was the open choice and is now decided (improvise-2yc2): it walks `cells` and projects insertion order onto records, deliberately paying O(cells) rather than O(records), because `cells`' order is already the answer `matching_positions` gives and a second answer to "what order is this store in" is the disease. It is derived per call rather than memoised on `RecordEntry`, since a stored position is a copy and a record deleted and re-created would hold its old one. Note what is *not* on offer: an unordered return — every iterator has an order, and a type that merely disclaims one still hands the caller hash order, which is the guard living one level away from the hazard and the shape this whole class of bug returns through. **The second real instance was `import::analyzer::analyze_records`** (improvise-7yx), which built its distinct values through a `HashSet`: `build_model` adds items in exactly that order and `IndexMap` preserves it, so the *same CSV imported twice* produced two different `.improv` files and two different row orders. Now first-seen order, pinned by a five-iteration test because each `HashSet` gets its own `RandomState`. Also answered here since: an **empty `partial`** matches every cell vacuously (improvise-hkt6) — it used to index `sets[0]` and panic, unreachable only because both callers guarded. **A group's fold reads through the same rule.** `Model::fold_stored` folds a collapsed group's stored values once, over every record the group covers, through `DataStore::spanned_values` → `spanned_positions` (improvise-yb4b) — one scan, not a lookup per member. `spanned_positions` chooses **by index-set size** between narrowing once by the base key and narrowing once per covered item of the smallest span, whichever visits fewer (improvise-jq0m), and **both arms return store order** — sorted positions, as `matching_positions` does. That is load-bearing, not tidy: a fold that returned hash or per-item order would reintroduce the `SUM` drift above, and the choice between the arms would then be visible in the last bits of a total. **A partial as wide as the widest stored cell needs no narrowing** (improvise-felo.13): `DataStore` keeps `arities` (how many stored cells have each width), and `exact_positions` answers a strictly-sorted partial at least that wide with one probe of `cells`, since only an equal cell can match; anything else falls through to the index. The answer is the same sorted positions. -19. **Commit paths for cell edits** split across `commit_cell_value` → `commit_regular_cell_value` / `commit_plain_records_edit` in `command/cmd/commit.rs`. Keep the synthetic-records branch in sync with the plain branch when changing behavior. **Which cells refuse an editor is one function, asked at both ends of an edit**: `cmd::core::uneditable(model, layout, row, col)` is the shared decline for **opening** an editor (`EnterEditAtCursor`) and for **committing** one (`CmdContext::committable_cell_key`) — a fold or other aggregate, a formula's result, or a `CellId::NonItem` (improvise-enz6, improvise-gk51). It reads the cell's `CellId` through `project::cell_id`, the same derivation the projection uses, and the browser's `resolve::declined_at` matches on the projected `CellId`, so the two front ends refuse the same cells. Asking only at open is the bug enz6 was: Enter-to-advance moved onto a collapsed group's row and committed a write to the group coordinate. **`CellId::NonItem { category, item }` is a stored value whose coordinate names no item of its category** — records mode can draw one, and no write can address it, so it is projected as its own variant rather than as a `Record` that the host then refuses (gk51 was exactly that mismatch). Its one wording is `improvise_protocol::non_item_refusal`. -20. **Minibuffer commits go through one command.** `commit-buffer ` (`CommitBuffer { kind: CommitKind }`) owns buffer→trim→bail-on-empty→dirty→status for all three; the varying parts are methods on `CommitKind`. The add effect is emitted **last** so its failure message wins over the command's optimistic status (improvise-6t3). `CommitKind::Item` carries its target category, resolved from `AppMode::ItemAdd` by `parse_commit_kind` at registry time — so `commit-buffer item` is not constructible from a script line. -21. **An abort stops the chain, never the bookkeeping — and ordering is what separates them.** Three effects set `abort_effects`, and they are not all saves: `Save` and `SaveAs` on a failed write or a missing path (improvise-8tk — `SaveAs` was the arm left out of the original fix, so `:w /bad/path . q` quit on a failed write), and `AddCategory` when `Model::add_category` refuses, because what follows assumes the category exists — from the `+cat` column (improvise-2zq) the rest of the batch would advance the cursor and re-open an editor on a column that was never created. A `Binding::Sequence` flattens into one effect vec, so the abort carries across sequence steps; that is a property of the *flattening*, and it is why improvise-0hb's proposal to apply effects between steps was rejected — `apply_effects` resets the flag per batch, so step 2 in a fresh batch would land the `Quit` after a failed `Save`, silently reopening a P1. `a_sequence_whose_first_step_aborts_does_not_run_the_second` is the guard. **The rule the aborts taught, which is the part to carry:** an abort at the *tail* of a batch also ate the mode restoration and the buffer clearing, stranding the user in `CommandMode`/`Editing`/`SavePrompt` — where `draw_bottom_bar` renders the minibuffer *instead of* the status line, so the error the effect had just written was never drawn and the operation appeared to do nothing at all. So **tidy-up goes at the head, where an abort cannot reach it**: `ExecuteCommand` fronts its batch with `ReturnToNormalUnlessMoved`, `CommitAndAdvance` with the mode change, `CommitSave` with the mode change (found by Copilot on PR #27). Ordering answers it without giving any effect a "runs anyway" flag, and it does not weaken gotcha 31's observed-not-predicted mode settling: whatever a later effect sets, it sets *after*, so the mode a `:` line ends in is unchanged in every case — the only difference is what happens when the batch never reaches the end. **An abort is not a rollback**; that is the hole clause 4 leaves (gotcha 47). -22. **The import wizard's back path is a history stack, not a reverse table.** `advance` pushes the step it leaves onto `ImportWizard::history`; `previous` pops it, so back always retraces the path actually taken through the conditional transitions (improvise-907). Preconditions live in `validation_error()`, checked on the step where the user can still fix them — never deferred to `build_model`. `NameModel` is terminal for `advance`: Enter there calls `build()`, which parks the workbook in `built` on `Done`, and `AcceptWizardModel` takes it from there. Going back from `Done` discards `built`. **`ResolveDuplicates` appears only when `ImportPipeline::collisions()` is non-empty**: `after_fields` routes there from `ReviewProposals` or `ConfigureDates`, so it comes after the field and date steps and before `DefineFormulas`, with `NameDuplicateCategory` between them when the answer is Separate. Enter there is `WizardOp::ChooseDuplicate` (sum, keep separate, or go back). `ImportPipeline::on_duplicate` defaults to `DuplicatePolicy::Refuse`, so a path that never asked — the headless `--no-wizard` import included — errors listing the collisions rather than summing them. **A collision is a cell written by two records** (a tuple *and* a measure), not a shared tuple: records with disjoint measures on one tuple write different cells and collide with nothing (improvise-9bit). Reasoning: `context/code/formula-and-io.md`, "Colliding records are a question, not a default". -23. **`app.layout` goes stale mid-batch.** `apply_effects` rebuilds only *after* the whole batch, so an effect that reads the layout after an earlier effect changed the grid needs a `RebuildLayout` effect in front of it — the command decides the order (see `EnterEditAtCursorCmd`, `CommitAndAdvance`). Read the cursor's cell text via `App::display_value_at_cursor()`; don't build a `CmdContext` inside `apply` (it reads `layout`, the memoised `grid_geometry()` and the cursor, and a stale layout there is the bug). **Since improvise-felo.13 the rebuild is skipped when nothing it reads has moved**: `App::refresh_layout` rebuilds only when `layout_stale` (any `EffectKind::Model(_)` in the batch, or `install_workbook`), the drawn view's name, or the drill snapshot's pointer (`LayoutBasis`) changed, so a cursor key rebuilds nothing, and the column widths and row-header width are measured once per layout (`App::grid_geometry`, dropped with the layout). Two of the three inputs are **observed** (the view name and the drill pointer); `layout_stale` is **declared** from `EffectKind::Model(_)` (`touches_model`), so gotcha 3's classification hole — a `ViewOnly` effect that changes what the layout reads — now also leaves `app.layout` stale, where before felo.13 every batch rebuilt. It is still not a hand-kept list of effects. A writer that bypasses effects and changes what the layout reads must call `rebuild_layout()` itself. Measurements: `context/test-evidence/layout-cost.md`. - -24. **The cursor is per-client and keyed by view.** `ViewState::cursors: HashMap` holds `selected`/`row_offset`/`col_offset`, keyed by view name; an absent key means the origin. Reach it through `App::cursor()` (a `Copy` snapshot), `App::cursor_mut()`, and `App::reset_cursor()` — never off `View`, which no longer has the fields (improvise-8uz). Two consequences. **Changing the grid's shape must reset the cursor explicitly**: `View::transpose_axes`/`cycle_axis` used to do it themselves, and now `TransposeAxes`/`CycleAxis` call `app.reset_cursor()` — a new geometry effect that renumbers rows or columns has to do the same. **Replacing the document must clear the map**: `LoadModel` and `AcceptWizardModel` call `cursors.clear()`, because the incoming workbook's view names have nothing to do with the outgoing one's. That clearing replaces `Workbook::normalize_view_state`, which is gone — a `View` can no longer carry a stale offset. The same clearing owes the **session mirrors**: `install_workbook` bumps `App::document_generation`, and `ClientSession::project` reseeds any mirror seeded at an earlier one (improvise-cqq). An effect cannot reach `Sessions`, so the generation is the mechanism and `Sessions::rebase` is only the client-facing half. The same is true of a **geometry** change, and it owes the mirrors *both halves*: `App::cursor_epoch` is bumped by `reset_cursor` and watched by `Sessions::adopt_cursor_epoch`, which resets every mirror **and** returns `Broadcast::ToAll(Command::ResetCursor)` (improvise-gvnb). Doing only the first is **strictly worse than doing nothing** — the diff then believes every client already holds origin-positioned content and sends nothing, so they stay stale *permanently* rather than until their next scroll. The trigger is the **epoch moving**, not the command variant, for the reason gotcha 39 watches `document_generation` instead of matching on `LoadModel`: which effects renumber the grid is a per-effect decision, so a table of variants would be a second description of it, free to disagree. What is broadcast is an **invalidation, never a position** — another client's transpose must not move where I am looking (that is UI and it is mine, improvise-g0f), only declare that where I was looking has stopped meaning anything. **Since improvise-ni6k no effect calls `reset_cursor` at all — the cursor is invalidated by OBSERVING the grid's frame.** `apply_effects` diffs the frame after every model effect, **inside the loop**, because a batch that reframes and then places the cursor loses the placement otherwise (four records-mode tests pin that position). There is no list of effects anywhere, so the mechanism is total over `EffectKind::Model(_)` by construction — a genuinely stronger shape than a hand-kept table, and the reason the old "a geometry effect that forgets `reset_cursor` is invisible in both directions" hazard is closed. - - **Two arms, and the asymmetry is the whole design (improvise-mcq4).** `LayoutGeometry` hashes only the **active** view — it is the held-key path, and other views cannot reframe there since `view_mut()` writes only the active view, with the category and item effects already covered by `view_shapes`. `CellData` hashes **every** view, because a cell write can prune a row out of a view the host is not on; narrowing that arm to one view measured as *free* and silently lost a multi-client invalidation, which is gotcha 24's own strictly-worse-than-nothing case. Cost of the wide arm: 0% on the held-key path, +5.7% on an 18k two-view `SetCell`, linear in view count, accepted. State the absolute as well as the ratio: the observation roughly doubles held-key latency on large pruned documents (3.04 → 5.69 ms at 18k) and 56 ms at 90,000 cells is 3.4 frames. - - **Totality is conditional on classification**, and nothing makes classification total: a `ViewOnly` effect that renumbers the grid is invisible to all of this (gotcha 3's hole, unchanged). And two known gaps are filed rather than fixed — a geometry effect immediately after a mid-batch `SwitchView` skips the check entirely, because there is no basis for the new view yet (improvise-9c9o; under-reporting, which gotcha 35 says is the expensive direction), and `Model(WholeWorkbook)` matches neither arm, which is benign only because all three such effects go through `install_workbook` → `rebase` → a fresh `Snapshot`. - -25. **`ViewState` and its field types are `Serialize`/`Deserialize`.** It is the wire payload of the client Snapshot (improvise-q08), so `AppMode`, `MinibufferConfig`, `MinibufColor`, `ViewFrame`, `KeyCode` and `KeyModifiers` all derive both (improvise-a2x), and `improvise-command` depends on `serde` for it. The constraint this puts on new state: **no borrowed-`'static` fields anywhere ViewState can reach** — `Deserialize` cannot produce one, and the error surfaces on every enclosing variant rather than at the offending field. `view_state_survives_a_json_round_trip` guards it. `ViewState` and `Snapshot` also derive `Clone` and `PartialEq`, so the bootstrap message can be compared in a round-trip test. - -26. **Axis presentation lives on `Axis`; its colour is a `Face`.** Four vocabularies hang off the enum in `view/axis.rs` — `plain_label()` ("Row"/"Col"/"Page"/"None"), `glyph()` (which `Display` appends, giving "Row ↕"), `short_label()` (three columns for the tile bar; `None` is "·", *not* a truncation), and `initial()` (one letter for the view panel; `None` is empty). Plus `next()` for the Row→Column→Page→None rotation. **`Axis` carries no colour**: `AxisColor` and `Axis::color()` are gone. A colour is a `Face::Axis(axis)` in `improvise-protocol/src/face.rs` (ADR 004, `context/code/protocol.md` §10), resolved by the `Theme`, and `draw.rs::face_style` is the one place a face becomes a ratatui `Style` — the tile bar and the category panel both call it, so neither keeps a table. `improvise-core` still names no ratatui type. Not covered here: `persistence/format.rs` maps `Axis` to the on-disk keywords ("row"/"column"/"page"), which is a format concern and stays in the io layer. - -27. **`Axis::Filter` constrains without being navigable.** Drilling fixes its coordinates on `Filter`, not `Page` (improvise-dz8): both restrict which records are visible, but only `Page` is something the user may move or page through. The split shows up as two different questions, and picking the wrong one is the trap: **"what data is visible" is `View::filtering_cats()`** (Page ∪ Filter — used by `GridLayout` and by `add-record-row` when it builds a key from the current constraints), while **"what can the user act on" is `categories_on(Axis::Page)`** (used by `[`/`]` page cycling) or `View::tileable_cats()` (used by all three tile-select sites). `Axis::next()` maps `Filter` to itself and no navigable axis rotates into it, so tile select cannot hand one out. The `.improv` grammar accepts `filter` and round-trips a selection with it — a Filter axis only ever occurs in the never-persisted `_Drill` view, but the alternative was writing it out as some other axis. **What a virtual view must *be* is model-side and has one home** (improvise-gd28): `view::{records_spine, records_steps, drill_steps}` return `Vec` — pure data naming no `Effect`, no `App` and no front end — and `cmd/grid.rs::steps_into_active_view` is the whole of *how the TUI gets there*, twenty lines a browser answers its own way. Before the split, the drill's filter placement and the records spine were spelled out twice in `cmd/grid.rs` with nothing keeping them in agreement. `DrillState` deliberately did **not** move: it holds an `Rc` and its `pending_edits` are uncommitted writes, so gotcha 30's session-local tag already places it. Reasoning: `context/code/core.md` §11. - -28. **The browser backend is sync on purpose, with an async seam.** `OpfsFs` (`improvise-io/src/opfs.rs`, wasm32 only) implements the same synchronous `Storage` trait as `PhysicalFs`, because `FileSystemFileHandle.createSyncAccessHandle()` yields a handle whose read/write/truncate/flush **are** blocking — available only inside a dedicated Web Worker, which is where improvise-9x6 puts the app. So `Storage` never became async. What is still async splits out into four explicit methods the worker awaits from its message loop: `open()`, `attach(path)` (must precede any read/write of that document — acquiring a handle is async even though using it is not), `refresh()` (repopulates the index `list` answers from), and `commit_deletes()`. Consequence: **`delete` is visible immediately but lands in OPFS at the end of the batch** — it drops the name from the index and queues a `removeEntry`. Handles are exclusive per file, so `attach` closes any handle it replaces and `detach` releases the lock. `storage_contract` in `storage.rs` is the behaviour every backend owes; it runs against both host backends and is the written-down spec `OpfsFs` must satisfy in a browser, where the harness cannot reach. **And the premise is dedicated-worker-only, which is narrower than this paragraph used to read.** `createSyncAccessHandle` is spec'd `[Exposed=DedicatedWorker]`, and measured in a module `SharedWorker` on Chromium and Firefox it is **absent** — not rejected, not flagged, not transient: `getDirectory()` and `getFileHandle()` both succeed and the method is simply not on the object (improvise-12ha). So `OpfsFs` cannot be hosted by a `SharedWorker` as written, and "a browser worker" in this paragraph means a *dedicated* one everywhere it appears. `OpfsFs::open`'s own guard does not enforce that — it `dyn_into`s `WorkerGlobalScope`, which a `SharedWorkerGlobalScope` **satisfies**, so a shared worker sails past a check whose message names the right requirement and fails later as a raw `TypeError` deep in the glue (improvise-7gip). The consequence for multi-tab is that the machinery everyone reaches for first is the storage seam, not `Sessions`: `Sessions`, the `Addressed` filter and the one `frame` pair are all ready, and none of that helps, because the no is entirely on the handle. - -29. **"Which view is active" is stored twice, and the copies are now allowed to differ.** `Workbook::active_view` is the document's **default** — what a freshly connected client opens, and what `.improv` writes as `Initial View`. `ViewState::active_view` is **authoritative for that client**, and it is what keys `ViewState::cursors`. The second copy exists because `reduce_view` runs on a client that holds no `Workbook` (improvise-cqi); the *split* is improvise-jytm answering improvise-g0f, on the user's ruling that **the model is shared and the UI is not**. Two clients may sit on different views of one document. Three things follow. **`App::resolve_view` is the one answer** to what a view name names in this document, falling back to the default so a slice naming a deleted view cannot panic in `rebuild_layout`; `view()`, `view_mut()`, `view_name()`, `project::grid_in` and `ClientSession::view_in` all go through it, and a second copy of that fallback was the first thing review found. **The guard was replaced, not deleted** — `the_two_active_view_names_never_diverge` became `the_documents_default_view_outlives_this_clients_switches` plus `a_fresh_session_opens_the_documents_default_view`, because a deliberate split with no guard is how one silently regresses into the other. And **a deleted view is the case the split creates**: `DeleteView` can take away the view another client is on, so `Sessions::adopt_deleted_views` moves that mirror to the default and tells it, and `effect::DeleteView` does the same for the host — observed ("does this mirror still name a view the document has"), never predicted from the command variant. **Testing any of it needs a bystander that can be moved**: both audience tests first sat theirs on the default, where a wrongly-addressed relay folds to a no-op, and review demonstrated the vacuity by injecting `ToAll` twice. One gap remains: `Initial View` no longer follows the TUI user (improvise-azhb). The other — a view-scoped `ModelCommand` naming no view — is **closed** (improvise-lltt), and this paragraph described it as an open gap for a day after: every view-scoped variant carries `view: String` and `model_effects` maps it to `ViewTarget::Named`, so a connected client's `SetAxis` lands on the view it names. `ViewTarget::Active` resolves through `App::view_mut()` and is therefore the *host's* view — but **the wire never constructs one**, which is what closed it. Follow mode is deliberately not foreclosed. - -30. **Two tests gate `ViewState` membership.** `ViewState` is the *portable* session slice — the one thing serialized into the Snapshot a newly-connected client gets (improvise-q08). A field may live there only if it is **portable** (plain data from the allowlisted crates, no `Rc`/`Arc`) and **snapshot-safe** (a fresh client inheriting the value is correct). `wizard` fails both — `ImportWizard` is an `improvise-io` type, and a half-open dialog is not something to inherit — and `drill_state` fails both — it holds an `Rc`, and `pending_edits` are uncommitted writes belonging to whoever staged them. Both live on `App` under the **session-local** residue tag (improvise-h0b). The `ViewState: Send` static assertion in `app.rs`'s tests is the compile-time proxy for portability: `Rc` is not `Send`, so a re-added shared pointer fails the build (the `Deserialize` derive from gotcha 25 now rejects it too, serde's `rc` feature being off). `Send` does *not* catch the origin-crate or snapshot-safety halves — check those by hand. - -31. **`:` commands settle their mode by observation, not prediction.** `ExecuteCommand` ends every batch with `ReturnToNormalUnlessMoved(ctx.mode)`, which drops to `Normal` only if the mode is still what the batch started in. There is no `Effect::changes_mode()` any more: it was a per-type constant, and `ViewBack`/`ViewForward`/`StartImportWizard` swap the mode only on some runs, so no up-front answer was right — `:import` opened the wizard and was snapped straight back out (improvise-toc). A new effect that changes the mode needs no annotation; just don't return the batch's starting mode from a success path. - -32. **There is no aggregate render cache — each widget folds its own stream.** `improvise-protocol::render` holds one cache type per widget (`GridCache`, `CategoryTreeCache`, `FormulaCache`, `ViewListCache`, `TileBarCache`), each implementing `WidgetCache` with its own `Update` type, and a `Projection` is one such update addressed to one widget (improvise-rkeq). `reduce_view` folds only `ViewState` now — it no longer takes a cache, and `Command::Project` passes through it untouched. Two consequences. **The overlap between caches is deliberate**: the tile bar and the category panel both carry `(category, axis)` and are updated by separate projections, so neither is woken by the other's changes; do not "fix" it by sharing a variant. **Bootstrap is a *compaction*, not a replay**: `Snapshot { view, seed: Vec }` seeds a new client with a batch of the same projections the steady stream carries, so there is no whole-state type that could disagree with the incremental one. The seed is the server's *current* state, not the head of the history — `GridCache::seed()` builds it, and the property that earns it is `GridCache::folded(&cache.seed()) == *cache`. Nothing is defined for the help overlay (its content is static and `help_page` is already in `ViewState`) or for the import wizard, which now has a shape and **deliberately no projection**: `WizardCache` is built by `project::wizard_cache` and drawn locally, because a half-open dialogue is addressed to precisely the session that opened it, never to every client (improvise-764 — the rendering-side twin of the argument that removed `AcceptWizardModel` from `ModelCommand`, gotcha 38). **A cache without a `Projection` is the shape session-local residue takes when it still has to be drawn by two different front ends.** It lives in `improvise-protocol` rather than `improvise-command` for the `flatten_cat_tree` reason: when the wasm client grows a wizard it must draw *these* rows, not a second derivation. - -33. **The grid's labels frame its cells, and `apply` enforces it — the fold is not commutative.** Three consequences in `improvise-protocol/src/render.rs`. (a) A `GridUpdate::Cells` that overtakes the `RowFrame`/`ColFrame` naming its positions is **dropped**, because `fit_to_labels` truncates to the labels it has; a client must not reorder the stream, and `a_cell_update_that_overtakes_its_labels_is_lost` pins it. (b) A label change that *renames* a position blanks the cell under it: `fit_to_labels` preserves by POSITION, which is only right where the label did not change, so `apply` calls `invalidate_relabelled` itself rather than trusting a producer to send `GridUpdate::Invalidated` first. `Invalidated` therefore now means only "contents stale, frame unchanged". (c) **every arm that touches labels *or* widths re-fits** — `col_widths` is re-fitted with the columns — a new column is drawn at its label's own character count until `ColWidths` catches up — because a renderer zips widths with labels and a length mismatch indexes off the end. Fitting on the label arms alone is what let a late `ColWidths` put the mismatch straight back — a short update keeps each unheard-of column at its label's width, and a long one is dropped as having overtaken the frame that names it, exactly as (a) holds `Cells`. `GridCache` also carries `totals` and `page_coords` (improvise-n10). `totals` is framed by `col_frame` on the same terms as the cells and widths and re-fits with them, with one asymmetry: an **empty** strip means "this grid has no totals" (records mode, or an empty grid) and is therefore *not* padded, where a non-empty one is — padding it would draw a blank total row under a records grid that has none. `Invalidated` blanks the totals too: they are contents, and a stale sum under fresh cells is the one number a user takes at face value. `page_coords` is framed by nothing (it names categories, not positions) and survives a reshape. Note the fold tests: asserting `fold(prefix)+fold(rest) == fold(whole)` is vacuous (it is the definition of a left fold), so the tests that earn their place are the order-sensitivity one and the compaction-equivalence one — see `doctrine(fold_tests_must_test_order_not_the_axiom)`. - -34. **A formula target is unquoted, and pipe quoting has one home.** `parse_formula` strips the pipes off a target exactly as it does off an `Expr::Ref` — the target names a `_Measure` item, so a target that kept its delimiters made `add_formula` create an item literally named `|Total Revenue|`: a ghost column the user's real item never matched, and `Expr::Ref("Total Revenue")` never resolved to. Both the TUI formula minibuffer (`AddFormula`) and `.improv` load go through that one call, so both were affected. The writer emits `Formula::raw`, not `target`, so file text is unaffected either way — which is exactly why the round-trip tests never saw it, and why the fix repairs existing files on next load with no migration. Pipe quoting has exactly **one home**: `improvise_formula::quote`, owning both halves — `quote_name`/`is_bare_name`, `pipe_quote` (unconditional, for a `.improv` text cell that must not read back as a number) and `unescape_pipe_body`. `improvise-gp` re-exports it, `improvise-io` calls it from `persistence/{format,parse}.rs`, `parser.rs::unquote_pipe` is a shim over it, and io's own copy is **deleted** (improvise-i2m5). **The keyword guard is canonical — user-confirmed, so do not revert it as over-caution**: an item literally named `SUM` is written `|SUM|`. Bare keywords do parse, but only because `agg_call` needs a `(` and PEG backtracking falls through to `ref_expr` — rule ordering, not a stated rule — and a `## Formulas` line carries formula text verbatim, so a name `format_md` writes is read back by `parse_formula`: one namespace, one spelling rule. One thing the guard does not cover, and one that used to be an exception and no longer is. **A section header takes its name as the whole line, totally** (improvise-pczv): `header_name = { pipe_quoted ~ &NEWLINE | line_name }` — the pipe-quoted arm needs the lookahead, or `|A|B` would parse as `A` and silently drop `B`, and `line_name` catches everything else, so a header name can no longer fail to parse. Four rules share it (`model_name`, `initial_view`, `category_section`, `view_section`), which is why it is a rule and not four copies. Note a header is still *written* raw where a reference is quoted — the grammar accepts both arms, so the quoting asymmetry between a header and a reference is a writer convention, not a format one. And the two former *unquoting* copies differed only on a trailing lone backslash, unreachable from the grammar, where io returned it and `parser.rs` panicked — the surviving `unescape_pipe_body` is total. Two `## Category:` sections naming one category are **one** category, concatenated by `ParsedFile::push_category` in **pass 1** (improvise-349m) — merging in pass 2 was not idempotent, because `build_category` creates its own section's item-groups before its own `>` lines while `format_md` writes every item before every `>` line. No grammar change was needed: this is the tree-walker, not the format, and duplicate headers stay legal. And **`WHERE` now carries `IS`'s word boundary** — both are compound-atomic with `sp+` (improvise-r08u), so `SUM(Rev WHEREAS x = y)` is a syntax error rather than a `WHERE` on a category `AS`. The guard is `sp+` and **not** the `!bare_char` lookahead the ticket proposed, because the grammar-walking generator reads the grammar through `pest_meta` and **skips predicates**, emitting `WHEREa=""` — a rule whose only guard is a lookahead is a rule the generator can violate. It costs one spelling: a category may no longer abut the keyword. - -35. **`Dirty` over-reports on purpose, and the asymmetry is the reason.** `Dirty::of` maps an `EffectKind` to the widgets that kind can reach, and the two directions do not cost the same: over-reporting pays one redundant `Projection`, under-reporting leaves a client's cache wrong **forever**, with nothing in the stream to correct it. `LayoutGeometry` therefore includes `formulas`, because `RemoveCategory` wears it and `Model::remove_category` ends by dropping every formula targeting the category — a client's formula panel otherwise kept showing a formula the document no longer had. It now coincides with `Dirty::EVERYTHING` and is still spelled out, because the two are derived from different effects and need not stay equal. The derivation now lives in `context/code/command.md`, and the doc comment on `Dirty::of` points there rather than restating it; the reason `LayoutGeometry` is spelled out rather than written as `EVERYTHING` is a two-line comment inside the match body. It went stale once already — claiming `AddCategory` was `CellData` after it was reclassified — so reclassifying an effect means re-reading the prose, not just the comment. - -36. **A client's command has one door, and the door takes the `App`.** `Sessions::apply_checked(app, id, Checked<'_>)` is the only way a connected client's `Command` reaches the server — `Host` calls *that* (improvise-ahdf). `Sessions::apply` is not `pub`, delegates to `apply_checked` after `Checked::against`, and has no production caller: its call sites are tests, which is why it takes a raw `&Command`. `ClientSession::fold` and the mirror's `ViewState` are private, so a transport cannot reach the mirror at all. `apply_switch` now **takes a `SessionId`** — it is addressed to exactly one session, which is the opposite of what this gotcha used to say — moves only that session's mirror, and dirties nothing. The one thing it still asks the document is whether the view exists, because a mirror keys `ViewState::cursors` and is read by `grid_in`. The audience is conditional: an accepted switch relays **nothing** (the asker folded it, and no other client's view moves), a refused one is `Broadcast::To(asker, …)` — improvise-6c6q, where `ToOthers` told every client but the one that was wrong. `adopt_active_view` and `Broadcast::ToOthers` are **deleted**; the variants are `ToAll` and `To(SessionId, _)`, the second because the deleted-view case needs a one-session audience that is *not* the asker. The paragraph's warning about the "what do the others hear" helper survives and is why the 6c6q test drives the asker's own `ViewState` — that helper is itself deleted. What survives unchanged: the model is shared and the UI is not, and `Broadcast` is a `Vec` because one command can owe two relays with different audiences. **The trap this paragraph used to describe is gone with `ToOthers`, and the lesson it taught is not.** It was copying `ToOthers` from `apply_switch` into `apply_model`, invisible from the bystander's side because `adopt_active_view` moved the asker's mirror either way — so the drift sat between the server's picture of the asker and the asker's own slice, where a test written through a "what do the others hear" helper looks away by construction. improvise-6c6q closed the mirror-image hole on the refused path. Two rules survive the mechanism: **assert the `Broadcast` itself**, since the audience is the message rather than something to infer from folded state, and **seat a bystander somewhere a wrong relay can move it from** — a bystander already on the relayed value cannot see a wrongly *widened* audience either, because the wrong relay folds to a no-op. Review demonstrated that vacuity twice on PR #48. A `Vec` rather than an `Option` because one command can owe two relays with different audiences, and an `Option` would have had to drop one silently. Everything else is that client's alone and never touches the `App`'s own view slice: **the model is shared, the UI is not.** Folding `SwitchView` into the mirror alone was the bug — `viewport()` then read the cursor under the new view's key and the next `project()` built that client a grid out of the *other* view. - -37. **`gp-tree` returns data and never prints, and a fit takes two things to reproduce.** Two rules with one root — the crate is a library a TUI, a wasm worker or a CLI may each host. (a) A decision the user should hear about comes back as **data**: `Dataset::warnings` carries the columns `preprocess` dropped for being entirely missing (improvise-d44s), and `fit.rs` hands back pooled `(prediction, actual)` pairs and a score per feature rather than tables. `examples/fit_csv.rs` owns every `println!`, and that is the only reason its output stayed byte-identical across these moves — adding an `eprintln!` to either module is the regression. (b) **The seed is necessary and not sufficient**: `data::Model` records `seed: Option` *and* `config: Option`, because the same seed under a different population/generations/elite/penalty lands on a different tree (improvise-ksst). Both are `Option` for one reason — every `FitConfig` value and the seed `0` are legal, so nothing can double as "unknown". Note `#[serde(default)]` on an `Option` field is **documentation, not the mechanism**: serde defaults it anyway, so deleting the attribute fails no test, and a backward-compatibility test that only deletes it is not discriminating. Aim such a test at an absent field loading as the *default* instead of as unknown. The GA loop itself is `fit::fit` (improvise-0ai); `breed` remains one fitness-agnostic generation for callers whose fitness is not "predict this column". (c) **A scoring function takes one slice of rows.** `Row` pairs a feature vector with what it predicts, so the parallel `x`/`y` lengths that `zip` used to truncate silently cannot be built (improvise-dkqn, `doctrine(an_invariant_is_made_unrepresentable_not_validated)` — the same answer `prune_axis_items` got, and the reason no `Result` entered five signatures). The pairing is made at the boundary that *builds* the data: `preprocess` walks one row index and the rows live inside `Target`, because a `Dataset` carrying `rows` and `target` separately just moves the zip up one level into every caller. The empty row set is one shape, answered once in `mean_over`. Changing any of this needs **bit-level** evidence: `tests/scoring_known_answers.rs` pins every path as f32 bit patterns plus a `next_u64` witness that catches a changed RNG call order, because the CLI prints three or four decimals and byte-identical output alone cannot see a moved accumulation. (d) **A fit is storable only when the document can *name* its features.** `preprocess` invents `Region=North`, `Cost.missing`, `when.year`; those are **encoding artefacts, not items of the document**, and `Expr::Ref` resolves through `Model::find_item_category` — so a tree splitting on one emitted a well-formed formula whose every cell was `Error("ref:…")`, silently (improvise-8xhy). `fitted_formula` therefore takes the `&Model` and refuses, listing what it could not resolve; `formula_text` still spells it, being the tree's own text. **The check asks the document, never the name's shape**, because a measure literally named `Region=North` is a legal document and the two are indistinguishable as strings — which also means `.missing` and the date parts are covered with no second copy of `preprocess`'s expansion rules to drift from. **This is an interim** (improvise-4zl7): a `WHERE` predicate needs a *scalar coordinate predicate* the formula language does not have — `Filter` gates a formula or an aggregate, it is not an `Expr` — would still leave `.missing` and the date parts, and must first answer what such a predicate means where `context.get(cat)` is `None`, since `Formula::filter` falls through to `aggregate_raw` and a categorical fit would silently degrade in every view that aggregates over the category. **improvise-zv8t closed the categorical half of the accident, and the close cost a document.** `fit_measure` handed `preprocess` an `Overrides::default()`, so it never said which header columns are document categories *although the very next line told `Features::of`* — and `preprocess` infers `Numeric → Date → Categorical` from the values, so a `Year` category with items `2024`/`2025` was a *numeric column* and a category of ISO dates a *date column*, both refused with `Year IS 2024` spellable all along. It now forces the same `header[..category_columns]` slice to `Forced::Categorical`. Two things follow. The **date decline is narrower than it read** — it covers a genuine date *measure*, never a category misread as one. And the forcing **widens the encoding**: a hundred-item numeric category one-hots past `MAX_FEATURES` and refuses a fit that previously succeeded by never splitting on it (improvise-e8no — and **not** to be closed by making the forcing conditional on cardinality, which is inference by another name and makes the *encoding* depend on a budget rather than on what the document says). The `Artefact` arms stay: `crate::dataset` is `pub` and takes the caller's `Overrides`, so the roles are still reachable. **Forcing by name while budgeting by index defeats the budget** — a category and a measure may legally share a name, and `no_ambiguous_column_names` ran *after* `preprocess`, so the expansion the preflight exists to prevent was allocated and only then refused; `as_categories` now returns `Result`, so the ordering is a compile error rather than a convention. The test that missed all this is the shape to remember: `improvise_eval` **fabricates** a `_Measure` item per feature name, so it asserted the emitter and `gp_tree::eval` agree against a model that could not exist. It is kept, with its scope now stated, and the property it could not state is asserted separately against a real document. - -38. **The wire vocabulary is derived twice, by the same recipe.** `Command`'s view half is the `EffectKind::ViewOnly` effects targeting the portable slice (`decision(improvise-cqi, command_set_derived_not_invented)`); its model half, `ModelCommand`, is the `EffectKind::Model(..)` effects — **28** of them, brace-matched on `c1772ca9` (2026-09-24) — minus `ApplyAndClearDrill` and `AcceptWizardModel`, leaving **26** variants (improvise-kmgs). The two added since this paragraph counted 26 and 24 are `ReplaceFormula` (improvise-km71) and `RenameRecordCoordinate` (improvise-fdk1); `SwitchView`, `ViewBack` and `ViewForward` left the model half with improvise-jytm. Every count in the rest of this paragraph is dated, so recount rather than cite it. **Completeness is now checked, not counted**: `ui/reduce/vocabulary_is_complete.rs` requires every effect to be on the wire under its own name or listed in `NOT_ON_THE_WIRE` with a verdict — `Command`, `Frame`, `Off` with its argument, or `Gap` naming the ticket that owns it (`SaveAs` and `ExportCsv` are gaps, improvise-2mqz). It is the forward twin of `vocabulary_is_composed.rs`, and it is what the enumeration below lacked. `SwitchView` no longer needs subtracting separately: it is not a model effect any more. `ViewBack`/`ViewForward` left for a reason this paragraph **stated wrongly until 2026-09-21**: it said they went for the *same* reason as the other two exclusions, session-local residue. That is false, and the falseness mattered. `ViewState::view_back_stack` **is in `ViewState`**, which is the portable slice — so by the view half's own rule (`decision(improvise-cqi)`: the `ViewOnly` effects targeting the portable slice) they were always `Command`s and were simply **missing**. The cost of the mis-statement was a live divergence nobody could see: `effect::SwitchView` pushes the leaving frame and clears the forward stack, while `reduce_view`'s arm set `active_view` **and nothing else** — so **no client ever had a back history at all**, and that, not the drill commit the `declined()` comment blames, is why `view-back` was dead in a browser (improvise-gd28, PR #115). **The lesson is about derivations, not about these two variants**: a derivation stated as a rule is only as good as the last time somebody enumerated it *against* the rule. This one had been checked for correctness — every listed variant belongs — and never for **completeness**. Note the obvious `awk` extraction reports **30**: it picks up `HelpPageSet`, which is `ViewOnly`, because that is the last `pub struct` before `#[cfg(test)]` and the test module mentions `ModelChange::` — brace-match the `impl Effect` blocks instead (`doctrine(derive_then_verify)`). Three things follow. **A `ModelCommand` does not mutate**: `ui/reduce.rs::model_effects` maps it back to the effect it came from and `apply_command` runs that through `App::apply_effects`, so effects stay the only mutation site and the `Dirty` is the effect's own `EffectKind`, never a second classification beside gotcha 35's table. **Two model effects are deliberately off the wire.** `ApplyAndClearDrill` and `AcceptWizardModel` were once kept on the grounds that the *command* is portable even where the operand is not; **that was the wrong criterion** (improvise-3x3u). A `Command::Model` is routed with no `SessionId`, which is right exactly when the change is addressed to *no one* session — and these are addressed to precisely one, never the sender: their operands are `drill_state.pending_edits` and the built wizard workbook, session-local residue (gotcha 30) private to whoever staged them (improvise-tom1). Unaddressed, B's `ApplyAndClearDrill` commits A's uncommitted typing and B's `AcceptWizardModel` swaps the document out of a dialogue B never opened — both demonstrated against `Sessions::apply` before removal. So **28** − these two = **26** on `c1772ca9` (it was 26 − 2 = 24 before km71 and fdk1), and the older spelling of this sum (`29 − SwitchView − these two = 26`) was itself a survival of the same staleness this paragraph documents: it subtracted `SwitchView` and forgot its two siblings, so it contradicted the count at the head of the paragraph and the enum’s own 24 variants (found by ChatGPT Codex on PR #116). The effects are untouched and still reached by the key path. Not permanent: multi-client drill wants the *addressed* form, which is where ADR 001 layer 2 goes anyway. Guarded by `no_model_command_commits_another_clients_session_local_state`, plus a companion asserting both effects still exist and are still model changes so the first cannot pass by naming nothing. **`Sessions::apply_checked` routes the model half to `apply_model`**, which relays `adopt_cursor_epoch`'s `ResetCursor` and `adopt_deleted_views`'s per-client switch; a document-replacing `LoadModel` bumps the generation and lands in the rebase arm. The former claim here — that `ViewBack`, `ViewForward` and `LoadModel` all end in `App::switch_to_view` so every mirror takes the accepted active view — was wrong on all three clauses after improvise-jytm and is deleted. A keymap lookup still cannot return a `Command`, and not for want of vocabulary: a binding names an *intent* resolved against `CmdContext` (`move-cursor down`) while a `Command` is the *resolved mutation* (`SetSelected { row, col }`) — improvise-gxi's AC 4 asks for something the split forbids. - -39. **`Applied` is an enum because a document swap is not a diff.** `Sessions::apply_checked` returns `Applied::Diff { dirty, broadcast }`, `Applied::Rebased { snapshots }` or — since ADR 001 layer 1 — `Applied::Rejected { reason }`, and the enum *is* the mechanism: a host that wants the `Dirty` must match, and matching names the case where a `Snapshot` is owed. The server's mirrors self-heal — `ClientSession::project` reseeds on the `document_generation` bump — but `ViewState::cursors` lives on the **client**, keyed by the outgoing document's view names (gotcha 24), and only a fresh `Snapshot` replaces it. `Command::Model` put the three `WholeWorkbook` mutations on the wire (gotcha 38), so the host that must send the snapshots is no longer the host that asked. `apply_model` therefore calls `rebase` itself rather than instructing the host to, exactly as `apply_switch` adopts the accepted view itself. **The trigger is the generation moving, not the command variant**: a `LoadModel` that cannot read its path leaves the document alone, and reseeding on the variant would cost a client its cursors for a *failed* load — `a_load_that_fails_is_not_a_rebase` guards that arm. A `rebase: bool` beside `dirty` was rejected, because the host that forgets to call `rebase` is the host that forgets to read the flag. The **third** case is a *normal outcome*, not an error, and it is in the enum for the same reason the other two are: a host that wants the `Dirty` must match, and matching names it. `ServerMsg::Rejected` is the matching frame, addressed to **the asker alone** — nothing happened, so nobody else is owed anything — and the client keeps it as a `Rejection` rather than a sentence, because force needs to know *which* record and *which* kind (gotcha 43). - **Fixture warning.** When testing a document swap, the incoming document's active view must have the **same name** as the outgoing one. A rename makes `apply_model` broadcast a `SwitchView`; the client folds it, and the stale cursor is then read under a key nothing has written — right *by accident*, with the bug invisible. `other_document()` renames, and a first version of this test passed because of it; `a_document_shaped_differently_under_the_same_view_name()` keeps the name and changes only the shape, so there is nothing to hide behind. - -40. **A batch's basis is three tiers, and only two of them can be read off the effect.** `App` carries `document_generation` (bumped by `install_workbook`), `layout_version` (per *view shape*) and `record_versions` (per *record*) — ADR 001's stale-basis check, and shared document state, so never on `ViewState` (gotcha 30). **The unit is the record**: `record_of(key)` is the coordinate tuple minus `_Measure` (`record_row_identity`), because renaming a category coordinate in wide records mode moves every measure cell in the row (gotcha 1) — a cell-level unit is violated by one legal edit, so the unit must be at least as coarse as the coarsest legal write. **Only the `Dirty` is read off the `EffectKind`; both lower tiers are *observed* around the loop** — the source says so at `apply_effects` and this paragraph said the opposite until improvise-ilvl's audit. The `EffectKind` decides only *whether to look*: `touches_model` arms the shape snapshot, `writes_cells` the record fingerprints. `record_versions` **must** be observed, because an `EffectKind` says *that* a batch writes cells, never *which*; `layout_version` is observed for a better reason than necessity — `observe_reshape` diffs `view_shapes` before and after and bumps only where a view's shape actually moved (improvise-pdvm/nynk), so it is a **fact about the document** rather than a claim about the batch. Two consequences the declared form did not have. An **idempotent** `set-axis` bumps nothing, and batching two real changes into one line or two gives the same `2`. And the gotcha 21 argument is subsumed rather than restated: an aborted batch moved no shape, so there is nothing to not-claim — which is why observing once **after** the loop is correct where accumulating `Dirty` inside it is. **`view_shape` hashes what is drawn, not occupancy** (its doc comment: "Drawn, not configured") — axes, the drawn item names minus hidden ones, folds, page selections and the `prune_empty` *flag* — so an occupancy write does **not** bump `layout_version`: a `CellData`-only batch that prunes a row out of a view moves the cursor epoch (the frame observation) and not `layout_version`, as `context/code/command.md` records (the question that raised was improvise-6nsd, closed). The arming is wider than the tier (`touches_model` takes the shape snapshot), but a bump needs a shape hash to have moved. Three further consequences. A write landing the value already there **bumps nothing**, which is correct — no basis was invalidated. A `LayoutGeometry` effect that drops cells **will** bump their records when it shares a batch with a `CellData` one: an over-bump costing at most an unnecessary rejection, the same direction `Dirty::of` is deliberately sloppy in. And only *raw* data is fingerprinted — formula results live in `Model::formula_cache`, and a record whose derived value moved because someone edited elsewhere has not itself been written. The diff collects into a **set**: chaining `before.keys()` with `after.keys()` counts any rewritten record twice, a bug that survived one test run. - -41. **`Storage` is not a VFS, and that closes the SQLite question.** The trait is whole-document `read`/`write` bytes (gotcha 2, `invariant(storage_is_two_ops)`) — no offsets, seeking, handles or locking — so an `sqlite3_vfs`, which needs `xRead(offset,len)`/`xWrite`/`xTruncate`/`xLock`, has nothing to bind to. SQLite *can* still sit behind it, as an in-memory database serialized to a blob (`rusqlite` `serialize`/`deserialize_read_exact`, proven with a probe before declining — declining on a capability that actually exists would have been the wrong decline). It is worth nothing: measured on 18,000 cells the SQLite container is **22% larger** than `.improv`, **4.7× larger gzipped**, ~**160×** larger at the floor (24 KB against ~150 bytes), and still rewritten whole on every save, so the one dimension SQLite normally wins — incremental page writes — is unavailable. improvise-9ix was declined on that evidence. Two things generalise past SQLite. **A second serialization format is a second implementation of the grammar**, obliged to track `improv.pest` forever; and **its own round-trip test cannot detect drift from `.improv`**, because it never compares the two — that missing cross-format equality test is what made the ticket look cheap. If a structured-query story is ever wanted it is a one-way `export_sqlite` beside `export_csv`, native-only and feature-gated: `libsqlite3-sys`' C build fails under this repo's nix cc-wrapper for `wasm32-unknown-unknown`, which would break the `wasm` CI gate the day it landed. - -42. **The browser client's render decisions are made in Rust, and the frames it speaks are nobody's but the protocol's.** `improvise-wasm-client/src/render.rs` builds a `RenderModel` in which every question with a *silent* wrong answer is already answered, because the DOM half (`web/app.js`) is untested JavaScript and a wrong answer there shows up as a wrong pixel rather than a failing test. Three such answers. **`editable` is a fact about the `CellId` variant** — `Record` only, per ADR 002 — and the trap is the third case: `None` means "no projected cell", **not** "aggregate", so conflating them either hides a cell or offers a write the type system exists to make unconstructible. **The selection is placed by reconciling the scroll offset**, because `GridCache` is viewport-indexed while `ViewState::selected` is absolute. `render.rs` does it by *adding* — `r + cursor.row_offset == sel_row` — rather than subtracting, which is why there is no underflow path and why a selection above the window simply never matches. Omitting the offset entirely looks right until the first scroll, so the test that earns its place uses **nonzero and unequal** offsets: at `selected=(5,1)`, `row_offset=4`, `col_offset=1` it separates "reconciles" from "ignores" *and* from "swaps the two offsets". **The category panel is `viewmodel::flatten_cat_tree`**, never a second walk of `CategoryTreeCache` — that second walk was improvise-edp. The page is four string calls: `Client::viewport` (the handshake — the `Snapshot` seeds the *visible* cells, so one built at extent zero is an empty grid and a resize is a fan-out with nothing dirty), `on_frame` (one frame, one fold, arrival order — gotcha 33 holds *inside* a batch, so never coalesce), `render_model`, `typed_cell`. - - **One envelope pair, two deployments (improvise-xks6).** `improvise_protocol::frame` owns `ClientFrame { Viewport, Commands(Vec<_>), Save, Key }` and `ServerFrame { Snapshot(Box<_>), Commands(Vec<_>), Rejected, Accepted }` (as of `c1772ca9`; this line named a singleton `Command` and no `Save`/`Accepted` for some time after both changed); `improvise-ws-server` and `improvise-wasm-client` each `pub use` them and **neither defines one**. There used to be **three** pairs for two deployments — the socket's server side, a hand-kept *mirror* of it in the browser bundle, and a third written for the worker (improvise-tm6) before the projection architecture landed. The question that had to be answered before any of it moved was whether the **worker** and the **socket** want the same vocabulary, and the answer is **yes**: every difference was a *date* rather than a decision, and the test applied to each was whether the **argument** for it names the thing they differ in. None did. `Viewport` is deployment-independent — the extent is not in `ViewState` and no effect produces it, so no derivation reaches it, and a worker hosting the same `Sessions` owes `set_extent` the same answer; it is the handshake on both because a `Snapshot` seeds the *visible* cells. `Key` is deployment-independent too — a binding names an *intent* resolved against a `CmdContext` a thin client does not hold (gotcha 38), which says nothing about what the bytes travel over; the socket lacked it because key forwarding was never built on it (improvise-bpq2), and `Host::handle` now answers `ClientFrame::Key` with an explicit empty vec that **says so**, where the frame used to just fail to parse in `next_client_msg`. And `Commands(Vec<_>)` batches where a singleton cannot, because a fan-out must arrive whole and in sequence (gotcha 33) — a property of the **fold**, not of the transport; an in-process `postMessage` interleaves exactly as a socket does. The worker pair was not merely different but **incoherent by omission**: written before ADR 001, it could carry `Command::Model { basis, .. }` and had no `Rejected`, so a client speaking it composed a claim that could never be answered. That incoherence — not the difference — is the tell for staleness. - - **The mirror is gone, and so are the two tests that held it honest**; that deletion was the ticket's measure of success. `the_socket_framing_matches_the_server` round-tripped hand-listed fixtures through the other crate's types to catch a **rename**; `every_server_variant_is_one_this_crate_mirrors` matched exhaustively to catch a **variant added on the server and never mirrored**. With one definition both are **type identity**: a rename is a compile error at every call site in both crates, and the un-mirrored variant is not a constructible state. That is stronger than either test, and it is the shape to insist on — **if a consolidation leaves one of its compatibility tests alive, the consolidation is not finished.** What survives is `the_retired_spellings_do_not_parse` in `frame.rs`, aimed at the one thing still possible (the retired `Bootstrap`/`Command` spellings returning as serde **aliases**, which would be two homes again wearing one type), and it keeps the injected-fault discipline this paragraph earned the hard way: **assert the mutated input is still well-formed** before feeding it in, and **assert the error kind** — serde's `unknown variant` — not merely that an error occurred, because a rewrite that also breaks the JSON makes the check pass against an enum that really does have the variant. End-to-end compatibility is now `tests/browser_round_trip.rs`, which hands the wasm client's own serialized frame to a real `Host::handle`; `improvise-ws-server` stays a **dev**-dependency of the browser bundle for exactly that. - - **What the harness does and does not reach is gotcha 48's subject** — `web/app.js` is no longer unrun (`harness/tests/shell.spec.js` executes it under Chromium, and its standalone half under Firefox), improvise-hwpi is closed, and the quiet-failure and never-reads-a-key claims that used to stand here went with it. What remains unexercised is named there: the grouped and multi-level DOM, the CSS and the `ResizeObserver` extent arithmetic. What a careful read *did* establish about the last of these: the observer cannot feed back into itself (`#grid` is a flex child with `overflow: auto`, so `replaceChildren` cannot resize it), and `ROW_PX`/`COL_PX` under-estimate, so the worst case is a row of over-fetch. What *is* verified end to end is the Rust half: `tests/browser_round_trip.rs` drives improvise-1ey's stated scope against a real `Host` over a real `App` touching the client only through the four string calls. One deployment note that is not a bug: `examples/demo.improv` leaves `Customer`/`Date` on the None axis, so **every** cell is an aggregate and nothing is editable — correct behaviour, bad demo. Pick a model with no regular category on None. - -43. **A batch carries its basis, and the server had to publish it first.** `Command::Model { basis, cmd }` is a **struct variant** so the basis cannot be forgotten, and `Sessions::apply_model` checks it *before* applying anything — a stale batch is never applied, not even partially (improvise-xwi6/fkfg, ADR 001 layer 1). Four things to know. **`Basis` has no `Default` and no public constructor**: a zero-valued basis is not "no claim", it is the claim that *the document is fresh*, which in a new `App` is true — so a default would be a forgeable bypass rather than an absence. `BasisCache::claim` is the only door a client has and it returns `None` until the server has published one. Note what that does and does not buy: `Basis` derives `Deserialize` because it must cross the wire, and its fields are public, so **a hand-rolled client can state any tiers it likes**. This is an *anti-staleness coordination check, not a security boundary* — `RecordUnstated` closes **omission**, not **invention**, and nothing here tries to close invention, because a client that forges a current basis is indistinguishable from one genuinely caught up. "The only constructor" is a statement about clients written against this crate, which is still worth having: it makes the honest path the easy one. **Which records are checked is the server's decision** — `reduce.rs::records_named`, an exhaustive match with one or-pattern for the variants that name no record, so a new record-writing variant fails the build. Reading the list off the *client's* claim would be a **force by omission**: the record tier skipped, with no flag to remove and nothing in the message saying so. A claim that omits a record the command writes gets `Rejection::RecordUnstated`, a case of its own because "you lost a race" and "you were never in one" are different things to tell a user. **`Rejection` keeps structural and data staleness apart**, which is the whole reason ADR 002 put a maintained `RecordHash` index in `DataStore`: `RecordGone` means nothing resolves and there is nothing to force (**force never creates structure**), `RecordMoved` means it resolves at another version and force is one keystroke — which is also why the wasm client stores the `Rejection` itself rather than collapsing it into `status_msg`, since a string cannot carry which record or which kind. **And the ADR omitted the publication.** It said a batch carries the basis it was composed against and never said how a client learns one; nothing in the projection stream carried any of the three tiers, so the check as written was satisfiable only by forging. `Projection::Basis` + `BasisCache` close that — the server publishes the two document tiers plus the counters for the records *this client's grid names*, diffed against what that client was last told and seeded in the bootstrap — and it is emitted **after** the grid updates in a batch, because the counters describe the cells those updates carry: a basis arriving first would let a client claim a version for a value it had not been shown, the mirror image of gotcha 33. That ordering was documented as load-bearing in three places and **held up only by the order of two statements** until `a_published_basis_follows_the_grid_updates_it_describes` pinned it (proved by injecting the swap) — the same job `a_cell_update_that_overtakes_its_labels_is_lost` does for the mirror-image rule. Two things are unchecked and deliberately so: `Command::SwitchView` carries no basis (it sits outside `Command::Model`; the workbook refuses a view it does not have, so it cannot corrupt, but it is a hole in the rule), and `apply_command` does not check at all — **the check is a property of the session door, not of the mutation path**, which is right because a host holding the `App` is the truth rather than a claimant. That second one **was** a door rather than an exemption, and needed a type to stop being one (improvise-ondj): both took a wire `Command` and their `Command::Model { cmd, .. }` arm **discarded the basis with `..`**, while `ui/reduce.rs`'s own module doc pointed every future host — the browser worker included — at exactly those two names. **The struct variant closed omission *from the message*; it did nothing to close ignoring *on arrival*, which is the half a struct variant is usually reached for.** So both now take `Checked<'_>`, a private-field newtype over a `&Command` whose only constructor is `Checked::against(&App, &Command) -> Result<_, Rejection>` — also the only caller of `stale_basis`. `Sessions::apply_checked` receives the proof and `Checked::against` is answered **once, before the dispatch**, so `apply_switch`/`apply_model` receive the proof rather than each remembering to ask: an invariant made unrepresentable rather than validated, the same answer `prune_axis_items` and `gp-tree`'s `Row` got. Visibility was the weaker option and was rejected for a concrete reason — `pub(crate)` closes only the out-of-crate path, and `reduce_full` had **no production caller anywhere in the workspace**, so it would have become dead code. It has since been **deleted** — which is the argument’s own conclusion arriving, not a contradiction of it; the surviving sibling is `reduce_key`. This paragraph named it in the present tense for some time after, so read every function name here as a claim to check. **The regression guard is therefore the signature**: the test that reproduced the bug no longer compiles, and what remains asserts the constructor refuses and that refusing leaves the document untouched. One residual hole, documented on the type and filed: a `Checked` does not name *which* `App` it was checked against, so a multi-document host could in principle check one and apply to another. That is also why `ui::reduce::caught_up(&App, cmd)` is not a bypass: it takes `&App`, so nothing reachable from a wire can call it. **Since improvise-955n the frame, not the command, is what is answered — and only two of the three tiers hoist.** `CheckedBatch::against` runs before anything in a `ClientFrame::Commands` is applied. `document_generation` and `layout_version` describe the *document*, so answering them once for the frame is exactly right, and a first version hoisted all three — which was a lost update: the record tier asks "is the client's picture of **this record** current", and which records are at stake is a property of the *command* (`records_named`). Skipping it for later commands let `[SetCell(North), SetCell(South)]` overwrite a South somebody else had moved, silently, with the frame reported accepted, and invisible to the other two tiers because a `SetCell` is `CellData` and moves neither. So every mutation answers for its own records **against the frame's basis** — the one the first basis-carrying command stated, never each command's own field, or a later command could claim a fresher picture than the frame was composed under. The one exemption is stated over a *property* rather than a list of commands: a record that **neither resolves in the document nor is claimed by the basis** is one the frame is about to create (`[AddItem(East), SetCell(East)]`, which `BasisCache::claim` cannot possibly claim for), and only that combination is exempt; a record that does not resolve but *is* claimed is still `RecordGone`. That the `Basis` inside a *later* `Command::Model` is ignored rather than compared is an honest interim: clause 1 says the **frame** states one basis, and moving the field from `Command::Model` to `ClientFrame::Commands` would make that unrepresentable-otherwise instead of a convention — a protocol change that belongs with clause 4. Two things worth keeping from the rejected design: **arity is not a property of a variant** (a future `Command::RunScript` is one variant performing unboundedly many mutations), so no count over the vocabulary can enforce "one mutation per frame"; and **the transaction boundary is not the frame**, because one command may already be a chain, which is why clause 4's mechanism belongs at `App::apply_effects`. - -44. **The TUI grid draws from its cache, and since improvise-anr9 the cache frames it too.** `GridWidget` holds no `&Model` and no `&View`: cell text, aggregate-ness (`CellId::Aggregate`, ADR 002), column widths, the page bar, the totals strip and now the **frame** all come off `GridCache`. The TUI still builds that cache over the **whole** grid (offsets zero), so a cache position is an absolute position — the browser's `Viewport` is a real window, the TUI's is not, and the two clients therefore use it differently; column totals **must** be projected rather than client-derived, because the sum runs over every row *including the ones a viewport scrolled past*. The frame is an `AxisFrame` per axis — `cats` (the categories, outermost first) plus `entries: Vec` interleaving `Group(FrameGroup)` headers and `GroupEnd { level }` markers with `Item { tuple, collapsed }` label *tuples*, exactly as `GridLayout::row_items` interleaves them. That replaced a `Vec` built by `GridLayout::row_label`, which slash-joined the tuple — the exact rendering `two_row_categories_produce_cross_product_labels` says must not appear — so the browser drew `Food/Alice` where the TUI drew two header columns, with no groups and no collapse indicators. It is **one** type and not a `labels` plus a `levels` beside it: two answers to "what are the labels" is the disease (improvise-edp), and this is the second cure, which cost a *type* change rather than an addition. Three things to know before touching it. **`leaf_labels()` is not "the labels"** — it is the innermost label, what a one-line summary calls a row by; a draw loop wants `header_labels(first_rendered)`, `groups()` and `shares_ancestry(sel)`, each of which is one pass, because `item`/`label`/`shows_level`/`group_of` scan `entries` from the start and asking them per position is quadratic in the frame *and grows with the TUI's absolute scroll offset* (found three times on anr9; the per-position forms survive for one-off lookups and say so). **The width rule has one home**, `improvise_protocol::header_level_widths` — widest label per level, plus a gap, clamped — reached by `AxisFrame::header_widths` and by `ui::viewport::compute_row_header_width`, which borrows level tuples off a `GridLayout` rather than building a frame on the keypress path. **A `cats` change re-identifies every position**: a cell's identity is `cats` zipped with its tuple, so `invalidate_relabelled` blanks the whole axis when the categories move, not merely the positions whose tuples differ, and blanks the **totals** on a recategorization of *either* axis — a column total is a sum over the rows, so re-identifying the rows leaves a number answering coordinates the grid no longer has (gotcha 33(b): the rule lives in `apply`, not in a producer's memory). What is still **not** on the cache is `record_col`, and that is correct: it keys pending drill edits and is session-local (gotcha 1). `body_from(idx)` takes an argument only because the two front ends scroll differently — a browser's frame is already windowed and passes `0`, the TUI's covers the whole grid and passes its row offset — not because the rule differs. `group_of` answers group **membership**, and since improvise-e265 membership is **the innermost open run**: `FrameGroup` and `FrameEntry::GroupEnd` each carry a `level` (an index into `cats`), runs **nest**, a header at level L replaces L and closes every deeper run, and a `GroupEnd { level }` closes L and deeper. Before e265 it was the nearest preceding header unless a level-less `GroupEnd` intervened (improvise-gd83, PR #77), which could not express one run inside another. `GroupEnd` draws nothing and exists only so membership is not inherited. `AxisFrame` deserialises through `UncheckedAxisFrame` and **refuses** a `level >= cats.len()` — a level naming no category is a malformed frame, not a clamp. And `FrameEntry::Item` is `{ tuple, collapsed }`: `collapsed` is the **layout's own fold flag**, read by `AxisFrame::folds`, so a client asks the frame whether a position folds rather than re-deriving it from the groups (improvise-4k1j). Both of the gaps this sentence used to list are **closed**, and it named them as open for a day after: `group_runs` coalesces by **header identity** (`std::ptr::eq`), not by name, so two runs under two headers equal as values are two runs (improvise-h0iv); and `AxisFrame::first` carries **no** `serde(default)` — its doc comment says why, since zero is a claim and not an absence (improvise-1wl0). `doctrine(a_stale_list_of_what_is_missing_is_worse_than_no_list)` is about exactly this paragraph, twice now. And `project::grid_of` used to evaluate every cell per frame where the widget evaluated only the visible ones; since improvise-felo.13 the host's own layout keeps a `GridSummary` (whole-grid widths, column formats and totals, `ui/viewport.rs::grid_summary`, held in `App::grid_geometry()`), and `grid_in` evaluates only a `grid_window` over it. The browser's window therefore costs its own cells. A session on another view, or a rederived model, still takes the whole-grid `grid_body` path. `grid_window` equals `grid_body` where the summary is current. Since improvise-jytm `grid_in` may additionally **derive a `GridLayout`** for a session on a view the host is not on; the host's layout is reused where the names agree, which is the common case and the only one that can carry a drill snapshot. - - **`RecordHash` is document-scoped; `AggregateHash` is layout-scoped and cannot be otherwise** (improvise-gd28). A `RecordHash` resolves through `DataStore`'s maintained index — but **only when the command names a view the document has** (improvise-wpxr; Codex P2 on #281). `SetCell`/`ClearCell` carry `view`, and the lookup is **strict**: a view the document lacks drew nothing, stored or not, so it is `Rejection::RecordGone` rather than a `resolve_view` fallback. An **empty** record — one the store holds no cell for, which the grid still drew — resolves against **that named view's layout**, the only place it exists. `RenameRecordCoordinate` is the exception and resolves in the **store alone**, since only a record holding cells can be renamed: `protocol::named_records` answers each command's records as `NamedRecord::Drawn { view, record }` or `NamedRecord::Stored(record)`, one exhaustive match. Every one of these questions goes through `App::grids()`, a `Grids` borrow that memoises **one derived `GridLayout` per view for as long as the borrow lives** (improvise-kimw) — hold one across a check, never across a mutation, since the layouts it holds describe the model it borrowed. An aggregate has no existence outside the layout that derives it, so there is no index to fall back to and none can be built — `App::resolve_aggregate` therefore answers only for the **host's** active view, which since improvise-jytm is not the client's. **That, and not ADR 001 layer 2, is what blocks browser drill.** The ticket's blocking argument was right about the *commit* (`ApplyAndClearDrill` is addressed to one session, gotcha 38) and wrong about the *entry*: entering a drill needs no commit, and `CreateView`/`SetAxis`/`SetPageSelection` are already unaddressed `ModelCommand`s that name their view. The gap is one level down from where it was looked for — **the mutation and its operand have different scopes**: the mutation configures the shared `_Drill` and is addressed to nobody, while the operand exists only inside one session's layout. So the fix is not a new mechanism but the `view: String` convention every other view-scoped variant already follows, resolved against a layout derived for that name (improvise-obtu). Two things travel with it: editing *inside* a drill grid from a browser does not follow from it, because `edit_or_drill` declines records-mode editing out loud before reaching a cell (improvise-fdk1, since closed, is the separate records-coordinate rename on the wire — `RenameRecordCoordinate`, gotcha 38 — and did not unblock drill editing); and closing it makes improvise-t219 live, `_Drill` being one shared view in `Workbook::views` that two clients drilling would contend for — the same masking relationship improvise-6uva has with improvise-up8g. Guarded by `an_aggregate_a_session_was_shown_does_not_resolve_against_the_hosts_view`, whose **control** is the load-bearing half — the same surrogate resolves once the host is moved onto that view, so a blanket `None` cannot pass. - -45. **One session loop, two transports — and the browser saves itself.** `Host` is `App` + `Sessions` + `handle(id, ClientFrame) -> Vec` and lives in `improvise_command::ui::host`, **not** in `improvise-ws-server`, which re-exports it. It moved for improvise-9x6: the worker hosts the same authoritative `App` over a `MessagePort` and cannot depend on tokio, and a second copy would have been exactly the mirror improvise-xks6 had just deleted for the frames — a drifting `Applied`/`Broadcast` match is invisible until a client sees it. `registry` stayed on the server, because a worker hosts **one** document. What the worker adds is only what a `MessagePort` differs in: **one client by construction** (the `Addressed` filter is honoured anyway, against a future `SharedWorker`), and **a handshake with no close** — `ClientFrame::Viewport` must still come first, but a pre-handshake frame is *dropped* rather than closing a connection that does not exist. - - **Saving is a frame, and dirtiness is read off the classification the effect already carries.** `effect::Save` is `ViewOnly` (gotcha 3) — correctly, since it writes *this host's* `file_path` through *this host's* `Storage` — so the derivation that builds `ModelCommand` from the model effects cannot name it, and for a while that meant no client on any transport could ask its host to save. `ClientFrame::Save` closes it (improvise-368h): a **request** carrying no path, because a client cannot see the host's storage, answered `ServerFrame::Rejected(NoDocumentPath | SaveFailed)` to the asker alone and **silently on success** — nothing moved, so no widget is dirty. It is a frame and not a `Command` for one concrete reason: as a `Command` it put a runtime-fallible operation inside a batch `ClientFrame::Commands` promises is all-or-nothing, so `[Save, SetCell]` reported a failed write and then applied the edit. It also fails the test every `Command` passes — it mutates no shared state and no client's slice, so `reduce_view` has nothing to fold and `Checked` has no basis to answer for. **And `MarkDirty` is gone.** It was a separate effect the *command* layer composed beside each mutation — reasonable on the key path, unreachable from anywhere else, because improvise-kmgs derived `ModelCommand` from the effects **one at a time** and so put `SetCell` on the wire without its companion (improvise-gv9r): a derivation correct per element and wrong for a set, whose symptom was that "save if dirty" never fired. `App::apply_effects` now raises the flag from the effect's `EffectKind`, which makes omitting it unrepresentable rather than validated — **declared** for `WholeWorkbook` (a replaced document differs from the file its path names, so a `LoadModel` that cannot read its path still raises it, costing one redundant write) and **observed** for `CellData` and `LayoutGeometry`, where `document_content`/`document_config` fingerprint the document before and after, so a write that leaves it as it was raises nothing (improvise-748f). The observation is read inside the loop, and `Save` clears the flag in loop order, so a batch that edits and then saves ends clean. The worker's persist trigger is therefore the **flag**, not a table of command variants: an observation rather than a second description of which commands change the document, for the reason `cursor_epoch` beats matching on `CycleAxis` (gotcha 24) and `document_generation` beats matching on `LoadModel` (gotcha 39). It is also strictly tighter — a refused batch raises nothing, and neither does a write that changes nothing, so the "was anything rejected" guard went with it. A browser still never *sends* `ClientFrame::Save`, and that is the answer rather than a gap: there is no file the user owns and no Ctrl-S semantics, the document *is* the OPFS entry, so the worker persists on its own behalf through the one existing `Save` effect. The cost that remains: `Storage` is whole-document (gotcha 41), so OPFS rewrites the file per accepted model command. - - **The async seam does not survive being boxed.** `OpfsFs::open`/`attach`/`refresh`/`commit_deletes` are on the struct, not on `Storage`. `App::storage` is a `Box` and a trait object can name none of the four, so **handing the backend to the `App` puts the async seam permanently out of reach**; the host must hold the concrete backend *and* give the `App` a `Storage`, which means sharing one. That is `impl Storage for Rc`, and it lives in **`improvise-io`** rather than in each worker (improvise-hmws): `impl Storage for Rc` is two foreign types in any other crate, so improvise-9x6 wrote a forwarding newtype that every future worker would have written again. `Rc` and not `Arc` because the one caller is a Web Worker and wasm32 has no other thread. **The flat root is no longer load-bearing**: `OpfsFs::list` joins `dir` like the other two backends, so `storage_contract` takes `dir` as a parameter and the three agree at *every* directory rather than only at `Path::new("")` where joining is the identity (improvise-vhcz). What the index holds is **documents, not entries**, and the split between its three writers is the rule: `attach` indexes nothing (what it creates is zero bytes), `Storage::write` indexes the name and un-queues any pending delete of it, and `refresh` rebuilds from OPFS skipping zero-byte entries — counting an entry it cannot measure as a document, since the one cause of that is another tab holding the exclusive handle (improvise-vvl3). - - **What no test here can see** has shrunk, and precisely. `harness/` (improvise-vvl3/xf1) runs `improvise_io::storage::contract::storage_contract` against a real `OpfsFs` inside a real dedicated Web Worker under Chromium and Firefox, and the `browser` job in CI can fail the build — so **`OpfsFs` is no longer compile-checked-and-never-run**, and two bugs fell out of its first execution. That is why `storage_contract` stopped being a `#[cfg(test)]` helper and became a public function returning `Result`: a browser must run *that* sequence, not a second transcription of it, and a failure has to travel out of a worker as a string. Still unseen: **Safari** (Playwright's WebKit has no OPFS, so xf1's criterion is met for two engines of three; `opfs.spec.js` skips on a runtime probe and an unexpected skip on any other engine is a failure). **`bindings.rs` is NOT in that list, and said it was for months.** `shell.spec.js`'s "the standalone deployment" loads the app with no `?ws=`, which spawns the worker, which imports the glue, which calls `bindings.rs::start` and `OpfsFs::open` — and the spec then asserts a grid draws, so the whole chain has run in CI since improvise-vvl3 (Chromium, and Firefox since 2026-09-20). The claim survived in **five** places (this line, the File Inventory, two `kb.pl` facts and improvise-bck's grooming note) because each was written before the harness landed and none was re-read after; the spec's own comment says the chain plainly. It is the reason to distrust a "never run" written in prose: nothing checks it, and `kb.pl`'s CI gate checks consistency rather than truth. What was already verified stays: `tests/post_message.rs` drives a real `Worker` over a real `App` on nothing but the JSON a `MessagePort` carries. Build note unchanged: the worker bundle needs `--enable-nontrapping-float-to-int` **as well as** `--enable-bulk-memory`, because it carries the whole `App` and so chrono's date arithmetic. - -46. **The page picks a peer, and the demo was built against an assertion.** `web/app.js` holds two transports and no protocol knowledge: `?ws=` selects `improvise-ws-server`, its absence spawns `new Worker('./worker.js?doc=&seed=', { type: "module" })` (improvise-djm). `type: "module"` is mandatory — `worker.js` `import`s the bundle — and there is **no `MessageChannel`**: a dedicated worker's own `postMessage`/`onmessage` *is* the port, so the ticket's channel wording was a date rather than a decision. The websocket path is **kept**, because the ws-server is how a TUI-hosted document is shared and nothing else in the browser can reach it; the cost is contained because the two differ only in *where strings come from* — handshake, fold, repaint and keyboard are all above the seam — and **the presence of the parameter is the whole selection mechanism**, a second knob being a second thing to get wrong. `#status` composes **two** facts, the transport's and the document's, because a single source let a repaint silently erase a connection error. **A worker that rejects in `start()` fires no `error` on the main thread** — `worker.js` catches its own — so the guard is a **first-frame watchdog**, deliberately *not* per-transport: "the peer never answered the handshake" is the one failure that means the same thing over a socket and over a port, and a check aimed at each peer's own failure modes would have missed exactly this one. - - **The demo is a model built to satisfy a check, not an axis flipped and hoped for.** `examples/demo.improv` leaves `Customer`/`Date` on None, so every cell is an `Aggregate` and nothing is editable — and the naive fix does not work either, since `Product × Region` by `Date_Month × Measure` still aggregates. `web/demo.improv` puts **exactly one category on each visible axis** (`Product` row, `_Measure` column, `Quarter` page, nothing regular on None) and carries `Profit = Revenue - Cost`, so the payoff is on one screen with no keystroke. **Settability is asked of `grid_of` as a `CellId` variant**, never re-derived from the axes — a second derivation could agree with itself while disagreeing with `render.rs::editable`, which is the whole point of ADR 002 — and the assertion was proved discriminating by pointing it at `examples/demo.improv`, where it names the aggregate at (0,0). **One category per axis is a property of the demo, not of the format**: when this was written `GridCache` slash-joined a multi-level tuple, so a second category on an axis put that known gap on the front page — improvise-anr9 has since closed it (gotcha 44), so what remains is the demo's own simplicity rather than a rendering limit. `web/*.improv` is un-ignored in `.gitignore` (`*.improv` is ignored repo-wide) and the CI assemble step copies it: leaving it out is an empty grid, not a 404 the page could report. One wart it now puts on the front page: a formula cell **is** a `Record`, so it is offered as editable and a write to it reverts on recompute — pre-existing and matching the TUI, newly visible. - -47. **Keys become commands in the browser, and the server must not know the keymap.** The user's ruling on improvise-bpq2, following from "the model is shared, the UI is not" — and it buys two clients different bindings over one document. What made it urgent is **not** latency. `a_key_resolved_on_the_host_moves_state_no_client_can_hear` shows nothing in the projection stream carries the host's own `ViewState`, so host-side resolution **diverges**; and `Host::handle` answers `ClientFrame::Key` with an explicit empty vec, so **an upstream key is a black hole, not a slow path** — every `KeyOutcome::Upstream` names a binding that does nothing at all in a browser. That test still holds and was not flipped: bpq2 closes the divergence by *avoiding* it. **The method is the ticket's** — enumerate the **bindings**, not the commands (`doctrine(binding_dump_diff)`) — and where a client cannot compose the `Command`, **enrich the projection** rather than moving resolution back. - - Three things that cost. **`GridExtent` on `GridCache`** — the whole grid, of which the labels are one window — because a viewport-indexed cache cannot tell "the last visible row" from "the last row". It is deliberately **not framed by the labels** (`fit_to_labels` leaves it alone) and its batch position is not load-bearing the way `RowFrame`-before-`Cells` is, since it addresses no position; the producer reads `GridLayout::row_count()`, and reading it off the projected ranges is **invisible in the TUI** (which builds at offsets zero, gotcha 44) and is exactly the browser bug. **The window is the client's own fact**, recorded by `Client::set_viewport` from what it announced — *not derivable*, since the row frame's own length is the window **clipped to the grid** and so differs precisely at the bottom edge where the scroll arithmetic matters. `Client::viewport` (associated, no recording) still exists and is the one that quietly turns cursor keys off; `web/app.js` does call `set_viewport`. **`ClientFrame::Commands(Vec)`** replaces the singleton, because one cursor key is `SetSelected` plus the offsets `viewport_effects` returns and a host that folds the selection but not the offset projects the wrong window. It is **answered whole, then applied whole** (improvise-955n): `CheckedBatch::against` checks the frame before anything runs, so there is no longer a moment at which part of a frame has landed and the rest has been refused, and `Applied::Rejected` is unreachable through `Host::commands` — still matched, because a host that wants the `Dirty` must name every case (gotcha 39). The contract is four clauses and **all four are now delivered** — clause 4 by improvise-yqpd on 2026-09-21, whose mechanism is `App::snapshot`/`restore` plus a `Sessions` clone taken in `Host::mutating_frame` and put back by `undo_frame`, deliberately *not* inside `apply_effects`, which runs on every keypress. **This paragraph said it was undelivered for a day after it landed**, and an orchestrator brief repeated that from here, so read the clause list as a claim to check. A delivered clause 4 is also what makes the CLIENT's rollback well-defined (improvise-nxah): a refusal means the host applied none of the frame and restored every mirror, so a client keeping its optimistic fold is the only thing left out of step. What rolls back there is `ViewState`, never a cache — a `Command::Model` is a no-op in `reduce_view`, so the optimistic fold never reached one. The old text, kept because the hazard it names is still the right shape to fear: one basis per frame, checked once, applied in order with each command seeing the last one's state, and all-or-nothing. was that the fourth was open because `App::apply_effects` aborts without undoing (gotcha 21 — an abort is not a rollback), so `[RemoveItem("North"), SetCell(the North record)]` would land the removal, resolve the write to nothing, and report the frame accepted having applied half of itself — partial *and* silent. That is what `yqpd` closed, and it closed it **outside** `apply_effects` for the reason the old text implies: the abort lives on the keypress path and must stay cheap. The old note that it was *unreachable* from any client here, the only multi-command producer being `keys::commit`, still reads correctly as a caution — **reachability is a reason not to rush a fix and not a reason not to notice one**, the lost update `CheckedBatch` fixed having had the same reachability argument and none of the documentation, which is how it shipped. - - Two corrections to the ticket's own gap list, both worth more than the code. **Panel cursor movement was never a gap**: improvise-edp had already made `viewmodel::flatten_cat_tree` the one home both ends call, so it is the *shared* derivation, not a second one — worth re-checking the "known gaps" in other old tickets against what has landed since. And the "unmodified characters only" rule was **mis-scoped, not wrong**: it was about `KeyPattern::AnyChar`, which is bound only in the text-entry modes, so `Ctrl-d` in Normal has an exact binding and mirroring it is a faithful subset. Each resolver mirrors its `Cmd` **line for line, including where siblings disagree** — `Move` and `MoveTileCursor` emit even when they clamp, `MovePanelCursor` returns nothing when the cursor does not move. Tidying that into one rule would make the client a second derivation, which is what `tests/lockstep.rs` exists to rule out. What still needs the host is **one list**, `resolve::NEEDS_THE_HOST` in `improvise-protocol/src/resolve.rs` (improvise-uln5) — this line named a list in `keys.rs` and a test called `the_normal_mode_keys_that_still_need_the_host_are_these` for some time after both had gone. Three tests hold it, one per direction: `the_command_names_that_still_need_the_host_are_these` (no **false** entry: each name is bound and really is silent), `a_help_row_the_browser_cannot_run_is_drawn_marked` in `wasm-client/render.rs` (the help overlay **marks** those rows, improvise-uln5), and `every_silent_binding_is_on_the_list` (every binding of **every mode a browser can reach** acts, declines out loud, or is on the list — improvise-oyaa, improvise-s2gb). The last replaced a check read off the help rows, which could not see a binding no help row names. Closing a gap therefore means deleting its entry, and the first test fails until you do. - - **What the browser client decides before a frame leaves it** (all in `improvise-protocol`, so both ends share it). Five rules, each bought by a bug: - - **Every client cache read goes through `projected_at`**, which subtracts the frame's own origin (`AxisFrame::first`) before indexing. A position outside the window the cache holds is **unknown**, never the cell at the same cache index (improvise-7mro) — once the client scrolled ahead of its cache, an index read answered a different cell's value. `unprojected` names that state, and a cell op on it waits rather than guessing. - - **The keyboard commit writes `ClientState::opened`**, not the cursor. `OpenedCell` captures the record, the `OpenedTarget` (`Value { measure }` or `Coordinate { category }`), the record's version, the `document_generation` and the view **at open**, so a commit lands on the cell the editor opened on however the cursor moved since (improvise-8xq0), and claims the basis it was **opened** against rather than one read at commit, which silently overwrote a value another client wrote while the editor was open (improvise-xz2z). A retry after a crossing `LayoutMoved` must match the **whole** `OpenedTarget`, not the record alone, or it wrote whatever cell was under the cursor by then (improvise-8phw for values, improvise-9ww8 for renames). - - **`refused_here` exempts a captured editor** (improvise-3h9w): with `opened` set, `commit-cell-edit` is answered by the capture, so the cursor's current cell cannot refuse a commit aimed elsewhere. - - **A decline that must stop a binding is `Step::Declined`**, not a `SetStatus` beside the binding's other steps: Enter's welded `clear-buffer` otherwise still ran and threw away the typed text (improvise-joed), and a touched editor over a declined cell went `Upstream` with no word at all (improvise-ik6l). - - **`ClientState::notify` queues a client-side notice at the tail of `in_flight`**, as if it were a sent frame, so a rebase that rebuilds the view from `settled` and replays `in_flight` replays the notice too instead of erasing it (improvise-n0pb). `settle_status` runs on **every** path that rebuilds the view (bootstrap, accept, refuse, relay), and assigns a message the editor open as it arrives, so an old message loses its owner once that editor closes (improvise-0dbk). `EditorSession`, the key it is owned by, lives in `improvise-protocol/src/lib.rs` beside `ViewState` so both front ends ask it (improvise-su4u, improvise-1x9h). - -48. **The browser JavaScript is argued *and now partly executed*, and the two rules it can break are both about *when*.** `web/app.js` and `web/worker.js` were syntax-checked, `.d.ts`-checked for call arity and read — never run — and every bug in them was found by reading. `harness/tests/shell.spec.js` is the first execution of either: it runs `app.js` under Chromium against a mocked peer, and it exists because PR #21 declined to add regression tests for its own close-listener fix on the ground that there was nowhere to put them. That decline is only honest because the assertions live there now, and **a change to `web/*.js` belongs in that spec directory** rather than in a second ad-hoc surface beside it. Playwright rather than a headless-DOM shim because what goes wrong in these files is a real Worker, a real WebSocket, real OPFS and real event ordering, none of which jsdom has. Two classes of bug still follow from *when*, both found by review. **An `async` `message` listener does not serialize.** `worker.js`'s `handle()` awaits `server.flush()`, so the event loop may start the next message while an earlier one is suspended, interleaving two `on_message`/`flush` pairs and letting a later frame's replies **overtake** an earlier frame's — which gotcha 33 answers by *dropping* the overtaken update, not by replaying it. Every frame is therefore chained onto **one tail promise** and the listener is deliberately not `async`; that also turns "frames that arrived before `start()` finished" into an ordinary case of the ordering rule instead of a drain loop, which had the same race twice over because `server` was assigned *before* the loop ran. **A watchdog must be disarmed, not pre-empted.** The handshake watchdog exists for one condition — the peer never answered — and every *loud* failure (worker `error`/`messageerror`, socket `error`) has already written something more precise; the timer then fired 8s later and replaced it, so `setShellFailure` clears it. Its neighbour, the socket's `close` listener overwriting a precise `error` with "disconnected", **is fixed** (improvise-taxx, PR #21) — and the ticket's own proposed mechanism was wrong: `readyState` cannot discriminate, an `opened` flag is the only thing that can. PR #21's *other* half is the one that keeps getting lost: it routes **both** close arms through `setShellFailure`, which disarms the watchdog, so a genuine mid-session disconnect is **not** overwritten eight seconds later. A bug report said it was, measured it honestly against a pre-#21 tree, and then "confirmed" it survived #21 by simulating the `opened` flag **and not the routing** — a simulated fix that is not the fix you named reproduces a bug the real code does not have, and reads exactly like a confirmation. The ticket was filed and closed invalid; the property is now asserted by `a socket that opened and then dropped reads as disconnected, and stays that way`, which runs past the 8s window, rather than argued in a comment that was wrong twice. - - **What the harness still cannot see**, and why the argued-not-tested discipline stays: the grouped and multi-level DOM, the CSS, and the `ResizeObserver` extent arithmetic. What a careful read established about the last of these — the observer cannot feed back into itself (`#grid` is a flex child with `overflow: auto`, so `replaceChildren` cannot resize it) and `ROW_PX`/`COL_PX` under-estimate, so the worst case is a row of over-fetch — is still a read and not a test. - - **A third thing no test here can see, which is why the docs must be read as code.** Three comments in one range said "not built yet" about work that had landed *in that same range* — `README.md`'s browser section, `app.js`'s keyboard note, and `Host::handle`'s `Key` arm — each because a follow-up commit updated **one copy of an argument and not its siblings**. When a fact is stated in more than one place, changing it is a grep, not an edit. - - **An evidence heading may be module-qualified, and must be when its leaf is not unique.** `scripts/check-comment-runs.py --check-evidence` takes its Rust names from `cargo test --workspace -- --list` minus the `--ignored` twin, as **full module paths**, and **refuses** an ambiguous leaf naming every candidate rather than warning. Six leaf names collide in this workspace. Note neither half suffices alone: rename one of a colliding pair and the leaf is unique again and resolves silently to the survivor, so qualification is the fix and the refusal is only what forces someone to write it (improvise-rm4s). The residual, filed: `--list` does not name the binary, so two crates sharing a module path are indistinguishable (improvise-sxsf) — **measured latent** (1444 names, zero collisions) and the proposed per-package fix **declined twice over**: 18× the cost (45.2s against 2.4s, and `--check-evidence` needs the `--ignored` twin, so 22 cargo invocations), and *a package is not a binary* — `improvise` alone has five integration test binaries beside its lib, and an integration test's top-level `fn` lists with **no module path at all** (60 of 1444 names are bare), so `-p improvise` prefixes all six identically. Per-**target** is the only shape that would close it. - -49. **A websocket handshake is exempt from the same-origin policy, and that is a server-side problem.** There is no CORS preflight, so any page a user visits can open a socket to `improvise-ws-server` *from that user's own allowed address* and read and rewrite every document it holds — cross-site websocket hijacking (improvise-kc7n). **No network control can see it**: an IP allow-list, a private network and loopback binding all answer "may this **client** reach the endpoint", and the honest answer is yes; only `Origin` says what is **driving** the client. **Loopback is the least protected case, not the safest** — no proxy in front of it, and loopback is not a boundary a browser enforces. `src/origin.rs` is the whole decision and `transport::admitted` its one caller; it refuses the **upgrade** with a 403 rather than accepting a socket and ignoring it, and `serve` **takes** the policy so a host cannot forget to apply it. Three arms are decided rather than defaulted: **loopback allowed** (the shipped local recipe serves its page from `http://localhost:8080`, so the one case worth optimising for must need no flag), **absent allowed** (no browser omits `Origin`, so absent is a non-browser client an address check already answers — and refusing it breaks `tests/socket.rs`, the repo's own instrument), and **two refused** (`HeaderMap::get` is first-wins, so a proxy that *appends* rather than sets leaves two, and answering from the first lets the second be anything). The deployed Pages origin is deliberately **not** hard-coded: it can only reach a plain-`ws://` server through a TLS proxy anyway, so one flag is an honest cost on an already multi-step path where a baked-in default would be a permanent hole for every other deployment. The residual hole is **stated rather than guarded** — a proxy that *strips* `Origin` turns the check off, which is why the proxy's own allow-list stays in `README.md` as defence in depth. **None of this is authentication**: it answers what page is driving a client, never who is connecting, and improvise-kpnr is still open. It correctly does **not** move to `improvise-command` beside `Host` (gotcha 45) — a `MessagePort` has no handshake and no headers, and a worker and its page are the same origin by construction, so this is genuinely transport-only and not a mirror waiting to drift. - - -50. **A composed TUI row measures in cells and names which side yields.** `draw.rs::fill_line` takes a `Keeps`, because the three rows that share a line genuinely disagree about it: the title bar keeps the **right** (the `?:help :q quit` escape hatch outranks an unbounded document name — improvise-t14d), while `minibuffer_line` and `draw_status` keep the **left**, which is what they already did, since right-priority there would let a long status message eat the text the user is editing. So the ticket's "give the right segment priority" was correct only for the title bar, and a global rule would have been the wrong shape. Widths go through `ui::text::truncate`, never `str::len` — and the ticket framed *that* half wrongly too: the title's own `·` separator is two bytes and one cell, so a byte count mis-padded **every** document, not only one with a non-ASCII name. - - **One key table on screen, and it is the one derived from the keymap** (improvise-rr8n). `TileBar` now advertises no keys at all: its literal read `Ctrl+↑↓←→ to move tiles` where those keys are bound to `enter-tile-select`, and the derived status line is a strict superset (`Space`, `h`/`l`, `Tab` appear only there). The ticket asked for the opposite — rewrite the literal, suppress the derived line — and was inverted deliberately, because suppressing a derived superset to preserve a literal that had already rotted keeps the stale table and deletes information. The **import wizard goes the other way**, and the asymmetry is the rule rather than an exception: its status line *is* derived, but from the **mode** keymap while the real keys are per-step inside `HandleWizardKey`, so there the derived table is the one that lies and `draw_status` suppresses the fallback for that one mode. **Keep whichever table is derived from the thing it describes.** Two blind spots make this class invisible: `every_hint_key_resolves_in_its_mode` **cannot see a string literal in a widget** (the same gap improvise-wk18 names for `help.rs`), and the widget's own characterization test renders at **80 columns**, where the hint does not fit and is never drawn — so the one test pinning that output was structurally incapable of seeing either hint. - - **A scroll is measured where it is drawn** (improvise-gtn7.1). The keypress path counted `visible_cols` from `term_width` while `draw_content` gave the grid `term_width` minus the 32-column side panel, so with any panel open the cursor scrolled to a column never drawn. Now `viewport::grid_width(term_width, side_panel_open)` is the one answer, reached through `App::grid_width`, and both `draw_content` and `App::cmd_context` read it; `App::side_panel_open` is the one answer to whether a panel took its column. The tile bar had the same shape: `first_tile` chose its scroll against a width that began one or two columns before where the tiles were drawn, and both now start at `tiles_left(start)`. The grid bug was latent from `b5418f2e`, when `compute_visible_cols` took `term_width`, and was exposed by `50e4a372`, which opened the category panel by default. Still open: columns of mixed width, where a `visible_cols` count taken at the old offset is applied at the new one (improvise-gtn7.2). - - -51. **The command language reads pipes, refuses surplus, and checks names when it applies (improvise-31oe.2, 31oe.3).** Reasoning: `context/code/cli.md` §14 and §15. (a) A `:` argument is read through `improvise_formula::quote`, so `create-view |By area|` names `By area`; a **surplus argument is refused** rather than ignored, and `cmd::core::only_when_omitted` gates the *cursor fallbacks* of `clear-cell`, `yank`, `paste` and `drill-into-cell` so a coordinate typed by the user is never overridden by the cursor. (b) `set-axis`, `set-page`, `toggle-group`, `hide-item`, `show-item` and `switch-view` return `RequireKnown(Needs)` (`effect.rs`) ahead of the real effect, built by `effect_cmds.rs::unless_unknown`. It is `ViewOnly`, writes nothing, is **off the wire** (listed in `vocabulary_is_complete.rs`) and checks the **document at apply time** — a `:` line builds every segment before any runs, so a check in `execute` refused `create-view Second . switch-view Second` — then refuses and aborts the batch (gotcha 21). (c) A refused name therefore stops the rest of a `:` line, while headless `improvise cmd` rebuilds the context per segment and runs each as its own batch (improvise-31oe.14, open). - -52. **A date column imports as one hierarchical category, and the import refuses what it cannot keep (improvise-8jk6.3, 8jk6.6, d0n6).** Reasoning: `context/code/formula-and-io.md` ("Import: …") and `cli.md` §13. The default `DateShape` nests Year > Quarter > Month (the chosen levels) in ONE category named after the column and opens folded to its years; `DateShape::Split` is the old flat split, chosen by `--split-date ` headless or `s` on the wizard's date step. `Plan::spelling_clash` refuses a raw, unparsed date spelled like a generated item or group (`2024`, `2024-01`) because the two would silently merge — asked at the wizard's ConfigureDates step **and** again at build time, so a headless import cannot reach it either. Headless, `--strict` turns a partial skip into a refusal before the build, and a folder whose CSVs are all header-only (no records) is refused as an import that kept no record. +1. **Records mode is wide format.** A row is a coordinate tuple; columns are `[coords…, +cat, raw measures…, +]`. Editing a category column moves every measure cell in the row. `_Dim` carries a column *index*, resolved by `GridLayout::record_col`. `GridLayout::new` does **not** prune; `with_frozen_records(m, v, None)` does — production code wants the latter. `prune_axis_items` takes a predicate and re-marks survivors. +2. **Effects never touch `std::fs`.** Documents go through `App::storage: Box`. Autosave and the headless final write still bypass it (improvise-31oe.11). +3. **Every effect declares an `EffectKind`; there is no default.** `ViewOnly` never reaches other clients; `Model(CellData | LayoutGeometry | WholeWorkbook)` does. The view lives inside the workbook, so axis/page/format are `LayoutGeometry` and cursor/scroll/`SwitchView` are `ViewOnly`; `Add/RemoveCategory` are `LayoutGeometry`. `EffectKind` is too coarse to check the `ModelCommand` mapping — `every_model_command_names_the_effect_of_the_same_name` does. +4. **Commands never mutate.** `&CmdContext` is read-only; return effects. +5. **`CellKey` is always sorted.** Use the constructors. +6. **No `Model::add_item`.** Use the `Workbook` facade (`wb.add_item`) or go through the category. +7. **Virtual categories.** `_Measure` items = explicit ∪ formula targets (`measure_item_names()`). Cell aggregation (`apply_agg`) and formula `SUM/AVG` (`eval_agg`) differ on empty input; non-numeric values reduce by unanimity, else `Error("mixed")` — never dropped. `evaluate_aggregated` takes `Aggregation::{Exact, OverHidden}`. +8. **Display rounding is view-only.** `format_f64` only in rendering; never feed a rounded value back into eval. +9. **Formula eval is fixed-point.** `recompute_formulas` iterates to stability; cycles become `Error("circular")`. +10. **Drilling a formula cell** strips the `_Measure` coordinate so the raw feeding records come back. +11. **Minibuffer keymaps come from `TextEntry`** (eight text-entry modes); the buffer a prompt names and the keys edit are one string by construction. Read/write a runtime-named buffer with `read_buffer`/`write_buffer`. `SearchMode` is hand-written. +12. **Panel keymaps come from `PanelKeymap`**; `Panel::ALL`/`name()`/`key()` drive the toggle keys. +13. **A prefix key is stored as an address** (`ViewState::transient_prefix`), resolved by `Keymaps::prefix_keymap`; prefixes must stay flat (`prefix_keymaps_are_flat`). `ViewState` stays free of `improvise-command` types. +14. **Keybindings are per-mode** (`ModeKey::from_app_mode`); minibuffer Enter/Esc are `Binding::Sequence`. `every_hint_key_resolves_in_its_mode` checks which keys a hint names, not what they do. +15. **Registry names come from prototypes, not strings** — except the `Move` literals and four aliases, guarded by tests; a wrong-variant prototype shows up as a *duplicate* registration. +16. **`effect_cmd!` / `simple_effect!`** wrap one effect; `simple_effect!`'s kind argument is mandatory. Six wrappers return `unless_unknown(...)` (gotcha 51). Mode entry is `enter-mode `; no per-mode shim commands. +17. **One float epsilon** (`FLOAT_EQ_EPSILON = 1e-10`) for `=`/`!=` *and* the div/0 guard, in both evaluators. +18. **Order is data.** `IndexMap` order is persisted; an index of `HashSet`s is for lookup, never iteration (`DataStore::matching_*` sort to store order; `analyze_records` is first-seen order). A group's fold reads through the same rule (`spanned_positions`). +19. **Cell-edit commit paths split** (`commit_regular_cell_value` / `commit_plain_records_edit`); keep both in sync. `cmd::core::uneditable` is the one decline, asked at open *and* commit. `CellId::NonItem` is a stored value naming no item. +20. **Minibuffer commits go through `commit-buffer `**; the add effect is emitted last so its failure message wins. +21. **An abort stops the chain, never the bookkeeping.** `Save`/`SaveAs`/`AddCategory` set `abort_effects`; tidy-up (mode restore, buffer clear) goes at the *head* of a batch. An abort is not a rollback (gotcha 47). +22. **The import wizard's back path is a history stack.** Preconditions live in `validation_error()`; `ResolveDuplicates` appears only when `collisions()` is non-empty; headless defaults to `DuplicatePolicy::Refuse`. A collision is a cell written by two records. +23. **`app.layout` goes stale mid-batch** — put a `RebuildLayout` effect before any effect that reads the layout after a grid change. The rebuild is skipped when nothing it reads moved (felo.13); a writer bypassing effects must call `rebuild_layout()`. +24. **The cursor is per-client, keyed by view** (`ViewState::cursors`; use `App::cursor()/cursor_mut()`). Replacing the document clears the map and bumps `document_generation`. Cursor invalidation is *observed* from the grid frame (`cursor_epoch`), not called by effects; two arms (`LayoutGeometry` hashes the active view, `CellData` every view). +25. **`ViewState` and its field types derive `Serialize`/`Deserialize`** — no borrowed-`'static` fields anywhere it reaches. +26. **Axis presentation lives on `Axis`; colour is a `Face`** resolved by the `Theme`; `draw.rs::face_style` is the one translation to ratatui. +27. **`Axis::Filter` constrains without being navigable.** Visible data = `filtering_cats()`; user-actionable = `categories_on(Page)` / `tileable_cats()`. What a virtual view must be is model-side (`records_steps`, `drill_steps`). +28. **The browser backend is sync on purpose, with an async seam** (`OpfsFs`; `open/attach/refresh/commit_deletes`). It needs a *dedicated* worker — `createSyncAccessHandle` is absent in a `SharedWorker`. +29. **"Active view" is stored twice and may differ:** `Workbook::active_view` is the default, `ViewState::active_view` is authoritative per client. `App::resolve_view` is the one fallback; `Sessions::adopt_deleted_views` handles a deleted view. +30. **Two tests gate `ViewState` membership:** portable (no `Rc`/`Arc`) and snapshot-safe. `wizard` and `drill_state` live on `App` as session-local residue. +31. **`:` commands settle their mode by observation** (`ReturnToNormalUnlessMoved`), not by a per-effect constant. +32. **There is no aggregate render cache.** One cache per widget (`WidgetCache`), a `Projection` is one update to one widget; bootstrap is a *compaction* (`Snapshot { view, seed }`). The wizard has a cache but deliberately no projection. +33. **The grid's labels frame its cells; the fold is not commutative.** A `Cells` update that overtakes its frame is dropped; `apply` invalidates relabelled positions itself; every arm touching labels or widths re-fits. Don't coalesce a batch. +34. **A formula target is unquoted; pipe quoting has one home** (`improvise_formula::quote`). The keyword guard is canonical. `header_name` is total; duplicate `## Category:` sections are one category; `WHERE` needs `sp+`. +35. **`Dirty` over-reports on purpose** — a redundant projection is cheap, a missed one leaves a client wrong forever. `LayoutGeometry` includes `formulas`. +36. **A client's command has one door:** `Sessions::apply_checked(app, id, Checked)`. Assert the `Broadcast` itself, and seat a bystander where a wrong relay can move it. +37. **`gp-tree` returns data and never prints; a fit needs seed *and* `FitConfig`.** A scorer takes `Row` slices. `fitted_formula` refuses features the document cannot name; categories are forced `Categorical` (zv8t). +38. **The wire vocabulary is derived twice** (`Command` view half, `ModelCommand` model half). Completeness is checked (`vocabulary_is_complete.rs`). `ApplyAndClearDrill` and `AcceptWizardModel` are deliberately off the wire (session-local operands). Recount, don't cite. +39. **`Applied` is an enum (`Diff | Rebased | Rejected`)** because a document swap is not a diff; the trigger is the generation moving, not the variant. A swap fixture must keep the view *name*. +40. **A batch's basis is three tiers** (`document_generation`, `layout_version`, `record_versions`); the unit is the record; lower tiers are *observed*, not read off the effect. +41. **`Storage` is not a VFS;** SQLite was declined on measurement (a second serialization is a second grammar). +42. **Browser render decisions are made in Rust** (`RenderModel`: `editable` from the `CellId` variant, selection reconciled with offsets, `flatten_cat_tree`). `improvise_protocol::frame` is the one envelope pair for both deployments. +43. **A batch carries its basis, and the server publishes it** (`Projection::Basis`). `Basis` has no `Default` or public constructor; `Checked`/`CheckedBatch` make "ignoring the basis" unrepresentable; `Rejection` keeps `RecordGone` / `RecordMoved` / `RecordUnstated` apart. Anti-staleness, not security. The check is a property of the session door (`apply_command` does not check), and only two of the three tiers hoist per frame — the record tier is asked per command (`records_named`) against the frame's basis. Read gotcha 43 before touching it. +44. **The TUI grid draws from `GridCache`**, which frames it too (`AxisFrame`; width rule `header_level_widths`). Use the one-pass accessors, not per-position scans. `RecordHash` is document-scoped, `AggregateHash` layout-scoped. +45. **One session loop (`Host`), two transports.** Saving is a request frame (`ClientFrame::Save`); dirtiness is *declared* for `WholeWorkbook` and *observed* for `CellData`/`LayoutGeometry` (`MarkDirty` is gone). `impl Storage for Rc` lives in `improvise-io`. `harness/` runs `OpfsFs` for real; Safari is unseen. +46. **The page picks a peer:** `?ws=` → socket, else a module `Worker`. `#status` composes two facts; the first-frame watchdog is per-peer-agnostic. The demo model is built so every cell is a `Record`. +47. **Keys become commands in the browser;** the server must not know the keymap. `GridExtent`, `set_viewport`, `Commands(Vec)`, `resolve::NEEDS_THE_HOST` and the five client rules (`projected_at`, `opened`, `refused_here`, `Step::Declined`, `notify`). +48. **The browser JS is partly executed** (`harness/tests/shell.spec.js`); changes to `web/*.js` belong there. Serialize frames on one tail promise; disarm, don't pre-empt, a watchdog. Evidence headings may be module-qualified (and must be when the leaf collides). +49. **A websocket handshake is exempt from same-origin policy** — `src/origin.rs` refuses the upgrade (loopback and absent allowed, two refused). Not authentication. +50. **A composed TUI row measures in cells and names which side yields** (`fill_line` + `Keeps`). Keep whichever key table is *derived from what it describes*. Scrolls are measured where drawn (`grid_width`). +51. **The command language reads pipes, refuses surplus, and checks names when applied** (`only_when_omitted`; `RequireKnown`/`unless_unknown`). +52. **A date column imports as one hierarchical category;** `--split-date` restores flat. `Plan::spelling_clash` refuses what would silently merge; `--strict` refuses partial skips. --- -## File Inventory +## File Map -Line counts are a **dated snapshot**, not a maintained fact: measured 2026-09-24 at `c1772ca9`, and **70 of 105 had drifted past 10%** in the three days since the 2026-09-21 refresh — eleven had doubled or more (`client.rs` 255 → 943, `lockstep.rs` 1126 → 4064). Treat them as "is this file big", never as a figure to cite; re-derive with `wc -l`. The `ws-server/` and `wasm-client/` entries use a shorthand prefix, so a refresh must map it to `crates/improvise-/{src,}/` by hand or they drift silently. Test counts are informational — run `cargo test --workspace` for live numbers. Files under 100 lines and render-only widgets omitted. +Per-crate entry points. Per-file notes, test counts and a dated line-count snapshot are in +`context/file-inventory.md`; re-derive sizes with `wc -l`. -### `improvise-core` (`crates/improvise-core/src/`) -``` -model/types.rs 3560 / 112t Model, formula eval (fixed-point), CRUD -model/cell.rs 1358 / 47t CellKey (sorted), CellValue, DataStore (interned + index), - live_records (insertion order — gotcha 18) -model/record.rs 226 / 8t RecordHash / CellId surrogates: the opaque identities a - thin client names a cell by (ADR 002) -model/category.rs 310 / 9t Category, Item, Group, CategoryKind -model/symbol.rs 89 / 3t Symbol interning -view/layout.rs 2604 / 57t GridLayout, drill, records mode, records_columns; a private - per-axis position index (`row_pos`/`col_pos`, rebuilt by `reindex`) - so a data position resolves in O(1) (improvise-felo.13) -view/types.rs 817 / 43t View config (axes, pages, hidden, collapsed, format) -model/id.rs 290 slugify + IdAllocator: how a generated id is made (ADR 007 - clause 4, core.md §20). Nothing outside the file calls it yet -view/axis.rs 190 / 5t Axis enum + label/next methods (no colour: that is a Face) -workbook.rs 413 / 22t Workbook: Model + cross-view ops, item facade -format.rs 519 / 40t format_f64, parse_number_format (display only) -``` - -### `improvise-formula` (`crates/improvise-formula/src/`) -``` -parser.rs 1394 / 79t pest grammar + tokenizer → Formula AST -quote.rs 160 / 7t quote_name / is_bare_name / pipe_quote / unescape_pipe_body: - the one home, both halves (improvise-i2m5). improvise-gp - re-exports it; improvise-io calls it from - persistence/{format,parse}.rs -formula.pest 108 PEG grammar for formula text -ast.rs 130 Expr, BinOp, AggFunc, Formula, Filter -``` - -### `improvise-io` (`crates/improvise-io/src/`) -``` -storage.rs 334 / 13t Storage trait (read/write/list/delete), PhysicalFs, MemoryFs, - `impl Storage for Rc` (improvise-hmws), and - contract::storage_contract — public, not cfg(test), because - a browser must run that sequence (harness/) -opfs.rs 289 / 0t OpfsFs: Storage over the browser's Origin Private File - System. wasm32 only. Run by harness/, not by cargo test -persistence/improv.pest 179 PEG grammar — single source of truth -persistence/mod.rs 441 / 11t document lifecycle: save/load/gzip/legacy-JSON over Storage; - facade re-exporting format_md/parse_md/export_csv* -persistence/format.rs 401 / 11t format_md: Workbook → .improv text -persistence/parse.rs 1684 / 46t parse_md: pest tree-walk → Workbook (+ grammar generator) -persistence/export.rs 243 / 5t export_csv / render_csv. Builds its layout with - with_frozen_records, NOT new — `new` skips prune_empty - (improvise-iot6) -persistence/roundtrip_tests.rs - 1236 / 62t format_md ↔ parse_md characterization (tests only) -import/wizard.rs 1967 / 62t ImportPipeline + ImportWizard. `Plan::place` is the one - placement pass, shared by `collisions()` and the build so - the two cannot disagree; `DuplicatePolicy` / `Collision` - (improvise-cgqt) -import/analyzer.rs 434 / 17t Field kind detection (Category/Measure/Time/Skip); - analyze_records is first-seen order, not HashSet (gotcha 18) -import/csv_parser.rs 590 / 26t CSV parsing, multi-file merge -``` - -### `improvise-protocol` (`crates/improvise-protocol/src/`) -``` -lib.rs 619 / 2t ViewState, ViewCursor, ViewFrame, AppMode (17), Panel, - MinibufferConfig, MinibufColor, HELP_PAGE_COUNT -reduce.rs 749 / 10t Command + ModelCommand (the wire format), HelpPage, reduce_view -render.rs 2157 / 46t CellDisplay, WidgetCache trait + one cache per widget - (GridCache/CategoryTreeCache/FormulaCache/ViewListCache/ - TileBarCache), AxisFrame/FrameEntry/FrameGroup and - header_level_widths (the grid frame, gotcha 44), - GridCache::seed (the bootstrap compaction), Projection, - bootstrap-only Snapshot -keymap.rs 975 / 0t KeyPattern, Binding, Keymap, ModeKey (17), TextEntry, - PanelKeymap, Keymaps::default_keymaps/prefix_keymap — the - shared table both front ends look up. Tests live with - dispatch in improvise-command/src/command/keymap.rs -viewmodel.rs 170 / 5t flatten_cat_tree — derivations between a cache and a - renderer, called by both front ends (improvise-edp) -face.rs 682 Face (a role: Axis, WizardField, Mode*, MinibufferOf…), Theme, - AnsiSlot — appearance as data a front end applies (ADR 004, - protocol.md §10); draw.rs::face_style is the TUI's one translation -key.rs 96 / 3t KeyCode, KeyModifiers, KeyEvent (moved from command/) -frame.rs 156 / 3t ClientFrame/ServerFrame — the ONE envelope pair, both - deployments (improvise-xks6). Nothing else defines one. - ClientFrame::Save is a request, not a Command (gotcha 45) -basis.rs 257 / 6t Basis (the three tiers), Rejection, BasisCache — ADR 001 - layer 1. Basis has no Default and no public constructor -help.rs 1241 / 14t HelpPage content and Hint { keys: Vec<(KeyCode, KeyModifiers)>, - command, caption }. `compact` drops a chord by its MODIFIERS - (anything beyond shift), never by parsing a label it just - formatted (improvise-thpe) — the guard is the signature -resolve.rs 4883 / 88t the browser's key resolvers: a BrowserKey to the Commands it - composes, mirroring each Cmd line for line INCLUDING where - siblings disagree (gotcha 47). The largest file in the crate -client.rs 943 / 15t ClientState: ViewState + one cache per widget, and the fold; - in_flight, `opened` (OpenedCell), notify, settle_status - (gotcha 47). EditorSession is in lib.rs beside ViewState -``` - -### `gp-tree` / `improvise-gp` (`crates/gp-tree/`, `crates/improvise-gp/`) -``` -gp-tree/src/lib.rs 575 / 8t Tree, Spec, eval, simplify, render, breed, Population, - SplitMix64 (vendored seeded Rng; seed() keeps the seed so - a fit can be shown and recorded, pinned to the published - reference vectors) -gp-tree/src/data.rs 1000 / 21t raw string rows -> f32 feature vectors: per-column type - inference, one-hot, mean-imputation + col.missing flag, - ColumnSpec/encode_row, Dataset (incl. warnings), - Model = trees + plan + seed + FitConfig -gp-tree/src/fit.rs 961 / 32t the search: FitConfig (Default = the CLI's numbers), fit() - -> simplified champion, cv_regress/cv_classify, - permutation_importance, folds/standardize/mse. - Returns data; never prints -gp-tree/examples/fit_csv.rs 583 CLI over the above: flags, demo dataset, all printing -gp-tree/tests/scoring_known_answers.rs - 250 / 6t f32 bit patterns + a next_u64 witness: the evidence any - change to the scoring paths needs (gotcha 37c) -improvise-gp/src/lib.rs 978 / 25t formula_text / fitted_formula: a Tree as IF(...) formula - text, validated back through parse_formula. A threshold is - the SHORTEST decimal that reads back as the same f64 (not - the exact expansion), so a row on a split boundary routes - the same in f64 as in f32 eval; non-finite values panic -improvise-gp/src/dataset.rs 506 / 15t a View's records -> the feature matrix a fit consumes — - the improvise-side counterpart of preprocess. Records mode - is already a feature matrix (gotcha 1) -improvise-gp/src/fit.rs 1778 / 35t one fit end to end: a View's records in, an improvise - Formula and the evidence for believing it out. Not in a - command, because none of it is terminal-specific -``` - -### Command layer (`crates/improvise-command/src/command/`) -``` -cmd/core.rs 475 / 2t Cmd trait, CmdContext, CmdRegistry, parse helpers -cmd/registry.rs 903 / 13t default_registry() — all registrations -cmd/navigation.rs 507 / 12t Move, EnterAdvance, Page* -cmd/cell.rs 724 / 20t ClearCellCommand, YankCell, PasteCell, TransposeAxes, SaveCmd -cmd/commit.rs 1304 / 27t CommitAndAdvance, CommitBuffer(CommitKind), CommitExport, - CommitSave -cmd/effect_cmds.rs 519 / 6t effect_cmd! macro, 25+ simple wrappers, `unless_unknown` (gotcha 51) -cmd/fit.rs 293 / 8t fit-measure: evolve a tree over the active view and store it - as an ordinary `## Formulas` line — nothing GP-specific is - left in the document afterwards -cmd/grid.rs 988 / 22t ToggleGroupAtCursor, ViewNavigate, DrillIntoCell, TogglePruneEmpty -cmd/mode.rs 469 / 13t EnterMode, Quit, EditOrDrill, EnterTileSelect -cmd/panel.rs 936 / 24t Panel toggle/cycle/cursor, formula/category/view panels -cmd/search.rs 202 / 4t SearchNavigate, SearchOrCategoryAdd, ExitSearchMode -cmd/text_buffer.rs 399 / 14t AppendChar, PopChar, CommandModeBackspace, ExecuteCommand -cmd/tile.rs 170 / 6t MoveTileCursor, TileAxisOp -cmd/wizard.rs 244 / 6t HandleWizardKey — per-step import-wizard key tables -keymap.rs 1353 / 44t KeymapSet, Keymap::dispatch/dispatch_transient, effects_of, - SetTransientPrefix — what a thin client cannot have. The - table itself is improvise-protocol/src/keymap.rs -parse.rs 350 / 23t Script/command-line parser (prefix syntax) -``` - -### Semantic UI state (`crates/improvise-command/src/ui/`) -``` -ui/effect.rs 4020 / 99t Effect trait. 63 effects: 53 `impl Effect for` blocks, 9 - simple_effect! expansions, and SetTransientPrefix in - command/keymap.rs. The 10 outside the brace-match are all - ViewOnly; 28 of the 53 are Model(_) (gotcha 38, h3y7) -ui/app.rs 5558 / 134t App, ModelState, ViewState + dispatch methods, Panel, handle_key, dispatch_key, document_generation, autosave; apply_effects declares `dirty` for WholeWorkbook and observes it for CellData/LayoutGeometry (gotcha 45) -ui/project.rs 6383 / 118t Dirty, Viewport, grid_of/category_rows/formula_texts/ - view_summaries/tile_entries, grid_updates (the diff), - ClientSession + Sessions (connect/apply/rebase/project_all) -ui/reduce.rs 1217 / 24t Checked / CheckedBatch / apply_command / model_effects / - reduce_key — one door from a Command or a keypress to the - authoritative App. `reduce_full` is GONE (improvise-h3y7) -ui/host.rs 3410 / 67t Host: one App + one Sessions; handle, commands, project_all, - rebase. Lives here, not in ws-server (gotcha 45) -ui/cat_tree.rs 174 / 8t Category tree flattener for panel -ui/viewport.rs 259 compute_col_widths / compute_row_header_width / visible_cols - (pure geometry; the width rule itself is the protocol's); - grid_width, SIDE_PANEL_WIDTH, MIN_GRID_WIDTH (improvise-gtn7.1); - GridSummary / grid_summary / grid_window (improvise-felo.13) -``` - -### TUI rendering, draw, main (`src/ui/`, `src/draw.rs`, `src/main.rs`) -``` -ui/grid.rs 905 / 14t GridWidget (ratatui): no &Model, no &View — the frame comes - off GridCache (gotcha 44). Its module doc is the best-written - statement of that rule in the repo -ui/help.rs 277 5-page help overlay (render only) -ui/import_wizard_ui.rs 829 / 9t Import wizard rendering. `Panel` is the one writer, and - every write is clipped to the panel's rows and width -ui/text.rs 169 truncate and wrap, both measured in terminal cells, never bytes -draw.rs 1122 / 20t TUI event loop, frame composition; `content_tests` draws a - whole frame to check the selection beside a side panel -main.rs 428 CLI entry (clap): open, import, cmd, script -# other ui/*.rs are small panel renderers — skip unless changing layout/style -tests/grid_frame_agreement.rs - 479 / 6t the two front ends draw the same grid frame (improvise-anr9) -tests/records_wide_smoke.rs 67 / 1t -tests/shipped_documents_load.rs 333 / 8t every TRACKED .improv/.improv.gz loads, and every record - it projects is writable. "Shipped" is read off `git ls-files`, - not a .gitignore reading (improvise-c0aj), and the walk's git - is scrubbed of every hostile GIT_* setting the caller may carry -tests/web_demo_model.rs 83 / 2t every visible cell in web/demo.improv is a Record -``` - -### Browser host and client (`crates/improvise-ws-server/src/`, `crates/improvise-wasm-client/src/`, `web/`) -``` -ws-server/registry.rs 616 / 14t Registry of Hosts keyed by DocumentId; ClientId {document, - session} — a SessionId is unique only within its document. - A document is never evicted while the server runs (ADR 001) -ws-server/host.rs 4 / 0t a re-export shim only. The session loop is - improvise-command/src/ui/host.rs (gotcha 45) -ws-server/origin.rs 132 / 6t OriginPolicy: which origins may open a socket. Loopback by - default, --allow-origin widens, absent allowed, two refused - (improvise-kc7n). The only decision transport::admitted makes -ws-server/transport.rs 330 / 4t tokio + tungstenite plumbing, plus the Origin check at the - handshake — a refused upgrade, not an ignored socket -ws-server/main.rs 190 / 5t CLI entry: serve a file on a port; --bind, --allow-origin -ws-server/tests/socket.rs 389 / 6t the one test that uses a real socket: handshake, JSON - framing, SessionId routing, which document a URL asked for -wasm-client/lib.rs 455 / 8t Client: the wasm-bindgen surface (init, on_message, on_frame, - viewport/set_viewport, render_model, set_cell_at, typed_cell) -wasm-client/keys.rs 187 / 5t BrowserKey ONLY — a KeyboardEvent flattened to an improvise - key. It does NOT resolve: `resolve` and KeyOutcome live in - improvise-protocol/src/resolve.rs, shared by both front ends -wasm-client/render.rs 1249 / 21t RenderModel: editable / selected / cat tree, decided here -wasm-client/socket.rs 547 / 13t the fold (fold_frame/handle_frame), the two compositions - and typed() text→CellValue. Defines no enums -# state.rs is GONE — ClientState moved to improvise-protocol/src/client.rs -wasm-client/tests/browser_round_trip.rs 2136 / 34t improvise-1ey's scope against a real Host -wasm-client/tests/lockstep.rs 4064 / 78t host and client agree on a keypress -web/app.js 816 string calls only, no protocol knowledge, no framework. - Run by harness/tests/shell.spec.js (Chromium; the standalone half on Firefox too) — see - gotcha 48 for what that does and does not reach -web/worker.js 86 three string calls, no protocol knowledge. RUN by - shell.spec.js's no-?ws path, with bindings.rs (gotcha 45) -web/shell.js 399 everything app.js DECIDES that is not painting: no DOM, - no browser API, one `node --test` away. Owns socketUrl, - gridPlan, frameCostOf, documentTitle — so a chunk of the - browser's render decisions is NOT in the file gotcha 42 - and gotcha 46 name. Reasoning: context/code/clients.md -web/serial.js 19 the one rule worker.js exists to keep: one frame at a - time, in arrival order (gotcha 48's async-listener race), - extracted so it is testable without worker globals -web/index.html, style.css the page; legibility only -web/build.sh 38 wasm-pack build → web/pkg (gitignored). Needs - wasm-opt = ["-O", "--enable-bulk-memory"] or the bundled - wasm-opt rejects LLVM's memory.copy -web/assemble.sh 112 assembles the deployable site (copies web/*.improv) -web/demo.improv 65 the document the site ships and seeds from. Every cell a - Record — asserted (gotcha 46) -``` - -### Browser worker host (`crates/improvise-worker-server/`) -``` -improvise-worker-server/src/worker.rs 765 / 17t Worker: one port, one client; handshake, Addressed - filter, persist trigger (model_state.dirty). Every decision, every test -improvise-worker-server/src/vfs.rs 348 / 8t documents/open: seeding is a Storage::write -improvise-worker-server/src/bindings.rs 185 / 0t wasm32 shim: OpfsFs awaits + postMessage. No Rust test - reaches it, but harness/tests/shell.spec.js DOES: the - no-?ws path runs start()/OpfsFs::open under Chromium - and Firefox. - Called NEVER RUN here for months: see gotcha 45 -improvise-worker-server/src/lib.rs 51 / 1t the module doc that states exactly what is and is not - covered — the one correct copy of that fact -improvise-worker-server/tests/post_message.rs 407 / 8t main-thread traffic, strings only -``` - -### Browser harness (`harness/`) — not a workspace member - -``` -harness/playwright.config.js 50 projects: chromium (every spec), firefox (opfs, plus - shell.spec's standalone half: grepInvert @mocked-peer), - webkit (opfs only, which skips on a runtime probe: no - OPFS). CI runs `--project=chromium --project=firefox` only: - nixpkgs' WebKit aborts inside the runner VMs (ci.yml) -harness/tests/shell.spec.js 345 the first execution of web/app.js. A change to web/*.js - belongs here (gotcha 48). Playwright serves harness/serve/, - which harness/build.sh ASSEMBLES: edit web/*.js, rebuild, - or the specs run the old page -harness/tests/opfs.spec.js 103 drives the OPFS contract page and reads back its result -harness/opfs-contract/src/lib.rs 470 runs improvise_io::storage::contract::storage_contract - against a real OpfsFs inside a real Web Worker -harness/www/opfs/{index.html,worker.js} the page and worker that host it -harness/build.sh 50 assembles harness/serve/ (web/assemble.sh, the fixtures, - and the contract crate's wasm-pack build) — gitignored -# gated by the `browser` job in .github/workflows/ci.yml, which can fail the build -``` - -### Examples -``` -examples/gen-grammar.rs Grammar-walking random .improv generator (pest_meta) -examples/pretty-print.rs Parse stdin, print formatted .improv -crates/improvise-core/examples/category_width_bench.rs - The same records at 12 and at 24 regular categories - (improvise-felo.1; context/code/core.md) -``` - -### Context docs -``` -.claude/skills/code-review/ Review skill: checks a change against the doctrine in these docs + kb.pl -scripts/check-comment-runs.py The two-line comment cap AND --check-evidence. The flag is - ADDITIVE since improvise-ijql — every invocation that can - exit 0 answers the cap — guarded by a subprocess self-test, - the only level a dispatch bug is visible at. Reasoning: - context/code/tooling.md §2 -scripts/check-ci-graph.py pages-deploy must `needs` every job in ci.yml, and every - suite `checkers.sh --list` knows must run in some CI job - (improvise-dak5). It REFUSES what it cannot read rather than - parsing YAML: the values it accepts are an allowlist - (tooling.md). --self-test; run in the `test` job. README - states the rule and lists no jobs (improvise-x7cq) -scripts/checkers.sh The one entry point for every *-test.sh suite: scrubs the - env to an allowlist, runs under the bash that runs it (no - version floor since #297). `ci_suites` is the table of which - suites each CI job runs (`--ci `); ci.yml names none -scripts/pr-watch Waits on a PR's checks (the pr-watch skill). Exit 5: the PR - is already MERGED or CLOSED, so there is nothing to wait for -scripts/assemble-stamp.sh The cache-bust stamp gate web/assemble.sh calls, extracted - so its refusals can be exercised without a web/pkg -scripts/assemble-stamp-test.sh Its check_the_checker, run first in the `build` CI job -context/code/tooling.md The repository's own checkers: why an evidence heading's - names come from cargo rather than from the source, and - why a leaf name is not an identity -context/test-evidence/tooling.md Evidence for the gates in scripts/ and for the tests that - hold CI's shape -scripts/nix-seed-flake-inputs.sh Seeds locked flake inputs over git so `nix develop` works in cloud sessions -prolog-sessions/kb.pl Executable knowledge base: invariants, doctrine, decisions (CI-checked) -context/adr/007-stable-identities.md Stable identities: ids unique across the document are the - keys and names are display-only; gensym suffixes apply to - generated ids only, and hand-written ids are canonical. - Accepted 2026-09-25. (There is no ADR 005 or 006 file: - improvise-l2b8 and improvise-ilvl claim those numbers.) -context/adr/004-faces-and-themes.md Faces: appearance as UI-agnostic data a front end - applies, emacs-style. Accepted 2026-09-22. -context/adr/003-live-backed-models.md A model may be backed by a live query source — - improvise as a DB client. Accepted, NOT a priority. -context/adr/002-naming-a-cell.md How a client names a cell: an opaque surrogate on - CellDisplay, whose absence means the cell is an - aggregate. Records why _Index cannot serve. -context/adr/001-concurrent-edits.md Concurrent edits between clients: record-level atomicity, - optimistic basis check per batch, advisory reservations. - Records why a causal clock was rejected. -context/cloud-agents.md Cloud-session environment: tracker sync, bd pitfalls, what is blocked -context/design-principles.md Architectural principles & testing doctrine -context/plan.md Show HN launch plan -context/repo-map.md This file -docs/design-notes.md Product vision & non-goals -``` +| Area | Where | Entry points | +|---|---|---| +| Core model | `crates/improvise-core/src/` | `model/{types,cell,record,category,id}.rs`, `view/{layout,types,axis}.rs`, `workbook.rs`, `format.rs` | +| Formula | `crates/improvise-formula/src/` | `parser.rs`, `formula.pest`, `ast.rs`, `quote.rs` | +| I/O | `crates/improvise-io/src/` | `storage.rs`, `opfs.rs`, `persistence/{improv.pest,format,parse,export,mod}.rs`, `import/{wizard,analyzer,csv_parser}.rs` | +| Protocol | `crates/improvise-protocol/src/` | `lib.rs` (ViewState), `reduce.rs`, `render.rs`, `keymap.rs`, `face.rs`, `frame.rs`, `basis.rs`, `resolve.rs`, `client.rs`, `viewmodel.rs`, `help.rs` | +| Command layer | `crates/improvise-command/src/command/` | `cmd/{core,registry,navigation,cell,commit,effect_cmds,fit,grid,mode,panel,search,text_buffer,tile,wizard}.rs`, `keymap.rs`, `parse.rs` | +| Semantic UI state | `crates/improvise-command/src/ui/` | `effect.rs`, `app.rs`, `project.rs`, `reduce.rs`, `host.rs`, `viewport.rs`, `cat_tree.rs` | +| TUI + CLI | `src/` | `draw.rs`, `main.rs`, `ui/{grid,help,import_wizard_ui,text}.rs`; integration tests in `tests/` | +| GP | `crates/{gp-tree,improvise-gp}/` | `gp-tree/src/{lib,data,fit}.rs`, `improvise-gp/src/{lib,dataset,fit}.rs` | +| Socket host | `crates/improvise-ws-server/src/` | `registry.rs`, `origin.rs`, `transport.rs`, `main.rs`; `host.rs` is a re-export shim | +| Worker host | `crates/improvise-worker-server/src/` | `worker.rs`, `vfs.rs`, `bindings.rs` (wasm32 shim) | +| Browser client | `crates/improvise-wasm-client/src/`, `web/` | `lib.rs`, `keys.rs`, `render.rs`, `socket.rs`; `web/{app,shell,worker,serial}.js`, `index.html`, `demo.improv`, `build.sh`, `assemble.sh` | +| Browser harness | `harness/` | Playwright (`tests/shell.spec.js`, `tests/opfs.spec.js`), `opfs-contract/`, `build.sh` — not a workspace member; its own CI job | +| Examples | `examples/`, `crates/*/examples/` | `gen-grammar.rs`, `pretty-print.rs`, `category_width_bench.rs`, `fit_csv.rs` | +| Scripts | `scripts/` | `checkers.sh` (every `*-test.sh` suite), `check-comment-runs.py`, `check-ci-graph.py`, `bd-audit-ids`, `kb`, `pr-watch`, `pr-reviews`, `build-doc-site.mjs` | +| Context docs | `context/` | `design-principles.md`, `gotchas.md`, `file-inventory.md`, `code/.md` (arguments), `test-evidence/` (bug evidence by test name), `adr/` (001–004, 007), `cloud-agents.md`; `prolog-sessions/kb.pl` is the KB; `docs/design-notes.md` is product vision | --- ## Wasm -The browser bundle is `improvise-core`, `improvise-formula`, `improvise-io`, -`improvise-protocol`, `improvise-command`, `improvise-wasm-client` and -`improvise-worker-server`. All seven -`cargo check --target wasm32-unknown-unknown` clean with no feature flags -(improvise-ywd), gated by the `wasm` job in `.github/workflows/ci.yml`. That -job gates **eight** crates, not seven: `improvise-gp` was added (improvise-z5eg) -explicitly *not* as a bundle member but as a candidate, because staying -wasm-clean is free now and expensive to retrofit — so the bundle is seven and -the gate is eight, and neither number is the other one's typo. -**A feature that adds a dependency to a bundle crate must be checked with `cargo tree --edges normal`** — the default output includes **dev** edges, so a plain `cargo tree -p improvise-protocol --target wasm32-unknown-unknown` reports a dev-only `serde_json` as *present* and reads exactly like a leak. Check per-crate too, because the `wasm` CI job invokes each crate separately and feature unification cannot reach across that (improvise-26dm added a `test-support` feature on `improvise-protocol` on those terms; it is off in every shipped build). - -`improvise-ws-server` is the host and is native only — tokio, tungstenite and -`Box` — so `improvise-wasm-client` cannot depend on it in anger. It -no longer needs to: both ends `pub use` `improvise_protocol::frame` and neither -defines a pair (gotcha 42). The **dev**-dependency remains, for -`tests/browser_round_trip.rs`, which hands the client's own serialized frame to -a real `Host::handle`. The root `improvise` crate is -the terminal binary and does **not** build for wasm — it depends on -crossterm; that is correct, not a gap. - -`cargo check` passing is not the same as running. Two known runtime hazards -it cannot see: `Instant::now()` panics on wasm (hence `web-time` in -`improvise-command`), and any `std::fs` call compiles but fails in a browser -— improvise-fuj closed the effects that bypassed `Storage`; autosave and the headless final write still do (improvise-31oe.11). +The browser bundle is `improvise-{core,formula,io,protocol,command,wasm-client,worker-server}`; all seven +`cargo check --target wasm32-unknown-unknown` clean with no feature flags, and the `wasm` CI job also gates +`improvise-gp` (a candidate, not a member). `improvise-ws-server` and the root crate are native-only. +Check new bundle dependencies with `cargo tree --edges normal` (the default output includes dev edges), +per crate. `cargo check` cannot see runtime hazards: `Instant::now()` panics on wasm (`web-time`), and +`std::fs` compiles but fails in a browser. Long form: `context/file-inventory.md`, appendix. ## CLI @@ -673,39 +261,20 @@ improvise cmd 'add-cat Region' -f model.improv # headless command(s) improvise script setup.txt -f model.improv # run script file ``` -Import flags: `--category`, `--measure`, `--time`, `--skip`, `--extract`, `--axis`, `--formula`, `--name`, `--split-date ` (a date column imports as ONE hierarchical category by default; this restores the flat split, and `--extract` still picks the components — improvise-8jk6.3), `--strict` (with `--no-wizard`, any skipped record is a refusal and nothing is written, where the default writes and warns with a count — improvise-8jk6.6), and `--on-duplicate refuse|sum|separate` (default `refuse`; applies with `--no-wizard`, where the wizard would otherwise ask — improvise-cgqt). A headless import also refuses a `--split-date` that names a non-date field and an `--axis` that names a category the import did not build (exit 1). - ---- +Import flags: `--category --measure --time --skip --extract --axis --formula --name`, `--split-date ` +(flat date split; default is one hierarchical category), `--strict` (refuse any skipped record), and +`--on-duplicate refuse|sum|separate` (default `refuse`). Details: `context/code/cli.md`, gotchas 51–52. ## Testing — the short version -Full guidance lives in `context/design-principles.md` §7. Quick reminders: +Full guidance: `context/design-principles.md` §7. -- **Two numbers, not one** — the ambiguity between them is what let the old - "suite runs in <2s" drift: **no single test binary over 2 seconds**, and the - workspace wall clock is the sum. Measured at `63a5c13`: **1464 tests over 35 - binaries** (1 ignored), longest single binary **0.6–1.0s across two runs on one machine** — - the spread being its own argument for reading your own run. Excludes compilation, - and excludes `harness/` (Playwright, minutes, its own CI job). *Re-measure* rather - than re-assert — `cargo test --workspace` prints a `finished in` per binary, and - **the portable signal is the ratio, not the seconds**. **Not the ratio to the - median, though: it is degenerate.** Most binaries print `0.00s` at two decimals, - so the median is zero and every ratio to it is undefined (improvise-i5z6, 39 - binaries). Compare the slowest binary to the next few slowest instead: an outlier - is one that sits well clear of them *in every run*. That is how i5z6 was read — - `improvise-formula`'s lib longest of 39 in all three runs, at 0.91–1.95s against a - runner-up's 0.50–0.92s. Both outliers so far had one cause: a generator re-parsing - its pest grammar per case. `improvise-io` (improvise-8shu, 1.42s → 0.21s) and - `improvise-formula` (improvise-i5z6, 0.84–0.98s → 0.06–0.07s) each read it once through a - `OnceLock` now. A new grammar-walking strategy should do the same from the start. - The full argument is `context/design-principles.md` §7. -- Commands: build a `CmdContext`, call `execute`, assert on returned effects. No terminal needed. -- Property tests (`proptest`, default 256 cases) cover invariants: CellKey sort, axis consistency, save/load roundtrip, `parse(format(parse(generate())))` stability. -- **Bug-fix workflow**: write a failing test *before* the fix (regression guard). Record the bug in `context/test-evidence/`, **keyed by test name** — *not* in the test's doc comment, which the two-line comment cap forbids and `scripts/check-comment-runs.py` enforces. A heading may be module-qualified and must be when its leaf is not unique. -- Coverage target ~80% line/branch on logic code; skip ratatui render paths. - ---- +- **No single test binary over 2 seconds**; the workspace wall clock is the sum. Re-measure with `cargo test --workspace` (a `finished in` per binary) and compare the slowest binary to the next few — a ratio, not seconds, and not a ratio to the median (it is zero). +- Commands: build a `CmdContext`, call `execute`, assert on the returned effects. No terminal. +- Property tests (`proptest`, default 256 cases) for invariants and round-trips; a grammar-walking strategy must read its pest grammar once (`OnceLock`). +- **Bug-fix workflow:** write the failing test first. Record the bug in `context/test-evidence/`, keyed by test name — not in a doc comment (two-line cap, `scripts/check-comment-runs.py`). +- Coverage target ~80% on logic code; skip ratatui render paths. ## Key dependencies -ratatui 0.30, crossterm 0.29, clap 4.6 (derive), serde/serde_json, indexmap 2, anyhow, chrono 0.4, pest + pest_derive, flate2 (gzip), csv, enum_dispatch. Dev: proptest, tempfile, pest_meta. +ratatui 0.30, crossterm 0.29, clap 4.6 (derive), serde/serde_json, indexmap 2, anyhow, chrono 0.4, pest, flate2, csv, enum_dispatch. Dev: proptest, tempfile, pest_meta. diff --git a/context/test-evidence/core.md b/context/test-evidence/core.md index 53eb3eb8..61210545 100644 --- a/context/test-evidence/core.md +++ b/context/test-evidence/core.md @@ -2103,3 +2103,122 @@ fails on that rival (no `▶ Overview`), because the active view is a key The browser half of the same three panels. It also asserts that the view row is still `active` when found by key. + +## Displays in the grid frame, page bar, item rows and CSV (ADR 007 clause 2, improvise-cq1q.12) + +The same test-only way to make a display differ from its key (`set_display`, +`set_item_display`, `set_group_display`). Each rival below was injected, the +workspace run, and the named tests failed; the revert was re-run green. + +### view::layout::tests::a_layout_shows_displays_and_still_addresses_by_keys + +Rivals: an item's display pushed as its key (`display.push(item_name)`); a page +axis's `selected_display` set to `selected`. Both fail on the display asserts, +while `cell_key`, `row_group_for` and `row_run_start` stay on keys. + +### view::layout::tests::a_folded_group_is_shown_under_the_groups_display_and_keyed_by_its_name + +A folded row's tuple holds the group name where an item name would be, so the +display must come from the group. Not a distinct injection; it covers the arm +that the item test cannot reach. + +### view::layout::tests::a_records_grid_shows_displays_in_its_headers_and_coordinates + +Rival: `coordinate_display` returns the item key. Fails here and in +`ui::project::tests::a_records_coordinate_shows_its_display_and_edits_its_key`. + +### ui::project::tests::the_frame_shows_displays_and_its_key_tuples_still_address_the_cells + +The rival the ticket names: `axis_frame` writes `tuple: display.clone()`. The +test rebuilds every cell's `CellKey` from the frame's key tuples, its `cats` and +the page bar, and compares it to `layout.cell_key`, so it fails (and so does the +records test, which looks the Region column up by its key tuple). Also fails when +a group header's display is its key. + +### ui::project::tests::each_view_summary_shows_its_own_display_and_its_categories_displays + +Rival: every summary takes the ACTIVE view's display. The second view "Plan" has +a display unlike both the active view's ("Overview") and its own key, so the +fault fails here by name. + +### ui::project::tests::an_item_row_shows_its_display_and_keeps_its_key + +Rival: the item row's display is its key. The unrenamed item "West" must show its +key, and `_Measure`'s "Revenue" shows "Sales". + +### ui::project::tests::the_page_bar_carries_displays_beside_the_keys_a_page_command_names + +Rival: `selected_display` is the key. Fails here, in the layout test, in the TUI +grid test and in the CSV test. + +### render::tests::a_header_draws_displays_and_blanks_a_repeat_by_key + +Rival: suppression compares display tuples. West shares East's DISPLAY ("Same") +and must still draw, so comparing displays blanks it. + +### render::tests::a_header_is_as_wide_as_what_it_shows + +Rival: `header_widths` measures the key tuples. Fails (key "E" against display +"Eastern Seaboard"), as does the TUI grid test. + +### render::tests::a_frame_with_ragged_displays_is_refused + +Asserts the control parses, then that a `cat_displays` shorter than `cats` and a +display tuple shorter than its key tuple are each refused with their own message. + +### resolve::tests::hiding_a_displayed_row_item_names_its_key + +Rival: the hide resolver reads the first DISPLAY of the row frame. It composes +`HideItem { item: "Eastern Seaboard" }` and the test fails on `"East"`. + +### resolve::tests::picking_a_displayed_page_item_still_names_its_key + +Picks index 1 of a page axis whose displays are not its keys and requires the +command to name `Q2`. The index is the only thing that travels from the menu. + +### render::tests::the_grid_page_bar_and_item_rows_paint_displays + +The browser half. Rival: the cycle's `items` relays the keys, which fails here. + +### ui::grid::tests::the_frame_and_page_bar_draw_displays_not_keys + +The TUI half. Rival: the group header draws `group.name`; fails on the missing +"▼ Bay". The key assertions use strings that appear only as keys. + +### ui::category_panel::tests::an_item_row_draws_its_display_not_its_key + +The TUI item row. Not separately injected: its draw path is the one line the +`CatTreeEntry::Item` pattern changed. + +### persistence::export::tests::export_csv_writes_displays_in_every_header_and_label + +Rival: the column header uses `col_label` (the key). The exact output pins the +corner cell `Kind (Who=Alicia)`, the column header and the row label. + +### view::layout::tests::a_page_axis_with_ragged_displays_is_refused + +From Copilot's finding on #336: `item_displays` is a parallel list on a wire type +that derives `Deserialize`. The control parses a well-formed axis first; then a +shorter list and an absent one beside `items: Some(..)` are each refused with the +"item displays" message. Without the `try_from` the same JSON parses. + +### ui::viewport::tests::a_column_is_as_wide_as_the_display_its_header_draws + +From the Opus review of #336: `header_level_seed` could measure keys and the +suite stayed green. The fixture's column display ("Januaryyyyy", 11) is far longer +than its key ("Jan") and the cell, so the 5-column minimum cannot hide it. Rival: +the seed reads `tuple` instead of `display`; the width is 5, not 12, and this test +fails. + +### ui::viewport::tests::the_row_header_is_as_wide_as_the_display_its_rows_draw + +Rival: `compute_row_header_width` reads `tuple`. That width drives `visible_cols` +on the keypress path (the improvise-gtn7.1 shape: a scroll measured elsewhere than +it is drawn), so a key measurement lets the cursor scroll to an undrawn column. +The display "Eastern Seaboard" against the key "E" gives 17, not the 4 minimum. + +### render::tests::a_column_group_header_is_painted_by_its_display + +Rival: `col_groups` labels a run with `g.name`. The browser test for group displays +covered row groups only; this one folds a column frame whose group "Q1" is shown as +"First quarter" and fails on the key. diff --git a/context/test-evidence/numtower.md b/context/test-evidence/numtower.md new file mode 100644 index 00000000..c57949a3 --- /dev/null +++ b/context/test-evidence/numtower.md @@ -0,0 +1,468 @@ +# Test evidence: `numtower` + +What each test guards, which rival it was run against, and where that rival failed. The +entries are keyed by exact test name, and a rename orphans its entry, which +`scripts/check-comment-runs.py --check-evidence` catches. The reasoning these tests defend +is in `context/code/numtower.md`. The decisions are ADR 005's. + +**How the rivals are kept.** Every property in `crates/numtower/tests/properties.rs` is a +checker that takes the implementation as a parameter. The property's own test runs it on +the real code. A companion test, named `_catches_`, runs the same checker +on the rival, with a deterministic seed, and asserts two things: +- the run fails; +- the failure message carries **the property's own assertion text**, so a rival that + fails in setup, panics, or fails some other assertion does not count (`assert_caught` + in `tests/oracle/mod.rs`). + +So the rival is watched failing on every run of the suite, not once by hand. Two rivals +were also injected into the crate's source, and the run was recorded. Those runs are noted +under their entries. + +**The oracle** is `num-rational`'s `BigRational`, a dev-dependency feature. The crate +itself never uses `BigRational`, so the oracle shares no rational type with the code under +test. "Correctly rounded" is checked in the review's interval form (`is_nearest`). The +exact value must lie inside the round-to-nearest-even interval of the returned f64, with +the endpoints included only when the f64's mantissa is even. Comparing against +`BigRational::to_f64` would have been circular in the way ADR 005's option (d) section +describes. + +--- + +## `crates/numtower/tests/properties.rs` + +### an_exact_result_is_never_a_rounded_one + +Every exact pairing of arms and every operation (`+ − × ÷`) is checked against the oracle. +When the result claims to be exact, its value must be the exact value. The generator +spans 96-bit mantissas, scales 0–28 and `i128` fractions. A fixed case pins ADR 005's own +example, `1000 + 0.3333333333333333333333333333`. `rust_decimal` returns that at scale +25, rounded, and the crate must not. + +**Injected in the source.** In `arith.rs::decimal_sum`, the scale filter was deleted, so +`checked_add`'s result was trusted. Three tests failed: this one, +`contagion_is_total`, and `a_recomputed_result_keeps_the_stored_scale_clause_4_names`. +The revert was followed by a full rerun. + +### an_exact_result_is_never_a_rounded_one_catches_a_silently_rounding_add + +The rival is `rust_decimal`'s own `checked_add`, trusted as exact. That is the one +outcome ADR 005 clause 4 exists to prevent. + +### contagion_is_total + +Every pairing of the three arms, NaN and the infinities included, through `+ − × ÷`: +- an inexact operand makes the result inexact; +- an exact pair gives an exact result **iff** the oracle value fits a stored arm, and + otherwise the correctly rounded f64 of that value; +- division refuses iff the divisor is exactly zero, at either level. + +"Total" means no pairing is missing. A pairing the code never handled would panic, and the +runner reports a panic as a failure. + +### contagion_is_total_catches_an_integral_inexact_kept_exact + +The rival is a "harmless" promotion: an integral f64 is treated as exact, so `2 + 3.0` is +exact. It is caught on the seed pair `(2, Inexact(3.0))`. That pair is in the generator on +purpose. Integral f64s are rare among random floats, so without the seed the rival was +caught only by luck. + +### pow_is_exact_for_an_exact_base_and_integer_exponent_when_it_fits + +Bases from every arm, exponents −6 to 6. An exact base gives an exact result iff the +oracle's power fits, and otherwise the correctly rounded value. `0^-n` refuses. + +### pow_decides_its_size_before_computing_and_honours_the_ceiling + +- `POW(1/3, 10^9)` returns at once, inexact. Its lower bound is past the ceiling, so + nothing is allocated. Without the size check the test does not finish. +- `(1/3)^6000` is computed, but is 9,510 bits and demotes. That is the ceiling's + exception. +- `1.05^28` is exact and `1.05^29` is not, the option (d) table's k = 29. +- Also pinned: `(-1.0)^(10^20)`, `0^0`, the scale of `1.50^2`, and that a non-integer + exponent is `powf`. + +### the_f64_conversion_agrees_with_parsing_the_decimal_text + +The generator spans **96-bit mantissas and scales 0–28**, as ADR 005 requires. The oracle +text is built digit by digit, not by `rust_decimal`'s `Display`, which is what the crate +itself calls. So the test does not compare the crate's path with itself. The review's +case `63294.607176721429540940386554` is pinned. + +**Injected in the source.** `num.rs::decimal_to_f64` was replaced with `rust_decimal`'s +`ToPrimitive::to_f64`, and three tests failed: +- this one, minimal case `-0.10806611601966961` at scale 17; +- `the_number_languages_gate_and_parse_agree`, on `92240.01133710762224`; +- `exact_mixed_comparison_is_transitive`. + +The third was not expected. The comparison's fast path decides by the rounded value, and +that is sound only if the rounding is correct. + +### the_f64_conversion_agrees_with_parsing_the_decimal_text_catches_rust_decimal_to_f64 + +The rival is `rust_decimal`'s `to_f64`, which is not correctly rounded (ADR 005, B2). The +test also asserts that the rival misses on the review's case, so the pinned case is known +to discriminate. + +### rust_decimal_to_f64_agrees_on_fifteen_digit_mantissas_so_a_narrow_generator_is_vacuous + +This test **passes the rival** on purpose. With mantissas of at most 15 digits, +`rust_decimal`'s `to_f64` agrees with the text every time, at this test's 256 cases as +in ADR 005's 200,000 of 200,000. So a test whose generator stays in that range proves +nothing about the conversion, and this test is why the generator above spans 96 bits. It +ran at 4,096 cases until Copilot pointed out, on PR #337, that the repository's budget is +256 (`context/design-principles.md` §7). The 200,000 is the ADR's measurement, and this +test only pins its direction. + +If `rust_decimal` ever fixes its conversion, this test still passes. What fails then is +`the_f64_conversion_agrees_with_parsing_the_decimal_text_catches_rust_decimal_to_f64`, +whose rival would agree. That is the signal to re-read both. An earlier version of this +entry named this test instead, which was wrong. + +### a_fractions_conversion_lies_inside_the_round_to_nearest_even_interval + +Reduced fractions with numerator and denominator widths from 20 to 126 bits, both signs: +the review's independent form. It also checks the product of two such fractions, which +reaches the bigint path and is stored. That covers `ratio_to_f64` on values wider than +`i128`. + +### a_fractions_conversion_lies_inside_the_round_to_nearest_even_interval_catches_an_f64_division + +The rival is `n as f64 / d as f64`, which rounds three times. + +### exact_mixed_comparison_is_transitive + +The generator is a **crowd**: values crowded around one f64 `x`. +- `x` itself, and its two neighbours; +- the shortest decimal that spells `x`, and that decimal ± `k·10^-25`; +- a third of the shortest decimal. + +Every triple is checked for transitivity, and every pair for antisymmetry. Random values +would almost never put two exacts and an inexact on one f64, which is exactly where the +rival breaks. + +### exact_mixed_comparison_is_transitive_catches_comparing_through_f64 + +The rival is revision 1's rule: convert the exact operand to f64, then compare. It fails +as ADR 005 clause 6 says it does. `Exact(s) = Inexact(x)` and `Inexact(x) = Exact(s + +10^-25)`, while the two exacts differ. The marker is `comparison: not transitive`, so it +is the transitivity assertion that fails, not antisymmetry. + +### comparison_agrees_with_the_oracle_at_every_pairing_of_arms + +The differential half of comparison: the crowd, plus mixed random pools of decimals, +fractions and finite f64s, against `BigRational`'s order. NaN is unordered, and an +infinity orders past every exact value. + +### min_and_max_prefer_the_exact_value_and_do_not_depend_on_order + +Question 11's tie-break, plus the two the crate adds: the larger scale between equal +decimals, and IEEE's `-0.0` for `MIN` and `0.0` for `MAX`. Each pair is asserted in both +argument orders, the order-independence property the tie-breaks exist for. + +### the_accumulator_is_independent_of_order + +Random **weighted** term lists, from every arm, are summed in one order and in a shuffled +order. The results must agree in **arm, value and scale**. The terms include fractions +over a pool of fourteen non-decimal denominators, so groups receive several terms and +lists often hold more than K of them, and arbitrary fractions too. ADR 005 clause 8's +seed case, `[0.30000000000000004, 1e12, −1e12]`, is one of the generator's arms and is +also pinned exactly. It caught the review's F12 (a group's old f64 not retracted), where +the stale values make the past-K read depend on the order of updates. + +**On weights** (the review of `302594bf`, B2). Until then every `NumSum` property called +`insert` and `retract` only, so every weight was ±1. That was the cause the answer to +Codex's P1 had itself named, and the generators had not changed. Two faults passed the +whole suite: F7 (the decimal partition ignored the weight's magnitude) and F8 (a group +multiplied by `weight.signum()`). Weights now come from `weight()`: +- the units 1, −1 and 0; +- −3 to 3; +- magnitudes up to 2^40; +- `±i64::MAX` and 2^30 − 1. + +`i64::MIN` is left out so a weight always negates. + +### the_accumulator_is_independent_of_order_catches_decimal_addition + +The rival is clause 8's own rival, a fold with `Decimal`'s `+`, demoting to the tower +where it overflows. On the seed case, one order rounds `0.30000000000000004 + 1e12` to +scale 16 and the other does not. + +### the_accumulator_retracts_exactly + +Adding weighted `xs` and `ys`, then adding each `y` again with its weight negated in +reverse order, must equal a fresh sum of `xs` in arm, value and scale. A group whose +numerator returns to zero must be dropped, or a retracted denominator would still count +toward K. With weights it catches F7 and F12 as well. + +### the_accumulator_reads_the_exact_sum_or_its_stated_roundings + +The differential test of the read against a `BigRational` oracle, over weighted terms. It +states the value of all three branches: +- **No inexact term, at most K live groups.** The read is exact iff the exact total fits, + and is then that total. Otherwise it is the total's correctly rounded f64. +- **Inexact terms present.** The exact total is rounded once, then the weighted f64s are + added and the result is rounded once more. +- **Past K groups.** The nearest f64 of Σ round(live group) + round(decimal partition) + + Σ w·f. + +**Until the review of `302594bf` (B1) the past-K branch asserted only that the read was +inexact, and this entry claimed the test stated every branch. Both were wrong.** Three +faults passed the whole suite: +- F12: a group's old f64 was not retracted from `group_floats`; +- F14: the past-K read dropped the decimal partition; +- F15: the past-K read dropped every inexact term. + +The test now has the reviewer's oracle, at the repository's 256-case budget. 34ab0fc3 ran +it at 512, and Copilot pointed out the budget. At 256, every fault in the table below +still failed by name, and F7, F8, F12, F14 and F15 failed in four of four passes. Half the lists come from the +fourteen-denominator pool, up to 40 terms long, so groups receive several terms and K is +passed often. **It also counts how many cases reach each branch and fails if any branch +is reached fewer than 20 times**, so a generator change cannot silently starve one. + +The oracle counts **live** groups, those whose summed numerator is nonzero. The first +version counted denominators ever seen, and that disagrees with the state-based count +whenever terms cancel. Inexact presence is likewise per bit pattern, by net weight. + +**Every fault in the review's script, re-run after the fix.** Each was injected into the +source and `cargo test -p numtower` was run on a confirmed rebuild. Each failed by name: + +| fault | failed | +|---|---| +| F1 decimal sum's scale filter dropped | `a_recomputed_result_keeps_the_stored_scale_clause_4_names`, `an_exact_result_is_never_a_rounded_one`, `contagion_is_total` | +| F2 `rust_decimal`'s `to_f64` | `exact_mixed_comparison_is_transitive`, `comparison_agrees_with_the_oracle_at_every_pairing_of_arms`, `the_f64_conversion_agrees_with_parsing_the_decimal_text`, `the_number_languages_gate_and_parse_agree` | +| F3 mixed comparison through f64 | `exact_mixed_comparison_is_transitive`, `comparison_agrees_with_the_oracle_at_every_pairing_of_arms`, `an_interim_compares_exactly_without_storing` | +| F4 one net inexact count | `the_accumulator_keeps_inexact_terms_whose_weights_cancel`, `the_accumulator_retracts_exactly`, this test | +| F5 product, not lcm | `sum::tests::the_grouped_read_combines_over_the_lcm_and_reduces_once` | +| F6 `ExactSum` rounded per limb | `exact_sum_is_the_exact_weighted_total_rounded_once`, `more_than_k_denominators_read_inexact_even_when_the_total_fits`, `the_accumulator_keeps_inexact_terms_whose_weights_cancel`, this test | +| F7 decimal weight magnitude ignored | this test, `the_accumulator_retracts_exactly` | +| F8 group weight as its sign | this test | +| F9 decimal product's scale filter dropped | `a_typed_value_and_the_same_value_computed_are_equally_exact`, `an_exact_result_is_never_a_rounded_one`, `contagion_is_total` | +| F10 `ExactSum`'s low weight digit only | `exact_sum_is_the_exact_weighted_total_rounded_once`, this test | +| F11 `POW` sign lost | `a_huge_odd_exponent_keeps_the_sign_of_a_negative_base` | +| F12 old group f64 kept | this test, `the_accumulator_is_independent_of_order`, `the_accumulator_retracts_exactly` | +| F13 mixed tie not checked exactly | `comparison_agrees_with_the_oracle_at_every_pairing_of_arms`, `exact_mixed_comparison_is_transitive`, `an_interim_compares_exactly_without_storing` | +| F14 past K, decimal partition dropped | this test | +| F15 past K, inexact terms dropped | this test | + +- **F13's text now occurs twice.** `Interim`'s comparison added a second copy of the + replaced line, so the review's script refuses to inject it. It was injected at its + original site, in `cmp.rs::mixed`. +- **The five former survivors were run twice and failed both times.** F12's set varied + between runs: `the_accumulator_retracts_exactly` failed in one run and not the other, + and this test failed in both. + +### more_than_k_denominators_read_inexact_even_when_the_total_fits + +`1/3 + 1/7 + … + 1/31` is nine distinct denominators. The exact total fits an `i128` +fraction, yet the read is inexact. That is question 13's second exception, the K cutoff +overriding representability. Eight of them read as an exact `Fraction`. The read past K is +asserted to be **each group rounded, then the total rounded once**. The first version +asserted one rounding of the exact total, and it failed: clause 8's maintained f64 +accumulator of group values rounds twice by design. + +### exact_sum_is_the_exact_weighted_total_rounded_once + +ADR 009's `ExactSum` against the oracle. Weights are drawn from small values, from the +whole `i64` range and up to 2^40. A whole-range weight exercises all three base-2^30 +digits, which ADR 009's chunk loop could not reach in reasonable time. Values include +subnormals. + +### division_decides_fit_directly + +ADR 005 clause 5's three seed cases, as both property inputs and pinned assertions: +- `1 / 2^40` and `1 / 5^30` are fractions; +- `1e-28 / 0.5` is the exact decimal `2e-28`. + +Random decimal pairs must also be an exact decimal iff the quotient fits one, an exact +fraction iff it fits only that, and otherwise inexact. The test also pins `AVG`'s example +(`70/3`), `1/3 × 3 = 1`, and division by an inexact `-0.0`. + +### division_decides_fit_directly_catches_multiplying_back + +The rival is multiplying back with **revision 6's** scale test (demote on the hint). It +fails on `1e-28 / 0.5`, as ADR 005 says. An experiment, not committed, ran multiplying +back with `Num`'s own `*`, which carries revision 7's recompute. It agreed with the direct +test on all 5,120 random cases and the seeds. The ADR's argument that multiplying back is +incomplete holds only under the earlier rule (`context/code/numtower.md`, "Division"). + +### the_number_languages_gate_and_parse_agree + +The generated lexemes have four sources: +- structured lexemes: an optional sign, up to 40 digits, an optional fraction, and an + optional exponent of up to 4 digits, so the empty parts give `1.`, `.5` and `.`; +- junk over the alphabet of digits, `eE+-._/aifnIN` and space; +- `n/d` lexemes; +- the named spellings: `inf`, `NaN`, `Infinity`, `1_000`, `+.5`, `e5`. + +For each lexeme, the parse succeeds iff `f64::from_str` reads a finite value. The exact +value parsed must round to the f64 `f64::from_str` reads. `is_number` and `parse_number` +must agree in both typings. + +### the_number_languages_gate_and_parse_agree_catches_from_str_exact + +The rival is `rust_decimal`'s `from_str_exact`. ADR 005 clause 3 names it as the parse +whose language differs from the gate's: it rejects `1.5e3` and accepts `1_000`. + +### the_number_language_seed_cases + +Every seed case ADR 005's migration step 1 lists: +- `1.5e3` is the exact `1500`; +- `1_000` is text; +- `-0.0` is the exact `0.0`, with no sign; +- `3/4` is text when the type is inferred, and `0.75` when it is forced; +- `1e999`, `-1e999`, `nan`, `inf` and `Infinity` are text; +- `1e-999` is an inexact zero; +- `1e-30` is the exact `1/10^30`; +- `0.12345678901234567890123456789` is an exact fraction, not a rounded decimal; +- `2^120` is an exact `n/1`; +- `1e-40` is inexact. + +### a_typed_value_and_the_same_value_computed_are_equally_exact + +For random decimal pairs (and the seed `1e-15 × 1e-15`), the exact product is spelled as +`e-` and parsed. The parse must have the arm and the value of the computed +product. An earlier draft built the text only when the reduced denominator was a power of +ten, and rejected most cases. A reduced decimal product's denominator is `2^a·5^b`, so the +text now scales by the smallest power of ten that makes the product an integer. + +### a_typed_value_and_the_same_value_computed_are_equally_exact_catches_demoting_on_the_hint + +The rival is revision 6's multiplication, which demotes whenever the returned scale drops. +`1e-15 × 1e-15` then becomes an f64, while the typed `1e-30` is the exact fraction. That +is the asymmetry revision 6's N2 was about. + +### a_recomputed_result_keeps_the_stored_scale_clause_4_names + +ADR 005 clause 4's example: `1.2500000000000000 × 4.0000000000000000` is `5` at scale 28, +not scale 0 and not 32. The test also pins that a common-path sum keeps its returned scale +(`12.50 + 1.5 = 14.00`), that `1000 + 0.333…` goes to the fraction arm, and that an exact +zero result has no sign. + +### an_intermediate_beyond_i128_comes_back_exact + +The point of question 13 (b). `a = 10^30/7`; `a × a` does not fit `i128`, yet `(a × a) / +a` through `Interim` is `a`, exactly. The same product stored through `Num` is inexact +and correctly rounded. + +### the_canonical_text_reads_back_as_the_same_number + +Every arm's `Display` text reads back through `from_canonical` as the same arm, value and +scale. The test pins the four spellings ADR 005 clause 11 tabulates, and that canonical +exact text never reads back as an inexact value. + +### the_accumulator_keeps_inexact_terms_whose_weights_cancel + +ChatGPT Codex, P1 on PR #337, against `edbb50c4`. `NumSum` decided "is an inexact term +present" from the **net** inexact weight. Adding `1.0` with weight 1 and `2.0` with weight +−1 nets to zero while the accumulator holds −1.0. So `read()` took the exact branch and +returned the exact partition alone, dropping the inexact term silently. + +This test **failed at `edbb50c4`**, on its first assertion: the read of `[1.0 ×1, 2.0 ×−1, +0.5]` claimed to be exact. Two things made the bug invisible to the existing properties: +- every generated term was inserted with weight +1; +- the retraction test retracts the very values it inserted, so per-value presence and net + presence agree there by construction. + +The test also asserts that retracting both inexact terms returns the exact `0.5`, so the +fix cannot simply make every sum that ever saw an f64 inexact. + +### a_huge_odd_exponent_keeps_the_sign_of_a_negative_base + +Copilot on PR #337, against `edbb50c4`. When the size check demotes a power to `powf`, the +exponent becomes an f64. Above 2^53 it rounds to an even value, so `(-2)^(2^53 + 1)` +returned `+inf` where the sign must be negative, and the negative exponent gave `+0.0`. +This test **failed at `edbb50c4`** on its first assertion. It also checks a fraction base, +and that an even exponent stays positive, so a fix that always negates cannot pass. + +### i128_min_is_an_exact_fraction_numerator + +Copilot on PR #337, against `edbb50c4`. `Num::fit` rejected `i128::MIN` as a numerator +because `to_i128` filtered it for both halves, so `Num::ratio(i128::MIN, 1)` became inexact +although `Ratio` holds it. This test **failed at `edbb50c4`** on its first +assertion. The oracle's `fits` had the same symmetric-range assumption, so no property +could see the bug. Both are fixed, and `fraction()` now generates `i128::MIN / d`. + +Allowing the value exposes a second hazard: it cannot be negated in `i128`, and the old +`negate` computed `-n`, which panics in a debug build. So the test also negates +`i128::MIN / 3` (an inexact, correctly rounded result), takes it through `Interim` twice +and back, divides it, compares it and reads its canonical text. + +### a_negative_power_has_its_minimal_scale + +Copilot on PR #337, and B3 of the review of `302594bf`. The unit-base arm of `POW` +padded every result to `s·|n|`, so `1.0^-2` was `1.00` and `(-1.0)^-3` was `-1.000`, +while `2.0^-2` was `0.25`. `context/code/numtower.md` already said "a negative power … +has its minimal scale", so the code contradicted the crate's stated rule. This test +**failed at `302594bf`** on its first assertion. It also pins that a positive power keeps +`s·n` (`(-1.0)^3 = -1.000`), so a fix that stripped every scale cannot pass. + +The broader guard is the scale assertion now in +`pow_is_exact_for_an_exact_base_and_integer_exponent_when_it_fits`, which had checked +value and exactness but never scale. Its oracle, `power_scale`, is the rule restated: +- start from the minimal scale; +- pad toward `min(s·k, 28)` for a decimal base with `k > 0` only, while the mantissa + fits 96 bits. + +Unit bases (`1.0`, `-1.0`, `1.000`, `-1`, `0.00`) are among its seeds. It **failed at +`302594bf`** with the minimal case `1.0^-1 = 1.0`. The injection G1 (a negative power +keeps the base's scale) fails both tests. + +### exact_sums_are_equal_when_their_values_are + +Copilot on PR #337, and B3 of the review. `ExactSum` derived `PartialEq` over its raw +limbs and its `pending` carry counter. This test **failed at `302594bf`** on its first +assertion, `0.5 + 0.5 == 1.0`. It also asserts: +- insert then retract equals an empty sum; +- weights `(2^30 − 1, 2)` added in either order are equal; +- different values are unequal, and a counted NaN is part of the value, so an `==` that + always answers true cannot pass. + +Injection G2, equality back on the representation, fails it. + +### a_forced_fraction_never_reads_as_infinity + +A should-fix from the review of `302594bf`. A forced `n/d` with a 400-digit `n` read as +`Inexact(inf)`, though clause 3 says a finite lexeme never reads as infinity. This test +**failed at `302594bf`**. It also asserts: +- `is_number` agrees with the parse; +- a negative one is text too; +- an underflowing `1/9…9` is still the inexact zero, so a fix that refused every inexact + forced fraction cannot pass. + +Injection G3 fails it. + +### an_interim_compares_exactly_without_storing + +New API, from a should-fix in the review: `Interim` had no comparison and no zero test +while step 3 is told to evaluate through it. For two wide products `a`, it compares `a` +against: +- another wide product; +- `a ± 1/(3·10^30)`; +- `a` itself; +- the f64 nearest `a`, and that f64's upper neighbour. + +Each comparison is made in both directions, against the `BigRational` order. It also +asserts `a − a` is zero and `a` is not. Injection G4, storing before comparing, fails it, +and so do F3 and F13, because the stored arms delegate to `Num::cmp_exact`. + +### an_interim_compares_exactly_without_storing_catches_storing_first + +The rival is the comparison the review warned step 3 would otherwise have to make: store +both operands, then compare. `a` and `a ± 1/(3·10^30)` store to one f64, so the rival +calls them equal. + +--- + +## `crates/numtower/src/sum.rs` + +### sum::tests::the_grouped_read_combines_over_the_lcm_and_reduces_once + +The combine must return the **lcm** as its denominator, and the reduced result must equal +the `BigRational` sum. Group numerators are up to `i128 × i64`, and the denominators share +factors (3, 77, 2^20, 3^40), so the lcm and the product differ. + +### sum::tests::the_grouped_read_combines_over_the_lcm_and_reduces_once_catches_the_product + +The rival combines over the product of the denominators. Its value is right, so only the +lcm assertion can see it. That is why the property asserts the denominator before the +value. diff --git a/crates/improvise-command/src/command/cmd/panel.rs b/crates/improvise-command/src/command/cmd/panel.rs index d15dbbd3..14f08c97 100644 --- a/crates/improvise-command/src/command/cmd/panel.rs +++ b/crates/improvise-command/src/command/cmd/panel.rs @@ -777,6 +777,7 @@ impl Cmd for FilterToItem { Some(CatTreeEntry::Item { cat_name, item_name, + .. }) => { vec![ Box::new(effect::SetAxis { @@ -821,6 +822,7 @@ impl Cmd for DeleteCategoryAtCursor { Some(CatTreeEntry::Item { cat_name, item_name, + .. }) => { let mut effects: Vec> = vec![ Box::new(effect::RemoveItem { diff --git a/crates/improvise-command/src/ui/app.rs b/crates/improvise-command/src/ui/app.rs index 47425e2b..ce17b26d 100644 --- a/crates/improvise-command/src/ui/app.rs +++ b/crates/improvise-command/src/ui/app.rs @@ -106,7 +106,9 @@ fn frame_of(layout: &crate::view::layout::GridLayout) -> u64 { group_name.hash(&mut h); collapsed.hash(&mut h); } - crate::view::layout::AxisEntry::DataItem { tuple, collapsed } => { + crate::view::layout::AxisEntry::DataItem { + tuple, collapsed, .. + } => { 1u8.hash(&mut h); tuple.hash(&mut h); collapsed.hash(&mut h); diff --git a/crates/improvise-command/src/ui/cat_tree.rs b/crates/improvise-command/src/ui/cat_tree.rs index b8db1ea7..19892233 100644 --- a/crates/improvise-command/src/ui/cat_tree.rs +++ b/crates/improvise-command/src/ui/cat_tree.rs @@ -62,6 +62,7 @@ mod tests { CatTreeEntry::Item { cat_name, item_name, + .. } if cat_name == MEASURE_CATEGORY => Some(item_name), _ => None, }) @@ -145,6 +146,7 @@ mod tests { let item = CatTreeEntry::Item { cat_name: "Region".into(), item_name: "North".into(), + display: "North".into(), }; assert_eq!(header.cat_name(), "Region"); assert_eq!(item.cat_name(), "Region"); diff --git a/crates/improvise-command/src/ui/project.rs b/crates/improvise-command/src/ui/project.rs index 257c4b95..1c0c0efa 100644 --- a/crates/improvise-command/src/ui/project.rs +++ b/crates/improvise-command/src/ui/project.rs @@ -14,12 +14,12 @@ use improvise_core::workbook::Workbook; use improvise_io::import::analyzer::{DateComponent, DateShape, FieldKind}; use improvise_io::import::wizard::{DuplicatePolicy, ImportWizard, WizardStep}; use improvise_protocol::{ - AxisFrame, Basis, BasisCache, CategoryRow, CategoryTreeCache, CategoryTreeUpdate, CellDisplay, - CellId, Command, CreateKind, DocumentCache, FormulaCache, FormulaEntry, FrameEntry, FrameGroup, - GridCache, GridExtent, GridUpdate, Projection, RecordCell, Rejection, Snapshot, TileBarCache, - TileEntry, ViewListCache, ViewState, ViewSummary, WidgetCache, WizardCache, WizardChoice, - WizardCollision, WizardDateField, WizardFieldKind, WizardProposal, WizardStepKind, basis, - reduce_view, + AxisFrame, Basis, BasisCache, CategoryItem, CategoryRow, CategoryTreeCache, CategoryTreeUpdate, + CellDisplay, CellId, Command, CreateKind, DocumentCache, FormulaCache, FormulaEntry, + FrameEntry, FrameGroup, GridCache, GridExtent, GridUpdate, Projection, RecordCell, Rejection, + Snapshot, TileBarCache, TileEntry, ViewAxis, ViewListCache, ViewState, ViewSummary, + WidgetCache, WizardCache, WizardChoice, WizardCollision, WizardDateField, WizardFieldKind, + WizardProposal, WizardStepKind, basis, reduce_view, }; use crate::ui::app::App; @@ -157,8 +157,16 @@ pub fn grid_in(app: &App, view_name: &str, viewport: Viewport) -> GridCache { }; GridCache { - row_frame: axis_frame(&layout.row_cats, &layout.row_items, rows.clone()), - col_frame: axis_frame(&layout.col_cats, &layout.col_items, cols.clone()), + row_frame: axis_frame( + (&layout.row_cats, layout.row_cat_displays(model)), + &layout.row_items, + rows.clone(), + ), + col_frame: axis_frame( + (&layout.col_cats, layout.col_cat_displays(model)), + &layout.col_items, + cols.clone(), + ), col_widths: cols .clone() .map(|c| body.col_widths.get(c).copied().unwrap_or(0)) @@ -222,7 +230,11 @@ pub(crate) fn rederived_for( /// One axis's frame, windowed to the data indices `window` names. Three edge rules, /// all about group headers — see `context/code/command.md`. -fn axis_frame(cats: &[String], items: &[AxisEntry], window: std::ops::Range) -> AxisFrame { +fn axis_frame( + (cats, cat_displays): (&[String], Vec), + items: &[AxisEntry], + window: std::ops::Range, +) -> AxisFrame { let total = items .iter() .filter(|e| matches!(e, AxisEntry::DataItem { .. })) @@ -242,6 +254,7 @@ fn axis_frame(cats: &[String], items: &[AxisEntry], window: std::ops::Range { @@ -249,6 +262,7 @@ fn axis_frame(cats: &[String], items: &[AxisEntry], window: std::ops::Range { + AxisEntry::DataItem { + tuple, + display, + collapsed, + } => { let idx = data_idx; data_idx += 1; if window.contains(&idx) { @@ -276,6 +294,7 @@ fn axis_frame(cats: &[String], items: &[AxisEntry], window: std::ops::Range()`, which called a coordinate `2024` a number. - let text = value.as_ref().map(|v| v.to_string()).unwrap_or_default(); + let text = layout + .coordinate_display(&app.model_state.workbook.model, row, col) + .unwrap_or_else(|| value.as_ref().map(|v| v.to_string()).unwrap_or_default()); return CellDisplay { edit_text: Some(value.as_ref().map(CellValue::edit_text).unwrap_or_default()), text, @@ -436,11 +458,6 @@ fn non_item(category: &str, item: &str) -> CellId { } } -/// What a person sees for a category key; the key itself when no category holds it. -fn category_display<'a>(model: &'a Model, name: &'a str) -> &'a str { - model.category(name).map_or(name, |c| c.display()) -} - /// The categories the category panel draws: **every** category, virtual ones /// included, with their items and their axis (improvise-vjms, gotcha 7). pub fn category_rows(model: &Model, view: &View) -> Vec { @@ -449,8 +466,15 @@ pub fn category_rows(model: &Model, view: &View) -> Vec { .into_iter() .map(|name| CategoryRow { name: name.to_string(), - display: category_display(model, name).to_string(), - items: model.effective_item_names(name), + display: model.category_display(name).to_string(), + items: model + .effective_item_names(name) + .into_iter() + .map(|item| CategoryItem { + display: model.item_display(name, &item).to_string(), + name: item, + }) + .collect(), // `try_axis_of`, not `axis_of`: a view that has never seen a category // has no entry and `axis_of` panics. Unseen shows as `None`. axis: view.try_axis_of(name).unwrap_or(Axis::None), @@ -473,8 +497,8 @@ pub fn formula_entries(app: &App) -> Vec { /// Every view and where each category sits in it. Which view is *active* is /// per-client and travels as `Command::SwitchView`, so it is not here. pub fn view_summaries(app: &App) -> Vec { - app.model_state - .workbook + let workbook = &app.model_state.workbook; + workbook .views .iter() .map(|(name, view)| ViewSummary { @@ -483,7 +507,11 @@ pub fn view_summaries(app: &App) -> Vec { axes: view .category_axes .iter() - .map(|(cat, axis)| (cat.clone(), *axis)) + .map(|(cat, axis)| ViewAxis { + category: cat.clone(), + display: workbook.model.category_display(cat).to_string(), + axis: *axis, + }) .collect(), }) .collect() @@ -501,7 +529,7 @@ pub fn tiles_of(model: &Model, view: &View) -> Vec { .into_iter() .map(|cat| TileEntry { category: cat.to_string(), - display: category_display(model, cat).to_string(), + display: model.category_display(cat).to_string(), axis: view.try_axis_of(cat).unwrap_or(Axis::None), kind: model .category(cat) @@ -1873,6 +1901,205 @@ mod tests { assert_ne!(summary.name, "Overview"); } + /// A category, an item, a group, a page category, a measure and two views, + /// each shown under a name that is not its key (improvise-cq1q.12). + fn app_shown_otherwise() -> App { + let mut wb = Workbook::new("Test"); + wb.add_category("Region").unwrap(); + wb.add_category("Quarter").unwrap(); + wb.add_item_in_group("Region", "East", "Coastal").unwrap(); + wb.add_item("Region", "West").unwrap(); + wb.add_item("Quarter", "Q1").unwrap(); + wb.add_item("Quarter", "Q2").unwrap(); + wb.add_item("_Measure", "Revenue").unwrap(); + wb.active_view_mut().set_axis("Region", Axis::Row); + wb.active_view_mut().set_axis("_Measure", Axis::Column); + wb.active_view_mut().set_axis("Quarter", Axis::Page); + wb.active_view_mut().set_expand_depth("Region", 1); + for region in ["East", "West"] { + for quarter in ["Q1", "Q2"] { + wb.model.set_cell( + CellKey::new(vec![ + ("Region".into(), region.into()), + ("Quarter".into(), quarter.into()), + ("_Measure".into(), "Revenue".into()), + ]), + CellValue::Number(1.0), + ); + } + } + wb.create_view("Plan").set_axis("Region", Axis::Column); + wb.active_view_mut().set_display("Overview"); + wb.views.get_mut("Plan").unwrap().set_display("Budget"); + let region = wb.model.category_mut("Region").unwrap(); + region.set_display("Sales Area"); + region.set_item_display("East", "Eastern Seaboard"); + region.set_group_display("Coastal", "Shoreline"); + let quarter = wb.model.category_mut("Quarter").unwrap(); + quarter.set_display("Period"); + quarter.set_item_display("Q1", "First"); + quarter.set_item_display("Q2", "Second"); + wb.model + .category_mut("_Measure") + .unwrap() + .set_item_display("Revenue", "Sales"); + let mut app = App::new(wb, None); + app.rebuild_layout(); + app + } + + const ALL: Viewport = Viewport { + row_offset: 0, + col_offset: 0, + rows: 20, + cols: 20, + }; + + #[test] + fn the_frame_shows_displays_and_its_key_tuples_still_address_the_cells() { + let app = app_shown_otherwise(); + let grid = grid_of(&app, ALL); + assert_eq!(grid.row_frame.cat_displays, vec!["Sales Area"]); + assert_eq!( + grid.row_frame.cats, + vec!["Region"], + "the key stays beside it" + ); + assert_eq!( + grid.col_frame.header_labels(0), + vec![vec!["Sales".to_string()]] + ); + let header = grid + .row_frame + .entries + .iter() + .find_map(|e| match e { + FrameEntry::Group(g) => Some(g), + _ => None, + }) + .expect("East sits under Coastal"); + assert_eq!( + (header.name.as_str(), header.display.as_str()), + ("Coastal", "Shoreline") + ); + assert_eq!( + grid.row_frame.header_labels(0)[0], + vec!["Eastern Seaboard".to_string()] + ); + // The frame's KEY tuples, joined with the page bar, are the cell's address. + let page: Vec<(String, String)> = grid + .page_coords + .iter() + .map(|p| (p.cat.clone(), p.selected.clone())) + .collect(); + for (r, row) in grid.row_frame.items().enumerate() { + for (c, col) in grid.col_frame.items().enumerate() { + let coords = grid + .row_frame + .cats + .iter() + .cloned() + .zip(row.iter().cloned()) + .chain(grid.col_frame.cats.iter().cloned().zip(col.iter().cloned())) + .chain(page.iter().cloned()) + .collect(); + assert_eq!( + Some(CellKey::new(coords)), + app.layout.cell_key(r, c), + "({r},{c}) must be addressed by keys" + ); + } + } + } + + #[test] + fn the_page_bar_carries_displays_beside_the_keys_a_page_command_names() { + let app = app_shown_otherwise(); + let bar = grid_of(&app, ALL).page_coords; + let quarter = &bar[0]; + assert_eq!( + (quarter.cat.as_str(), quarter.selected.as_str()), + ("Quarter", "Q1") + ); + assert_eq!(quarter.cat_display, "Period"); + assert_eq!(quarter.selected_display, "First"); + assert_eq!( + quarter.items.as_deref(), + Some(&["Q1".to_string(), "Q2".into()][..]) + ); + assert_eq!( + quarter.item_displays.as_deref(), + Some(&["First".to_string(), "Second".into()][..]) + ); + } + + #[test] + fn an_item_row_shows_its_display_and_keeps_its_key() { + let app = app_shown_otherwise(); + let rows = category_rows(&app.model_state.workbook.model, app.view()); + let region = rows.iter().find(|r| r.name == "Region").unwrap(); + let east = region.items.iter().find(|i| i.name == "East").unwrap(); + assert_eq!(east.display, "Eastern Seaboard"); + let west = region.items.iter().find(|i| i.name == "West").unwrap(); + assert_eq!(west.display, "West", "an unrenamed item shows its key"); + let measure = rows.iter().find(|r| r.name == "_Measure").unwrap(); + assert_eq!(measure.items[0].display, "Sales"); + } + + #[test] + fn each_view_summary_shows_its_own_display_and_its_categories_displays() { + let app = app_shown_otherwise(); + let summaries = view_summaries(&app); + let by_key = |key: &str| summaries.iter().find(|s| s.name == key).unwrap(); + assert_eq!(by_key("Default").display, "Overview"); + assert_eq!(by_key("Plan").display, "Budget", "not the active view's"); + let plan_region = by_key("Plan") + .axes + .iter() + .find(|a| a.category == "Region") + .unwrap(); + assert_eq!(plan_region.display, "Sales Area"); + assert_eq!(plan_region.axis, Axis::Column); + let line = improvise_protocol::view_axis_summary(by_key("Plan")); + assert_eq!( + line, "C:Sales Area,Period", + "the line draws displays, not keys" + ); + } + + #[test] + fn a_records_coordinate_shows_its_display_and_edits_its_key() { + let mut app = app_shown_otherwise(); + let view = app.model_state.workbook.active_view_mut(); + view.set_axis("_Index", Axis::Row); + view.set_axis("_Dim", Axis::Column); + app.rebuild_layout(); + let grid = grid_of(&app, ALL); + assert!(grid.records); + let col = grid + .col_frame + .items() + .position(|t| t == &vec!["Region".to_string()]) + .expect("a column per category, found by its key"); + assert_eq!( + grid.col_frame.header_labels(0)[col], + vec!["Sales Area".to_string()] + ); + let shown: Vec<&str> = grid.cells.iter().map(|r| r[col].text.as_str()).collect(); + assert!(shown.contains(&"Eastern Seaboard"), "{shown:?}"); + let east = grid + .cells + .iter() + .map(|r| &r[col]) + .find(|c| c.text == "Eastern Seaboard") + .unwrap(); + assert_eq!( + east.edit_text.as_deref(), + Some("East"), + "the editor seeds the key" + ); + } + /// A pivot-shaped default view plus a records-shaped second one, so a /// switch between them crosses the boundary `settled_mode` watches. fn app_with_a_records_shaped_second_view() -> App { @@ -2481,6 +2708,7 @@ mod tests { row_frame: AxisFrame { first: 0, cats: vec!["Region".into()], + cat_displays: vec!["Region".into()], entries: vec![FrameEntry::item(vec!["North".into()])], }, col_frame: AxisFrame::flat(["A", "B"]), @@ -2493,6 +2721,7 @@ mod tests { row_frame: AxisFrame { first: 0, cats: vec!["Department".into()], + cat_displays: vec!["Department".into()], entries: mirror.row_frame.entries.clone(), }, ..mirror.clone() diff --git a/crates/improvise-command/src/ui/viewport.rs b/crates/improvise-command/src/ui/viewport.rs index 01d499b0..d0a2eaca 100644 --- a/crates/improvise-command/src/ui/viewport.rs +++ b/crates/improvise-command/src/ui/viewport.rs @@ -73,7 +73,9 @@ pub fn grid_body( let (key, value) = cell_value(layout, &eval, records, ri, ci); let fmt = plan.at(key.as_ref()); let text = match records { - true => value.as_ref().map(|v| v.to_string()).unwrap_or_default(), + true => layout + .coordinate_display(model, ri, ci) + .unwrap_or_else(|| value.as_ref().map(|v| v.to_string()).unwrap_or_default()), false => fmt.value(value.as_ref()), }; widen(&mut widths, ci, text.width()); @@ -168,7 +170,7 @@ fn cell_value( fn header_level_seed(layout: &GridLayout, n: usize) -> Vec { let mut widths = vec![0u16; n]; let items = layout.col_items.iter().filter_map(|e| match e { - AxisEntry::DataItem { tuple, .. } => Some(tuple), + AxisEntry::DataItem { display, .. } => Some(display), AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None, }); for (ci, levels) in items.enumerate().take(n) { @@ -197,7 +199,7 @@ pub fn compute_col_widths(model: &Model, layout: &GridLayout, plan: &FormatPlan) /// keypress-path caller that borrows tuples — see `context/code/command.md`. pub fn compute_row_header_width(layout: &GridLayout) -> u16 { let items = layout.row_items.iter().filter_map(|e| match e { - AxisEntry::DataItem { tuple, .. } => Some(tuple), + AxisEntry::DataItem { display, .. } => Some(display), AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None, }); header_level_widths(items, layout.row_cats.len().max(1)) @@ -251,6 +253,51 @@ mod tests { use crate::view::Axis; use crate::workbook::Workbook; + /// One row and one column, each item shown under a display far longer than its + /// key and than the cell, so the 5-column minimum cannot hide a key measurement. + fn long_displays() -> Workbook { + let mut wb = Workbook::new("T"); + wb.add_category("Type").unwrap(); + wb.add_category("Month").unwrap(); + wb.add_item("Type", "E").unwrap(); + wb.add_item("Month", "Jan").unwrap(); + wb.model.set_cell( + CellKey::new(vec![ + ("Type".into(), "E".into()), + ("Month".into(), "Jan".into()), + ]), + CellValue::Number(1.0), + ); + wb.model + .category_mut("Type") + .unwrap() + .set_item_display("E", "Eastern Seaboard"); + wb.model + .category_mut("Month") + .unwrap() + .set_item_display("Jan", "Januaryyyyy"); + wb + } + + #[test] + fn a_column_is_as_wide_as_the_display_its_header_draws() { + let wb = long_displays(); + let layout = GridLayout::new(&wb.model, wb.active_view()); + let plan = FormatPlan::of(&wb.model, wb.active_view()); + let widths = compute_col_widths(&wb.model, &layout, &plan); + assert_eq!(widths, vec!["Januaryyyyy".len() as u16 + 1]); + } + + #[test] + fn the_row_header_is_as_wide_as_the_display_its_rows_draw() { + let wb = long_displays(); + let layout = GridLayout::new(&wb.model, wb.active_view()); + assert_eq!( + compute_row_header_width(&layout), + "Eastern Seaboard".len() as u16 + 1 + ); + } + /// Q1 = {Jan, Feb} collapsed, Q2 = {Apr}; F0 = Revenue and F(i) = F(i-1) twice. fn folded_chain() -> Workbook { let mut wb = Workbook::new("T"); diff --git a/crates/improvise-core/examples/retraction_drift.rs b/crates/improvise-core/examples/retraction_drift.rs new file mode 100644 index 00000000..407a7ab5 --- /dev/null +++ b/crates/improvise-core/examples/retraction_drift.rs @@ -0,0 +1,458 @@ +//! How far a SUM maintained by insert/retract deltas drifts from the eager fold, +//! in f64, and what an exact accumulator costs. ADR 009 (improvise-felo.4), clause 8. + +use std::hint::black_box; +use std::time::Instant; + +use improvise_core::format::format_f64; +use improvise_core::model::cell::{CellKey, CellValue}; +use improvise_core::model::types::{Aggregation, Model}; + +const MEASURE: &str = "_Measure"; + +/// SplitMix64: seeded, so a run is reproducible and a second seed is a new sample. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// A user-entered decimal as an exact count of 1e-5 units, its magnitude spread +/// over `decades` decades. The f64 the store holds is `typed(units)`. +fn decimal(rng: &mut Rng, decades: u32) -> i64 { + let scale = 10i64.pow(rng.below(u64::from(decades)) as u32); + let cents = rng.below(100_000) as i64; + let sign = if rng.below(4) == 0 { -1 } else { 1 }; + sign * cents * scale +} + +fn typed(units: i64) -> f64 { + units as f64 / 1e5 +} + +/// The exact decimal sum, rounded to f64 once: what ADR 005's arithmetic would show. +fn exact(units: &[i64]) -> f64 { + units.iter().map(|&u| i128::from(u)).sum::() as f64 / 1e5 +} + +fn key(row: usize) -> CellKey { + CellKey::new(vec![ + ("Row".to_string(), format!("r{row}")), + (MEASURE.to_string(), "Amount".to_string()), + ]) +} + +/// `cells` rows of one measure; the eager SUM is the one the grid draws. +fn document(rng: &mut Rng, cells: usize, decades: u32) -> (Model, Vec) { + let mut m = Model::new("drift"); + m.add_category("Row").expect("under the limit"); + let values: Vec = (0..cells).map(|_| decimal(rng, decades)).collect(); + for (row, v) in values.iter().enumerate() { + m.category_mut("Row") + .expect("just added") + .add_item(format!("r{row}")); + m.set_cell(key(row), CellValue::Number(typed(*v))); + } + (m, values) +} + +fn eager(m: &Model) -> f64 { + let total = CellKey::new(vec![(MEASURE.to_string(), "Amount".to_string())]); + m.evaluate_aggregated_f64(&total, Aggregation::OverHidden) +} + +/// Neumaier's compensated running sum: the cheap mitigation, measured beside the naive one. +#[derive(Default)] +struct Compensated { + sum: f64, + carry: f64, +} + +impl Compensated { + fn add(&mut self, x: f64) { + let t = self.sum + x; + self.carry += match self.sum.abs() >= x.abs() { + true => (self.sum - t) + x, + false => (x - t) + self.sum, + }; + self.sum = t; + } + + fn value(&self) -> f64 { + self.sum + self.carry + } +} + +const LIMB_BITS: u32 = 32; +/// Every finite f64 is `mant * 2^(shift - 1074)` with `shift <= 2045` and `mant < 2^53`. +const LIMBS: usize = 67; +/// The most weight one limb absorbs between carries: each add moves a limb by < 2^32. +const CHUNK: i64 = 1 << 30; + +/// An exact sum of f64s: a signed fixed-point integer in units of 2^-1074, wide enough +/// for every finite f64, with non-finite values counted so they can be retracted too. +#[derive(Clone)] +struct ExactSum { + limbs: [i64; LIMBS], + pending: u64, + nan: i64, + pos_inf: i64, + neg_inf: i64, +} + +impl ExactSum { + fn new() -> Self { + ExactSum { + limbs: [0; LIMBS], + pending: 0, + nan: 0, + pos_inf: 0, + neg_inf: 0, + } + } + + /// Add `x` with any `weight` (a consolidated Z-set weight). Exact: no rounding here. + fn add(&mut self, x: f64, weight: i64) { + match (x.is_nan(), x.is_infinite()) { + (true, _) => self.nan += weight, + (false, true) if x > 0.0 => self.pos_inf += weight, + (false, true) => self.neg_inf += weight, + (false, false) => { + let mut left = weight; + while left != 0 { + let chunk = left.clamp(-CHUNK, CHUNK); + self.add_finite(x, chunk); + left -= chunk; + } + } + } + } + + /// Bounded so each limb stays under 2^62: `|weight| <= CHUNK` and `pending` + /// counts magnitude, not calls. + fn add_finite(&mut self, x: f64, weight: i64) { + if self.pending + weight.unsigned_abs() > CHUNK as u64 { + propagate(&mut self.limbs); + self.pending = 0; + } + let bits = x.to_bits(); + let biased = ((bits >> 52) & 0x7ff) as u32; + let fraction = bits & ((1 << 52) - 1); + let (mant, shift) = match biased { + 0 => (fraction, 0), + e => (fraction | (1 << 52), e - 1), + }; + let sign = if x.is_sign_negative() { + -weight + } else { + weight + }; + let wide = u128::from(mant) << (shift % LIMB_BITS); + let k = (shift / LIMB_BITS) as usize; + for i in 0..3 { + let part = ((wide >> (LIMB_BITS * i as u32)) & 0xffff_ffff) as i64; + self.limbs[k + i] += sign * part; + } + self.pending += weight.unsigned_abs(); + } + + /// The exact sum rounded to the nearest f64, ties to even: the one rounding. + fn value(&self) -> f64 { + if self.nan > 0 || (self.pos_inf > 0 && self.neg_inf > 0) { + return f64::NAN; + } + if self.pos_inf > 0 { + return f64::INFINITY; + } + if self.neg_inf > 0 { + return f64::NEG_INFINITY; + } + let mut limbs = self.limbs; + propagate(&mut limbs); + let negative = limbs[LIMBS - 1] < 0; + if negative { + limbs.iter_mut().for_each(|l| *l = -*l); + propagate(&mut limbs); + } + let magnitude = round_to_f64(&digits(&limbs)); + match negative { + true => -magnitude, + false => magnitude, + } + } +} + +/// Carry each limb into the next, leaving every limb but the top in [0, 2^32). +fn propagate(limbs: &mut [i64; LIMBS]) { + for k in 0..LIMBS - 1 { + let carry = limbs[k] >> LIMB_BITS; + limbs[k] -= carry << LIMB_BITS; + limbs[k + 1] += carry; + } +} + +/// A non-negative normalized accumulator as 32-bit digits, the top limb split out. +fn digits(limbs: &[i64; LIMBS]) -> Vec { + let mut out: Vec = limbs.iter().map(|&l| l as u64).collect(); + let mut top = out.pop().unwrap_or(0); + loop { + out.push(top & 0xffff_ffff); + top >>= LIMB_BITS; + if top == 0 { + break; + } + } + out +} + +fn bit(d: &[u64], pos: usize) -> u64 { + (d[pos / 32] >> (pos % 32)) & 1 +} + +/// Round an exact multiple of 2^-1074 to the nearest f64, ties to even. +fn round_to_f64(d: &[u64]) -> f64 { + let Some(k) = d.iter().rposition(|&x| x != 0) else { + return 0.0; + }; + let msb = 32 * k + 63 - d[k].leading_zeros() as usize; + if msb < 53 { + let int = d[0] | (d[1] << 32); + return int as f64 * f64::from_bits(1); + } + let mut mant = (0..53).fold(0u64, |m, i| m | (bit(d, msb - 52 + i) << i)); + let round = bit(d, msb - 53) == 1; + let sticky = (0..msb - 53).any(|p| bit(d, p) == 1); + let mut top = msb; + if round && (sticky || mant & 1 == 1) { + mant += 1; + if mant == 1 << 53 { + mant >>= 1; + top += 1; + } + } + let exp = top as i64 - 52 - 1074; + if exp > 971 { + return f64::INFINITY; + } + let scale = match exp >= -1022 { + true => f64::from_bits(((exp + 1023) as u64) << 52), + false => f64::from_bits(1u64 << (exp + 1074)), + }; + mant as f64 * scale +} + +fn exact_over(values: &[i64]) -> f64 { + let mut acc = ExactSum::new(); + values.iter().for_each(|&v| acc.add(typed(v), 1)); + acc.value() +} + +fn ulps(a: f64, b: f64) -> u64 { + (a.to_bits() as i64).abs_diff(b.to_bits() as i64) +} + +struct Sums { + exact: f64, + eager: f64, + naive: f64, + comp: f64, + xs_maintained: f64, + xs_fresh: f64, +} + +/// Each running sum against the eager fold (the differential oracle) and against the +/// exact decimal sum. `flip`: the gap reaches `FLOAT_EQ_EPSILON`, so `=` can answer differently. +fn report(label: &str, cycles: usize, s: &Sums) { + let shown = |v: f64| format_f64(v, true, 2); + println!( + "{label:<22} {cycles:>8} {:>10.2e} {:>6} {:>10.2e} {:>6} {:>5} {:>10.2e} {:>10.2e} {:>10.2e} {:>7} {:>10.2e}", + (s.naive - s.eager).abs(), + ulps(s.naive, s.eager), + (s.comp - s.eager).abs(), + ulps(s.comp, s.eager), + (s.naive - s.eager).abs() >= 1e-10, + (s.eager - s.exact).abs(), + (s.naive - s.exact).abs(), + (s.comp - s.exact).abs(), + s.xs_maintained.to_bits() == s.xs_fresh.to_bits(), + (s.xs_maintained - s.exact).abs(), + ); + if shown(s.naive) != shown(s.eager) || shown(s.eager) != shown(s.exact) { + println!( + " shown at ,.2: naive {} eager {} exact {} maintained-exact-sum {}", + shown(s.naive), + shown(s.eager), + shown(s.exact), + shown(s.xs_maintained) + ); + } +} + +/// Each cycle retracts one live value and inserts a fresh one at the same key. +fn random_updates(seed: u64, cells: usize, decades: u32, checkpoints: &[usize]) { + let mut rng = Rng(seed); + let (mut m, mut values) = document(&mut rng, cells, decades); + let mut naive = eager(&m); + let mut comp = Compensated::default(); + comp.add(naive); + let mut xs = ExactSum::new(); + values.iter().for_each(|&v| xs.add(typed(v), 1)); + let mut done = 0; + for &at in checkpoints { + while done < at { + let row = rng.below(cells as u64) as usize; + let new = decimal(&mut rng, decades); + naive = naive - typed(values[row]) + typed(new); + comp.add(-typed(values[row])); + comp.add(typed(new)); + xs.add(typed(values[row]), -1); + xs.add(typed(new), 1); + values[row] = new; + m.set_cell(key(row), CellValue::Number(typed(new))); + done += 1; + } + let sums = Sums { + exact: exact(&values), + eager: eager(&m), + naive, + comp: comp.value(), + xs_maintained: xs.value(), + xs_fresh: exact_over(&values), + }; + report(&format!("{cells} cells, {decades} dec"), at, &sums); + } +} + +/// A large value arrives, small ones accumulate under it, then it is retracted. +fn absorbed_by_a_large_value() { + let mut m = Model::new("absorb"); + m.add_category("Row").expect("under the limit"); + m.category_mut("Row").expect("just added").add_item("r0"); + let mut naive = 0.0; + let mut comp = Compensated::default(); + let mut xs = ExactSum::new(); + m.set_cell(key(0), CellValue::Number(1e17)); + naive += 1e17; + comp.add(1e17); + xs.add(1e17, 1); + for row in 1..=1000 { + m.category_mut("Row") + .expect("just added") + .add_item(format!("r{row}")); + m.set_cell(key(row), CellValue::Number(0.01)); + naive += 0.01; + comp.add(0.01); + xs.add(0.01, 1); + } + m.clear_cell(&key(0)); + naive -= 1e17; + comp.add(-1e17); + xs.add(1e17, -1); + let mut fresh = ExactSum::new(); + (0..1000).for_each(|_| fresh.add(0.01, 1)); + let sums = Sums { + exact: 10.0, + eager: eager(&m), + naive, + comp: comp.value(), + xs_maintained: xs.value(), + xs_fresh: fresh.value(), + }; + report("1e17, 1000 x 0.01, -1e17", 1001, &sums); +} + +/// Nanoseconds per operation over `n` values, best of three runs. +fn per_op(n: usize, mut op: impl FnMut() -> f64) -> f64 { + (0..3) + .map(|_| { + let t = Instant::now(); + black_box(op()); + t.elapsed().as_secs_f64() * 1e9 / n as f64 + }) + .fold(f64::INFINITY, f64::min) +} + +fn cost() { + let n = 10_000_000; + let mut rng = Rng(7); + let values: Vec = (0..n).map(|_| typed(decimal(&mut rng, 9))).collect(); + let updates = 3 * n; + let plain = per_op(updates, || { + values + .iter() + .fold(0.0, |s, &v| black_box(s + v) - black_box(v) + v) + }); + let exact_add = per_op(updates, || { + let mut xs = ExactSum::new(); + values.iter().for_each(|&v| { + xs.add(black_box(v), 1); + xs.add(black_box(v), -1); + xs.add(v, 1); + }); + xs.limbs[0] as f64 + }); + let mut xs = ExactSum::new(); + values.iter().for_each(|&v| xs.add(v, 1)); + let read = per_op(1, || xs.value()); + println!(); + println!("cost, {n} values x (insert, retract, insert), best of 3, per single update:"); + println!(" plain f64 add/sub {plain:>8.2} ns per update"); + println!(" exact accumulator {exact_add:>8.2} ns per update"); + println!(" exact read (round) {read:>8.0} ns per read"); + println!( + " state per group {} bytes, against 8 for an f64", + std::mem::size_of::() + ); +} + +/// A consolidated Z-set weight can be large; the accumulator must stay exact at 2^33. +fn large_weight() { + let x = f64::from_bits((1u64 << 32) - 1); + let mut xs = ExactSum::new(); + xs.add(x, 1 << 33); + let want = ((1u64 << 32) - 1) as f64 * 2f64.powi(33) * f64::from_bits(1); + println!(); + println!( + "large weight: (2^32-1) x 2^-1074 at weight 2^33 reads {:e}, exact {:e}, equal {}", + xs.value(), + want, + xs.value().to_bits() == want.to_bits() + ); +} + +fn main() { + println!( + "{:<22} {:>8} {:>10} {:>6} {:>10} {:>6} {:>5} {:>10} {:>10} {:>10} {:>7} {:>10}", + "case", + "cycles", + "naive-eag", + "ulp", + "comp-eag", + "ulp", + "flip", + "eag-exact", + "nai-exact", + "cmp-exact", + "xs=fresh", + "xs-exact" + ); + let checkpoints = [1_000, 10_000, 100_000, 1_000_000]; + for seed in [1, 2] { + random_updates(seed, 1_000, 3, &checkpoints); + random_updates(seed, 18_000, 3, &checkpoints); + random_updates(seed, 18_000, 9, &checkpoints); + } + absorbed_by_a_large_value(); + large_weight(); + cost(); +} diff --git a/crates/improvise-core/src/model/category.rs b/crates/improvise-core/src/model/category.rs index ac08ae37..01c5d8e5 100644 --- a/crates/improvise-core/src/model/category.rs +++ b/crates/improvise-core/src/model/category.rs @@ -38,18 +38,26 @@ impl Item { #[derive(Debug, Clone)] pub struct Group { pub name: String, + /// What a person sees; the key until a rename (ADR 007 clause 2). + display: String, /// Parent group name for nested hierarchies pub parent: Option, } impl Group { pub fn new(name: impl Into) -> Self { + let name = name.into(); Self { - name: name.into(), + display: name.clone(), + name, parent: None, } } + pub fn display(&self) -> &str { + &self.display + } + pub fn with_parent(mut self, parent: impl Into) -> Self { self.parent = Some(parent.into()); self @@ -180,6 +188,34 @@ impl Category { .is_some() } + /// What a person sees for an item: its display, or the key where the + /// category holds no such item (a `_Measure` formula target). + pub fn item_display<'a>(&'a self, item: &'a str) -> &'a str { + self.items.get(item).map_or(item, Item::display) + } + + /// What a person sees for a group; the key where no `Group` was declared + /// (a group may exist only as an item's `group`). + pub fn group_display<'a>(&'a self, group: &'a str) -> &'a str { + self.groups + .iter() + .find(|g| g.name == group) + .map_or(group, Group::display) + } + + /// Changes a group's display and nothing else. A group may exist only as an + /// item's `group`, so one is declared (as a root) when none is yet. + pub fn set_group_display(&mut self, group: &str, display: impl Into) { + match self.groups.iter_mut().find(|g| g.name == group) { + Some(g) => g.display = display.into(), + None => { + let mut declared = Group::new(group); + declared.display = display.into(); + self.groups.push(declared); + } + } + } + pub fn with_kind(mut self, kind: CategoryKind) -> Self { self.kind = kind; self @@ -307,6 +343,21 @@ mod tests { assert!(!c.set_item_display("West", "Western"), "no such item"); } + #[test] + fn a_group_item_and_missing_key_fall_back_to_the_key_they_were_asked() { + let mut c = cat(); + c.add_item_in_group("East", "Coastal"); + assert_eq!(c.group_display("Coastal"), "Coastal", "no group declared"); + c.set_group_display("Coastal", "Shoreline"); + assert_eq!(c.group_display("Coastal"), "Shoreline"); + assert_eq!( + c.group_chain("Coastal"), + vec!["Coastal"], + "the key is unmoved" + ); + assert_eq!(c.item_display("Nowhere"), "Nowhere", "a formula target"); + } + /// improvise-2lh: call sites ask `of_name` rather than compare literals, /// so both directions of the mapping must agree. #[test] diff --git a/crates/improvise-core/src/model/types.rs b/crates/improvise-core/src/model/types.rs index 4068121f..eeb6f71a 100644 --- a/crates/improvise-core/src/model/types.rs +++ b/crates/improvise-core/src/model/types.rs @@ -525,6 +525,17 @@ impl Model { self.categories.get(name) } + /// What a person sees for a category key; the key itself when none holds it. + pub fn category_display<'a>(&'a self, name: &'a str) -> &'a str { + self.category(name).map_or(name, |c| c.display()) + } + + /// What a person sees for an item of `cat`; the key where `cat` has no such + /// item, as for a formula target in `_Measure` (core.md §21). + pub fn item_display<'a>(&'a self, cat: &str, item: &'a str) -> &'a str { + self.category(cat).map_or(item, |c| c.item_display(item)) + } + /// `group` and its ancestors in `cat`, root first and owned: the chain a /// `View` folds by (core.md §19). Just `[group]` where `cat` is unknown. pub fn group_chain(&self, cat: &str, group: &str) -> Vec { diff --git a/crates/improvise-core/src/view/layout.rs b/crates/improvise-core/src/view/layout.rs index 5b4c93fa..37a0c133 100644 --- a/crates/improvise-core/src/view/layout.rs +++ b/crates/improvise-core/src/view/layout.rs @@ -212,6 +212,8 @@ pub enum AxisEntry { depth: usize, cat_name: String, group_name: String, + /// What the group is called on screen; `group_name` stays the key (core.md §21). + group_display: String, /// The view's answer for this group, so no reader re-derives it. collapsed: bool, }, @@ -220,17 +222,32 @@ pub enum AxisEntry { GroupEnd { level: usize, depth: usize }, /// `collapsed` is not derivable from `tuple`: a group and an item may share a /// name. Set for a folded group and for everything expanded beneath one. - DataItem { tuple: Vec, collapsed: bool }, + DataItem { + tuple: Vec, + /// Parallel to `tuple`; the tuple is never replaced (core.md §21). + display: Vec, + collapsed: bool, + }, } impl AxisEntry { - /// An ordinary, uncollapsed data item. + /// An ordinary, uncollapsed data item shown under its own keys. pub fn item(tuple: Vec) -> Self { AxisEntry::DataItem { + display: tuple.clone(), tuple, collapsed: false, } } + + /// An ordinary, uncollapsed data item whose levels show other names. + pub fn item_shown(tuple: Vec, display: Vec) -> Self { + AxisEntry::DataItem { + tuple, + display, + collapsed: false, + } + } } /// The data-item tuples on an axis, skipping group headers. Every public @@ -270,6 +287,14 @@ fn data_item<'a>(items: &'a [AxisEntry], pos: &[usize], idx: usize) -> Option<&' } } +/// The names the data item at `idx` is shown under, one per level. +fn data_display<'a>(items: &'a [AxisEntry], pos: &[usize], idx: usize) -> Option<&'a Vec> { + match items.get(*pos.get(idx)?)? { + AxisEntry::DataItem { display, .. } => Some(display), + AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None, + } +} + /// Whether the data item at `idx` folds a collapsed group. fn folds(items: &[AxisEntry], pos: &[usize], idx: usize) -> bool { match pos.get(idx).and_then(|at| items.get(*at)) { @@ -317,6 +342,19 @@ fn axis_label(items: &[AxisEntry], pos: &[usize], idx: usize) -> String { .unwrap_or_default() } +fn cat_displays(model: &Model, cats: &[String]) -> Vec { + cats.iter() + .map(|c| model.category_display(c).to_string()) + .collect() +} + +/// [`axis_label`] as a person reads it: the displays, not the keys. +fn axis_display_label(items: &[AxisEntry], pos: &[usize], idx: usize) -> String { + data_display(items, pos, idx) + .map(|v| v.join("/")) + .unwrap_or_default() +} + /// Visual index of the nth data entry (skipping group headers). fn data_index_to_visual(pos: &[usize], data_idx: usize) -> Option { pos.get(data_idx).copied() @@ -452,12 +490,55 @@ fn innermost_group(items: &[AxisEntry], idx: usize) -> Option<(String, String)> /// One category constraining which records are visible: what it is fixed to, /// and — when the user may page it — what `[`/`]` cycle. core.md §7. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(try_from = "UncheckedPageAxis")] pub struct PageAxis { pub cat: String, + /// What the bar shows for `cat`; the key stays what a command names (core.md §21). + pub cat_display: String, pub selected: String, + /// What the bar shows for `selected`. + pub selected_display: String, /// `Some` exactly for `Axis::Page`. A `Filter` axis constrains without /// being navigable, so a client can compose no cycle for one (gotcha 27). pub items: Option>, + /// Parallel to `items`: what the menu shows for each. `[`/`]` and a click + /// compose a key from `items` and never from here. + pub item_displays: Option>, +} + +/// A [`PageAxis`] as the wire delivers it, before `item_displays` is checked +/// against `items`: a parallel list a client must not see ragged. +#[derive(serde::Deserialize)] +struct UncheckedPageAxis { + cat: String, + cat_display: String, + selected: String, + selected_display: String, + items: Option>, + item_displays: Option>, +} + +impl TryFrom for PageAxis { + type Error = String; + + fn try_from(raw: UncheckedPageAxis) -> Result { + let keys = raw.items.as_ref().map(Vec::len); + let shown = raw.item_displays.as_ref().map(Vec::len); + match keys == shown { + true => Ok(Self { + cat: raw.cat, + cat_display: raw.cat_display, + selected: raw.selected, + selected_display: raw.selected_display, + items: raw.items, + item_displays: raw.item_displays, + }), + false => Err(format!( + "page axis {}: {keys:?} items but {shown:?} item displays", + raw.cat + )), + } + } } /// The page axes as the plain coordinate pairs `matching_cells` and @@ -469,6 +550,18 @@ pub fn page_constraints(axes: &[PageAxis]) -> Vec<(String, String)> { } impl PageAxis { + /// An axis shown under its own keys, as it is wherever nothing is renamed. + pub fn plain(cat: &str, selected: &str, items: Option>) -> Self { + Self { + cat: cat.to_string(), + cat_display: cat.to_string(), + selected: selected.to_string(), + selected_display: selected.to_string(), + item_displays: items.clone(), + items, + } + } + /// The items to cycle and the index `selected` sits at, for the two /// callers that page. `None` where the axis is not navigable. pub fn cycle(&self) -> Option<(&[String], usize)> { @@ -560,9 +653,13 @@ impl GridLayout { .map(String::from) .or_else(|| items.first().cloned())?; let navigable = view.try_axis_of(cat) == Some(Axis::Page); + let shown = |item: &str| model.item_display(cat, item).to_string(); Some(PageAxis { cat: cat.clone(), + cat_display: model.category_display(cat).to_string(), + selected_display: shown(&selected), selected, + item_displays: navigable.then(|| items.iter().map(|i| shown(i)).collect()), items: navigable.then_some(items), }) }) @@ -608,7 +705,12 @@ impl GridLayout { let record_cols = Self::records_columns(model); let col_items: Vec = record_cols .iter() - .map(|c| AxisEntry::item(vec![Self::record_col_label(c)])) + .map(|c| { + AxisEntry::item_shown( + vec![Self::record_col_label(c)], + vec![Self::record_col_display(model, c)], + ) + }) .collect(); Self { @@ -671,6 +773,25 @@ impl GridLayout { } } + /// The same header as a person reads it: a category's or a measure's display. + fn record_col_display(model: &Model, col: &RecordCol) -> String { + match col { + RecordCol::Category(name) => model.category_display(name).to_string(), + RecordCol::Measure(name) => model.item_display(MEASURE_CATEGORY, name).to_string(), + RecordCol::NewMeasure | RecordCol::NewCategory => Self::record_col_label(col), + } + } + + /// The text a records-mode coordinate cell shows: the item's display, where + /// `records_display` is the item's key. `None` for every other column. + pub fn coordinate_display(&self, model: &Model, row: usize, col: usize) -> Option { + let RecordCol::Category(category) = self.record_col(col)? else { + return None; + }; + let item = self.record_row(row)?.coords.get(category)?; + Some(model.item_display(category, item).to_string()) + } + /// What the records-mode column at `col` holds; `None` outside records /// mode or past the last column. pub fn record_col(&self, col: usize) -> Option<&RecordCol> { @@ -787,6 +908,24 @@ impl GridLayout { axis_label(&self.col_items, &self.col_pos, col) } + /// [`GridLayout::row_label`] as a person reads it (ADR 007): the displays. + pub fn row_display_label(&self, row: usize) -> String { + axis_display_label(&self.row_items, &self.row_pos, row) + } + + pub fn col_display_label(&self, col: usize) -> String { + axis_display_label(&self.col_items, &self.col_pos, col) + } + + /// What each row-axis category is called on screen, outermost first. + pub fn row_cat_displays(&self, model: &Model) -> Vec { + cat_displays(model, &self.row_cats) + } + + pub fn col_cat_displays(&self, model: &Model) -> Vec { + cat_displays(model, &self.col_cats) + } + /// The value of a pivot key as THIS grid reads it: aggregated over its None /// axis, and a fold over the members it draws (core.md §14). One evaluation. pub fn evaluate(&self, model: &Model, key: &CellKey) -> Option { @@ -815,7 +954,9 @@ impl GridLayout { plan: &crate::format::FormatPlan, ) -> String { if self.is_records_mode() { - self.records_display(row, col).unwrap_or_default() + self.coordinate_display(model, row, col) + .or_else(|| self.records_display(row, col)) + .unwrap_or_default() } else { self.cell_key(row, col) .and_then(|key| { @@ -922,7 +1063,7 @@ fn expand_category( model: &Model, view: &View, cat_name: &str, - prefix: Vec, + (prefix, shown): (Vec, Vec), prefix_collapsed: bool, ) -> Vec { let Some(cat) = model.category(cat_name) else { @@ -930,6 +1071,7 @@ fn expand_category( }; // The prefix holds one item per outer category, so its length is this level. let mut runs = CategoryRuns { + model, level: prefix.len(), cat_name, view, @@ -956,15 +1098,21 @@ fn expand_category( runs.enter(&chain[..fold_depth.map_or(chain.len(), |d| d + 1)]); let mut tuple = prefix.clone(); + let mut display = shown.clone(); match fold { Some(g) => { folded.push(g); tuple.push(g.to_string()); + display.push(cat.group_display(g).to_string()); + } + None => { + tuple.push(item_name.to_string()); + display.push(cat.item_display(item_name).to_string()); } - None => tuple.push(item_name.to_string()), } runs.out.push(AxisEntry::DataItem { tuple, + display, collapsed: prefix_collapsed || fold.is_some(), }); } @@ -977,6 +1125,7 @@ fn expand_category( /// One category's group runs as `expand_category` emits them: the chain of groups /// open, root first, and the markers that move it from item to item. struct CategoryRuns<'a> { + model: &'a Model, level: usize, cat_name: &'a str, view: &'a View, @@ -995,6 +1144,7 @@ impl<'a> CategoryRuns<'a> { .take_while(|(open, want)| open == want) .count(); let level = self.level; + let category = self.model.category(self.cat_name); for depth in (shared + 1..self.open.len()).rev() { self.out.push(AxisEntry::GroupEnd { level, depth }); } @@ -1011,6 +1161,9 @@ impl<'a> CategoryRuns<'a> { depth, cat_name: self.cat_name.to_string(), group_name: group.to_string(), + group_display: category + .map_or(*group, |c| c.group_display(group)) + .to_string(), collapsed: self .view .is_group_collapsed(self.cat_name, &chain[..=depth]), @@ -1031,9 +1184,11 @@ fn cross_product(model: &Model, view: &View, cats: &[String]) -> Vec result = result .into_iter() .flat_map(|entry| match entry { - AxisEntry::DataItem { tuple, collapsed } => { - expand_category(model, view, cat_name, tuple, collapsed) - } + AxisEntry::DataItem { + tuple, + display, + collapsed, + } => expand_category(model, view, cat_name, (tuple, display), collapsed), marker @ (AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. }) => { vec![marker] } @@ -1073,7 +1228,7 @@ impl GridEval<'_> { #[cfg(test)] mod tests { use super::{ - AxisEntry, GridLayout, NEW_CATEGORY_COL, NEW_MEASURE_COL, RecordCol, data_items, + AxisEntry, GridLayout, NEW_CATEGORY_COL, NEW_MEASURE_COL, PageAxis, RecordCol, data_items, prune_axis_items, synthetic_record_info, }; use crate::model::Aggregation; @@ -1119,6 +1274,7 @@ mod tests { depth: 0, cat_name: "C".into(), group_name: "G".into(), + group_display: "G".into(), collapsed: false, }, AxisEntry::item(vec!["B".into()]), @@ -1146,6 +1302,7 @@ mod tests { depth: 0, cat_name: "C".into(), group_name: "G".into(), + group_display: "G".into(), collapsed: false, }, AxisEntry::item(vec!["A".into()]), @@ -1542,6 +1699,145 @@ mod tests { ); } + /// A category, two items, a group, a page category and a measure, each shown + /// under a name that is not its key. The test-only way to make them differ. + fn shown_otherwise() -> Workbook { + let mut wb = Workbook::new("T"); + for cat in ["Month", "Type", "Region", "_Measure"] { + let _ = wb.add_category(cat); + } + wb.add_item_in_group("Month", "Jan", "Q1").unwrap(); + wb.add_item_in_group("Month", "Apr", "Q2").unwrap(); + wb.add_item("Type", "Food").unwrap(); + wb.add_item("Region", "East").unwrap(); + wb.add_item("Region", "West").unwrap(); + wb.add_item("_Measure", "Revenue").unwrap(); + wb.active_view_mut().set_expand_depth("Month", 1); + wb.active_view_mut().set_axis("Type", Axis::Column); + wb.active_view_mut().set_axis("Region", Axis::Page); + wb.active_view_mut().set_axis("_Measure", Axis::None); + let month = wb.model.category_mut("Month").unwrap(); + month.set_display("Period"); + month.set_item_display("Jan", "January"); + month.set_group_display("Q1", "First quarter"); + let region = wb.model.category_mut("Region").unwrap(); + region.set_display("Sales Area"); + region.set_item_display("East", "Eastern Seaboard"); + region.set_item_display("West", "Pacific"); + wb.model + .category_mut("Type") + .unwrap() + .set_item_display("Food", "Groceries"); + wb + } + + #[test] + fn a_layout_shows_displays_and_still_addresses_by_keys() { + let wb = shown_otherwise(); + let layout = GridLayout::new(&wb.model, wb.active_view()); + assert_eq!(layout.row_label(0), "Jan", "the key"); + assert_eq!(layout.row_display_label(0), "January"); + assert_eq!(layout.col_display_label(0), "Groceries"); + assert_eq!(layout.row_cat_displays(&wb.model), vec!["Period"]); + assert!(matches!( + &layout.row_items[0], + AxisEntry::GroupHeader { group_name, group_display, .. } + if group_name == "Q1" && group_display == "First quarter" + )); + // Addressing is untouched: the cell, its group and its run start are keys. + assert_eq!( + layout.cell_key(0, 0), + Some(CellKey::new(vec![ + ("Month".into(), "Jan".into()), + ("Region".into(), "East".into()), + ("Type".into(), "Food".into()), + ])) + ); + assert_eq!(layout.row_group_for(0), Some(("Month".into(), "Q1".into()))); + assert_eq!(layout.row_run_start("Month", "Q1", &[]), Some(0)); + let page = &layout.page_coords[0]; + assert_eq!( + (page.cat.as_str(), page.selected.as_str()), + ("Region", "East") + ); + assert_eq!(page.cat_display, "Sales Area"); + assert_eq!(page.selected_display, "Eastern Seaboard"); + assert_eq!( + page.items.as_deref(), + Some(&["East".to_string(), "West".into()][..]) + ); + assert_eq!( + page.item_displays.as_deref(), + Some(&["Eastern Seaboard".to_string(), "Pacific".into()][..]) + ); + } + + #[test] + fn a_page_axis_with_ragged_displays_is_refused() { + let good = PageAxis::plain("Q", "Q1", Some(vec!["Q1".into(), "Q2".into()])); + let wire = serde_json::to_value(&good).expect("a page axis serializes"); + assert_eq!( + serde_json::from_value::(wire.clone()).unwrap(), + good + ); + for bad in [serde_json::json!(["Only one"]), serde_json::json!(null)] { + let mut ragged = wire.clone(); + ragged["item_displays"] = bad; + let err = serde_json::from_value::(ragged).unwrap_err(); + assert!(err.to_string().contains("item displays"), "{err}"); + } + } + + #[test] + fn a_folded_group_is_shown_under_the_groups_display_and_keyed_by_its_name() { + let mut wb = shown_otherwise(); + wb.active_view_mut().toggle_group_collapse("Month", &["Q1"]); + let layout = GridLayout::new(&wb.model, wb.active_view()); + assert_eq!(layout.row_label(0), "Q1"); + assert_eq!(layout.row_display_label(0), "First quarter"); + assert!(layout.row_folds_a_group(0)); + } + + #[test] + fn a_records_grid_shows_displays_in_its_headers_and_coordinates() { + let mut wb = shown_otherwise(); + wb.model.set_cell( + CellKey::new(vec![ + ("Month".into(), "Jan".into()), + ("Region".into(), "East".into()), + ("Type".into(), "Food".into()), + ("_Measure".into(), "Revenue".into()), + ]), + CellValue::Number(1.0), + ); + wb.model + .category_mut("_Measure") + .unwrap() + .set_item_display("Revenue", "Sales"); + let view = wb.active_view_mut(); + view.set_axis("_Index", Axis::Row); + view.set_axis("_Dim", Axis::Column); + let layout = GridLayout::new(&wb.model, wb.active_view()); + assert!(layout.is_records_mode()); + let col_of = |label: &str| (0..layout.col_count()).find(|&c| layout.col_label(c) == label); + let month = col_of("Month").expect("a column per category"); + assert_eq!(layout.col_display_label(month), "Period"); + assert_eq!( + layout.col_display_label(col_of("Revenue").unwrap()), + "Sales" + ); + assert_eq!(layout.records_display(0, month).as_deref(), Some("Jan")); + assert_eq!( + layout.coordinate_display(&wb.model, 0, month).as_deref(), + Some("January") + ); + assert_eq!( + layout.coordinate_display(&wb.model, 0, col_of("Revenue").unwrap()), + None, + "a measure column holds a value, not a coordinate" + ); + } + /// The inversion of `collapsed_group_has_header_but_no_data_items`: a /// collapsed group is ONE row. Evidence: context/test-evidence/core.md. #[test] @@ -1874,10 +2170,12 @@ mod tests { depth: 0, cat_name: "Month".into(), group_name: "Q1".into(), + group_display: "Q1".into(), collapsed: true, }, AxisEntry::DataItem { tuple: vec!["Q1".into()], + display: vec!["Q1".into()], collapsed: true }, AxisEntry::GroupEnd { level: 0, depth: 0 }, @@ -2719,6 +3017,7 @@ mod group_membership { depth: 0, cat_name: "Region".into(), group_name: "Upper".into(), + group_display: "Upper".into(), collapsed: false, }, AxisEntry::item(vec!["North".into()]), @@ -2727,6 +3026,7 @@ mod group_membership { depth: 0, cat_name: "Nowhere".into(), group_name: "Unreachable".into(), + group_display: "Unreachable".into(), collapsed: false, }, AxisEntry::item(vec!["South".into()]), @@ -2746,6 +3046,7 @@ mod group_membership { depth: 0, cat_name: "Month".into(), group_name: group.into(), + group_display: group.into(), collapsed: false, }; let items = vec![ @@ -2878,6 +3179,7 @@ mod hierarchy { depth, cat_name: "Date".into(), group_name: group.into(), + group_display: group.into(), collapsed, } } diff --git a/crates/improvise-io/src/persistence/export.rs b/crates/improvise-io/src/persistence/export.rs index 44f4a68d..fb5fdf91 100644 --- a/crates/improvise-io/src/persistence/export.rs +++ b/crates/improvise-io/src/persistence/export.rs @@ -41,11 +41,11 @@ pub fn render_csv(workbook: &Workbook, view_name: &str) -> Result> { .from_writer(Vec::new()); // Header row - let row_header = layout.row_cats.join("/"); + let row_header = layout.row_cat_displays(model).join("/"); let page_label: Vec = layout .page_coords .iter() - .map(|p| format!("{}={}", p.cat, p.selected)) + .map(|p| format!("{}={}", p.cat_display, p.selected_display)) .collect(); let header_prefix = if page_label.is_empty() { row_header @@ -56,13 +56,13 @@ pub fn render_csv(workbook: &Workbook, view_name: &str) -> Result> { if !header_prefix.is_empty() { header.push(header_prefix); } - header.extend((0..layout.col_count()).map(|ci| layout.col_label(ci))); + header.extend((0..layout.col_count()).map(|ci| layout.col_display_label(ci))); wtr.write_record(&header)?; // Data rows for ri in 0..layout.row_count() { let mut record: Vec = Vec::new(); - let row_label = layout.row_label(ri); + let row_label = layout.row_display_label(ri); if !row_label.is_empty() { record.push(row_label); } @@ -106,6 +106,34 @@ mod tests { ); } + #[test] + fn export_csv_writes_displays_in_every_header_and_label() { + let mut m = Workbook::new("T"); + for cat in ["Type", "Month", "Payer"] { + m.add_category(cat).unwrap(); + } + m.add_item("Type", "Food").unwrap(); + m.add_item("Month", "Jan").unwrap(); + m.add_item("Payer", "Alice").unwrap(); + m.model.set_cell( + coord(&[("Type", "Food"), ("Month", "Jan"), ("Payer", "Alice")]), + CellValue::Number(1.0), + ); + let ty = m.model.category_mut("Type").unwrap(); + ty.set_display("Kind"); + ty.set_item_display("Food", "Groceries"); + m.model + .category_mut("Month") + .unwrap() + .set_item_display("Jan", "January"); + let payer = m.model.category_mut("Payer").unwrap(); + payer.set_display("Who"); + payer.set_item_display("Alice", "Alicia"); + let bytes = super::render_csv(&m, "Default").unwrap(); + let text = String::from_utf8(bytes).unwrap(); + assert_eq!(text, "Kind (Who=Alicia),January\nGroceries,1.00\n"); + } + /// Bug improvise-1cz: bare-comma joining and an ignored number format /// corrupted every row. Evidence: `context/test-evidence/formula-and-io.md`. #[test] diff --git a/crates/improvise-protocol/src/client.rs b/crates/improvise-protocol/src/client.rs index 6b8cefd2..379f9000 100644 --- a/crates/improvise-protocol/src/client.rs +++ b/crates/improvise-protocol/src/client.rs @@ -531,7 +531,7 @@ mod tests { live.fold(&Command::Project(Projection::ViewList(vec![ViewSummary { name: "Default".into(), display: "Default".into(), - axes: vec![("Region".into(), Axis::Row)], + axes: vec![crate::ViewAxis::plain("Region", Axis::Row)], }]))); live.fold(&Command::SetSelected { row: 1, col: 1 }); diff --git a/crates/improvise-protocol/src/lib.rs b/crates/improvise-protocol/src/lib.rs index 71dabf12..b9b880e4 100644 --- a/crates/improvise-protocol/src/lib.rs +++ b/crates/improvise-protocol/src/lib.rs @@ -34,12 +34,12 @@ pub use reduce::{ records_named, reduce_view, }; pub use render::{ - AxisFrame, CategoryRow, CategoryTreeCache, CategoryTreeUpdate, CellDisplay, CellId, CreateKind, - DocumentCache, FormulaCache, FormulaEntry, FrameEntry, FrameGroup, GridCache, GridExtent, - GridUpdate, MAX_ROW_HEADER_W, MIN_ROW_HEADER_W, Projection, RecordCell, Snapshot, TileBarCache, - TileEntry, ViewListCache, ViewSummary, WidgetCache, WizardCache, WizardChoice, WizardCollision, - WizardDateField, WizardFieldKind, WizardProposal, WizardStepKind, header_level_widths, - non_item_refusal, opening_tile, + AxisFrame, CategoryItem, CategoryRow, CategoryTreeCache, CategoryTreeUpdate, CellDisplay, + CellId, CreateKind, DocumentCache, FormulaCache, FormulaEntry, FrameEntry, FrameGroup, + GridCache, GridExtent, GridUpdate, MAX_ROW_HEADER_W, MIN_ROW_HEADER_W, Projection, RecordCell, + Snapshot, TileBarCache, TileEntry, ViewAxis, ViewListCache, ViewSummary, WidgetCache, + WizardCache, WizardChoice, WizardCollision, WizardDateField, WizardFieldKind, WizardProposal, + WizardStepKind, header_level_widths, non_item_refusal, opening_tile, }; pub use resolve::{ Click, KeyOutcome, OpenedEditor, Request, click, edited, read_buffer, resolve, typed, diff --git a/crates/improvise-protocol/src/reduce.rs b/crates/improvise-protocol/src/reduce.rs index 051a738d..c9a1c83e 100644 --- a/crates/improvise-protocol/src/reduce.rs +++ b/crates/improvise-protocol/src/reduce.rs @@ -557,7 +557,7 @@ mod tests { Command::Project(Projection::ViewList(vec![ViewSummary { name: "Default".into(), display: "Default".into(), - axes: vec![("Region".into(), Axis::Row)], + axes: vec![crate::ViewAxis::plain("Region", Axis::Row)], }])), Command::Project(Projection::TileBar(vec![TileEntry { category: "Region".into(), diff --git a/crates/improvise-protocol/src/render.rs b/crates/improvise-protocol/src/render.rs index 0e9e7ef3..10420abc 100644 --- a/crates/improvise-protocol/src/render.rs +++ b/crates/improvise-protocol/src/render.rs @@ -217,7 +217,10 @@ pub struct FrameGroup { /// The group's depth in its category's hierarchy, 0 for a root. Runs nest /// by `(level, depth)`: `context/code/protocol.md` §3a (improvise-felo.6). pub depth: usize, + /// The key; what `toggle-group` and a run lookup name. pub name: String, + /// What the header shows (ADR 007); never an address. + pub display: String, pub collapsed: bool, } @@ -235,16 +238,21 @@ pub enum FrameEntry { /// One label per header level, outermost first. Data indices — the /// `row`/`col` of every other field on [`GridCache`] — count only these. Item { + /// The keys: the coordinate this position names, never replaced by `display`. tuple: Vec, + /// Parallel to `tuple`, one name per level: what the header shows (ADR 007). + display: Vec, /// The layout's own fold flag: `context/code/protocol.md` (improvise-4k1j). collapsed: bool, }, } impl FrameEntry { - /// An ordinary, unfolded item, as `AxisEntry::item` is on the layout side. + /// An ordinary, unfolded item shown under its own keys, as + /// `AxisEntry::item` is on the layout side. pub fn item(tuple: Vec) -> Self { FrameEntry::Item { + display: tuple.clone(), tuple, collapsed: false, } @@ -259,6 +267,8 @@ pub struct AxisFrame { /// The categories on this axis, outermost first. Empty is a real layout /// (one nameless row or column), not a gap — hence [`AxisFrame::levels`]. pub cats: Vec, + /// Parallel to `cats`: what each level's category is called on screen. + pub cat_displays: Vec, /// The **absolute** data index of this frame's first item. Zero is a claim /// and not an absence, so it is NOT `serde(default)` — §3a, improvise-1wl0. pub first: usize, @@ -270,6 +280,7 @@ pub struct AxisFrame { #[derive(Deserialize)] struct UncheckedAxisFrame { cats: Vec, + cat_displays: Vec, first: usize, entries: Vec, } @@ -278,6 +289,22 @@ impl TryFrom for AxisFrame { type Error = String; fn try_from(raw: UncheckedAxisFrame) -> Result { + if raw.cat_displays.len() != raw.cats.len() { + return Err(format!( + "{} category displays for {} categories", + raw.cat_displays.len(), + raw.cats.len() + )); + } + let ragged = raw.entries.iter().find_map(|e| match e { + FrameEntry::Item { tuple, display, .. } if tuple.len() != display.len() => { + Some((tuple.len(), display.len())) + } + _ => None, + }); + if let Some((keys, shown)) = ragged { + return Err(format!("{shown} displays for a tuple of {keys} keys")); + } let beyond = raw .entries .iter() @@ -294,6 +321,7 @@ impl TryFrom for AxisFrame { )), None => Ok(Self { cats: raw.cats, + cat_displays: raw.cat_displays, first: raw.first, entries: raw.entries, }), @@ -307,6 +335,7 @@ impl AxisFrame { pub fn flat>(labels: impl IntoIterator) -> Self { Self { cats: Vec::new(), + cat_displays: Vec::new(), first: 0, entries: labels .into_iter() @@ -324,6 +353,14 @@ impl AxisFrame { }) } + /// The display tuples, parallel to [`AxisFrame::items`]: what is drawn. + pub fn displays(&self) -> impl Iterator> { + self.entries.iter().filter_map(|e| match e { + FrameEntry::Item { display, .. } => Some(display), + FrameEntry::Group(_) | FrameEntry::GroupEnd { .. } => None, + }) + } + /// The innermost label of each data item. **Not "the labels"** — a draw /// loop wants [`AxisFrame::header_labels`] and would draw these wrong. pub fn leaf_labels(&self) -> Vec { @@ -391,12 +428,13 @@ impl AxisFrame { /// **One pass, and that is the point** — what a draw loop wants (§3). pub fn header_labels(&self, first_rendered: usize) -> Vec> { let items: Vec<&Vec> = self.items().collect(); + let shown: Vec<&Vec> = self.displays().collect(); let levels = self.levels(); (0..items.len()) .map(|idx| { (0..levels) .map(|level| match shows(&items, idx, first_rendered, level) { - true => items[idx].get(level).cloned().unwrap_or_default(), + true => shown[idx].get(level).cloned().unwrap_or_default(), false => String::new(), }) .collect() @@ -499,7 +537,7 @@ impl AxisFrame { /// The width of each header level, in display cells. Derived rather than /// projected, because the frame is the whole input (§3, improvise-1125). pub fn header_widths(&self) -> Vec { - header_level_widths(self.items(), self.levels()) + header_level_widths(self.displays(), self.levels()) } } @@ -617,7 +655,7 @@ impl GridCache { if self.col_widths.len() < cols { // Materialized once: `AxisFrame::item` scans, so asking it per // column would make a re-fit quadratic in the column count. - let items: Vec<&Vec> = self.col_frame.items().collect(); + let items: Vec<&Vec> = self.col_frame.displays().collect(); for levels in &items[self.col_widths.len()..cols] { // Display cells, the unit `header_widths` and the // authoritative `ColWidths` use — `chars().count()` is short. @@ -761,10 +799,34 @@ pub struct CategoryRow { pub name: String, /// What the panel shows; `name` stays what a command names (ADR 007). pub display: String, - pub items: Vec, + pub items: Vec, pub axis: Axis, } +/// One item row: the key a command names, and what the panel shows (ADR 007). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CategoryItem { + pub name: String, + pub display: String, +} + +impl CategoryItem { + /// An item shown under its own key. + pub fn plain(name: impl Into) -> Self { + let name = name.into(); + Self { + display: name.clone(), + name, + } + } +} + +impl From<&str> for CategoryItem { + fn from(name: &str) -> Self { + Self::plain(name) + } +} + /// The category panel's categories, items and axes (improvise-pca) — /// **every** category, and `_Measure`'s items are derived. See §3. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -786,7 +848,7 @@ pub enum CategoryTreeUpdate { Categories(Vec), Items { category: String, - items: Vec, + items: Vec, }, /// Every category whose axis moved, in one update. Axes(Vec<(String, Axis)>), @@ -861,8 +923,29 @@ pub struct ViewSummary { pub name: String, /// What the view panel shows; `name` stays what a command names. pub display: String, - /// `(category, axis)` for the categories this view displays. - pub axes: Vec<(String, Axis)>, + /// Where each category this view displays sits. + pub axes: Vec, +} + +/// One category's place in a view: the key a command names, what the panel +/// shows for it, and the axis. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ViewAxis { + pub category: String, + pub display: String, + pub axis: Axis, +} + +impl ViewAxis { + /// A category shown under its own key. + pub fn plain(category: impl Into, axis: Axis) -> Self { + let category = category.into(); + Self { + display: category.clone(), + category, + axis, + } + } } /// The view panel's copy of the view list (improvise-pca). Which view is @@ -1167,7 +1250,7 @@ mod tests { CategoryRow { name: name.into(), display: name.into(), - items: items.iter().map(|s| (*s).to_string()).collect(), + items: items.iter().map(|s| CategoryItem::plain(*s)).collect(), axis, } } @@ -1328,6 +1411,7 @@ mod tests { grid.apply(&GridUpdate::RowFrame(AxisFrame { first: 0, cats: vec!["Zone".into()], + cat_displays: vec!["Zone".into()], entries: grid.row_frame.entries.clone(), })); assert!(grid.totals.iter().all(|t| t.text.is_empty())); @@ -1343,11 +1427,7 @@ mod tests { #[test] fn the_page_bar_outlives_a_reshape() { let mut grid = framed_grid(); - let payer = PageAxis { - cat: "Payer".into(), - selected: "Alice".into(), - items: Some(vec!["Alice".into(), "Bob".into()]), - }; + let payer = PageAxis::plain("Payer", "Alice", Some(vec!["Alice".into(), "Bob".into()])); grid.apply(&GridUpdate::PageCoords(vec![payer.clone()])); grid.apply(&GridUpdate::RowFrame(AxisFrame::default())); grid.apply(&GridUpdate::ColFrame(AxisFrame::default())); @@ -1361,6 +1441,7 @@ mod tests { AxisFrame { first: 0, cats: vec!["Type".into(), "Who".into()], + cat_displays: vec!["Type".into(), "Who".into()], entries: vec![ FrameEntry::item(vec!["A".into(), "x".into()]), FrameEntry::item(vec!["A".into(), "y".into()]), @@ -1368,6 +1449,7 @@ mod tests { level: 0, depth: 0, name: "Late".into(), + display: "Late".into(), collapsed: true, }), FrameEntry::item(vec!["B".into(), "z".into()]), @@ -1375,6 +1457,75 @@ mod tests { } } + /// Two keys per level shown under displays: the outer pair share a display, + /// the inner pair share a KEY. Suppression must follow the keys. + fn shown_frame() -> AxisFrame { + let item = |tuple: [&str; 2], display: [&str; 2]| FrameEntry::Item { + tuple: tuple.map(String::from).to_vec(), + display: display.map(String::from).to_vec(), + collapsed: false, + }; + AxisFrame { + cats: vec!["Region".into(), "Month".into()], + cat_displays: vec!["Sales Area".into(), "Period".into()], + first: 0, + entries: vec![ + item(["East", "Jan"], ["Same", "January"]), + item(["East", "Feb"], ["Same", "February"]), + item(["West", "Jan"], ["Same", "January"]), + ], + } + } + + #[test] + fn a_header_draws_displays_and_blanks_a_repeat_by_key() { + let frame = shown_frame(); + assert_eq!( + frame.header_labels(0), + vec![ + vec!["Same".to_string(), "January".into()], + vec![String::new(), "February".into()], + vec!["Same".to_string(), "January".into()], + ], + "East repeats and blanks; West shares East's DISPLAY and still draws" + ); + let keys: Vec<&Vec> = frame.items().collect(); + assert_eq!(keys[2], &vec!["West".to_string(), "Jan".into()]); + assert_eq!(frame.label(0, 0), "East", "the lookup is still the key"); + } + + #[test] + fn a_header_is_as_wide_as_what_it_shows() { + let frame = AxisFrame { + cats: vec!["R".into()], + cat_displays: vec!["Region".into()], + first: 0, + entries: vec![FrameEntry::Item { + tuple: vec!["E".into()], + display: vec!["Eastern Seaboard".into()], + collapsed: false, + }], + }; + assert_eq!( + frame.header_widths(), + vec!["Eastern Seaboard".len() as u16 + 1] + ); + } + + #[test] + fn a_frame_with_ragged_displays_is_refused() { + let good = serde_json::to_value(shown_frame()).expect("a frame serializes"); + assert!(serde_json::from_value::(good.clone()).is_ok()); + let mut short_cats = good.clone(); + short_cats["cat_displays"] = serde_json::json!(["Only one"]); + let err = serde_json::from_value::(short_cats).unwrap_err(); + assert!(err.to_string().contains("category displays"), "{err}"); + let mut short_tuple = good; + short_tuple["entries"][0]["Item"]["display"] = serde_json::json!(["Same"]); + let err = serde_json::from_value::(short_tuple).unwrap_err(); + assert!(err.to_string().contains("displays for a tuple"), "{err}"); + } + /// Two runs of one group name under two different outer items, with nothing /// ungrouped between them — improvise-h0iv's reachable shape. fn repeated_group_frame() -> AxisFrame { @@ -1382,11 +1533,13 @@ mod tests { level: 1, depth: 0, name: "Q1".into(), + display: "Q1".into(), collapsed: false, }; AxisFrame { first: 0, cats: vec!["Region".into(), "Month".into()], + cat_displays: vec!["Region".into(), "Month".into()], entries: vec![ FrameEntry::Group(q1()), FrameEntry::item(vec!["North".into(), "Jul".into()]), @@ -1404,11 +1557,13 @@ mod tests { AxisFrame { first: 0, cats: vec!["Month".into()], + cat_displays: vec!["Month".into()], entries: vec![ FrameEntry::Group(FrameGroup { level: 0, depth: 0, name: "Q1".into(), + display: "Q1".into(), collapsed: false, }), FrameEntry::item(vec!["Jan".into()]), @@ -1428,11 +1583,13 @@ mod tests { level, depth: 0, name: name.into(), + display: name.into(), collapsed: true, }) }; let frame = AxisFrame { cats: vec!["Region".into(), "Month".into()], + cat_displays: vec!["Region".into(), "Month".into()], first: 0, entries: vec![ group(0, "Upper"), @@ -1473,11 +1630,13 @@ mod tests { level: 0, depth, name: name.into(), + display: name.into(), collapsed: false, }) }; let frame = AxisFrame { cats: vec!["Date".into()], + cat_displays: vec!["Date".into()], first: 0, entries: vec![ group(0, "2024"), @@ -1527,11 +1686,13 @@ mod tests { let frame = AxisFrame { first: 0, cats: vec!["Month".into()], + cat_displays: vec!["Month".into()], entries: vec![ FrameEntry::Group(FrameGroup { level: 0, depth: 0, name: "Q1".into(), + display: "Q1".into(), collapsed: true, }), FrameEntry::GroupEnd { level: 0, depth: 0 }, @@ -1698,6 +1859,7 @@ mod tests { let frame = AxisFrame { first: 0, cats: vec!["A".into(), "B".into()], + cat_displays: vec!["A".into(), "B".into()], entries: vec![FrameEntry::item(vec!["Clothing".into(), "日本".into()])], }; assert_eq!( @@ -1736,6 +1898,7 @@ mod tests { let frame = AxisFrame { first: 0, cats: Vec::new(), + cat_displays: Vec::new(), entries: vec![FrameEntry::item(Vec::new())], }; assert_eq!(frame.levels(), 1); @@ -1756,7 +1919,10 @@ mod tests { }, ]); assert_eq!(tree.category("Region").unwrap().items.len(), 2); - assert_eq!(tree.category("Product").unwrap().items, vec!["Shirts"]); + assert_eq!( + tree.category("Product").unwrap().items, + vec![CategoryItem::plain("Shirts")] + ); // An update for a category the cache has never heard of is dropped. tree.apply(&CategoryTreeUpdate::Items { @@ -1784,7 +1950,7 @@ mod tests { assert_eq!(tree.category("Product").unwrap().axis, Axis::Row); assert_eq!( tree.category("Region").unwrap().items, - vec!["North"], + vec![CategoryItem::plain("North")], "moving an axis does not disturb membership" ); } @@ -1813,7 +1979,7 @@ mod tests { let views = ViewListCache::folded(&[vec![ViewSummary { name: "Default".into(), display: "Default".into(), - axes: vec![("Region".into(), Axis::Row)], + axes: vec![ViewAxis::plain("Region", Axis::Row)], }]]); assert_eq!(views.views[0].name, "Default"); @@ -1869,6 +2035,7 @@ mod tests { let axis = |cat: &str, labels: [&str; 2]| AxisFrame { first: 0, cats: vec![cat.into()], + cat_displays: vec![cat.into()], entries: labels .iter() .map(|l| FrameEntry::item(vec![(*l).into()])) @@ -1895,6 +2062,7 @@ mod tests { let renamed = AxisFrame { first: 0, cats: vec!["Department".into()], + cat_displays: vec!["Department".into()], entries: grid.row_frame.entries.clone(), }; assert_eq!( @@ -1935,6 +2103,7 @@ mod tests { let renamed = AxisFrame { first: 0, cats: vec!["Metric".into()], + cat_displays: vec!["Metric".into()], entries: grid.col_frame.entries.clone(), }; grid.apply(&GridUpdate::ColFrame(renamed)); @@ -2215,7 +2384,7 @@ mod tests { Projection::ViewList(vec![ViewSummary { name: "Default".into(), display: "Default".into(), - axes: vec![("Region".into(), Axis::None)], + axes: vec![ViewAxis::plain("Region", Axis::None)], }]), Projection::TileBar(vec![TileEntry { category: "Region".into(), diff --git a/crates/improvise-protocol/src/resolve.rs b/crates/improvise-protocol/src/resolve.rs index ea0d009d..47f9befc 100644 --- a/crates/improvise-protocol/src/resolve.rs +++ b/crates/improvise-protocol/src/resolve.rs @@ -1613,6 +1613,7 @@ fn filter_to_item_at(state: &ClientState, index: usize) -> Option> Some(CatTreeEntry::Item { cat_name, item_name, + .. }) => Some(vec![ model( state, @@ -1648,6 +1649,7 @@ fn delete_at_cat_cursor(state: &ClientState) -> Option> { Some(CatTreeEntry::Item { cat_name, item_name, + .. }) => { let mut commands = vec![ model( @@ -2204,6 +2206,7 @@ mod tests { Some(CatTreeEntry::Item { cat_name: "Product".into(), item_name: "Shirts".into(), + display: "Shirts".into(), }), "row 4 is Product's ONLY item — no sibling above it and none below" ); @@ -2628,10 +2631,12 @@ mod tests { // Named axes, as a host's always are: `H` hides from `cats[0]`. Projection::Grid(GridUpdate::RowFrame(AxisFrame { cats: vec!["Region".into()], + cat_displays: vec!["Region".into()], ..AxisFrame::flat(["North", "South", "East"]) })), Projection::Grid(GridUpdate::ColFrame(AxisFrame { cats: vec!["_Measure".into()], + cat_displays: vec!["_Measure".into()], ..AxisFrame::flat(["Revenue", "Cost", "Profit"]) })), Projection::Grid(GridUpdate::Cells(cells)), @@ -2665,11 +2670,11 @@ mod tests { // produce, so the fixture projects both (improvise-e5pf). crate::WidgetCache::apply(&mut state.grid, &GridUpdate::Records(mode.is_records())); state.fold(&Command::ChangeMode(mode)); - state.grid.page_coords = vec![crate::PageAxis { - cat: "Quarter".into(), - selected: "Q2".into(), - items: Some(vec!["Q1".into(), "Q2".into(), "Q3".into()]), - }]; + state.grid.page_coords = vec![crate::PageAxis::plain( + "Quarter", + "Q2", + Some(vec!["Q1".into(), "Q2".into(), "Q3".into()]), + )]; // THREE of everything, every panel cursor seated in the MIDDLE: an edge // makes `move-panel-cursor` answer `Ignored` for the wrong reason. crate::WidgetCache::apply( @@ -3989,16 +3994,12 @@ mod tests { let mut state = in_mode(AppMode::Normal); crate::WidgetCache::apply(&mut state.basis, &published()); state.grid.page_coords = vec![ - crate::PageAxis { - cat: "Year".into(), - selected: "2024".into(), - items: Some(vec!["2024".into(), "2025".into()]), - }, - crate::PageAxis { - cat: "Quarter".into(), - selected: "Q4".into(), - items: Some(vec!["Q1".into(), "Q2".into(), "Q3".into(), "Q4".into()]), - }, + crate::PageAxis::plain("Year", "2024", Some(vec!["2024".into(), "2025".into()])), + crate::PageAxis::plain( + "Quarter", + "Q4", + Some(vec!["Q1".into(), "Q2".into(), "Q3".into(), "Q4".into()]), + ), ]; state } @@ -4047,11 +4048,10 @@ mod tests { #[test] fn a_filter_axis_is_projected_but_not_cycled() { let mut state = paged(); - state.grid.page_coords.push(crate::PageAxis { - cat: "Region".into(), - selected: "East".into(), - items: None, - }); + state + .grid + .page_coords + .push(crate::PageAxis::plain("Region", "East", None)); let moved = selections(resolve(&state, ev(KeyCode::Char(']')))); assert!( !moved.iter().any(|(cat, _)| cat == "Region"), @@ -4082,16 +4082,68 @@ mod tests { ); } + /// ADR 007: the bar shows displays and a pick composes the KEY at that index. + #[test] + fn picking_a_displayed_page_item_still_names_its_key() { + let mut state = paged(); + state.grid.page_coords[1].cat_display = "Period".into(); + state.grid.page_coords[1].item_displays = Some( + ["First", "Second", "Third", "Fourth"] + .map(String::from) + .to_vec(), + ); + let commands = click(&state, Click::PageItem { axis: 1, item: 1 }); + let [ + Command::Model { + cmd: ModelCommand::SetPageSelection { category, item, .. }, + .. + }, + ] = commands.as_slice() + else { + panic!("expected one page selection, got {commands:?}"); + }; + assert_eq!((category.as_str(), item.as_str()), ("Quarter", "Q2")); + } + + /// The frame shows an item under its display; hiding it names the key. + #[test] + fn hiding_a_displayed_row_item_names_its_key() { + let mut state = in_mode(AppMode::Normal); + crate::WidgetCache::apply(&mut state.basis, &published()); + state.grid.row_frame = crate::AxisFrame { + cats: vec!["Region".into()], + cat_displays: vec!["Sales Area".into()], + first: 0, + entries: vec![crate::FrameEntry::Item { + tuple: vec!["East".into()], + display: vec!["Eastern Seaboard".into()], + collapsed: false, + }], + }; + let KeyOutcome::Local(commands) = resolve(&state, ev(KeyCode::Char('H'))) else { + panic!("hiding resolves here"); + }; + let [ + Command::Model { + cmd: ModelCommand::HideItem { category, item, .. }, + .. + }, + ] = commands.as_slice() + else { + panic!("expected one hide, got {commands:?}"); + }; + assert_eq!((category.as_str(), item.as_str()), ("Region", "East")); + } + /// A Filter axis is on the bar and offers nothing to pick (gotcha 27). /// Offering a menu that cannot act is worse than offering none. #[test] fn a_filter_axis_offers_no_item_to_pick() { let mut state = paged(); - state.grid.page_coords.push(crate::PageAxis { - cat: "Region".into(), - selected: "East".into(), - items: None, - }); + state + .grid + .page_coords + .push(crate::PageAxis::plain("Region", "East", None)); assert!( click(&state, Click::PageItem { axis: 2, item: 0 }).is_empty(), "a Filter axis has no item list, so there is nothing a click could mean" diff --git a/crates/improvise-protocol/src/viewmodel.rs b/crates/improvise-protocol/src/viewmodel.rs index a145f645..0de1255d 100644 --- a/crates/improvise-protocol/src/viewmodel.rs +++ b/crates/improvise-protocol/src/viewmodel.rs @@ -19,8 +19,12 @@ pub enum CatTreeEntry { expanded: bool, axis: Axis, }, - /// Item row under a category. - Item { cat_name: String, item_name: String }, + /// Item row under a category: the keys a command names, and what is drawn. + Item { + cat_name: String, + item_name: String, + display: String, + }, } impl CatTreeEntry { @@ -56,10 +60,11 @@ pub fn flatten_cat_tree( axis: row.axis, }); if is_expanded { - for item_name in &row.items { + for item in &row.items { entries.push(CatTreeEntry::Item { cat_name: row.name.clone(), - item_name: item_name.clone(), + item_name: item.name.clone(), + display: item.display.clone(), }); } } @@ -76,8 +81,8 @@ pub fn view_axis_summary(summary: &ViewSummary) -> String { let cats: Vec<&str> = summary .axes .iter() - .filter(|(cat, a)| *a == axis && !cat.starts_with('_')) - .map(|(cat, _)| cat.as_str()) + .filter(|a| a.axis == axis && !a.category.starts_with('_')) + .map(|a| a.display.as_str()) .collect(); (!cats.is_empty()).then(|| format!("{}:{}", axis.initial(), cats.join(","))) }) @@ -97,7 +102,7 @@ mod tests { .map(|(name, items)| CategoryRow { name: (*name).to_string(), display: (*name).to_string(), - items: items.iter().map(|i| i.to_string()).collect(), + items: items.iter().map(|i| (*i).into()).collect(), axis: Axis::None, }) .collect(), @@ -136,6 +141,7 @@ mod tests { CatTreeEntry::Item { cat_name: "Region".into(), item_name: "South".into(), + display: "South".into(), } ); } diff --git a/crates/improvise-wasm-client/src/render.rs b/crates/improvise-wasm-client/src/render.rs index e99c3c64..9b32ec1d 100644 --- a/crates/improvise-wasm-client/src/render.rs +++ b/crates/improvise-wasm-client/src/render.rs @@ -259,7 +259,7 @@ pub fn render_model(state: &ClientState) -> RenderModel { let row = match entry { FrameEntry::GroupEnd { .. } => return Some(None), FrameEntry::Group(group) => RenderRow::Group { - label: group.name.clone(), + label: group.display.clone(), collapsed: group.collapsed, }, FrameEntry::Item { .. } => { @@ -340,12 +340,14 @@ fn page_axes(state: &ClientState) -> Vec { .page_coords .iter() .map(|axis| RenderPageAxis { - cat: axis.cat.clone(), - selected: axis.selected.clone(), + cat: axis.cat_display.clone(), + selected: axis.selected_display.clone(), cycle: axis.cycle().map(|(items, at)| RenderPageCycle { at, count: items.len(), - items: items.to_vec(), + // The menu shows displays; a pick travels as the index `at` and + // `items` order name, so no key is read back from here. + items: axis.item_displays.clone().unwrap_or_default(), }), }) .collect() @@ -481,8 +483,8 @@ impl RenderCatRow { expanded: Some(*expanded), selected, }, - CatTreeEntry::Item { item_name, .. } => Self { - label: item_name.clone(), + CatTreeEntry::Item { display, .. } => Self { + label: display.clone(), is_category: false, axis: String::new(), item_count: None, @@ -544,7 +546,7 @@ fn col_groups(cols: &AxisFrame) -> Vec { true => runs .into_iter() .map(|(group, span)| RenderColGroup { - label: group.map(|g| g.name.clone()), + label: group.map(|g| g.display.clone()), collapsed: group.is_some_and(|g| g.collapsed), span, }) @@ -594,6 +596,7 @@ mod tests { level: 0, depth: 0, name: name.into(), + display: name.into(), collapsed, }) } @@ -654,6 +657,7 @@ mod tests { let frame = AxisFrame { first: 0, cats: vec!["Month".into()], + cat_displays: vec!["Month".into()], entries: vec![ group("Q1", false), item("Jan"), @@ -879,6 +883,109 @@ mod tests { assert!(views[0].active, "the active view is still found by its key"); } + #[test] + fn the_grid_page_bar_and_item_rows_paint_displays() { + use improvise_protocol::{AxisFrame, CategoryItem, FrameGroup}; + let mut client = init(); + let state = client.state_mut(); + let shown = |key: &str, display: &str| FrameEntry::Item { + tuple: vec![key.into()], + display: vec![display.into()], + collapsed: false, + }; + state.fold(&Command::Project(Projection::Grid(GridUpdate::RowFrame( + AxisFrame { + cats: vec!["Region".into()], + cat_displays: vec!["Sales Area".into()], + first: 0, + entries: vec![ + FrameEntry::Group(FrameGroup { + level: 0, + depth: 0, + name: "Coastal".into(), + display: "Shoreline".into(), + collapsed: false, + }), + shown("East", "Eastern Seaboard"), + ], + }, + )))); + state.fold(&Command::Project(Projection::Grid(GridUpdate::ColFrame( + AxisFrame { + cats: vec!["_Measure".into()], + cat_displays: vec!["_Measure".into()], + first: 0, + entries: vec![shown("Revenue", "Sales")], + }, + )))); + state.fold(&Command::Project(Projection::Grid(GridUpdate::PageCoords( + vec![PageAxis { + cat: "Quarter".into(), + cat_display: "Period".into(), + selected: "Q1".into(), + selected_display: "First".into(), + items: Some(vec!["Q1".into(), "Q2".into()]), + item_displays: Some(vec!["First".into(), "Second".into()]), + }], + )))); + state.fold(&Command::Project(Projection::CategoryTree( + CategoryTreeUpdate::Categories(vec![CategoryRow { + name: "Region".into(), + display: "Sales Area".into(), + items: vec![CategoryItem { + name: "East".into(), + display: "Eastern Seaboard".into(), + }], + axis: Axis::Row, + }]), + ))); + state.view.expanded_cats.insert("Region".into()); + let model = client.model(); + assert!(matches!( + &model.rows[0], + RenderRow::Group { label, .. } if label == "Shoreline" + )); + assert!(matches!( + &model.rows[1], + RenderRow::Data { labels, .. } if labels == &vec!["Eastern Seaboard".to_string()] + )); + assert_eq!(model.col_levels, vec![vec!["Sales".to_string()]]); + let bar = &model.page_coords[0]; + assert_eq!( + (bar.cat.as_str(), bar.selected.as_str()), + ("Period", "First") + ); + assert_eq!(bar.cycle.as_ref().unwrap().items, vec!["First", "Second"]); + assert_eq!(model.categories[1].label, "Eastern Seaboard"); + } + + #[test] + fn a_column_group_header_is_painted_by_its_display() { + use improvise_protocol::{AxisFrame, FrameGroup}; + let mut client = init(); + let state = client.state_mut(); + state.fold(&Command::Project(Projection::Grid(GridUpdate::ColFrame( + AxisFrame { + cats: vec!["Month".into()], + cat_displays: vec!["Period".into()], + first: 0, + entries: vec![ + FrameEntry::Group(FrameGroup { + level: 0, + depth: 0, + name: "Q1".into(), + display: "First quarter".into(), + collapsed: false, + }), + item("Jan"), + ], + }, + )))); + let groups = client.model().col_groups; + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].label.as_deref(), Some("First quarter")); + } + fn client_with_panels() -> Client { let mut client = init(); client @@ -887,7 +994,10 @@ mod tests { ViewSummary { name: "Default".into(), display: "Default".into(), - axes: vec![("Region".into(), Axis::Row), ("_Dim".into(), Axis::Column)], + axes: vec![ + improvise_protocol::ViewAxis::plain("Region", Axis::Row), + improvise_protocol::ViewAxis::plain("_Dim", Axis::Column), + ], }, ViewSummary { name: "Plan".into(), @@ -1135,11 +1245,11 @@ mod tests { } fn page_axis(cat: &str, selected: &str, items: Option>) -> PageAxis { - PageAxis { - cat: cat.into(), - selected: selected.into(), - items: items.map(|i| i.into_iter().map(String::from).collect()), - } + PageAxis::plain( + cat, + selected, + items.map(|i| i.into_iter().map(String::from).collect()), + ) } /// improvise-p7lh: the page bar reached the client's cache and stopped diff --git a/crates/numtower/Cargo.toml b/crates/numtower/Cargo.toml new file mode 100644 index 00000000..54ca4e27 --- /dev/null +++ b/crates/numtower/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "numtower" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +description = "A three-level numeric tower: exact decimals, bounded exact fractions and inexact f64, with Scheme's contagion rules and exact, order-free sums." +keywords = ["decimal", "rational", "exact", "numeric-tower"] +categories = ["mathematics"] + +# ADR 005 clause 14: rust_decimal (std only), num-rational without its default +# features (for Ratio), and num-bigint always. No serde. +[dependencies] +rust_decimal = { version = "1.43", default-features = false, features = ["std"] } +num-rational = { version = "0.4.2", default-features = false } +num-bigint = { version = "0.4", default-features = false, features = ["std"] } +num-integer = { version = "0.1", default-features = false } +num-traits = { version = "0.2", default-features = false, features = ["std"] } + +[dev-dependencies] +num-rational = { version = "0.4.2", features = ["num-bigint-std"] } +proptest = "1" + +[[example]] +name = "bench" +test = false diff --git a/crates/numtower/examples/bench.rs b/crates/numtower/examples/bench.rs new file mode 100644 index 00000000..809a3be0 --- /dev/null +++ b/crates/numtower/examples/bench.rs @@ -0,0 +1,187 @@ +//! ADR 005 migration step 1's benchmark: `cargo run --release -p numtower --example bench`. +//! A microbenchmark, not the evaluator; what it does and does not show is in `context/code/numtower.md`. + +use std::hint::black_box; +use std::time::Instant; + +use numtower::{Decimal, ExactSum, Num, NumSum, Rounding, Typing, parse_number}; + +struct SplitMix64(u64); + +impl SplitMix64 { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} + +/// Typed two-place values from 0.01 to 999,999.99, as the ADR's appendix draws them. +fn typed_texts(seed: u64, n: usize) -> Vec { + let mut rng = SplitMix64(seed); + (0..n) + .map(|_| { + let cents = 1 + rng.next() % 99_999_999; + format!("{}.{:02}", cents / 100, cents % 100) + }) + .collect() +} + +fn nums(texts: &[String]) -> Vec { + texts + .iter() + .map(|t| parse_number(t, Typing::Inferred).unwrap()) + .collect() +} + +/// The best of `runs`, in nanoseconds per item. +fn best(runs: usize, items: usize, mut f: impl FnMut()) -> f64 { + (0..runs) + .map(|_| { + let t = Instant::now(); + f(); + t.elapsed().as_nanos() as f64 / items as f64 + }) + .fold(f64::INFINITY, f64::min) +} + +fn sums() { + println!("SUM over typed two-place values, ns per element, best of 20"); + for n in [18_000, 90_000] { + let texts = typed_texts(1, n); + let floats: Vec = texts.iter().map(|t| t.parse().unwrap()).collect(); + let decimals: Vec = texts.iter().map(|t| t.parse().unwrap()).collect(); + let values = nums(&texts); + let f64_fold = best(20, n, || { + black_box(black_box(&floats).iter().sum::()); + }); + let decimal_fold = best(20, n, || { + let s = black_box(&decimals) + .iter() + .fold(Decimal::ZERO, |a, b| a.checked_add(*b).unwrap()); + black_box(s); + }); + let num_fold = best(20, n, || { + let zero = Num::exact(Decimal::ZERO); + black_box(black_box(&values).iter().fold(zero, |a, b| &a + b)); + }); + let accumulator = best(20, n, || { + let mut s = NumSum::new(); + black_box(&values).iter().for_each(|v| s.insert(v)); + black_box(s.read()); + }); + println!( + " {n:>6} cells: f64 {f64_fold:.2}, rust_decimal {decimal_fold:.2}, Num + fold {num_fold:.2}, NumSum {accumulator:.2}" + ); + } +} + +fn operations() { + println!("Per operation over 18,000 typed pairs, ns, best of 20"); + let (a, b) = ( + nums(&typed_texts(21, 18_000)), + nums(&typed_texts(22, 18_000)), + ); + let decimal = + |v: &[Num]| -> Vec { v.iter().map(|n| n.to_string().parse().unwrap()).collect() }; + let (da, db) = (decimal(&a), decimal(&b)); + let pairs = a.len(); + let rd_add = best(20, pairs, || { + da.iter().zip(&db).for_each(|(x, y)| { + black_box(x.checked_add(*y)); + }) + }); + let rd_mul = best(20, pairs, || { + da.iter().zip(&db).for_each(|(x, y)| { + black_box(x.checked_mul(*y)); + }) + }); + let rd_div = best(20, pairs, || { + da.iter().zip(&db).for_each(|(x, y)| { + black_box(x.checked_div(*y)); + }) + }); + let add = best(20, pairs, || { + a.iter().zip(&b).for_each(|(x, y)| drop(black_box(x + y))) + }); + let mul = best(20, pairs, || { + a.iter().zip(&b).for_each(|(x, y)| drop(black_box(x * y))) + }); + let div = best(20, pairs, || { + a.iter() + .zip(&b) + .for_each(|(x, y)| drop(black_box(x.checked_div(y)))) + }); + let margin = best(20, pairs, || { + a.iter() + .zip(&b) + .for_each(|(rev, cost)| drop(black_box((rev - cost).checked_div(rev)))); + }); + let to_f64 = best(20, pairs, || { + a.iter().for_each(|x| { + black_box(x.to_f64()); + }) + }); + let places = best(20, pairs, || { + a.iter() + .for_each(|x| drop(black_box(x.round_to_places(2, Rounding::HalfAwayFromZero)))); + }); + println!(" rust_decimal checked: add {rd_add:.1}, mul {rd_mul:.1}, div {rd_div:.1}"); + println!(" Num: add {add:.1}, mul {mul:.1}, div {div:.1}"); + println!( + " Num (Rev - Cost) / Rev {margin:.1}, to_f64 {to_f64:.1}, round_to_places(2) {places:.1}" + ); +} + +/// K groups of near-`i128` denominators: the ADR's worst case for the exact read. +fn grouped_read() { + println!("NumSum read over K distinct 126-bit denominators, us per read, best of 200"); + for k in [2usize, 8] { + let mut rng = SplitMix64(0x5EED); + let mut sum = NumSum::new(); + for _ in 0..k { + let den = ((rng.next() as i128) << 62 | rng.next() as i128 >> 2) | 1 | (1 << 125); + let num = (rng.next() as i128) << 60 ^ rng.next() as i128; + let term = Num::ratio(num, den).unwrap(); + sum.add(&term, i64::MAX); + sum.add(&term, i64::MAX); + } + assert_eq!(sum.groups(), k); + let read = best(200, 1, || drop(black_box(sum.read()))); + println!( + " K = {k}: {:.1} us (exact: {})", + read / 1000.0, + sum.read().is_exact() + ); + } +} + +fn exact_sum() { + println!("ExactSum (f64 superaccumulator), best of 20"); + let floats: Vec = typed_texts(1, 1_000_000) + .iter() + .map(|t| t.parse().unwrap()) + .collect(); + let mut acc = ExactSum::new(); + let insert = best(20, floats.len(), || { + acc = ExactSum::new(); + floats.iter().for_each(|x| acc.add(*x, 1)); + }); + let read = best(20, 1, || { + black_box(acc.value()); + }); + println!( + " {insert:.2} ns per insert, {:.2} us per read", + read / 1000.0 + ); +} + +fn main() { + println!("numtower bench (release); name the machine when you quote these"); + sums(); + operations(); + grouped_read(); + exact_sum(); +} diff --git a/crates/numtower/src/arith.rs b/crates/numtower/src/arith.rs new file mode 100644 index 00000000..1b19a38b --- /dev/null +++ b/crates/numtower/src/arith.rs @@ -0,0 +1,445 @@ +//! Arithmetic and contagion (ADR 005 clauses 4 and 5, question 13 (b)). +//! The reasoning for each rule is in `context/code/numtower.md`, "Arithmetic". + +use std::ops::{Add, Mul, Neg, Sub}; + +use num_bigint::{BigInt, BigUint}; +use num_integer::Integer; +use num_traits::{One, Signed, ToPrimitive, Zero}; +use rust_decimal::Decimal; + +use crate::fixed::I256; +use crate::num::{MANTISSA_LIMIT, MAX_SCALE, Num, NumRef, decimal, pow10, small_ratio}; +use crate::ratio::BigQ; + +/// The widest an exact intermediate may grow, in bits of numerator or denominator. +pub const CEILING_BITS: u64 = 8192; + +/// A value inside one evaluation: exact intermediates are unbounded up to the ceiling. +#[derive(Clone, Debug)] +pub struct Interim(Inner); + +#[derive(Clone, Debug)] +enum Inner { + Stored(Num), + Wide(Box), +} + +/// The divisor was exactly zero, exact or inexact (clause 5). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DivisionByZero; + +impl From for Interim { + fn from(n: Num) -> Interim { + Interim(Inner::Stored(n)) + } +} + +impl Interim { + fn wide(q: BigQ) -> Interim { + Interim(Inner::Wide(Box::new(q))) + } + + /// The stored bound of question 13 (b): a wide value is the nearest f64. + pub fn store(self) -> Num { + match self.0 { + Inner::Stored(n) => n, + Inner::Wide(q) => Num::inexact(q.to_f64()), + } + } + + pub fn is_exact(&self) -> bool { + match &self.0 { + Inner::Stored(n) => n.is_exact(), + Inner::Wide(_) => true, + } + } + + fn operand(&self) -> Operand<'_> { + match &self.0 { + Inner::Stored(n) => Operand::of(n), + Inner::Wide(q) => Operand::Big(q), + } + } + + pub fn checked_div(&self, o: &Interim) -> Result { + divide(self.operand(), o.operand()) + } + + pub fn pow(&self, o: &Interim) -> Result { + power(self.operand(), o.operand()) + } + + /// Exactly zero; a wide value never is, because zero fits a stored arm. + pub fn is_zero(&self) -> bool { + match &self.0 { + Inner::Stored(n) => n.is_zero(), + Inner::Wide(_) => false, + } + } + + /// Clause 6's exact order, before the stored bound; `None` only when either is NaN. + pub fn cmp_exact(&self, o: &Interim) -> Option { + match (&self.0, &o.0) { + (Inner::Stored(a), Inner::Stored(b)) => a.cmp_exact(b), + _ => match (self.operand().big(), o.operand().big()) { + (Some(a), Some(b)) => Some(a.cmp_value(&b)), + (Some(a), None) => crate::cmp::exact_against_f64(&a, o.operand().float()), + (None, Some(b)) => { + crate::cmp::exact_against_f64(&b, self.operand().float()).map(|c| c.reverse()) + } + (None, None) => self.operand().float().partial_cmp(&o.operand().float()), + }, + } + } +} + +impl Num { + pub fn checked_div(&self, o: &Num) -> Result { + divide(Operand::of(self), Operand::of(o)).map(Interim::store) + } + + pub fn pow(&self, o: &Num) -> Result { + power(Operand::of(self), Operand::of(o)).map(Interim::store) + } +} + +macro_rules! binary_ops { + ($($trait:ident $method:ident $f:ident;)*) => {$( + impl $trait<&Interim> for &Interim { + type Output = Interim; + fn $method(self, o: &Interim) -> Interim { + $f(self.operand(), o.operand()) + } + } + impl $trait<&Num> for &Num { + type Output = Num; + fn $method(self, o: &Num) -> Num { + $f(Operand::of(self), Operand::of(o)).store() + } + } + )*}; +} + +binary_ops! { + Add add plus; + Sub sub minus; + Mul mul times; +} + +impl Neg for &Interim { + type Output = Interim; + fn neg(self) -> Interim { + negate(self.operand()) + } +} + +impl Neg for &Num { + type Output = Num; + fn neg(self) -> Num { + negate(Operand::of(self)).store() + } +} + +#[derive(Clone, Copy)] +enum Operand<'a> { + Dec(&'a Decimal), + Small(i128, i128), + Big(&'a BigQ), + Float(f64), +} + +impl<'a> Operand<'a> { + fn of(n: &'a Num) -> Operand<'a> { + match n.kind() { + NumRef::Exact(d) => Operand::Dec(d), + NumRef::Fraction(r) => Operand::Small(*r.numer(), *r.denom()), + NumRef::Inexact(x) => Operand::Float(x), + } + } + + /// The `i128` fast path's operands; `i128::MIN` cannot be negated, so it takes the bigint path. + fn parts(self) -> Option<(i128, i128)> { + match self { + Operand::Dec(d) => Some((d.mantissa(), pow10(d.scale()))), + Operand::Small(n, d) => (n != i128::MIN).then_some((n, d)), + Operand::Big(_) | Operand::Float(_) => None, + } + } + + fn big(self) -> Option { + match self { + Operand::Dec(d) => Some(BigQ::from_i128(d.mantissa(), pow10(d.scale()))), + Operand::Small(n, d) => Some(BigQ::from_i128(n, d)), + Operand::Big(q) => Some(q.clone()), + Operand::Float(_) => None, + } + } + + fn float(self) -> f64 { + match self { + Operand::Dec(d) => crate::num::decimal_to_f64(d), + Operand::Small(n, d) => crate::ratio::ratio_to_f64(&BigInt::from(n), &BigInt::from(d)), + Operand::Big(q) => q.to_f64(), + Operand::Float(x) => x, + } + } + + fn is_float(self) -> bool { + matches!(self, Operand::Float(_)) + } + + fn is_zero(self) -> bool { + match self { + Operand::Dec(d) => d.is_zero(), + Operand::Small(..) | Operand::Big(_) => false, + Operand::Float(x) => x == 0.0, + } + } +} + +fn inexact(x: f64) -> Interim { + Interim::from(Num::inexact(x)) +} + +/// An exact result in the first arm it fits; wide while under the ceiling, else demoted. +fn settle_wide(q: BigQ) -> Interim { + match Num::fit(&q) { + Some(n) => Interim::from(n), + None if q.bits() > CEILING_BITS => inexact(q.to_f64()), + None => Interim::wide(q), + } +} + +fn exact_or_wide( + a: Operand, + b: Operand, + small: impl Fn(i128, i128, i128, i128) -> Option<(i128, i128)>, + big: impl Fn(&BigQ, &BigQ) -> BigQ, +) -> Interim { + let fast = a + .parts() + .zip(b.parts()) + .and_then(|((n1, d1), (n2, d2))| small(n1, d1, n2, d2)); + fast.and_then(|(n, d)| small_ratio(n, d)) + .map(Interim::from) + .unwrap_or_else(|| match a.big().zip(b.big()) { + Some((x, y)) => settle_wide(big(&x, &y)), + None => unreachable!("both operands are exact here"), + }) +} + +fn plus(a: Operand, b: Operand) -> Interim { + match (a, b) { + (x, y) if x.is_float() || y.is_float() => inexact(x.float() + y.float()), + (Operand::Dec(x), Operand::Dec(y)) => decimal_sum(x, y), + _ => exact_or_wide( + a, + b, + |n1, d1, n2, d2| { + let n = n1.checked_mul(d2)?.checked_add(n2.checked_mul(d1)?)?; + Some((n, d1.checked_mul(d2)?)) + }, + BigQ::add, + ), + } +} + +fn minus(a: Operand, b: Operand) -> Interim { + plus(a, negate(b).operand()) +} + +fn times(a: Operand, b: Operand) -> Interim { + match (a, b) { + (x, y) if x.is_float() || y.is_float() => inexact(x.float() * y.float()), + (Operand::Dec(x), Operand::Dec(y)) => decimal_product(x, y), + _ => exact_or_wide( + a, + b, + |n1, d1, n2, d2| { + let (g1, g2) = (n1.gcd(&d2).max(1), n2.gcd(&d1).max(1)); + Some(( + (n1 / g1).checked_mul(n2 / g2)?, + (d1 / g2).checked_mul(d2 / g1)?, + )) + }, + BigQ::mul, + ), + } +} + +fn negate(a: Operand) -> Interim { + match a { + Operand::Dec(d) => Interim::from(Num::exact(-*d)), + Operand::Float(x) => inexact(-x), + Operand::Small(n, d) => n + .checked_neg() + .and_then(|m| small_ratio(m, d)) + .map(Interim::from) + .unwrap_or_else(|| settle_wide(BigQ::from_i128(n, d).neg())), + Operand::Big(q) => settle_wide(q.neg()), + } +} + +fn divide(a: Operand, b: Operand) -> Result { + match (a, b) { + (_, y) if y.is_zero() => Err(DivisionByZero), + (x, y) if x.is_float() || y.is_float() => Ok(inexact(x.float() / y.float())), + _ => Ok(exact_or_wide( + a, + b, + |n1, d1, n2, d2| { + let (g1, g2) = (n1.gcd(&n2).max(1), d1.gcd(&d2).max(1)); + Some(( + (n1 / g1).checked_mul(d2 / g2)?, + (d1 / g2).checked_mul(n2 / g1)?, + )) + }, + |x, y| { + x.div(y) + .unwrap_or_else(|| unreachable!("the divisor is not zero")) + }, + )), + } +} + +/// Clause 4's hint: a returned scale below the nominal one may mean rounding. +fn decimal_sum(x: &Decimal, y: &Decimal) -> Interim { + let nominal = x.scale().max(y.scale()); + match x.checked_add(*y).filter(|r| r.scale() == nominal) { + Some(r) => Interim::from(Num::exact(r)), + None => { + let align = |d: &Decimal| { + I256::from_i128(d.mantissa()) + .checked_mul_u128(pow10(nominal - d.scale()) as u128) + .expect("an aligned mantissa is below 2^190") + }; + let sum = align(x).checked_add(&align(y)).expect("below 2^191"); + settle_decimal_integer(sum, nominal, nominal) + } + } +} + +fn decimal_product(x: &Decimal, y: &Decimal) -> Interim { + let nominal = x.scale() + y.scale(); + match x.checked_mul(*y).filter(|r| r.scale() == nominal) { + Some(r) => Interim::from(Num::exact(r)), + None => { + settle_decimal_integer(I256::mul_i128(x.mantissa(), y.mantissa()), nominal, nominal) + } + } +} + +/// The recompute of clause 4: strip trailing zeros, fit 96 bits at scale <= 28, else a fraction. +fn settle_decimal_integer(m: I256, scale: u32, nominal: u32) -> Interim { + let (mut m, mut s) = (m, scale); + while s > 0 { + let (q, r) = m.div_rem_u64(10); + if r != 0 { + break; + } + m = q; + s -= 1; + } + let fits = m + .to_i128() + .filter(|v| v.unsigned_abs() < MANTISSA_LIMIT as u128); + match fits { + Some(v) if s <= MAX_SCALE => Interim::from(padded(v, s, nominal.min(MAX_SCALE))), + _ => settle_wide(BigQ::new(m.to_bigint(), BigInt::from(10).pow(s))), + } +} + +/// The stored scale of clause 4: pad toward `target` while the mantissa fits 96 bits. +pub(crate) fn padded(mantissa: i128, scale: u32, target: u32) -> Num { + let (mut m, mut s) = (mantissa, scale); + while s < target && (m * 10).unsigned_abs() < MANTISSA_LIMIT as u128 { + m *= 10; + s += 1; + } + decimal(m, s).expect("a padded mantissa still fits") +} + +/// The exponent of an exact integer-valued operand. +fn integer_value(b: Operand) -> Option { + match b { + Operand::Dec(d) if d.fract().is_zero() => { + Some(BigInt::from(d.mantissa() / pow10(d.scale()))) + } + Operand::Small(n, 1) => Some(BigInt::from(n)), + Operand::Big(q) if q.den.is_one() => Some(q.num.clone()), + _ => None, + } +} + +fn power(a: Operand, b: Operand) -> Result { + match (a.is_float(), integer_value(b)) { + (false, Some(n)) => exact_power(a, &n), + _ => Ok(inexact(a.float().powf(b.float()))), + } +} + +/// Clause 5's `POW`: the size is checked before anything is computed. +fn exact_power(a: Operand, n: &BigInt) -> Result { + let q = a.big().expect("an exact base"); + let base_scale = match a { + Operand::Dec(d) => d.scale(), + _ => 0, + }; + let k = n.magnitude(); + let target = match n.is_negative() { + true => 0, + false => k + .to_u32() + .and_then(|k| k.checked_mul(base_scale)) + .map_or(MAX_SCALE, |s| s.min(MAX_SCALE)), + }; + let unit = q.den.is_one() && q.num.magnitude().is_one(); + match (q.num.is_zero(), n.is_negative(), unit) { + (_, _, _) if n.is_zero() => Ok(Interim::from(padded(1, 0, 0))), + (true, true, _) => Err(DivisionByZero), + (true, false, _) => Ok(Interim::from(padded(0, 0, target))), + (false, _, true) => { + let negative = q.num.is_negative() && k.is_odd(); + Ok(Interim::from(padded( + if negative { -1 } else { 1 }, + 0, + target, + ))) + } + (false, negative, false) => { + let lower = |x: &BigInt| BigUint::from(x.bits().saturating_sub(1)) * k; + let too_big = lower(&q.num).max(lower(&q.den)) > BigUint::from(CEILING_BITS); + let odd_negative = q.num.is_negative() && k.is_odd(); + match (too_big, k.to_u32()) { + (false, Some(k)) => Ok(exact_integer_power(&q, k, negative, target)), + _ => Ok(inexact(signed_powf(a.float(), n, odd_negative))), + } + } + } +} + +/// `powf` on the magnitude, the sign taken from the exact exponent's parity. +fn signed_powf(base: f64, n: &BigInt, odd_negative: bool) -> f64 { + let magnitude = base.abs().powf(n.to_f64().unwrap_or(f64::INFINITY)); + match odd_negative { + true => -magnitude, + false => magnitude, + } +} + +fn exact_integer_power(q: &BigQ, k: u32, negative: bool, target: u32) -> Interim { + let (num, den) = (q.num.pow(k), q.den.pow(k)); + let value = match negative { + true => BigQ::new(den, num), + false => BigQ { num, den }, + }; + let settled = settle_wide(value); + match (&settled.0, negative) { + (Inner::Stored(n), false) => match n.kind() { + NumRef::Exact(d) => Interim::from(padded(d.mantissa(), d.scale(), target)), + _ => settled, + }, + _ => settled, + } +} diff --git a/crates/numtower/src/cmp.rs b/crates/numtower/src/cmp.rs new file mode 100644 index 00000000..81cac9b7 --- /dev/null +++ b/crates/numtower/src/cmp.rs @@ -0,0 +1,107 @@ +//! Exact comparison at every pairing of arms (clause 6), and the `MIN`/`MAX` +//! tie-breaks (clause 8). Why: `context/code/numtower.md`, "Comparison". + +use std::cmp::Ordering; + +use crate::fixed::I256; +use crate::num::{Num, NumRef}; +use crate::ratio::BigQ; + +impl Num { + /// The order of the two values, exactly; `None` only when either is NaN. + pub fn cmp_exact(&self, o: &Num) -> Option { + match (self.exact_parts(), o.exact_parts()) { + (Some(a), Some(b)) => Some(cross(a, b)), + (Some(_), None) => mixed(self, o.to_f64()), + (None, Some(_)) => mixed(o, self.to_f64()).map(Ordering::reverse), + (None, None) => self.to_f64().partial_cmp(&o.to_f64()), + } + } + + /// The smaller, NaN skipped; ties keep the most information (clause 8, question 11). + pub fn min_of<'a>(&'a self, o: &'a Num) -> &'a Num { + pick(self, o, Ordering::Less) + } + + pub fn max_of<'a>(&'a self, o: &'a Num) -> &'a Num { + pick(self, o, Ordering::Greater) + } +} + +impl PartialEq for Num { + fn eq(&self, o: &Num) -> bool { + self.cmp_exact(o) == Some(Ordering::Equal) + } +} + +impl PartialOrd for Num { + fn partial_cmp(&self, o: &Num) -> Option { + self.cmp_exact(o) + } +} + +/// `a.0/a.1` against `b.0/b.1`, denominators positive, in 256 bits. +fn cross(a: (i128, i128), b: (i128, i128)) -> Ordering { + I256::mul_i128(a.0, b.1).cmp(&I256::mul_i128(b.0, a.1)) +} + +/// An exact value against an f64: the rounded value decides unless it ties. +fn mixed(exact: &Num, x: f64) -> Option { + match (x.is_nan(), x.is_infinite()) { + (true, _) => None, + (false, true) => Some(if x > 0.0 { + Ordering::Less + } else { + Ordering::Greater + }), + (false, false) => match exact.to_f64().partial_cmp(&x)? { + Ordering::Equal => { + let q = exact.exact_q().expect("an exact operand"); + Some(q.cmp_value(&BigQ::from_f64(x))) + } + decided => Some(decided), + }, + } +} + +/// A wide exact value against an f64, with no fast path: the slow, total form of `mixed`. +pub(crate) fn exact_against_f64(q: &BigQ, x: f64) -> Option { + match (x.is_nan(), x.is_infinite(), x > 0.0) { + (true, _, _) => None, + (false, true, true) => Some(Ordering::Less), + (false, true, false) => Some(Ordering::Greater), + (false, false, _) => Some(q.cmp_value(&BigQ::from_f64(x))), + } +} + +fn pick<'a>(a: &'a Num, b: &'a Num, want: Ordering) -> &'a Num { + match (a.cmp_exact(b), is_nan(a), is_nan(b)) { + (_, true, _) => b, + (_, false, true) => a, + (Some(o), _, _) if o == want => a, + (Some(Ordering::Equal), _, _) => tie(a, b, want), + _ => b, + } +} + +fn is_nan(n: &Num) -> bool { + matches!(n.kind(), NumRef::Inexact(x) if x.is_nan()) +} + +/// Equal values: exact over inexact, the larger scale, and IEEE's signed zero. +fn tie<'a>(a: &'a Num, b: &'a Num, want: Ordering) -> &'a Num { + match (a.kind(), b.kind()) { + (NumRef::Inexact(_), NumRef::Exact(_) | NumRef::Fraction(_)) => b, + (NumRef::Exact(_) | NumRef::Fraction(_), NumRef::Inexact(_)) => a, + (NumRef::Exact(x), NumRef::Exact(y)) if y.scale() > x.scale() => b, + (NumRef::Inexact(x), NumRef::Inexact(y)) => { + let negative_wanted = want == Ordering::Less; + match y.is_sign_negative() == negative_wanted && x.is_sign_negative() != negative_wanted + { + true => b, + false => a, + } + } + _ => a, + } +} diff --git a/crates/numtower/src/fixed.rs b/crates/numtower/src/fixed.rs new file mode 100644 index 00000000..ba551181 --- /dev/null +++ b/crates/numtower/src/fixed.rs @@ -0,0 +1,234 @@ +//! Fixed-width two's-complement integers for the hot paths: no allocation. +//! Why they are hand-rolled: `context/code/numtower.md`, "The wide integers". + +use std::cmp::Ordering; + +use num_bigint::{BigInt, Sign}; + +/// A signed integer of `64 * N` bits, little-endian limbs, two's complement. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(crate) struct Fixed { + limbs: [u64; N], +} + +/// The 256-bit integer clauses 4–6 use, and the 320-bit decimal partition of clause 8. +pub(crate) type I256 = Fixed<4>; +pub(crate) type I320 = Fixed<5>; + +impl Fixed { + pub(crate) const ZERO: Self = Fixed { limbs: [0; N] }; + + pub(crate) fn from_i128(x: i128) -> Self { + let fill = if x < 0 { u64::MAX } else { 0 }; + let mut limbs = [fill; N]; + limbs[0] = x as u64; + limbs[1] = (x >> 64) as u64; + Fixed { limbs } + } + + pub(crate) fn is_negative(&self) -> bool { + self.limbs[N - 1] >> 63 == 1 + } + + pub(crate) fn is_zero(&self) -> bool { + self.limbs.iter().all(|&l| l == 0) + } + + fn wrapping_neg(&self) -> Self { + let mut out = [0u64; N]; + let mut carry = 1u64; + for (o, &l) in out.iter_mut().zip(&self.limbs) { + let (v, c) = (!l).overflowing_add(carry); + *o = v; + carry = c as u64; + } + Fixed { limbs: out } + } + + fn wrapping_add(&self, other: &Self) -> Self { + let mut out = [0u64; N]; + let mut carry = false; + for (o, (a, b)) in out.iter_mut().zip(self.limbs.iter().zip(&other.limbs)) { + let (s, c1) = a.overflowing_add(*b); + let (s, c2) = s.overflowing_add(carry as u64); + *o = s; + carry = c1 || c2; + } + Fixed { limbs: out } + } + + pub(crate) fn checked_add(&self, other: &Self) -> Option { + let sum = self.wrapping_add(other); + let same = self.is_negative() == other.is_negative(); + (!same || sum.is_negative() == self.is_negative()).then_some(sum) + } + + pub(crate) fn checked_neg(&self) -> Option { + let neg = self.wrapping_neg(); + (self.is_zero() || neg.is_negative() != self.is_negative()).then_some(neg) + } + + /// The magnitude as unsigned limbs, with the sign split out. + fn magnitude(&self) -> (bool, [u64; N]) { + match self.is_negative() { + true => (true, self.wrapping_neg().limbs), + false => (false, self.limbs), + } + } + + fn from_magnitude(negative: bool, limbs: [u64; N]) -> Option { + let unsigned = Fixed { limbs }; + match (negative, unsigned.is_negative()) { + (false, false) => Some(unsigned), + (true, _) => { + let neg = unsigned.wrapping_neg(); + (neg.is_negative() || neg.is_zero()).then_some(neg) + } + (false, true) => None, + } + } + + pub(crate) fn checked_mul_u128(&self, k: u128) -> Option { + let (negative, mag) = self.magnitude(); + let mut out = [0u64; N]; + for (j, kk) in [k as u64, (k >> 64) as u64].into_iter().enumerate() { + if kk == 0 { + continue; + } + let mut carry = 0u128; + for (i, &limb) in mag.iter().enumerate() { + let idx = i + j; + let prod = limb as u128 * kk as u128 + carry; + match idx < N { + true => { + let s = out[idx] as u128 + (prod as u64) as u128; + out[idx] = s as u64; + carry = (prod >> 64) + (s >> 64); + } + false if prod != 0 => return None, + false => carry = 0, + } + } + if carry != 0 { + return None; + } + } + Self::from_magnitude(negative, out) + } + + /// Divides the magnitude by `k`, returning the quotient (sign kept) and the remainder. + pub(crate) fn div_rem_u64(&self, k: u64) -> (Self, u64) { + let (negative, mag) = self.magnitude(); + let mut out = [0u64; N]; + let mut rem = 0u128; + for i in (0..N).rev() { + let cur = (rem << 64) | mag[i] as u128; + out[i] = (cur / k as u128) as u64; + rem = cur % k as u128; + } + let q = Self::from_magnitude(negative, out).unwrap_or(Self::ZERO); + (q, rem as u64) + } + + pub(crate) fn to_i128(self) -> Option { + let fill = if self.is_negative() { u64::MAX } else { 0 }; + let hi_ok = self.limbs[2..].iter().all(|&l| l == fill); + let v = (self.limbs[0] as u128 | ((self.limbs[1] as u128) << 64)) as i128; + (hi_ok && (v < 0) == self.is_negative()).then_some(v) + } + + pub(crate) fn to_bigint(self) -> BigInt { + let (negative, mag) = self.magnitude(); + let digits: Vec = mag + .iter() + .flat_map(|&l| [l as u32, (l >> 32) as u32]) + .collect(); + let sign = if negative { Sign::Minus } else { Sign::Plus }; + BigInt::from_slice(sign, &digits) + } + + /// The product of two `i128`s; needs `N >= 4`. + pub(crate) fn mul_i128(a: i128, b: i128) -> Self { + let negative = (a < 0) != (b < 0); + let (x, y) = (a.unsigned_abs(), b.unsigned_abs()); + let (x0, x1) = (x as u64 as u128, x >> 64); + let (y0, y1) = (y as u64 as u128, y >> 64); + let p00 = x0 * y0; + let p01 = x0 * y1; + let p10 = x1 * y0; + let p11 = x1 * y1; + let mid = (p00 >> 64) + (p01 as u64 as u128) + (p10 as u64 as u128); + let mut limbs = [0u64; N]; + limbs[0] = p00 as u64; + limbs[1] = mid as u64; + let hi = (mid >> 64) + (p01 >> 64) + (p10 >> 64) + p11; + limbs[2] = hi as u64; + limbs[3] = (hi >> 64) as u64; + let unsigned = Fixed { limbs }; + match negative { + true => unsigned.wrapping_neg(), + false => unsigned, + } + } +} + +impl Default for Fixed { + fn default() -> Self { + Self::ZERO + } +} + +impl Ord for Fixed { + fn cmp(&self, other: &Self) -> Ordering { + match (self.is_negative(), other.is_negative()) { + (true, false) => Ordering::Less, + (false, true) => Ordering::Greater, + _ => self.limbs.iter().rev().cmp(other.limbs.iter().rev()), + } + } +} + +impl PartialOrd for Fixed { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_256_bit_product_of_two_i128s_matches_bigint() { + let cases = [ + (i128::MAX, i128::MAX), + (i128::MAX, -i128::MAX), + (-1, 1), + (0, i128::MIN), + (123_456_789_012_345_678_901_234, -98_765_432_109_876_543_210), + ]; + for (a, b) in cases { + let want = BigInt::from(a) * BigInt::from(b); + assert_eq!(I256::mul_i128(a, b).to_bigint(), want, "{a} * {b}"); + } + } + + #[test] + fn checked_arithmetic_refuses_what_overflows_and_keeps_what_fits() { + let max = I256::from_i128(i128::MAX) + .checked_mul_u128(u128::MAX) + .unwrap(); + assert_eq!( + max.to_bigint(), + BigInt::from(i128::MAX) * BigInt::from(u128::MAX) + ); + assert!(max.checked_mul_u128(u128::MAX).is_none()); + let neg = max.checked_neg().unwrap(); + assert_eq!(neg.to_bigint(), -max.to_bigint()); + let (q, r) = neg.div_rem_u64(10); + assert_eq!(q.to_bigint() * 10 - BigInt::from(r), neg.to_bigint()); + assert_eq!(I256::from_i128(-5).to_i128(), Some(-5)); + assert_eq!(max.to_i128(), None); + assert!(I256::from_i128(-1) < I256::ZERO); + } +} diff --git a/crates/numtower/src/lang.rs b/crates/numtower/src/lang.rs new file mode 100644 index 00000000..b5d00960 --- /dev/null +++ b/crates/numtower/src/lang.rs @@ -0,0 +1,192 @@ +//! Clause 3's one number language: the gate, the exact parse, the forced `n/d` arm, +//! and the canonical text. Why each rule: `context/code/numtower.md`, "The number language". + +use std::fmt; + +use num_bigint::BigInt; +use num_traits::{One, Zero}; + +use crate::arith::padded; +use crate::num::{MAX_SCALE, Num, NumRef}; +use crate::ratio::BigQ; + +/// Whether the type of the text is being guessed or was forced (question 12). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Typing { + Inferred, + Forced, +} + +/// The gate: the strings `f64::from_str` accepts and reads as a finite value. +pub fn is_number(text: &str, typing: Typing) -> bool { + gate(text) || (typing == Typing::Forced && forced_fraction(text).is_some()) +} + +/// A number if the text is one, read exactly; `None` means the text is text. +pub fn parse_number(text: &str, typing: Typing) -> Option { + match (gate(text), typing) { + (true, _) => Lexeme::read(text).map(|lx| lx.value(text)), + (false, Typing::Forced) => forced_fraction(text), + (false, Typing::Inferred) => None, + } +} + +/// A forced `n/d`; one that rounds to infinity is text, as a finite lexeme never reads as one. +fn forced_fraction(text: &str) -> Option { + let n = Num::settle(&fraction_lexeme(text)?); + n.to_f64().is_finite().then_some(n) +} + +fn gate(text: &str) -> bool { + text.parse::().is_ok_and(f64::is_finite) +} + +/// `[+-]digits/digits` with a nonzero denominator, as an exact value. +fn fraction_lexeme(text: &str) -> Option { + let (n, d) = text.split_once('/')?; + let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()); + let unsigned = n.strip_prefix(['+', '-']).unwrap_or(n); + let ok = digits(unsigned) && digits(d); + let num: BigInt = ok.then(|| n.trim_start_matches('+').parse().ok())??; + let den: BigInt = d.parse().ok()?; + (!den.is_zero()).then(|| BigQ::new(num, den)) +} + +/// A lexeme of the gate's language: sign, significant digits, and the power of ten. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Lexeme { + pub(crate) negative: bool, + pub(crate) digits: String, + pub(crate) exp10: i64, +} + +/// Exponents beyond this cannot reach either exact arm, and saturate. +const EXP_SATURATE: i64 = 1 << 40; + +impl Lexeme { + /// `[+-]? (d+ (. d*)? | . d+) ([eE] [+-]? d+)?`, and nothing else. + pub(crate) fn read(text: &str) -> Option { + let (negative, rest) = match text.as_bytes().first()? { + b'-' => (true, &text[1..]), + b'+' => (false, &text[1..]), + _ => (false, text), + }; + let (mantissa, exponent) = match rest.find(['e', 'E']) { + Some(i) => (&rest[..i], Some(&rest[i + 1..])), + None => (rest, None), + }; + let (int, frac) = mantissa.split_once('.').unwrap_or((mantissa, "")); + let all_digits = |s: &str| s.bytes().all(|b| b.is_ascii_digit()); + let mantissa_ok = + all_digits(int) && all_digits(frac) && !(int.is_empty() && frac.is_empty()); + let exp = match exponent { + Some(e) => read_exponent(e)?, + None => 0, + }; + let digits = format!("{int}{frac}"); + mantissa_ok.then(|| Lexeme { + negative, + digits, + exp10: exp.saturating_sub(frac.len() as i64), + }) + } + + /// Clause 3's order: a decimal at the lexeme's scale, then a fraction, then the nearest f64. + fn value(&self, text: &str) -> Num { + let significant = self.digits.trim_start_matches('0'); + let stripped = significant.trim_end_matches('0'); + let implied_scale = (-self.exp10).clamp(0, MAX_SCALE as i64) as u32; + match stripped.is_empty() { + true => padded(0, 0, implied_scale), + false => { + let shift = self + .exp10 + .saturating_add((significant.len() - stripped.len()) as i64); + self.exact(stripped, shift, implied_scale) + .unwrap_or_else(|| Num::inexact(text.parse().unwrap_or(f64::NAN))) + } + } + } + + fn exact(&self, stripped: &str, shift: i64, implied_scale: u32) -> Option { + let leading = stripped.len() as i64 + shift; + let reachable = stripped.len() <= 180 && (-130..=40).contains(&leading) && shift >= -400; + let q = reachable.then(|| self.rational(stripped, shift))?; + Num::fit(&q).map(|n| match n.kind() { + NumRef::Exact(d) => padded(d.mantissa(), d.scale(), implied_scale), + _ => n, + }) + } + + pub(crate) fn rational(&self, stripped: &str, shift: i64) -> BigQ { + let m: BigInt = stripped.parse().unwrap_or_default(); + let m = if self.negative { -m } else { m }; + let ten = BigInt::from(10); + match shift >= 0 { + true => BigQ::new(m * ten.pow(shift as u32), BigInt::one()), + false => BigQ::new(m, ten.pow((-shift) as u32)), + } + } + + /// The exact value, whatever its size: for display rounding. + pub(crate) fn exact_value(&self) -> BigQ { + let stripped = self.digits.trim_start_matches('0'); + match stripped.is_empty() { + true => BigQ::new(BigInt::zero(), BigInt::one()), + false => self.rational(stripped, self.exp10), + } + } +} + +fn read_exponent(e: &str) -> Option { + let (negative, digits) = match e.as_bytes().first()? { + b'-' => (true, &e[1..]), + b'+' => (false, &e[1..]), + _ => (false, e), + }; + let ok = !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()); + let significant = digits.trim_start_matches('0'); + let magnitude = match significant.len() > 12 { + true => EXP_SATURATE, + false => significant.parse::().unwrap_or(0).min(EXP_SATURATE), + }; + ok.then_some(if negative { -magnitude } else { magnitude }) +} + +/// The shortest text that reads back as `x`; `None` for a non-finite value. +pub fn shortest_round_trip(x: f64) -> Option { + x.is_finite().then(|| format!("{x:?}")) +} + +const INEXACT_MARK: &str = "#i"; + +impl Num { + /// Reads every spelling `Display` writes: a decimal, `n/d`, `#i…`, `inf`, `-inf`, `nan`. + pub fn from_canonical(text: &str) -> Option { + match text { + "inf" => Some(Num::inexact(f64::INFINITY)), + "-inf" => Some(Num::inexact(f64::NEG_INFINITY)), + "nan" => Some(Num::inexact(f64::NAN)), + _ => match text.strip_prefix(INEXACT_MARK) { + Some(rest) if gate(rest) => rest.parse().ok().map(Num::inexact), + Some(_) => None, + None => parse_number(text, Typing::Forced).filter(Num::is_exact), + }, + } + } +} + +/// The canonical text: a decimal with its scale, `n/d`, `#i` and the shortest spelling, or a token. +impl fmt::Display for Num { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.kind() { + NumRef::Exact(d) => write!(f, "{d}"), + NumRef::Fraction(r) => write!(f, "{}/{}", r.numer(), r.denom()), + NumRef::Inexact(x) if x.is_nan() => f.write_str("nan"), + NumRef::Inexact(x) if x.is_infinite() => { + f.write_str(if x > 0.0 { "inf" } else { "-inf" }) + } + NumRef::Inexact(x) => write!(f, "{INEXACT_MARK}{x:?}"), + } + } +} diff --git a/crates/numtower/src/lib.rs b/crates/numtower/src/lib.rs new file mode 100644 index 00000000..458d3e9f --- /dev/null +++ b/crates/numtower/src/lib.rs @@ -0,0 +1,20 @@ +//! A three-level numeric tower: exact decimal, exact bounded fraction, inexact f64. +//! The decisions are ADR 005; the reasoning is `context/code/numtower.md`. + +mod arith; +mod cmp; +mod fixed; +mod lang; +mod num; +mod ratio; +mod round; +mod sum; + +pub use arith::{CEILING_BITS, DivisionByZero, Interim}; +pub use lang::{Typing, is_number, parse_number, shortest_round_trip}; +pub use num::{Num, NumRef}; +pub use round::Rounding; +pub use sum::{ExactSum, K, NumSum}; + +pub use num_rational::Ratio; +pub use rust_decimal::Decimal; diff --git a/crates/numtower/src/num.rs b/crates/numtower/src/num.rs new file mode 100644 index 00000000..1f0448f1 --- /dev/null +++ b/crates/numtower/src/num.rs @@ -0,0 +1,198 @@ +//! The stored number: an exact decimal, an exact bounded fraction, or an f64. +//! The arms are private so a value is always normalized: `context/code/numtower.md`. + +use num_bigint::BigInt; +use num_integer::Integer; +use num_rational::Ratio; +use num_traits::{One, ToPrimitive}; +use rust_decimal::Decimal; + +use crate::ratio::{BigQ, ratio_to_f64}; + +/// The most significant digits a `Decimal` holds, and its mantissa bound. +pub(crate) const MAX_SCALE: u32 = 28; +pub(crate) const MANTISSA_LIMIT: i128 = 1 << 96; + +/// A number of the tower (ADR 005 clause 2), always in its one normal form. +#[derive(Clone, Debug)] +pub struct Num(Repr); + +#[derive(Clone, Debug)] +enum Repr { + Exact(Decimal), + Fraction(Box>), + Inexact(f64), +} + +/// A borrowed view of a `Num`'s arm, for exhaustive matching by callers. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum NumRef<'a> { + Exact(&'a Decimal), + Fraction(&'a Ratio), + Inexact(f64), +} + +impl Num { + /// An exact decimal. Zero loses its sign: the sign of zero is an f64 artefact. + pub fn exact(d: Decimal) -> Num { + let mut d = d; + if d.is_zero() { + d.set_sign_positive(true); + } + Num(Repr::Exact(d)) + } + + pub fn inexact(x: f64) -> Num { + Num(Repr::Inexact(x)) + } + + /// The exact value `num / den`, normalized; `None` when `den` is zero. + pub fn ratio(num: i128, den: i128) -> Option { + (den != 0).then(|| Num::settle(&BigQ::from_i128(num, den))) + } + + fn fraction(num: i128, den: i128) -> Num { + Num(Repr::Fraction(Box::new(Ratio::new_raw(num, den)))) + } + + pub fn kind(&self) -> NumRef<'_> { + match &self.0 { + Repr::Exact(d) => NumRef::Exact(d), + Repr::Fraction(r) => NumRef::Fraction(r), + Repr::Inexact(x) => NumRef::Inexact(*x), + } + } + + pub fn is_exact(&self) -> bool { + !matches!(self.0, Repr::Inexact(_)) + } + + /// Exactly zero, at either level; the one zero predicate of clause 6. + pub fn is_zero(&self) -> bool { + match &self.0 { + Repr::Exact(d) => d.is_zero(), + Repr::Fraction(_) => false, + Repr::Inexact(x) => *x == 0.0, + } + } + + /// The correctly rounded f64 (clause 4): a decimal through its text, a fraction exactly. + pub fn to_f64(&self) -> f64 { + match &self.0 { + Repr::Exact(d) => decimal_to_f64(d), + Repr::Fraction(r) => ratio_to_f64(&BigInt::from(*r.numer()), &BigInt::from(*r.denom())), + Repr::Inexact(x) => *x, + } + } + + /// The exact value as a rational; `None` for an inexact number. + pub(crate) fn exact_parts(&self) -> Option<(i128, i128)> { + match &self.0 { + Repr::Exact(d) => Some((d.mantissa(), pow10(d.scale()))), + Repr::Fraction(r) => Some((*r.numer(), *r.denom())), + Repr::Inexact(_) => None, + } + } + + pub(crate) fn exact_q(&self) -> Option { + self.exact_parts().map(|(n, d)| BigQ::from_i128(n, d)) + } + + /// The stored bound: a decimal if it fits, else an `i128` fraction, else the nearest f64. + pub(crate) fn settle(q: &BigQ) -> Num { + Num::fit(q).unwrap_or_else(|| Num::inexact(q.to_f64())) + } + + /// `q` in an exact arm, at its minimal scale, if it fits one. + pub(crate) fn fit(q: &BigQ) -> Option { + decimal_of(q).or_else(|| { + let num = q.num.to_i128()?; + let den = q.den.to_i128()?; + Some(Num::fraction(num, den)) + }) + } +} + +/// `num / den` normalized without allocating; `None` hands the case to the bigint path. +pub(crate) fn small_ratio(num: i128, den: i128) -> Option { + let representable = num != i128::MIN && den != i128::MIN && den != 0; + representable.then(|| { + let g = num.gcd(&den); + let (n, d) = match den < 0 { + true => (-num / g, -den / g), + false => (num / g, den / g), + }; + let p = d.trailing_zeros(); + let mut rest = d >> p; + let mut q5 = 0; + while rest % 5 == 0 { + rest /= 5; + q5 += 1; + } + let k = p.max(q5); + let as_decimal = (rest == 1 && k <= MAX_SCALE) + .then(|| 2i128.pow(k - p) * 5i128.pow(k - q5)) + .and_then(|pad| n.checked_mul(pad)) + .and_then(|m| decimal(m, k)); + as_decimal.unwrap_or_else(|| Num::fraction(n, d)) + }) +} + +pub(crate) fn pow10(k: u32) -> i128 { + 10i128.pow(k) +} + +pub(crate) fn decimal_to_f64(d: &Decimal) -> f64 { + d.to_string().parse().unwrap_or(f64::NAN) +} + +/// A decimal from a mantissa and scale, if the mantissa fits 96 bits. +pub(crate) fn decimal(mantissa: i128, scale: u32) -> Option { + let fits = mantissa.unsigned_abs() < MANTISSA_LIMIT as u128 && scale <= MAX_SCALE; + fits.then(|| Num::exact(Decimal::from_i128_with_scale(mantissa, scale))) +} + +/// The direct test of clause 5: a reduced `n / (2^p 5^q)` is a decimal iff it fits. +fn decimal_of(q: &BigQ) -> Option { + let p = q.den.trailing_zeros().unwrap_or(0); + let mut rest = &q.den >> p as usize; + let five = BigInt::from(5); + let mut q5 = 0u64; + while rest.is_multiple_of(&five) && q5 <= MAX_SCALE as u64 { + rest /= &five; + q5 += 1; + } + let k = p.max(q5); + if !rest.is_one() || k > MAX_SCALE as u64 { + return None; + } + let pad = (BigInt::one() << (k - p) as usize) * five.pow((k - q5) as u32); + let mantissa = (&q.num * pad).to_i128()?; + decimal(mantissa, k as u32) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_num_is_24_bytes() { + assert_eq!(std::mem::size_of::(), 24); + } + + #[test] + fn a_fraction_that_is_a_fitting_decimal_is_stored_as_the_decimal() { + let half = Num::ratio(1, 2).unwrap(); + assert_eq!(half.kind(), NumRef::Exact(&Decimal::new(5, 1))); + let third = Num::ratio(-10, 3).unwrap(); + assert_eq!(third.kind(), NumRef::Fraction(&Ratio::new_raw(-10, 3))); + let tiny = Num::ratio(1, 1 << 40).unwrap(); + assert!(matches!(tiny.kind(), NumRef::Fraction(_)), "{tiny:?}"); + let mut minus_zero = Decimal::new(0, 2); + minus_zero.set_sign_negative(true); + assert!(minus_zero.is_sign_negative()); + let zero = Num::exact(minus_zero); + assert!(!zero.to_f64().is_sign_negative()); + assert!(Num::ratio(1, 0).is_none()); + } +} diff --git a/crates/numtower/src/ratio.rs b/crates/numtower/src/ratio.rs new file mode 100644 index 00000000..0c1a2d5c --- /dev/null +++ b/crates/numtower/src/ratio.rs @@ -0,0 +1,172 @@ +//! Exact rationals in bigints: the widened intermediate of question 13 (b), and +//! the one correctly rounded conversion of a rational to f64 (clauses 4 and 5). + +use std::cmp::Ordering; + +use num_bigint::{BigInt, BigUint, Sign}; +use num_integer::Integer; +use num_traits::{One, Signed, ToPrimitive, Zero}; + +/// A reduced fraction with a positive denominator. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct BigQ { + pub(crate) num: BigInt, + pub(crate) den: BigInt, +} + +impl BigQ { + pub(crate) fn new(num: BigInt, den: BigInt) -> BigQ { + let (num, den) = match den.is_negative() { + true => (-num, -den), + false => (num, den), + }; + let g = num.gcd(&den); + match g.is_zero() || g.is_one() { + true => BigQ { num, den }, + false => BigQ { + num: num / &g, + den: den / &g, + }, + } + } + + pub(crate) fn from_i128(num: i128, den: i128) -> BigQ { + BigQ::new(BigInt::from(num), BigInt::from(den)) + } + + pub(crate) fn add(&self, o: &BigQ) -> BigQ { + BigQ::new(&self.num * &o.den + &o.num * &self.den, &self.den * &o.den) + } + + pub(crate) fn neg(&self) -> BigQ { + BigQ { + num: -&self.num, + den: self.den.clone(), + } + } + + pub(crate) fn mul(&self, o: &BigQ) -> BigQ { + BigQ::new(&self.num * &o.num, &self.den * &o.den) + } + + /// `None` when `o` is zero. + pub(crate) fn div(&self, o: &BigQ) -> Option { + (!o.num.is_zero()).then(|| BigQ::new(&self.num * &o.den, &self.den * &o.num)) + } + + pub(crate) fn cmp_value(&self, o: &BigQ) -> Ordering { + (&self.num * &o.den).cmp(&(&o.num * &self.den)) + } + + pub(crate) fn bits(&self) -> u64 { + self.num.bits().max(self.den.bits()) + } + + pub(crate) fn to_f64(&self) -> f64 { + ratio_to_f64(&self.num, &self.den) + } + + /// The exact value of a finite f64. + pub(crate) fn from_f64(x: f64) -> BigQ { + let bits = x.to_bits(); + let biased = ((bits >> 52) & 0x7ff) as i64; + let fraction = bits & ((1u64 << 52) - 1); + let (mant, exp) = match biased { + 0 => (fraction, -1074), + e => (fraction | (1 << 52), e - 1075), + }; + let signed = match x.is_sign_negative() { + true => -BigInt::from(mant), + false => BigInt::from(mant), + }; + match exp >= 0 { + true => BigQ::new(signed << exp as usize, BigInt::one()), + false => BigQ::new(signed, BigInt::one() << (-exp) as usize), + } + } +} + +/// `num / den` rounded to the nearest f64, ties to even, subnormals and overflow included. +pub(crate) fn ratio_to_f64(num: &BigInt, den: &BigInt) -> f64 { + let negative = (num.sign() == Sign::Minus) != (den.sign() == Sign::Minus); + let a = num.magnitude(); + let b = den.magnitude(); + let magnitude = match a.is_zero() { + true => 0.0, + false => magnitude_to_f64(a, b), + }; + match negative { + true => -magnitude, + false => magnitude, + } +} + +fn magnitude_to_f64(a: &BigUint, b: &BigUint) -> f64 { + let shift = 54 - (a.bits() as i64 - b.bits() as i64); + let (n, d) = match shift >= 0 { + true => (a << shift as usize, b.clone()), + false => (a.clone(), b << (-shift) as usize), + }; + let (q, r) = n.div_rem(&d); + let qbits = q.bits() as i64; + let drop = (qbits - 53).max(shift - 1074); + if drop > qbits + 1 { + return 0.0; + } + let kept = (&q >> drop as usize).to_u64().unwrap_or(0); + let round = q.bit(drop as u64 - 1); + let sticky = !r.is_zero() || q.trailing_zeros().is_some_and(|t| (t as i64) < drop - 1); + let up = round && (sticky || kept & 1 == 1); + ldexp(kept + up as u64, drop - shift) +} + +/// `m * 2^e` for `m <= 2^53`, exact unless it overflows to infinity. +fn ldexp(m: u64, e: i64) -> f64 { + let top = e + 63 - m.leading_zeros() as i64; + match (m == 0, top > 1023) { + (true, _) => 0.0, + (false, true) => f64::INFINITY, + (false, false) => { + let half = e / 2; + m as f64 * pow2(half) * pow2(e - half) + } + } +} + +fn pow2(e: i64) -> f64 { + match e { + e if e >= -1022 => f64::from_bits(((e + 1023) as u64) << 52), + e => f64::from_bits(1u64 << (e + 1074)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rounding_to_f64_reaches_subnormals_ties_and_overflow() { + let one = BigInt::one(); + let min_sub = f64::from_bits(1); + assert_eq!(ratio_to_f64(&one, &(BigInt::one() << 1074)), min_sub); + assert_eq!(ratio_to_f64(&one, &(BigInt::one() << 1075)), 0.0); + let three = BigInt::from(3); + assert_eq!(ratio_to_f64(&three, &(BigInt::one() << 1076)), min_sub); + assert_eq!(ratio_to_f64(&one, &BigInt::from(3)), 1.0 / 3.0); + let tie = (BigInt::one() << 53) + 1; + assert_eq!(ratio_to_f64(&tie, &one), 9007199254740992.0); + let big = BigInt::one() << 1024; + assert_eq!(ratio_to_f64(&big, &one), f64::INFINITY); + assert_eq!(ratio_to_f64(&-big, &one), f64::NEG_INFINITY); + let max = BigQ::from_f64(f64::MAX); + assert_eq!(max.to_f64(), f64::MAX); + } + + #[test] + fn every_f64_converts_to_its_exact_value_and_back() { + for x in [0.1, -2.5e-310, 1e300, f64::MIN_POSITIVE, 123456.789, -7.0] { + assert_eq!(BigQ::from_f64(x).to_f64().to_bits(), x.to_bits(), "{x}"); + } + assert!(BigQ::from_f64(-0.0).num.is_zero()); + } +} diff --git a/crates/numtower/src/round.rs b/crates/numtower/src/round.rs new file mode 100644 index 00000000..c1a17633 --- /dev/null +++ b/crates/numtower/src/round.rs @@ -0,0 +1,107 @@ +//! Rounding any value to `places` for display, exactly (clause 12, "Format"). +//! An inexact value is rounded over its shortest round-trip spelling (improvise-felo.9). + +use num_bigint::BigInt; +use num_integer::Integer; +use num_traits::Signed; + +use crate::lang::{Lexeme, shortest_round_trip}; +use crate::num::Num; +use crate::ratio::BigQ; + +/// The rounding mode is the caller's policy, not the crate's. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Rounding { + HalfAwayFromZero, + HalfEven, +} + +impl Num { + /// Plain decimal text with exactly `places` fractional digits; `None` for a non-finite value. + pub fn round_to_places(&self, places: u8, mode: Rounding) -> Option { + let q = match self.exact_q() { + Some(q) => q, + None => Lexeme::read(&shortest_round_trip(self.to_f64())?)?.exact_value(), + }; + Some(render(round(&q, places, mode), places)) + } +} + +/// `q * 10^places`, rounded to an integer by `mode`. +fn round(q: &BigQ, places: u8, mode: Rounding) -> BigInt { + let scaled = q.num.abs() * BigInt::from(10).pow(places as u32); + let (whole, rest) = scaled.div_rem(&q.den); + let twice: BigInt = rest * 2; + let up = match twice.cmp(&q.den) { + std::cmp::Ordering::Greater => true, + std::cmp::Ordering::Equal => mode == Rounding::HalfAwayFromZero || whole.is_odd(), + std::cmp::Ordering::Less => false, + }; + let magnitude = whole + BigInt::from(up as u8); + match q.num.is_negative() { + true => -magnitude, + false => magnitude, + } +} + +fn render(units: BigInt, places: u8) -> String { + let digits = units.abs().to_string(); + let width = places as usize + 1; + let padded = format!("{digits:0>width$}"); + let (int, frac) = padded.split_at(padded.len() - places as usize); + let sign = if units.is_negative() { "-" } else { "" }; + match places { + 0 => format!("{sign}{int}"), + _ => format!("{sign}{int}.{frac}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lang::{Typing, parse_number}; + + fn typed(s: &str) -> Num { + parse_number(s, Typing::Inferred).unwrap() + } + + #[test] + fn a_typed_value_rounds_as_it_was_typed() { + let at2 = |n: &Num| n.round_to_places(2, Rounding::HalfAwayFromZero).unwrap(); + for (text, want) in [ + ("1.005", "1.01"), + ("1.015", "1.02"), + ("0.285", "0.29"), + ("-2.675", "-2.68"), + ] { + assert_eq!(at2(&typed(text)), want, "{text}"); + assert_eq!( + at2(&Num::inexact(text.parse().unwrap())), + want, + "inexact {text}" + ); + } + assert_eq!(at2(&Num::ratio(10, 3).unwrap()), "3.33"); + assert_eq!(at2(&Num::ratio(2, 3).unwrap()), "0.67"); + assert_eq!(at2(&typed("-0.001")), "0.00"); + let even = typed("0.125") + .round_to_places(2, Rounding::HalfEven) + .unwrap(); + assert_eq!(even, "0.12"); + assert_eq!( + typed("12.5") + .round_to_places(0, Rounding::HalfEven) + .unwrap(), + "12" + ); + assert_eq!( + Num::inexact(f64::NAN).round_to_places(2, Rounding::HalfEven), + None + ); + let wide = Num::ratio(1, 3) + .unwrap() + .round_to_places(40, Rounding::HalfEven) + .unwrap(); + assert_eq!(wide, format!("0.{}", "3".repeat(40))); + } +} diff --git a/crates/numtower/src/sum.rs b/crates/numtower/src/sum.rs new file mode 100644 index 00000000..366edbb5 --- /dev/null +++ b/crates/numtower/src/sum.rs @@ -0,0 +1,396 @@ +//! Order-free, retractable sums (clause 8): ADR 009's f64 `ExactSum`, and `NumSum`, +//! which adds the exact partitions. The design: `context/code/numtower.md`, "Sums". + +use std::collections::BTreeMap; + +use num_bigint::{BigInt, BigUint}; +use num_integer::Integer; +use num_traits::{One, Zero}; + +use crate::fixed::{I256, I320}; +use crate::num::{MAX_SCALE, Num, NumRef, pow10}; +use crate::ratio::{BigQ, ratio_to_f64}; + +/// The most non-decimal denominator groups a sum reads exactly (question 16: K = 8). +pub const K: usize = 8; + +const LIMB_BITS: u32 = 32; +/// A finite f64 is `mant * 2^(shift - 1074)`, `shift <= 2045`; weights add 60 bits. +const LIMBS: usize = 70; +const CHUNK: u64 = 1 << 30; + +/// An exact sum of weighted f64s, rounded once when read (ADR 009). +#[derive(Clone, Debug)] +pub struct ExactSum { + limbs: [i64; LIMBS], + pending: u64, + nan: i128, + pos_inf: i128, + neg_inf: i128, +} + +/// Equal sums, not equal representations: carries propagated, `pending` ignored. +impl PartialEq for ExactSum { + fn eq(&self, o: &ExactSum) -> bool { + let (mut mine, mut theirs) = (self.limbs, o.limbs); + propagate(&mut mine); + propagate(&mut theirs); + let counts = |s: &ExactSum| (s.nan, s.pos_inf, s.neg_inf); + mine == theirs && counts(self) == counts(o) + } +} + +impl Eq for ExactSum {} + +impl Default for ExactSum { + fn default() -> Self { + ExactSum { + limbs: [0; LIMBS], + pending: 0, + nan: 0, + pos_inf: 0, + neg_inf: 0, + } + } +} + +impl ExactSum { + pub fn new() -> ExactSum { + ExactSum::default() + } + + /// Adds `x` with a Z-set `weight`; the weight is split into base-2^30 digits. + pub fn add(&mut self, x: f64, weight: i64) { + match (x.is_nan(), x.is_infinite(), x > 0.0) { + (true, _, _) => self.nan += weight as i128, + (false, true, true) => self.pos_inf += weight as i128, + (false, true, false) => self.neg_inf += weight as i128, + (false, false, _) => { + let magnitude = weight.unsigned_abs(); + for j in 0..3 { + let digit = (magnitude >> (30 * j)) & (CHUNK - 1); + if digit != 0 { + let signed = if weight < 0 { + -(digit as i64) + } else { + digit as i64 + }; + self.add_digit(x, signed, 30 * j); + } + } + } + } + } + + fn add_digit(&mut self, x: f64, weight: i64, extra: u32) { + if self.pending + weight.unsigned_abs() > CHUNK { + propagate(&mut self.limbs); + self.pending = 0; + } + let bits = x.to_bits(); + let biased = ((bits >> 52) & 0x7ff) as u32; + let fraction = bits & ((1 << 52) - 1); + let (mant, shift) = match biased { + 0 => (fraction, 0), + e => (fraction | (1 << 52), e - 1), + }; + let sign = if x.is_sign_negative() { + -weight + } else { + weight + }; + let at = shift + extra; + let wide = u128::from(mant) << (at % LIMB_BITS); + let k = (at / LIMB_BITS) as usize; + for i in 0..3 { + let part = ((wide >> (LIMB_BITS * i as u32)) & 0xffff_ffff) as i64; + self.limbs[k + i] += sign * part; + } + self.pending += weight.unsigned_abs(); + } + + /// Adds another accumulator's contents. + pub fn absorb(&mut self, o: &ExactSum) { + let mut theirs = o.limbs; + propagate(&mut theirs); + propagate(&mut self.limbs); + for (mine, t) in self.limbs.iter_mut().zip(theirs) { + *mine += t; + } + propagate(&mut self.limbs); + self.pending = 0; + self.nan += o.nan; + self.pos_inf += o.pos_inf; + self.neg_inf += o.neg_inf; + } + + /// The exact sum rounded to the nearest f64, ties to even. + pub fn value(&self) -> f64 { + match (self.nan != 0, self.pos_inf != 0, self.neg_inf != 0) { + (true, _, _) | (false, true, true) => f64::NAN, + (false, true, false) => f64::INFINITY, + (false, false, true) => f64::NEG_INFINITY, + (false, false, false) => ratio_to_f64(&self.total(), &(BigInt::one() << 1074)), + } + } + + /// The finite part as an integer in units of 2^-1074. + fn total(&self) -> BigInt { + let mut limbs = self.limbs; + propagate(&mut limbs); + let digits: Vec = limbs[..LIMBS - 1].iter().map(|&l| l as u32).collect(); + let low = BigInt::from(BigUint::from_slice(&digits)); + (BigInt::from(limbs[LIMBS - 1]) << (LIMB_BITS as usize * (LIMBS - 1))) + low + } +} + +/// Carry each limb into the next, leaving every limb but the top in [0, 2^32). +fn propagate(limbs: &mut [i64; LIMBS]) { + for k in 0..LIMBS - 1 { + let carry = limbs[k] >> LIMB_BITS; + limbs[k] -= carry << LIMB_BITS; + limbs[k + 1] += carry; + } +} + +/// A `SUM` over any `Num`s: exact when every term is exact and at most K groups remain. +#[derive(Clone, Debug, Default)] +pub struct NumSum { + decimals: I320, + groups: BTreeMap, + group_floats: ExactSum, + inexact: ExactSum, + /// Net weight per f64 bit pattern: a read is inexact while any term is present. + inexact_terms: BTreeMap, +} + +impl NumSum { + pub fn new() -> NumSum { + NumSum::default() + } + + pub fn insert(&mut self, x: &Num) { + self.add(x, 1); + } + + pub fn retract(&mut self, x: &Num) { + self.add(x, -1); + } + + /// Adds `x` with a Z-set `weight`. Exact and associative: the order cannot matter. + pub fn add(&mut self, x: &Num, weight: i64) { + match x.kind() { + NumRef::Inexact(f) => { + self.inexact.add(f, weight); + let net = self.inexact_terms.entry(f.to_bits()).or_insert(0); + *net += weight as i128; + if *net == 0 { + self.inexact_terms.remove(&f.to_bits()); + } + } + NumRef::Exact(d) => { + let unit = pow10(MAX_SCALE - d.scale()) as u128; + self.add_scaled(d.mantissa(), unit, weight); + } + NumRef::Fraction(r) => match at_scale_28(*r.denom()) { + Some(unit) => self.add_scaled(*r.numer(), unit, weight), + None => self.add_to_group(*r.numer(), *r.denom(), weight), + }, + } + } + + fn add_scaled(&mut self, n: i128, unit: u128, weight: i64) { + let magnitude = weight.unsigned_abs() as u128; + let scaled = match unit.checked_mul(magnitude) { + Some(both) => I320::from_i128(n).checked_mul_u128(both), + None => I320::from_i128(n) + .checked_mul_u128(unit) + .and_then(|t| t.checked_mul_u128(magnitude)), + }; + let term = scaled + .and_then(|t| if weight < 0 { t.checked_neg() } else { Some(t) }) + .expect("a weighted term is below 2^284"); + self.decimals = self + .decimals + .checked_add(&term) + .expect("320 bits holds 2^36 terms"); + } + + fn add_to_group(&mut self, n: i128, d: i128, weight: i64) { + let old = self.groups.get(&d).copied().unwrap_or(I256::ZERO); + let new = old + .checked_add(&I256::mul_i128(n, weight as i128)) + .expect("256 bits holds 2^64 terms"); + let value = |num: I256| ratio_to_f64(&num.to_bigint(), &BigInt::from(d)); + if !old.is_zero() { + self.group_floats.add(value(old), -1); + } + match new.is_zero() { + true => { + self.groups.remove(&d); + } + false => { + self.group_floats.add(value(new), 1); + self.groups.insert(d, new); + } + } + } + + /// The current non-decimal denominator groups. + pub fn groups(&self) -> usize { + self.groups.len() + } + + /// One read: exact over at most K groups, else the f64 accumulators (clause 8). + pub fn read(&self) -> Num { + match (!self.inexact_terms.is_empty(), self.groups.len() > K) { + (false, false) => Num::settle(&self.exact_total()), + (_, true) => { + let mut acc = self.inexact.clone(); + acc.absorb(&self.group_floats); + acc.add(self.decimal_total().to_f64(), 1); + Num::inexact(acc.value()) + } + (true, false) => { + let mut acc = self.inexact.clone(); + acc.add(self.exact_total().to_f64(), 1); + Num::inexact(acc.value()) + } + } + } + + fn decimal_total(&self) -> BigQ { + BigQ::new(self.decimals.to_bigint(), BigInt::from(pow10(MAX_SCALE))) + } + + fn exact_total(&self) -> BigQ { + let decimal = (!self.decimals.is_zero()) + .then(|| (self.decimals.to_bigint(), BigInt::from(pow10(MAX_SCALE)))); + let terms: Vec<(BigInt, BigInt)> = decimal + .into_iter() + .chain( + self.groups + .iter() + .map(|(d, n)| (n.to_bigint(), BigInt::from(*d))), + ) + .collect(); + let (num, den) = combine_over_lcm(&terms); + BigQ::new(num, den) + } +} + +/// `Σ n/d` over the lcm of the denominators, not yet reduced: the read reduces once. +pub(crate) fn combine_over_lcm(terms: &[(BigInt, BigInt)]) -> (BigInt, BigInt) { + let lcm = terms.iter().fold(BigInt::one(), |l, (_, d)| l.lcm(d)); + let num = terms + .iter() + .fold(BigInt::zero(), |s, (n, d)| s + n * (&lcm / d)); + (num, lcm) +} + +/// `10^28 / d` when `d = 2^p 5^q` with `max(p, q) <= 28`: the term joins the decimal partition. +fn at_scale_28(d: i128) -> Option { + let p = d.trailing_zeros(); + let mut rest = d >> p; + let mut q = 0; + while rest % 5 == 0 { + rest /= 5; + q += 1; + } + (rest == 1 && p.max(q) <= MAX_SCALE).then(|| (pow10(MAX_SCALE) / d) as u128) +} + +#[cfg(test)] +mod tests { + use super::*; + use num_rational::BigRational; + use proptest::prelude::*; + use proptest::test_runner::{Config, RngAlgorithm, TestRng, TestRunner}; + + type Combine = fn(&[(BigInt, BigInt)]) -> (BigInt, BigInt); + + /// Up to K groups of wide numerators whose denominators share factors. + fn groups() -> impl Strategy> { + let shared = prop::sample::select(vec![1i128, 3, 7 * 11, 1 << 20, 3i128.pow(40)]); + let group = + (any::(), any::(), shared, 1i128..(1 << 60)).prop_map(|(n, w, s, d)| { + ( + BigInt::from(n) * BigInt::from(w), + BigInt::from(s) * BigInt::from(d), + ) + }); + prop::collection::vec(group, 1..=K) + } + + fn check_grouped_read(combine: Combine, deterministic: bool) -> Result<(), String> { + let config = Config { + cases: 256, + failure_persistence: None, + ..Config::default() + }; + let mut runner = match deterministic { + true => { + TestRunner::new_with_rng(config, TestRng::deterministic_rng(RngAlgorithm::ChaCha)) + } + false => TestRunner::new(config), + }; + let result = runner.run(&groups(), |terms| { + let (num, den) = combine(&terms); + let lcm = terms.iter().fold(BigInt::one(), |l, (_, d)| l.lcm(d)); + prop_assert_eq!(&den, &lcm, "the read did not combine over the lcm"); + let oracle: BigRational = terms + .iter() + .map(|(n, d)| BigRational::new(n.clone(), d.clone())) + .sum(); + let read = BigQ::new(num, den); + let got = BigRational::new(read.num.clone(), read.den.clone()); + prop_assert_eq!(got, oracle, "the read is not the sum"); + prop_assert!(read.num.gcd(&read.den).is_one(), "the read is not reduced"); + Ok(()) + }); + result.map_err(|e| e.to_string()) + } + + #[test] + fn the_grouped_read_combines_over_the_lcm_and_reduces_once() { + check_grouped_read(combine_over_lcm, false).unwrap(); + } + + fn over_the_product(terms: &[(BigInt, BigInt)]) -> (BigInt, BigInt) { + let den = terms.iter().fold(BigInt::one(), |p, (_, d)| p * d); + let num = terms + .iter() + .fold(BigInt::zero(), |s, (n, d)| s + n * (&den / d)); + (num, den) + } + + #[test] + fn the_grouped_read_combines_over_the_lcm_and_reduces_once_catches_the_product() { + let err = check_grouped_read(over_the_product, true).expect_err("the rival passed"); + assert!( + err.contains("the read did not combine over the lcm"), + "{err}" + ); + } + + #[test] + fn a_fraction_with_a_decimal_denominator_joins_the_decimal_partition() { + let mut s = NumSum::new(); + s.insert(&Num::ratio(10i128.pow(35), 1).unwrap()); + s.insert(&Num::ratio(3, 1 << 20).unwrap()); + assert_eq!(s.groups(), 0); + s.insert(&Num::ratio(1, 1i128 << 40).unwrap()); + s.insert(&Num::ratio(1, 3).unwrap()); + assert_eq!( + s.groups(), + 2, + "2^40 has scale 40, past 28: a group of its own" + ); + s.retract(&Num::ratio(1, 3).unwrap()); + assert_eq!( + s.groups(), + 1, + "a group whose numerator returns to zero is dropped" + ); + } +} diff --git a/crates/numtower/tests/oracle/mod.rs b/crates/numtower/tests/oracle/mod.rs new file mode 100644 index 00000000..6218812b --- /dev/null +++ b/crates/numtower/tests/oracle/mod.rs @@ -0,0 +1,189 @@ +//! The `BigRational` oracle and the generators the property tests share. +//! Each property is a checker taking the implementation, so a rival can be run through it. + +#![allow(dead_code)] + +use std::cmp::Ordering; + +use num_bigint::BigInt; +use num_rational::BigRational; +use num_traits::{One, Signed, ToPrimitive, Zero}; +use numtower::{Decimal, Num, NumRef}; +use proptest::prelude::*; +use proptest::test_runner::{Config, RngAlgorithm, TestCaseError, TestRng, TestRunner}; + +pub fn q(n: &Num) -> Option { + match n.kind() { + NumRef::Exact(d) => Some(BigRational::new( + d.mantissa().into(), + BigInt::from(10).pow(d.scale()), + )), + NumRef::Fraction(r) => Some(BigRational::new((*r.numer()).into(), (*r.denom()).into())), + NumRef::Inexact(x) => BigRational::from_float(x), + } +} + +/// Whether `v` fits a stored exact arm: a decimal at scale <= 28, or an `i128` fraction. +pub fn fits(v: &BigRational) -> bool { + let limit = BigInt::one() << 96; + let decimal = (0..=28u32).any(|s| { + let scaled = v * BigRational::from_integer(BigInt::from(10).pow(s)); + scaled.is_integer() && scaled.to_integer().abs() < limit + }); + let (min, max) = (BigInt::from(i128::MIN), BigInt::from(i128::MAX)); + decimal || (&min <= v.numer() && v.numer() <= &max && v.denom() <= &max) +} + +/// Whether `f` is the round-to-nearest-even f64 of `v`: the review's interval form. +pub fn is_nearest(f: f64, v: &BigRational) -> bool { + let exact = |x: f64| BigRational::from_float(x).expect("finite"); + let two = BigRational::from_integer(BigInt::from(2)); + let half_ulp_past = |x: f64, up: bool| { + let (next, inward) = match up { + true => (x.next_up(), x.next_down()), + false => (x.next_down(), x.next_up()), + }; + match next.is_finite() { + true => (exact(x) + exact(next)) / &two, + false => exact(x) + (exact(x) - exact(inward)) / &two, + } + }; + match f.is_finite() { + false => { + let edge = half_ulp_past(f64::MAX.copysign(f), f > 0.0); + (f > 0.0 && v >= &edge) || (f < 0.0 && v <= &edge) + } + true => { + let lo = half_ulp_past(f, false); + let hi = half_ulp_past(f, true); + let even = f.to_bits() & 1 == 0; + match even { + true => &lo <= v && v <= &hi, + false => &lo < v && v < &hi, + } + } + } +} + +/// A decimal's text built digit by digit, without `rust_decimal`'s `Display`. +pub fn decimal_text(mantissa: i128, scale: u32) -> String { + let digits = format!( + "{:0>width$}", + mantissa.unsigned_abs(), + width = scale as usize + 1 + ); + let (int, frac) = digits.split_at(digits.len() - scale as usize); + let sign = if mantissa < 0 { "-" } else { "" }; + format!("{sign}{int}.{frac}0") +} + +/// The correctly rounded f64 of an exact value, independent of `numtower`'s own. +pub fn independent_f64(n: &Num) -> f64 { + match n.kind() { + NumRef::Exact(d) => decimal_text(d.mantissa(), d.scale()).parse().unwrap(), + NumRef::Fraction(_) => q(n).unwrap().to_f64().unwrap(), + NumRef::Inexact(x) => x, + } +} + +/// Same arm, same value and same scale: what a fingerprint would see. +pub fn same(a: &Num, b: &Num) -> bool { + match (a.kind(), b.kind()) { + (NumRef::Exact(x), NumRef::Exact(y)) => { + x.mantissa() == y.mantissa() && x.scale() == y.scale() + } + (NumRef::Fraction(x), NumRef::Fraction(y)) => x == y, + (NumRef::Inexact(x), NumRef::Inexact(y)) => x.to_bits() == y.to_bits(), + _ => false, + } +} + +pub fn arm(n: &Num) -> &'static str { + match n.kind() { + NumRef::Exact(_) => "exact", + NumRef::Fraction(_) => "fraction", + NumRef::Inexact(_) => "inexact", + } +} + +pub fn mantissa96() -> impl Strategy { + let max = (1i128 << 96) - 1; + prop_oneof![ + -max..=max, + -1_000_000i128..1_000_000, + -(10i128.pow(15))..10i128.pow(15), + ] +} + +pub fn decimal() -> impl Strategy { + (mantissa96(), 0u32..=28).prop_map(|(m, s)| Num::exact(Decimal::from_i128_with_scale(m, s))) +} + +fn of_width(bits: u32) -> impl Strategy { + (any::(), any::()).prop_map(move |(r, neg)| { + let v = ((r >> (128 - bits)) | (1u128 << (bits - 1))) as i128; + if neg { -v } else { v } + }) +} + +pub fn fraction() -> impl Strategy { + let widths = (1u32..=126, 1u32..=126) + .prop_flat_map(|(a, b)| (of_width(a), of_width(b))) + .prop_map(|(n, d)| Num::ratio(n, d).unwrap()); + let edge = (1i128..i128::MAX).prop_map(|d| Num::ratio(i128::MIN, d | 1).unwrap()); + prop_oneof![8 => widths, 1 => edge] +} + +pub fn finite_f64() -> impl Strategy { + use proptest::num::f64::{NEGATIVE, NORMAL, POSITIVE, SUBNORMAL, ZERO}; + prop_oneof![ + (POSITIVE | NEGATIVE | NORMAL | SUBNORMAL | ZERO), + -1e6f64..1e6, + (-1000i64..1000).prop_map(|k| k as f64 / 8.0), + ] +} + +pub fn exact() -> impl Strategy { + prop_oneof![decimal(), fraction()] +} + +pub fn any_num() -> impl Strategy { + prop_oneof![ + 3 => exact(), + 1 => finite_f64().prop_map(Num::inexact), + 1 => prop_oneof![Just(f64::NAN), Just(f64::INFINITY), Just(f64::NEG_INFINITY)].prop_map(Num::inexact), + ] +} + +/// Runs a property; `deterministic` for a rival, so its failure is not luck. +pub fn run( + strategy: S, + cases: u32, + deterministic: bool, + test: impl Fn(S::Value) -> Result<(), TestCaseError>, +) -> Result<(), String> { + let config = Config { + cases, + failure_persistence: None, + ..Config::default() + }; + let mut runner = match deterministic { + true => TestRunner::new_with_rng(config, TestRng::deterministic_rng(RngAlgorithm::ChaCha)), + false => TestRunner::new(config), + }; + runner.run(&strategy, test).map_err(|e| e.to_string()) +} + +/// A rival must fail on the property's own assertion, never on something else. +pub fn assert_caught(result: Result<(), String>, marker: &str) { + let err = result.expect_err("the rival passed: the property cannot see it"); + assert!(err.contains(marker), "failed for another reason: {err}"); +} + +pub fn oracle_cmp(a: &Num, b: &Num) -> Option { + Some(q(a)?.cmp(&q(b)?)) +} + +pub fn integer(v: &BigRational) -> bool { + v.is_integer() && !v.numer().is_zero() +} diff --git a/crates/numtower/tests/properties.rs b/crates/numtower/tests/properties.rs new file mode 100644 index 00000000..5c5a8e5e --- /dev/null +++ b/crates/numtower/tests/properties.rs @@ -0,0 +1,1367 @@ +//! ADR 005 migration step 1's property tests, each beside the rival it must catch. +//! What each rival is and why it matters: `context/test-evidence/numtower.md`. + +mod oracle; + +use std::cmp::Ordering; +use std::str::FromStr; + +use num_bigint::BigInt; +use num_rational::BigRational; +use num_traits::{ToPrimitive, Zero}; +use numtower::{Decimal, Interim, Num, NumRef, NumSum, Typing, is_number, parse_number}; +use oracle::*; +use proptest::prelude::*; + +fn typed(s: &str) -> Num { + parse_number(s, Typing::Inferred).unwrap_or_else(|| panic!("{s} is a number")) +} + +fn dec(s: &str) -> Num { + Num::exact(Decimal::from_str(s).unwrap()) +} + +// --------------------------------------------------------------------------- +// An Exact result is never a rounded one. + +type BinOp<'a> = &'a dyn Fn(&Num, &Num) -> Option; + +fn exact_op(name: &str) -> fn(&BigRational, &BigRational) -> Option { + match name { + "+" => |a, b| Some(a + b), + "-" => |a, b| Some(a - b), + "*" => |a, b| Some(a * b), + _ => |a, b| (b.numer() != &BigInt::from(0)).then(|| a / b), + } +} + +fn check_never_rounded(name: &str, op: BinOp, deterministic: bool) -> Result<(), String> { + let oracle_op = exact_op(name); + run((exact(), exact()), 256, deterministic, |(a, b)| { + if let Some(r) = op(&a, &b).filter(Num::is_exact) { + let want = oracle_op(&q(&a).unwrap(), &q(&b).unwrap()); + prop_assert!( + q(&r) == want, + "an exact result was rounded: {a} {name} {b} = {r}" + ); + } + Ok(()) + }) +} + +fn real(name: &str) -> impl Fn(&Num, &Num) -> Option + '_ { + move |a: &Num, b: &Num| match name { + "+" => Some(a + b), + "-" => Some(a - b), + "*" => Some(a * b), + _ => a.checked_div(b).ok(), + } +} + +#[test] +fn an_exact_result_is_never_a_rounded_one() { + for name in ["+", "-", "*", "/"] { + check_never_rounded(name, &real(name), false).unwrap(); + } + let third = dec("0.3333333333333333333333333333"); + let sum = &dec("1000") + &third; + assert!(sum.is_exact(), "{sum}"); + assert_eq!(q(&sum), Some(q(&dec("1000")).unwrap() + q(&third).unwrap())); +} + +/// `rust_decimal`'s own `+`, trusted as exact: it rounds past 28 digits and says nothing. +fn silently_rounding_add(a: &Num, b: &Num) -> Option { + match (a.kind(), b.kind()) { + (NumRef::Exact(x), NumRef::Exact(y)) => { + Some(x.checked_add(*y).map_or_else(|| a + b, Num::exact)) + } + _ => Some(a + b), + } +} + +#[test] +fn an_exact_result_is_never_a_rounded_one_catches_a_silently_rounding_add() { + assert_caught( + check_never_rounded("+", &silently_rounding_add, true), + "an exact result was rounded", + ); +} + +// --------------------------------------------------------------------------- +// Contagion is total. + +fn check_contagion(add: BinOp, deterministic: bool) -> Result<(), String> { + let seed = Just((dec("2"), Num::inexact(3.0))); + let cases = prop_oneof![1 => seed, 8 => (any_num(), any_num())]; + run(cases, 256, deterministic, |(a, b)| { + let results = [ + ("+", add(&a, &b)), + ("-", Some(&a - &b)), + ("*", Some(&a * &b)), + ("/", a.checked_div(&b).ok()), + ]; + for (name, r) in results { + match (r, a.is_exact() && b.is_exact()) { + (None, _) => prop_assert!( + name == "/" && b.is_zero(), + "contagion: {a} {name} {b} refused" + ), + (Some(r), false) => { + prop_assert!(!r.is_exact(), "contagion: {a} {name} {b} = {r} is exact") + } + (Some(r), true) => { + let v = exact_op(name)(&q(&a).unwrap(), &q(&b).unwrap()).unwrap(); + prop_assert_eq!( + r.is_exact(), + fits(&v), + "contagion: {} {} {} = {}", + a, + name, + b, + r + ); + prop_assert!( + r.is_exact() || is_nearest(r.to_f64(), &v), + "contagion: {a} {name} {b} = {r}" + ); + } + } + } + prop_assert_eq!( + a.checked_div(&b).is_err(), + b.is_zero(), + "contagion: div/0 for {} / {}", + a, + b + ); + Ok(()) + }) +} + +#[test] +fn contagion_is_total() { + check_contagion(&|a, b| Some(a + b), false).unwrap(); +} + +/// Keeps an integral f64 exact, as a "harmless" promotion would. +fn integral_inexact_kept_exact(a: &Num, b: &Num) -> Option { + match (a.is_exact(), b.kind()) { + (true, NumRef::Inexact(x)) if x.is_finite() && x.fract() == 0.0 && x.abs() < 1e15 => { + Some(a + &Num::exact(Decimal::from(x as i64))) + } + _ => Some(a + b), + } +} + +#[test] +fn contagion_is_total_catches_an_integral_inexact_kept_exact() { + assert_caught( + check_contagion(&integral_inexact_kept_exact, true), + "contagion", + ); +} + +/// The scale a decimal power is stored at: minimal, padded toward `min(s·k, 28)` for k > 0 only. +fn power_scale(base: &Num, k: i32, v: &BigRational) -> u32 { + let scaled = |s: u32| v * BigRational::from_integer(BigInt::from(10).pow(s)); + let fits_at = + |s: u32| scaled(s).is_integer() && scaled(s).to_integer().magnitude().bits() <= 96; + let minimal = (0..=28) + .find(|&s| scaled(s).is_integer()) + .expect("a decimal result"); + let target = match (base.kind(), k > 0) { + (NumRef::Exact(d), true) => (d.scale() * k as u32).min(28), + _ => 0, + }; + (minimal..=target.max(minimal)) + .rev() + .find(|&s| fits_at(s)) + .unwrap_or(minimal) +} + +#[test] +fn pow_is_exact_for_an_exact_base_and_integer_exponent_when_it_fits() { + let units = + prop::sample::select(vec!["1.0", "-1.0", "1.000", "-1", "0.00", "2.0"]).prop_map(dec); + let bases = prop_oneof![4 => any_num(), 1 => units]; + run((bases, -6i32..=6), 256, false, |(a, k)| { + let r = a.pow(&Num::exact(Decimal::from(k))); + match (q(&a).filter(|_| a.is_exact()), r) { + (_, Err(_)) => prop_assert!(a.is_zero() && k < 0, "{a}^{k} refused"), + (None, Ok(r)) => prop_assert!(!r.is_exact(), "{a}^{k} = {r} is exact"), + (Some(v), Ok(r)) => { + let want = v.pow(k); + prop_assert_eq!(r.is_exact(), fits(&want), "{}^{} = {}", a, k, r); + prop_assert!( + !r.is_exact() || q(&r) == Some(want.clone()), + "{a}^{k} = {r}" + ); + prop_assert!( + r.is_exact() || is_nearest(r.to_f64(), &want), + "{a}^{k} = {r}" + ); + if let NumRef::Exact(d) = r.kind() { + let scale = power_scale(&a, k, &want); + prop_assert_eq!(d.scale(), scale, "power scale: {}^{} = {}", a, k, r); + } + } + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn a_negative_power_has_its_minimal_scale() { + let at = |base: &str, k: i64| dec(base).pow(&Num::exact(Decimal::from(k))).unwrap(); + assert!(same(&at("1.0", -2), &dec("1")), "{}", at("1.0", -2)); + assert!(same(&at("-1.0", -3), &dec("-1")), "{}", at("-1.0", -3)); + assert!(same(&at("2.0", -2), &dec("0.25"))); + assert!( + same(&at("-1.0", 3), &dec("-1.000")), + "a positive power keeps s·n" + ); +} + +#[test] +fn pow_decides_its_size_before_computing_and_honours_the_ceiling() { + let third = Num::ratio(1, 3).unwrap(); + let huge = third + .pow(&Num::exact(Decimal::from(1_000_000_000))) + .unwrap(); + assert!(!huge.is_exact() && huge.to_f64() == 0.0, "{huge}"); + let past = third.pow(&Num::exact(Decimal::from(6000))).unwrap(); + assert!( + !past.is_exact(), + "3^6000 has 9,510 bits, past the 8,192-bit ceiling" + ); + let growth = dec("1.05").pow(&Num::exact(Decimal::from(28))).unwrap(); + assert!(growth.is_exact(), "1.05^28 is exact under (d)"); + assert!( + !dec("1.05") + .pow(&Num::exact(Decimal::from(29))) + .unwrap() + .is_exact() + ); + let unit = dec("-1.0") + .pow(&Num::exact(Decimal::from_i128_with_scale( + 10i128.pow(20), + 0, + ))) + .unwrap(); + assert!( + same(&unit, &dec("1.0000000000000000000000000000")), + "{unit}" + ); + assert!(dec("0").pow(&dec("-1")).is_err()); + assert!(!dec("2").pow(&dec("0.5")).unwrap().is_exact()); + assert!(same(&dec("2").pow(&dec("-2")).unwrap(), &dec("0.25"))); + assert!(same(&dec("1.50").pow(&dec("2.0")).unwrap(), &dec("2.2500"))); + assert!(same(&dec("0").pow(&dec("0")).unwrap(), &dec("1"))); +} + +// --------------------------------------------------------------------------- +// The conversion to f64 agrees with str::parse of the decimal text. + +fn check_to_f64( + conv: &dyn Fn(&Decimal) -> f64, + range: i128, + deterministic: bool, +) -> Result<(), String> { + run((-range..=range, 0u32..=28), 256, deterministic, |(m, s)| { + let d = Decimal::from_i128_with_scale(m, s); + let want: f64 = decimal_text(m, s).parse().unwrap(); + prop_assert_eq!( + conv(&d).to_bits(), + want.to_bits(), + "to_f64 disagrees with the text of {}", + d + ); + Ok(()) + }) +} + +const FULL: i128 = (1 << 96) - 1; + +#[test] +fn the_f64_conversion_agrees_with_parsing_the_decimal_text() { + check_to_f64(&|d| Num::exact(*d).to_f64(), FULL, false).unwrap(); + let review = dec("63294.607176721429540940386554"); + assert_eq!( + review.to_f64(), + "63294.607176721429540940386554".parse::().unwrap() + ); +} + +fn crate_to_f64(d: &Decimal) -> f64 { + rust_decimal::prelude::ToPrimitive::to_f64(d).unwrap() +} + +#[test] +fn the_f64_conversion_agrees_with_parsing_the_decimal_text_catches_rust_decimal_to_f64() { + assert_caught(check_to_f64(&crate_to_f64, FULL, true), "to_f64 disagrees"); + let review = Decimal::from_str("63294.607176721429540940386554").unwrap(); + assert_ne!( + crate_to_f64(&review), + "63294.607176721429540940386554".parse::().unwrap() + ); +} + +#[test] +fn rust_decimal_to_f64_agrees_on_fifteen_digit_mantissas_so_a_narrow_generator_is_vacuous() { + let narrow = 10i128.pow(15) - 1; + let result = run((-narrow..=narrow, 0u32..=15), 256, true, |(m, s)| { + let d = Decimal::from_i128_with_scale(m, s); + prop_assert_eq!( + crate_to_f64(&d).to_bits(), + decimal_text(m, s).parse::().unwrap().to_bits() + ); + Ok(()) + }); + result.unwrap(); +} + +// --------------------------------------------------------------------------- +// A fraction's conversion lies inside the round-to-nearest-even interval. + +fn wide_fraction() -> impl Strategy { + ( + 20u32..=126, + 20u32..=126, + any::(), + any::(), + any::(), + ) + .prop_map(|(a, b, x, y, neg)| { + let n = ((x >> (128 - a)) | (1 << (a - 1))) as i128; + let d = ((y >> (128 - b)) | (1 << (b - 1))) as i128; + Num::ratio(if neg { -n } else { n }, d).unwrap() + }) +} + +fn check_fraction_conversion( + conv: &dyn Fn(&Num) -> f64, + deterministic: bool, +) -> Result<(), String> { + run( + (wide_fraction(), wide_fraction()), + 256, + deterministic, + |(x, y)| { + prop_assert!( + is_nearest(conv(&x), &q(&x).unwrap()), + "outside the interval: {x}" + ); + let wide = (&Interim::from(x.clone()) * &Interim::from(y.clone())).store(); + let exact = q(&x).unwrap() * q(&y).unwrap(); + prop_assert!( + wide.is_exact() || is_nearest(wide.to_f64(), &exact), + "outside the interval: {x} * {y}" + ); + Ok(()) + }, + ) +} + +#[test] +fn a_fractions_conversion_lies_inside_the_round_to_nearest_even_interval() { + check_fraction_conversion(&Num::to_f64, false).unwrap(); +} + +fn divide_in_f64(x: &Num) -> f64 { + match x.kind() { + NumRef::Fraction(r) => *r.numer() as f64 / *r.denom() as f64, + _ => x.to_f64(), + } +} + +#[test] +fn a_fractions_conversion_lies_inside_the_round_to_nearest_even_interval_catches_an_f64_division() { + assert_caught( + check_fraction_conversion(÷_in_f64, true), + "outside the interval", + ); +} + +// --------------------------------------------------------------------------- +// Exact mixed comparison is transitive. + +/// Values crowded around one f64: equal-looking decimals, the f64 and its neighbours. +fn crowd() -> impl Strategy> { + (-1e6f64..1e6, 1i64..9).prop_map(|(x, k)| { + let shortest = typed(&format!("{x:?}")); + let nudge = Num::exact(Decimal::new(k, 25)); + vec![ + &shortest + &nudge, + Num::inexact(x), + &shortest - &nudge, + shortest.clone(), + Num::inexact(x.next_up()), + Num::inexact(x.next_down()), + shortest.checked_div(&Num::exact(Decimal::from(3))).unwrap(), + ] + }) +} + +type Cmp<'a> = &'a dyn Fn(&Num, &Num) -> Option; + +fn check_transitive(cmp: Cmp, deterministic: bool) -> Result<(), String> { + run(crowd(), 256, deterministic, |xs| { + for a in &xs { + for b in &xs { + let reversed = cmp(b, a).map(Ordering::reverse); + prop_assert_eq!( + cmp(a, b), + reversed, + "comparison: not antisymmetric: {} vs {}", + a, + b + ); + for c in &xs { + let (ab, bc, ac) = (cmp(a, b), cmp(b, c), cmp(a, c)); + let chained = ab.is_some_and(|o| o != Ordering::Greater) + && bc.is_some_and(|o| o != Ordering::Greater); + let strict = ab == Some(Ordering::Less) || bc == Some(Ordering::Less); + let want = if strict { + Ordering::Less + } else { + Ordering::Equal + }; + prop_assert!( + !chained || ac == Some(want), + "comparison: not transitive: {a}, {b}, {c}" + ); + } + } + } + Ok(()) + }) +} + +#[test] +fn exact_mixed_comparison_is_transitive() { + check_transitive(&Num::cmp_exact, false).unwrap(); + assert_ne!(dec("0.1"), Num::inexact(0.1)); +} + +#[test] +fn comparison_agrees_with_the_oracle_at_every_pairing_of_arms() { + let finite = prop_oneof![exact(), finite_f64().prop_map(Num::inexact)]; + let pools = prop_oneof![crowd(), prop::collection::vec(finite, 2..6)]; + run(pools, 256, false, |xs| { + for a in &xs { + for b in &xs { + prop_assert_eq!(a.cmp_exact(b), oracle_cmp(a, b), "{} vs {}", a, b); + } + } + Ok(()) + }) + .unwrap(); + assert_eq!(Num::inexact(f64::NAN).cmp_exact(&dec("1")), None); + assert_eq!( + dec("1").cmp_exact(&Num::inexact(f64::INFINITY)), + Some(Ordering::Less) + ); +} + +/// Revision 1's rule: the exact operand converted to f64, then compared. +fn compare_through_f64(a: &Num, b: &Num) -> Option { + match a.is_exact() == b.is_exact() { + true => a.cmp_exact(b), + false => a.to_f64().partial_cmp(&b.to_f64()), + } +} + +#[test] +fn exact_mixed_comparison_is_transitive_catches_comparing_through_f64() { + assert_caught( + check_transitive(&compare_through_f64, true), + "comparison: not transitive", + ); +} + +#[test] +fn min_and_max_prefer_the_exact_value_and_do_not_depend_on_order() { + let half = dec("0.50"); + let pairs = [ + (half.clone(), Num::inexact(0.5)), + (dec("1.5"), dec("1.50")), + (Num::inexact(0.0), Num::inexact(-0.0)), + (Num::inexact(f64::NAN), dec("2")), + ]; + for (a, b) in &pairs { + assert!(same(a.min_of(b), b.min_of(a)), "min of {a}, {b}"); + assert!(same(a.max_of(b), b.max_of(a)), "max of {a}, {b}"); + } + assert!(same(half.min_of(&Num::inexact(0.5)), &half)); + assert!(same(dec("1.5").max_of(&dec("1.50")), &dec("1.50"))); + assert!( + Num::inexact(0.0) + .min_of(&Num::inexact(-0.0)) + .to_f64() + .is_sign_negative() + ); + assert!( + !Num::inexact(-0.0) + .max_of(&Num::inexact(0.0)) + .to_f64() + .is_sign_negative() + ); + assert!(same(Num::inexact(f64::NAN).min_of(&dec("2")), &dec("2"))); +} + +#[test] +fn the_accumulator_keeps_inexact_terms_whose_weights_cancel() { + let mut s = NumSum::new(); + s.add(&Num::inexact(1.0), 1); + s.add(&Num::inexact(2.0), -1); + s.insert(&dec("0.5")); + let read = s.read(); + assert!(!read.is_exact(), "an inexact term is still present: {read}"); + assert_eq!(read.to_f64(), -0.5); + s.add(&Num::inexact(1.0), -1); + s.add(&Num::inexact(2.0), 1); + assert!(same(&s.read(), &dec("0.5")), "every inexact term retracted"); +} + +#[test] +fn a_huge_odd_exponent_keeps_the_sign_of_a_negative_base() { + let odd = Num::exact(Decimal::from(9_007_199_254_740_993i64)); + let up = dec("-2").pow(&odd).unwrap(); + assert!(!up.is_exact() && up.to_f64() == f64::NEG_INFINITY, "{up}"); + let down = dec("-2").pow(&-&odd).unwrap(); + assert!( + !down.is_exact() && down.to_f64() == 0.0 && down.to_f64().is_sign_negative(), + "{down}" + ); + let third = Num::ratio(-1, 3).unwrap().pow(&odd).unwrap(); + assert!(third.to_f64().is_sign_negative(), "{third}"); + let even = &odd + &dec("1"); + assert_eq!(dec("-2").pow(&even).unwrap().to_f64(), f64::INFINITY); +} + +#[test] +fn i128_min_is_an_exact_fraction_numerator() { + let min = Num::ratio(i128::MIN, 1).unwrap(); + assert!( + matches!(min.kind(), NumRef::Fraction(r) if *r.numer() == i128::MIN), + "{min}" + ); + let third = Num::ratio(i128::MIN, 3).unwrap(); + assert!(matches!(third.kind(), NumRef::Fraction(_)), "{third}"); + let negated = -&third; + let want = -q(&third).unwrap(); + assert!( + !negated.is_exact() && is_nearest(negated.to_f64(), &want), + "{negated}" + ); + let back = + &(&Interim::from(third.clone()) * &Interim::from(dec("-1"))) * &Interim::from(dec("-1")); + assert!( + same(&back.store(), &third), + "through the wide path and back" + ); + let halved = third.checked_div(&dec("2")).unwrap(); + assert_eq!( + q(&halved), + Some(q(&third).unwrap() / BigRational::from_integer(BigInt::from(2))) + ); + assert_eq!( + min.cmp_exact(&Num::ratio(i128::MIN + 1, 1).unwrap()), + Some(Ordering::Less) + ); + assert_eq!( + Num::from_canonical(&min.to_string()).map(|n| same(&n, &min)), + Some(true) + ); +} + +// --------------------------------------------------------------------------- +// The accumulator is independent of order. + +fn term() -> impl Strategy { + let pooled = ( + -1000i128..1000, + prop::sample::select(vec![3i128, 7, 12, 49, 1 << 40, 3 * (1 << 33)]), + ) + .prop_map(|(n, d)| Num::ratio(n, d).unwrap()); + prop_oneof![ + 4 => decimal(), + 2 => pooled, + 1 => fraction(), + 1 => finite_f64().prop_map(Num::inexact), + ] +} + +/// Z-set weights: zero, unit, small, wide and both extremes (never `i64::MIN`, so it negates). +fn weight() -> impl Strategy { + prop_oneof![ + 3 => Just(1i64), + 1 => Just(-1i64), + 1 => Just(0i64), + 2 => -3i64..=3, + 2 => (-(1i64 << 40))..(1i64 << 40), + 1 => prop_oneof![Just(i64::MAX), Just(-i64::MAX), Just((1i64 << 30) - 1)], + ] +} + +/// Many non-decimal denominators, so a list often holds more than K groups, several terms each. +const POOL: [i128; 14] = [ + 3, + 7, + 11, + 12, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 49, + 1 << 40, + 3 * (1 << 33), +]; + +fn pooled_term() -> impl Strategy { + let pooled = (-1000i128..1000, prop::sample::select(POOL.to_vec())) + .prop_map(|(n, d)| Num::ratio(n, d).unwrap()); + prop_oneof![ + 3 => decimal(), + 4 => pooled, + 1 => fraction(), + 2 => finite_f64().prop_map(Num::inexact), + ] +} + +fn weighted(terms: impl Strategy) -> impl Strategy { + (terms, weight()) +} + +fn sum_all(xs: &[Num]) -> Num { + let mut s = NumSum::new(); + xs.iter().for_each(|x| s.insert(x)); + s.read() +} + +fn sum_weighted(xs: &[(Num, i64)]) -> Num { + let mut s = NumSum::new(); + xs.iter().for_each(|(x, w)| s.add(x, *w)); + s.read() +} + +type WeightedSum<'a> = &'a dyn Fn(&[(Num, i64)]) -> Num; + +fn check_order_free(sum: WeightedSum, deterministic: bool) -> Result<(), String> { + let seed = vec![ + (dec("0.30000000000000004"), 1), + (dec("1000000000000"), 1), + (dec("-1000000000000"), 1), + ]; + let cases = prop_oneof![ + 1 => Just(seed), + 4 => prop::collection::vec(weighted(term()), 0..24), + 2 => prop::collection::vec(weighted(pooled_term()), 0..40), + ]; + let shuffled = cases.prop_flat_map(|xs| (Just(xs.clone()), Just(xs).prop_shuffle())); + run(shuffled, 256, deterministic, |(xs, ys)| { + let (a, b) = (sum(&xs), sum(&ys)); + prop_assert!(same(&a, &b), "depends on order: {a} against {b}"); + Ok(()) + }) +} + +#[test] +fn the_accumulator_is_independent_of_order() { + check_order_free(&sum_weighted, false).unwrap(); + let seed = sum_all(&[ + dec("0.30000000000000004"), + dec("1000000000000"), + dec("-1000000000000"), + ]); + assert!(same(&seed, &dec("0.30000000000000004")), "{seed}"); +} + +/// Clause 8's rival: `Decimal`'s own `+`, demoting to f64 where it overflows. +fn decimal_fold(xs: &[(Num, i64)]) -> Num { + xs.iter().fold(Num::exact(Decimal::ZERO), |acc, (x, w)| { + let x = x * &Num::exact(Decimal::from(*w)); + match (acc.kind(), x.kind()) { + (NumRef::Exact(a), NumRef::Exact(b)) => { + a.checked_add(*b).map_or_else(|| &acc + &x, Num::exact) + } + _ => &acc + &x, + } + }) +} + +#[test] +fn the_accumulator_is_independent_of_order_catches_decimal_addition() { + assert_caught(check_order_free(&decimal_fold, true), "depends on order"); +} + +#[test] +fn the_accumulator_retracts_exactly() { + let terms = || prop::collection::vec(weighted(pooled_term()), 0..20); + run((terms(), terms()), 256, false, |(xs, ys)| { + let mut s = NumSum::new(); + xs.iter().chain(&ys).for_each(|(x, w)| s.add(x, *w)); + ys.iter().rev().for_each(|(y, w)| s.add(y, -w)); + let fresh = sum_weighted(&xs); + prop_assert!( + same(&s.read(), &fresh), + "{} after retraction, {} fresh", + s.read(), + fresh + ); + Ok(()) + }) + .unwrap(); +} + +/// The oracle's partition of a weighted list, exactly as clause 8 and option (d) state it. +struct Expected { + decimal_part: BigRational, + groups: std::collections::BTreeMap, + inexact: std::collections::BTreeMap, + inexact_total: BigRational, +} + +fn ratio_of(i: i128) -> BigRational { + BigRational::from_integer(BigInt::from(i)) +} + +fn rounded(v: &BigRational) -> BigRational { + BigRational::from_float(v.to_f64().unwrap()).unwrap() +} + +impl Expected { + fn of(xs: &[(Num, i64)]) -> Expected { + let zero = || ratio_of(0); + let mut e = Expected { + decimal_part: zero(), + groups: Default::default(), + inexact: Default::default(), + inexact_total: zero(), + }; + for (x, w) in xs { + let weighted = q(x).unwrap() * ratio_of(*w as i128); + match x.kind() { + NumRef::Inexact(f) => { + *e.inexact.entry(f.to_bits()).or_insert(0) += *w as i128; + e.inexact_total += weighted; + } + NumRef::Fraction(r) if !decimal_denominator(*r.denom()) => { + *e.groups.entry(*r.denom()).or_insert_with(zero) += weighted; + } + _ => e.decimal_part += weighted, + } + } + e + } + + fn live(&self) -> impl Iterator { + self.groups.values().filter(|v| !v.is_zero()) + } +} + +#[test] +fn the_accumulator_reads_the_exact_sum_or_its_stated_roundings() { + let arms = std::cell::Cell::new([0u32; 3]); + let lists = prop_oneof![ + prop::collection::vec(weighted(term()), 0..24), + prop::collection::vec(weighted(pooled_term()), 0..40), + ]; + run(lists, 256, false, |xs| { + let got = sum_weighted(&xs); + let e = Expected::of(&xs); + let present = e.inexact.values().any(|w| *w != 0); + let exact_total = e.live().fold(e.decimal_part.clone(), |s, g| s + g); + let mut seen = arms.get(); + match (present, e.live().count() <= numtower::K) { + (false, true) => { + seen[0] += 1; + prop_assert_eq!(got.is_exact(), fits(&exact_total), "{}", got); + prop_assert!( + !got.is_exact() || q(&got) == Some(exact_total.clone()), + "{got}" + ); + prop_assert!( + got.is_exact() || is_nearest(got.to_f64(), &exact_total), + "{got}" + ); + } + (true, true) => { + seen[1] += 1; + let total = rounded(&exact_total) + &e.inexact_total; + prop_assert!(!got.is_exact() && is_nearest(got.to_f64(), &total), "{got}"); + } + (_, false) => { + seen[2] += 1; + let groups = e.live().map(rounded).fold(ratio_of(0), |s, g| s + g); + let total = groups + rounded(&e.decimal_part) + &e.inexact_total; + let near = is_nearest(got.to_f64(), &total); + prop_assert!(!got.is_exact() && near, "past K groups: {got} for {total}"); + } + } + arms.set(seen); + Ok(()) + }) + .unwrap(); + let seen = arms.get(); + assert!( + seen.iter().all(|n| *n >= 20), + "every branch must be reached often: {seen:?}" + ); +} + +fn decimal_denominator(d: i128) -> bool { + let p = d.trailing_zeros(); + let mut rest = d >> p; + let mut q5 = 0; + while rest % 5 == 0 { + rest /= 5; + q5 += 1; + } + rest == 1 && p.max(q5) <= 28 +} + +#[test] +fn more_than_k_denominators_read_inexact_even_when_the_total_fits() { + let primes = [3i128, 7, 11, 13, 17, 19, 23, 29, 31]; + let terms: Vec = primes.iter().map(|&p| Num::ratio(1, p).unwrap()).collect(); + let eight = sum_all(&terms[..8]); + assert!(matches!(eight.kind(), NumRef::Fraction(_)), "{eight}"); + let nine = sum_all(&terms); + assert!(!nine.is_exact(), "{nine}"); + let exact: BigRational = terms.iter().map(|t| q(t).unwrap()).sum(); + assert!( + fits(&exact), + "the exact total fits, so the cutoff is what demotes" + ); + let each_rounded: BigRational = terms + .iter() + .map(|t| BigRational::from_float(q(t).unwrap().to_f64().unwrap()).unwrap()) + .sum(); + assert!( + is_nearest(nine.to_f64(), &each_rounded), + "each group rounded, then the sum once" + ); +} + +#[test] +fn exact_sum_is_the_exact_weighted_total_rounded_once() { + let weights = prop_oneof![-3i64..=3, any::(), (-(1i64 << 40))..(1i64 << 40)]; + run( + prop::collection::vec((finite_f64(), weights), 0..20), + 256, + false, + |terms| { + let mut acc = numtower::ExactSum::new(); + let mut exact = BigRational::from_integer(BigInt::from(0)); + for (x, w) in &terms { + acc.add(*x, *w); + exact += BigRational::from_float(*x).unwrap() + * BigRational::from_integer(BigInt::from(*w)); + } + prop_assert!( + is_nearest(acc.value(), &exact), + "{} for {:?}", + acc.value(), + terms + ); + Ok(()) + }, + ) + .unwrap(); +} + +// --------------------------------------------------------------------------- +// Division's direct fit test. + +fn check_division(div: BinOp, deterministic: bool) -> Result<(), String> { + let seeds = prop_oneof![ + Just((dec("1"), dec("1099511627776"))), + Just((dec("1"), dec("931322574615478515625"))), + Just((dec("0.0000000000000000000000000001"), dec("0.5"))), + ]; + let cases = prop_oneof![1 => seeds, 3 => (decimal(), decimal())]; + run(cases, 256, deterministic, |(a, b)| { + if let (Some(r), Some(v)) = (div(&a, &b), exact_op("/")(&q(&a).unwrap(), &q(&b).unwrap())) { + prop_assert_eq!( + r.is_exact(), + fits(&v), + "division fit: {} / {} = {}", + a, + b, + r + ); + prop_assert!( + !r.is_exact() || q(&r) == Some(v.clone()), + "division fit: {a} / {b} = {r}" + ); + let decimal = (0..=28u32).any(|s| { + let scaled = &v * BigRational::from_integer(BigInt::from(10).pow(s)); + scaled.is_integer() && scaled.to_integer().magnitude().bits() <= 96 + }); + prop_assert_eq!( + matches!(r.kind(), NumRef::Exact(_)), + decimal, + "division fit: {} / {} = {}", + a, + b, + r + ); + } + Ok(()) + }) +} + +#[test] +fn division_decides_fit_directly() { + check_division(&|a, b| a.checked_div(b).ok(), false).unwrap(); + let frac = |n: &Num| matches!(n.kind(), NumRef::Fraction(_)); + assert!( + frac(&dec("1").checked_div(&dec("1099511627776")).unwrap()), + "1/2^40" + ); + assert!( + frac(&dec("1").checked_div(&dec("931322574615478515625")).unwrap()), + "1/5^30" + ); + let tiny = dec("0.0000000000000000000000000001") + .checked_div(&dec("0.5")) + .unwrap(); + assert!( + same(&tiny, &dec("0.0000000000000000000000000002")), + "{tiny}" + ); + let avg = &(&(&dec("10.00") + &dec("20.00")) + &dec("40.00")); + assert!(frac(&avg.checked_div(&dec("3")).unwrap())); + assert!(same( + &dec("1").checked_div(&dec("4")).unwrap(), + &dec("0.25") + )); + let third = Num::ratio(1, 3).unwrap(); + assert!( + same(&(&third * &dec("3")), &dec("1")), + "1/3 * 3 is exactly 1" + ); + assert_eq!( + Num::inexact(1.0).checked_div(&Num::inexact(-0.0)), + Err(numtower::DivisionByZero) + ); +} + +/// Multiplying back, with revision 6's scale test: sound, and incomplete. +fn multiply_back(a: &Num, b: &Num) -> Option { + match (a.kind(), b.kind()) { + (NumRef::Exact(x), NumRef::Exact(y)) if !y.is_zero() => { + let candidate = x.checked_div(*y)?; + let back = candidate.checked_mul(*y)?; + let kept = back.scale() == candidate.scale() + y.scale() && back == *x; + Some(match kept { + true => Num::exact(candidate), + false => Num::inexact(a.to_f64() / b.to_f64()), + }) + } + _ => a.checked_div(b).ok(), + } +} + +#[test] +fn division_decides_fit_directly_catches_multiplying_back() { + assert_caught(check_division(&multiply_back, true), "division fit"); +} + +// --------------------------------------------------------------------------- +// The number language: the gate and the parse agree. + +fn lexeme() -> impl Strategy { + let digits = "[0-9]{0,40}"; + let structured = ( + prop::option::of("[+-]"), + digits, + prop::option::of(("\\.", digits)), + prop::option::of(("[eE]", prop::option::of("[+-]"), "[0-9]{0,4}")), + ) + .prop_map(|(sign, int, frac, exp)| { + let frac = frac.map(|(p, f)| format!("{p}{f}")).unwrap_or_default(); + let exp = exp + .map(|(e, s, d)| format!("{e}{}{d}", s.unwrap_or_default())) + .unwrap_or_default(); + format!("{}{int}{frac}{exp}", sign.unwrap_or_default()) + }); + prop_oneof![ + 4 => structured, + 1 => "[0-9eE+\\-._/aifnIN ]{0,10}", + 1 => "[+-]?[0-9]{1,30}/[0-9]{1,30}", + 1 => prop::sample::select(vec!["inf", "-inf", "nan", "NaN", "Infinity", "-infinity", "1_000", "+.5", "1.", ".", "e5"]).prop_map(String::from), + ] +} + +fn check_language( + parse: &dyn Fn(&str, Typing) -> Option, + deterministic: bool, +) -> Result<(), String> { + run(lexeme(), 256, deterministic, |s| { + let gate = s.parse::().is_ok_and(f64::is_finite); + let parsed = parse(&s, Typing::Inferred); + prop_assert_eq!( + parsed.is_some(), + gate, + "the gate and the parse disagree on {:?}", + s + ); + prop_assert_eq!( + is_number(&s, Typing::Inferred), + gate, + "the gate and the parse disagree on {:?}", + s + ); + if let Some(n) = &parsed { + let nearest: f64 = s.parse().unwrap(); + prop_assert!( + n.to_f64() == nearest, + "the gate and the parse disagree on the value of {s:?}: {n}" + ); + let v = q(n).unwrap(); + prop_assert!( + !n.is_exact() || is_nearest(nearest, &v), + "the gate and the parse disagree on {s:?}" + ); + } + let forced = parse(&s, Typing::Forced); + prop_assert_eq!( + forced.is_some(), + is_number(&s, Typing::Forced), + "the gate and the parse disagree on forced {:?}", + s + ); + Ok(()) + }) +} + +#[test] +fn the_number_languages_gate_and_parse_agree() { + check_language(&parse_number, false).unwrap(); +} + +/// `rust_decimal`'s exact parse: it refuses `1.5e3` and accepts `1_000`. +fn from_str_exact(s: &str, typing: Typing) -> Option { + match Decimal::from_str_exact(s) { + Ok(d) => Some(Num::exact(d)), + Err(_) if typing == Typing::Forced => parse_number(s, typing), + Err(_) => None, + } +} + +#[test] +fn the_number_languages_gate_and_parse_agree_catches_from_str_exact() { + assert_caught( + check_language(&from_str_exact, true), + "the gate and the parse disagree", + ); +} + +#[test] +fn the_number_language_seed_cases() { + let inferred = |s: &str| parse_number(s, Typing::Inferred); + let forced = |s: &str| parse_number(s, Typing::Forced); + assert!(same(&typed("1.5e3"), &dec("1500"))); + assert!(same(&typed("1.50"), &dec("1.50"))); + assert_eq!(inferred("1_000").map(|n| n.to_string()), None); + let zero = typed("-0.0"); + assert!(same(&zero, &dec("0.0")) && !zero.to_f64().is_sign_negative()); + assert!(inferred("3/4").is_none()); + assert!(same(&forced("3/4").unwrap(), &dec("0.75"))); + assert!(matches!( + forced("-10/3").unwrap().kind(), + NumRef::Fraction(_) + )); + assert!(forced("3/0").is_none()); + for text in ["1e999", "-1e999", "nan", "inf", "Infinity", "NaN", "-inf"] { + assert!( + inferred(text).is_none() && forced(text).is_none(), + "{text} is text" + ); + } + let underflow = typed("1e-999"); + assert!(!underflow.is_exact() && underflow.to_f64() == 0.0); + assert_eq!(typed("1e-30"), Num::ratio(1, 10i128.pow(30)).unwrap()); + assert!(matches!(typed("1e-30").kind(), NumRef::Fraction(_))); + let long = typed("0.12345678901234567890123456789"); + let want = BigRational::new( + BigInt::from(12345678901234567890123456789i128), + BigInt::from(10).pow(29), + ); + assert!(matches!(long.kind(), NumRef::Fraction(_)) && q(&long) == Some(want)); + let big = (1i128 << 120).to_string(); + let n = typed(&big); + assert!(matches!(n.kind(), NumRef::Fraction(r) if *r.denom() == 1 && *r.numer() == 1 << 120)); + assert!(!typed("1e-40").is_exact()); + assert!(!typed("1234567890123456789012345678901234567890").is_exact()); + assert!(same(&typed("0e99999999999999999999"), &dec("0"))); +} + +// --------------------------------------------------------------------------- +// A typed value and the same value computed are equally exact. + +fn small_decimal() -> impl Strategy { + let mantissa = prop_oneof![-1000i128..1000, mantissa96()]; + (mantissa, 0u32..=28).prop_map(|(m, s)| Num::exact(Decimal::from_i128_with_scale(m, s))) +} + +type Mul<'a> = &'a dyn Fn(&Num, &Num) -> Num; + +fn check_typed_equals_computed(mul: Mul, deterministic: bool) -> Result<(), String> { + let seed = Just((dec("0.000000000000001"), dec("0.000000000000001"))); + run( + prop_oneof![1 => seed, 3 => (small_decimal(), small_decimal())], + 256, + deterministic, + |(a, b)| { + let v = q(&a).unwrap() * q(&b).unwrap(); + let ten = |k: u32| BigRational::from_integer(BigInt::from(10).pow(k)); + let k = (0..=60u32) + .find(|&k| (&v * ten(k)).is_integer()) + .expect("a decimal product"); + let text = format!("{}e-{k}", (&v * ten(k)).to_integer()); + let parsed = typed(&text); + let computed = mul(&a, &b); + prop_assert_eq!( + arm(&parsed), + arm(&computed), + "typed and computed differ: {} against {} * {}", + text, + a, + b + ); + prop_assert!( + parsed.cmp_exact(&computed) == Some(Ordering::Equal), + "typed and computed differ: {text} against {computed}" + ); + Ok(()) + }, + ) +} + +#[test] +fn a_typed_value_and_the_same_value_computed_are_equally_exact() { + check_typed_equals_computed(&|a, b| a * b, false).unwrap(); + let computed = &typed("1e-15") * &typed("1e-15"); + assert!(same(&typed("1e-30"), &computed), "{computed}"); +} + +/// Revision 6's rule: demote whenever the returned scale drops. +fn demote_on_the_hint(a: &Num, b: &Num) -> Num { + match (a.kind(), b.kind()) { + (NumRef::Exact(x), NumRef::Exact(y)) => match x.checked_mul(*y) { + Some(r) if r.scale() == x.scale() + y.scale() => Num::exact(r), + _ => Num::inexact(a.to_f64() * b.to_f64()), + }, + _ => a * b, + } +} + +#[test] +fn a_typed_value_and_the_same_value_computed_are_equally_exact_catches_demoting_on_the_hint() { + assert_caught( + check_typed_equals_computed(&demote_on_the_hint, true), + "typed and computed differ", + ); +} + +// --------------------------------------------------------------------------- +// Clause 4's stored scale, question 13 (b)'s widening, and the canonical text. + +#[test] +fn a_recomputed_result_keeps_the_stored_scale_clause_4_names() { + let product = &dec("1.2500000000000000") * &dec("4.0000000000000000"); + assert!( + same(&product, &dec("5.0000000000000000000000000000")), + "{product}" + ); + let common = &dec("12.50") + &dec("1.5"); + assert!(same(&common, &dec("14.00")), "{common}"); + let rounded = &dec("1000") + &dec("0.3333333333333333333333333333"); + assert!(matches!(rounded.kind(), NumRef::Fraction(_)), "{rounded}"); + assert!(same(&(&dec("-1.5") + &dec("1.5")), &dec("0.0"))); +} + +#[test] +fn an_intermediate_beyond_i128_comes_back_exact() { + let a = Num::ratio(10i128.pow(30), 7).unwrap(); + let wide = &Interim::from(a.clone()) * &Interim::from(a.clone()); + assert!( + wide.is_exact(), + "an intermediate is unbounded up to the ceiling" + ); + let back = wide.checked_div(&Interim::from(a.clone())).unwrap().store(); + assert!(same(&back, &a), "{back}"); + let stored = (&a * &a).to_f64(); + assert!(!(&a * &a).is_exact() && is_nearest(stored, &(q(&a).unwrap() * q(&a).unwrap()))); +} + +#[test] +fn the_canonical_text_reads_back_as_the_same_number() { + run(any_num(), 256, false, |n| { + let text = n.to_string(); + let back = Num::from_canonical(&text); + prop_assert!( + back.as_ref() + .is_some_and(|b| same(b, &n) || (n.to_f64().is_nan() && b.to_f64().is_nan())), + "{text}" + ); + Ok(()) + }) + .unwrap(); + assert_eq!(dec("12.50").to_string(), "12.50"); + assert_eq!(Num::ratio(-10, 3).unwrap().to_string(), "-10/3"); + assert_eq!(Num::inexact(1.0 / 3.0).to_string(), "#i0.3333333333333333"); + assert_eq!(Num::inexact(f64::NEG_INFINITY).to_string(), "-inf"); + assert!(Num::from_canonical("#inan").is_none()); + assert!( + Num::from_canonical("1e-999").is_none(), + "canonical exact text never reads inexact" + ); + let ten35 = Num::ratio(10i128.pow(35), 1).unwrap(); + assert_eq!(ten35.to_string(), format!("{}/1", 10i128.pow(35))); +} + +#[test] +fn exact_sums_are_equal_when_their_values_are() { + let of = |terms: &[(f64, i64)]| { + let mut s = numtower::ExactSum::new(); + terms.iter().for_each(|(x, w)| s.add(*x, *w)); + s + }; + assert_eq!( + of(&[(0.5, 1), (0.5, 1)]), + of(&[(1.0, 1)]), + "0.5 + 0.5 = 1.0" + ); + assert_eq!( + of(&[(3.25, 1), (3.25, -1)]), + numtower::ExactSum::new(), + "insert then retract" + ); + let big = (1i64 << 30) - 1; + assert_eq!( + of(&[(1.5, big), (0.25, 2)]), + of(&[(0.25, 2), (1.5, big)]), + "order" + ); + assert_ne!(of(&[(1.0, 1)]), of(&[(1.0, 2)])); + assert_ne!( + of(&[(f64::NAN, 1)]), + of(&[]), + "a counted NaN is part of the value" + ); +} + +#[test] +fn a_forced_fraction_never_reads_as_infinity() { + let huge = format!("{}/1", "9".repeat(400)); + assert_eq!( + parse_number(&huge, Typing::Forced).map(|n| n.to_string()), + None + ); + assert!( + !is_number(&huge, Typing::Forced), + "the gate and the parse agree" + ); + let negative = format!("-{huge}"); + assert!(parse_number(&negative, Typing::Forced).is_none()); + let tiny = format!("1/{}", "9".repeat(400)); + let zero = parse_number(&tiny, Typing::Forced).unwrap(); + assert!( + !zero.is_exact() && zero.to_f64() == 0.0, + "an underflow is a finite zero, as 1e-999" + ); + assert!(is_number(&tiny, Typing::Forced)); +} + +type InterimCmp<'a> = &'a dyn Fn(&Interim, &Interim) -> Option; + +fn check_interim_order(cmp: InterimCmp, deterministic: bool) -> Result<(), String> { + let eps = Interim::from(Num::ratio(1, 3 * 10i128.pow(30)).unwrap()); + run( + ( + wide_fraction(), + wide_fraction(), + wide_fraction(), + wide_fraction(), + ), + 256, + deterministic, + |(x1, x2, y1, y2)| { + let a = &Interim::from(x1.clone()) * &Interim::from(x2.clone()); + let av = q(&x1).unwrap() * q(&x2).unwrap(); + let near = av.to_f64().unwrap(); + let eps_v = + BigRational::new(BigInt::from(1), BigInt::from(3) * BigInt::from(10).pow(30)); + let others = [ + ( + &Interim::from(y1.clone()) * &Interim::from(y2.clone()), + q(&y1).unwrap() * q(&y2).unwrap(), + ), + (&a + &eps, &av + &eps_v), + (&a - &eps, &av - &eps_v), + (a.clone(), av.clone()), + ( + Interim::from(Num::inexact(near)), + BigRational::from_float(near).unwrap(), + ), + ( + Interim::from(Num::inexact(near.next_up())), + BigRational::from_float(near.next_up()).unwrap(), + ), + ]; + for (b, bv) in &others { + prop_assert_eq!( + cmp(&a, b), + Some(av.cmp(bv)), + "interim order: {:?} vs {:?}", + a, + b + ); + prop_assert_eq!( + cmp(b, &a), + Some(bv.cmp(&av)), + "interim order: {:?} vs {:?}", + b, + a + ); + } + let twin = a.clone(); + prop_assert!( + !a.is_zero() && (&a - &twin).is_zero(), + "interim zero: {a:?}" + ); + Ok(()) + }, + ) +} + +#[test] +fn an_interim_compares_exactly_without_storing() { + check_interim_order(&Interim::cmp_exact, false).unwrap(); + assert_eq!( + Interim::from(Num::inexact(f64::NAN)).cmp_exact(&Interim::from(dec("1"))), + None + ); +} + +/// Storing first: what `=` would do if `Interim` could not compare. +fn store_then_compare(a: &Interim, b: &Interim) -> Option { + a.clone().store().cmp_exact(&b.clone().store()) +} + +#[test] +fn an_interim_compares_exactly_without_storing_catches_storing_first() { + assert_caught( + check_interim_order(&store_then_compare, true), + "interim order", + ); +} diff --git a/nix/ci.nix b/nix/ci.nix index ceb4f384..f8a56de9 100644 --- a/nix/ci.nix +++ b/nix/ci.nix @@ -46,6 +46,8 @@ "improvise-wasm-client" "improvise-worker-server" "improvise-gp" + # ADR 005 clause 14: nothing depends on it yet, and improvise-core will. + "numtower" ]; pFlags = crates: lib.concatMapStringsSep " " (c: "-p ${c}") crates; diff --git a/scripts/build-doc-site.mjs b/scripts/build-doc-site.mjs index 1cf88745..f8489795 100644 --- a/scripts/build-doc-site.mjs +++ b/scripts/build-doc-site.mjs @@ -34,6 +34,8 @@ const SECTIONS = [ ["docs/design-notes.md", "design-notes", "Design notes"], ["context/design-principles.md", "design-principles", "Design principles"], ["context/repo-map.md", "repo-map", "Repository map"], + ["context/gotchas.md", "gotchas", "Gotchas (long form)"], + ["context/file-inventory.md", "file-inventory", "File inventory (long form)"], ], }, { @@ -48,6 +50,7 @@ const SECTIONS = [ ["context/adr/005-exact-decimal-arithmetic.md", "adr/005-exact-decimal-arithmetic", "ADR 005 — Exact decimal arithmetic (proposed)"], ["context/adr/007-stable-identities.md", "adr/007-stable-identities", "ADR 007 — Stable identities"], ["context/adr/007-stable-identities-amendment-1.md", "adr/007-stable-identities-amendment-1", "ADR 007, amendment 1"], + ["context/adr/009-incremental-evaluation.md", "adr/009-incremental-evaluation", "ADR 009 — Incremental evaluation (proposed)"], ], }, { @@ -64,6 +67,7 @@ const SECTIONS = [ ["context/code/tui.md", "code/tui", "The TUI"], ["context/code/cli.md", "code/cli", "The headless CLI"], ["context/code/gp.md", "code/gp", "gp-tree and improvise-gp"], + ["context/code/numtower.md", "code/numtower", "numtower, the numeric tower"], ["context/code/kb-tool.md", "code/kb-tool", "The kb tool"], ["context/code/tooling.md", "code/tooling", "The repository's own checkers"], ], @@ -82,6 +86,7 @@ const SECTIONS = [ ["context/test-evidence/tui.md", "test-evidence/tui", "The TUI"], ["context/test-evidence/cli.md", "test-evidence/cli", "The headless CLI"], ["context/test-evidence/gp.md", "test-evidence/gp", "gp-tree and improvise-gp"], + ["context/test-evidence/numtower.md", "test-evidence/numtower", "numtower"], ["context/test-evidence/tooling.md", "test-evidence/tooling", "The repository's own gates"], ["context/test-evidence/layout-cost.md", "test-evidence/layout-cost", "Layout and keypress cost"], ["context/test-evidence/scale-bisect.md", "test-evidence/scale-bisect", "Scale bisect"], @@ -611,7 +616,7 @@ for (const section of SECTIONS) { for (const [src, slug, title] of section.docs) { const depth = slug.split("/").length - 1; const ctx = { depth, cited, unresolved, anchors: new Set() }; - if (slug === "repo-map") ctx.listItemId = (n) => (n >= 1 && n <= 200 ? `gotcha-${n}` : null); + if (slug === "repo-map" || slug === "gotchas") ctx.listItemId = (n) => (n >= 1 && n <= 200 ? `gotcha-${n}` : null); const md = readFileSync(join(root, src), "utf8"); const { html, toc } = renderMarkdown(md, ctx); diff --git a/scripts/repo-stats b/scripts/repo-stats index 6444d2a8..f96ab557 100755 --- a/scripts/repo-stats +++ b/scripts/repo-stats @@ -189,12 +189,9 @@ def read_rows(path): def gotchas(text): - """(number, lines, chars) for each numbered item of repo-map's Gotchas section.""" - out, cur, inside = [], None, False + """(number, lines, chars) for each numbered item of context/gotchas.md.""" + out, cur, inside = [], None, True for line in text.split("\n"): - if line.startswith("## "): - inside = line.startswith("## Gotchas") - continue if inside and line.startswith("---"): inside = False if not inside: @@ -214,13 +211,13 @@ def doc_sizes(repo, sha): "context").split("\n") if p.endswith(".md")] show = lambda p: git(repo, "show", f"{sha}:{p}") lines = {p: len(show(p).splitlines()) for p in sorted(files)} - gs = gotchas(show("context/repo-map.md")) if "context/repo-map.md" in files else [] + gs = gotchas(show("context/gotchas.md")) if "context/gotchas.md" in files else [] longest = max(gs, key=lambda g: g[2], default=None) evidence = {p: sum(1 for l in show(p).split("\n") if re.match(r"^#{3,4} ", l)) for p in sorted(files) if p.startswith("context/test-evidence/")} return { "context_lines": lines, - "repo_map": {"gotchas": len(gs), "longest": longest and + "gotchas": {"count": len(gs), "longest": longest and {"number": longest[0], "lines": longest[1], "chars": longest[2]}}, "test_evidence": {"entries": sum(evidence.values()), "per_file": evidence}, } @@ -270,9 +267,9 @@ def text(r, top): out += ["", TITLES[t]] + table(r["tables"][t]) out += ["", "Doc sizes (lines)"] out += [f" {n:>6} {path}" for path, n in r["docs"]["context_lines"].items()] - rm, te = r["docs"]["repo_map"], r["docs"]["test_evidence"] + rm, te = r["docs"]["gotchas"], r["docs"]["test_evidence"] lg = rm["longest"] - out.append(f" repo-map gotchas: {rm['gotchas']}" + ( + out.append(f" gotchas: {rm['count']}" + ( f"; longest is {lg['number']} ({lg['lines']} lines, {lg['chars']} chars)" if lg else "")) out.append(f" test-evidence entries: {te['entries']}") diff --git a/scripts/repo-stats-test.sh b/scripts/repo-stats-test.sh index 2e138dbd..65fbe5e2 100755 --- a/scripts/repo-stats-test.sh +++ b/scripts/repo-stats-test.sh @@ -76,10 +76,9 @@ put context/code/x.md 8 x3 mkdir -p "$repo/context/test-evidence" printf '### t_one\ntext\n#### t_two\n## not an entry\n### t_three\n' \ > "$repo/context/test-evidence/y.md" -cat > "$repo/context/repo-map.md" <<'EOF' -# Map - -## Gotchas (read before editing) +printf '# Map\n' > "$repo/context/repo-map.md" +cat > "$repo/context/gotchas.md" <<'EOF' +# Gotchas 1. **One.** short. 2. **Two.** a much longer gotcha, with plenty of words in it to be the longest one @@ -136,25 +135,25 @@ if run "$work/stats" text; then row "docs/*.md is other docs" by-area "other docs" '["2","0","0.000"]' row "context/code" by-area "context/code" '["8","0","0.000"]' row "context/test-evidence" by-area "context/test-evidence" '["5","0","0.000"]' - row "repo-map is its own area" by-area "repo-map" '["13","0","0.000"]' - row "other context" by-area "other context" '["2","0","0.000"]' + row "repo-map is its own area" by-area "repo-map" '["1","0","0.000"]' + row "other context" by-area "other context" '["13","0","0.000"]' row "kb.pl" by-area "kb.pl" '["5","0","0.000"]' row "scripts/CI takes .github, and a rename is a delete and an add" \ by-area "scripts/CI" '["15","7","0.467"]' row "web and harness, and its deletion ratio" by-area "web and harness" '["14","9","0.643"]' row "a lock file, .beads/ and site/ are generated" by-area "generated" '["14","0","0.000"]' row "the pre-window README counts nowhere" by-area "other" '["4","0","0.000"]' - row "prose is the context areas, other docs and agent docs" by-kind prose '["43","6","0.140"]' + row "prose is the context areas, other docs and agent docs" by-kind prose '["42","6","0.143"]' row "code is rust, scripts/CI and web" by-kind code '["82","45","0.549"]' - row "prose to code is prose over code" by-kind ProseToCode '["0.524","0.133",""]' + row "prose to code is prose over code" by-kind ProseToCode '["0.512","0.133",""]' is "weeks are columns, first-seen first" '.tables["area-by-week"][0][1:]' '["2026-W24","2026-W23"]' row "an area is split by ISO week" area-by-week "web and harness" '["3","11"]' is "every day's churn.csv total equals code-maat's abs-churn" \ '[.tables["by-day"][1:][]|.[1]==.[2]]|all' 'true' - row "a day's total is every file changed that day" by-day 2026-06-04 '["72","72"]' - is "gotchas end with their section" '.docs.repo_map.gotchas' '3' + row "a day's total is every file changed that day" by-day 2026-06-04 '["71","71"]' + is "gotchas end with their section" '.docs.gotchas.count' '3' is "the longest gotcha is measured over its lines" \ - '.docs.repo_map.longest|[.number,.lines]' '[2,2]' + '.docs.gotchas.longest|[.number,.lines]' '[2,2]' is "test-evidence entries are ### and #### headings" '.docs.test_evidence.entries' '3' is "coupling finds the 15 pairs of six files, and web/app.js with scripts/foo.sh" \ '.coupling|length' '16' diff --git a/src/ui/category_panel.rs b/src/ui/category_panel.rs index 70530163..cc892467 100644 --- a/src/ui/category_panel.rs +++ b/src/ui/category_panel.rs @@ -94,8 +94,8 @@ impl PanelContent for CategoryContent { ); } } - CatTreeEntry::Item { item_name, .. } => { - let label = format!(" · {item_name}"); + CatTreeEntry::Item { display, .. } => { + let label = format!(" · {display}"); buf.set_string(inner.x, y, &label, base_style); } } @@ -105,7 +105,7 @@ impl PanelContent for CategoryContent { #[cfg(test)] mod tests { use super::*; - use crate::protocol::CategoryRow; + use crate::protocol::{CategoryItem, CategoryRow}; use crate::ui::app::AppMode; use crate::ui::panel::Panel; use ratatui::{layout::Rect, widgets::Widget}; @@ -117,7 +117,7 @@ mod tests { name: name.to_string(), display: name.to_string(), axis: Axis::Row, - items: vec!["x".to_string()], + items: vec![CategoryItem::plain("x")], }], }; let area = Rect::new(0, 0, 30, 4); @@ -138,7 +138,7 @@ mod tests { name: "Region".to_string(), display: "Sales Area".to_string(), axis: Axis::Row, - items: vec!["x".to_string()], + items: vec![CategoryItem::plain("x")], }], }; let area = Rect::new(0, 0, 30, 4); @@ -154,6 +154,29 @@ mod tests { assert!(!row.contains("Region"), "{row}"); } + #[test] + fn an_item_row_draws_its_display_not_its_key() { + let cache = CategoryTreeCache { + categories: vec![CategoryRow { + name: "Region".to_string(), + display: "Region".to_string(), + axis: Axis::Row, + items: vec![CategoryItem { + name: "East".to_string(), + display: "Eastern Seaboard".to_string(), + }], + }], + }; + let expanded = std::collections::HashSet::from(["Region".to_string()]); + let area = Rect::new(0, 0, 30, 5); + let mut buf = Buffer::empty(area); + Panel::new(CategoryContent::new(&cache, &expanded), &AppMode::Normal, 0) + .render(area, &mut buf); + let text: String = buf.content().iter().map(|c| c.symbol()).collect(); + assert!(text.contains("Eastern Seaboard"), "{text}"); + assert!(!text.contains("East "), "the key is not drawn: {text}"); + } + /// improvise-8l0c — `context/test-evidence/tui.md`. #[test] fn the_axis_tag_sits_beside_a_multibyte_category_name() { diff --git a/src/ui/grid.rs b/src/ui/grid.rs index b4ccfbdc..9e03a96a 100644 --- a/src/ui/grid.rs +++ b/src/ui/grid.rs @@ -154,7 +154,7 @@ impl<'a> GridWidget<'a> { false => run_heads.get(ci).copied().flatten(), }; let label = match head { - Some(g) => format!("{} {}", indicator(g.collapsed), g.name), + Some(g) => format!("{} {}", indicator(g.collapsed), g.display), None => String::new(), }; buf.set_string( @@ -238,7 +238,7 @@ impl<'a> GridWidget<'a> { // consumed (improvise-gd83). FrameEntry::GroupEnd { .. } => continue, FrameEntry::Group(group) => { - let label = format!("{} {}", indicator(group.collapsed), group.name); + let label = format!("{} {}", indicator(group.collapsed), group.display); buf.set_string( area.x, y, @@ -454,7 +454,7 @@ impl<'a> Widget for GridWidget<'a> { .cache .page_coords .iter() - .map(|p| format!("{} = {}", p.cat, p.selected)) + .map(|p| format!("{} = {}", p.cat_display, p.selected_display)) .collect(); let page_str = format!(" [{}] ", page_info.join(" | ")); buf.set_string( @@ -884,6 +884,27 @@ mod tests { assert_eq!(text, GROUPED_FRAME, "\n--- got ---\n{text}\n--- end ---"); } + #[test] + fn the_frame_and_page_bar_draw_displays_not_keys() { + let mut m = grouped_model(); + let region = m.model.category_mut("Region").unwrap(); + region.set_item_display("East", "Eastward"); + region.set_group_display("Coastal", "Bay"); + let month = m.model.category_mut("Month").unwrap(); + month.set_item_display("Jan", "Janv"); + month.set_group_display("Q1", "Qa"); + let payer = m.model.category_mut("Payer").unwrap(); + payer.set_display("Who"); + payer.set_item_display("Alice", "Alicia"); + let text = buf_text(&render(&mut m, 60, 16)); + for shown in ["Who = Alicia", "Eastward", "▼ Bay", "Janv", "▼ Qa"] { + assert!(text.contains(shown), "{shown:?} missing:\n{text}"); + } + for key in ["Payer", "Alice]", "East ", "Coastal", "Jan ", "Q1"] { + assert!(!text.contains(key), "the key {key:?} was drawn:\n{text}"); + } + } + /// Re-pinned for improvise-0o6x: the leading `▼ Co…` is the ONE line that /// moved, and its absence was the bug. context/test-evidence/tui.md. const GROUPED_FRAME: &str = concat!( diff --git a/src/ui/view_panel.rs b/src/ui/view_panel.rs index c2081c49..bc961f62 100644 --- a/src/ui/view_panel.rs +++ b/src/ui/view_panel.rs @@ -99,7 +99,7 @@ mod tests { views: vec![ViewSummary { name: view_name.to_string(), display: view_name.to_string(), - axes: vec![("Region".to_string(), Axis::Row)], + axes: vec![crate::protocol::ViewAxis::plain("Region", Axis::Row)], }], }; let area = Rect::new(0, 0, 30, 4);