diff --git a/Cargo.toml b/Cargo.toml index 8da5e69f..1a5467a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,11 @@ name = "reshape_observation" harness = false test = false +[[bench]] +name = "scale_bisect" +harness = false +test = false + [dependencies] gp-tree = { path = "crates/gp-tree" } improvise-core = { path = "crates/improvise-core" } diff --git a/bench/edge.py b/bench/edge.py new file mode 100644 index 00000000..f5e199e5 --- /dev/null +++ b/bench/edge.py @@ -0,0 +1,8 @@ +import sys; sys.path.insert(0,'bench') +from scale_bisect import run +cfgs=[dict(k=k,prune=0) for k in (16,17,18,26,27)]+[dict(k=k,prune=1) for k in (7,11,12,13,14,21,22)]+[dict(k=1000,n=n,mode='records') for n in (1000,1200,1500,1800,2000,4600,14500)] +for c in cfgs: + c.setdefault('n',100000); c['ops']='open,key,window' + v=[run('/tmp/sb',c,timeout=200) for _ in range(3)] + f=lambda k:'/'.join(f"{r[k][1]:.1f}" if k in r else '-' for r in v) + print({k:c[k] for k in c if k!='ops'},'rows',v[0].get('rows'),'open',f('open'),'key',f('keypress_down'),'win',f('browser_grid_of_top'),'tuiframe',f('tui_frame_grid_of_whole'),flush=True) diff --git a/bench/full.py b/bench/full.py new file mode 100644 index 00000000..27d79581 --- /dev/null +++ b/bench/full.py @@ -0,0 +1,10 @@ +import sys, json; sys.path.insert(0,'bench') +from scale_bisect import run +cfgs=[dict(k=9,prune=0),dict(k=10,prune=0),dict(k=17,prune=0,groups=1),dict(k=26,prune=0,groups=1), + dict(k=8,prune=1),dict(k=13,prune=1,groups=1), + dict(k=1000,n=1300,mode='records'),dict(k=1000,n=4400,mode='records'),dict(k=1000,n=14000,mode='records')] +for c in cfgs: + c.setdefault('n',100000) + for rep in range(3): + r=run('/tmp/sb',c,timeout=300) + print(json.dumps({'cfg':c,'rep':rep,**r}),flush=True) diff --git a/bench/scale_bisect.py b/bench/scale_bisect.py new file mode 100644 index 00000000..9d576cfa --- /dev/null +++ b/bench/scale_bisect.py @@ -0,0 +1,38 @@ +"""Drive benches/scale_bisect one configuration per process; parse its lines. +usage: scale_bisect.py BIN k=.. n=.. [cats=..] [prune=..] [mode=..] [ops=..] [--timeout S] +""" +import subprocess, sys, re, json + +def run(binary, cfg, timeout=90): + args = [binary] + [f"{k}={v}" for k, v in cfg.items()] + try: + p = subprocess.run(args, capture_output=True, text=True, timeout=timeout) + out, status = p.stdout, "ok" + except subprocess.TimeoutExpired as e: + out, status = (e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or ""), "timeout" + res = {"status": status} + for line in out.splitlines(): + if line.startswith("REFUSED"): + res["status"] = "refused" + m = re.match(r"(\w+) min=([\d.]+) med=([\d.]+) max=([\d.]+)", line) + if m: + res[m[1]] = (float(m[2]), float(m[3]), float(m[4])) + for kv in re.findall(r"(\w+)=([\d.]+)", line): + if kv[0] in ("rows", "cols", "maxrss_mb", "improv_mb", "snapshot_kb", "rss_after_open_mb", "cells"): + res[kv[0]] = float(kv[1]) + m = re.match(r"json_encode_setcell_updates ms=(.*)", line) + if m: + res["json_setcell"] = m[1] + return res + +if __name__ == "__main__": + binary = sys.argv[1] + cfg, timeout = {}, 90 + rest = sys.argv[2:] + for a in rest: + if a.startswith("--timeout"): + timeout = int(a.split("=")[1]) + else: + k, v = a.split("=", 1) + cfg[k] = v + print(json.dumps(run(binary, cfg, timeout))) diff --git a/bench/sweep.py b/bench/sweep.py new file mode 100644 index 00000000..f0c166d6 --- /dev/null +++ b/bench/sweep.py @@ -0,0 +1,10 @@ +import sys; sys.path.insert(0,'bench') +from scale_bisect import run +import itertools +OPS=sys.argv[1]; +cfgs=eval(sys.argv[2]) +cols=sys.argv[3].split(',') +for c in cfgs: + c=dict(c); c['ops']=OPS + r=run('/tmp/sb',c,timeout=int(sys.argv[4]) if len(sys.argv)>4 else 90) + print(c, r['status'], 'rows',r.get('rows'), *[f"{k}={r[k][1]:.1f}" if k in r else f"{k}=-" for k in cols], f"rss={r.get('maxrss_mb')}", flush=True) diff --git a/benches/scale_bisect.rs b/benches/scale_bisect.rs new file mode 100644 index 00000000..9b559514 --- /dev/null +++ b/benches/scale_bisect.rs @@ -0,0 +1,266 @@ +//! One configuration per process: build a sparse document, time the real paths. +//! `scale_bisect k=1000 n=100000 cats=3 prune=1 mode=grid`; see context/test-evidence/scale-bisect.md. + +use std::collections::{HashMap, HashSet}; +use std::io::Write; +use std::time::Instant; + +use improvise::model::cell::{CellKey, CellValue}; +use improvise::view::Axis; +use improvise::workbook::Workbook; +use improvise_command::ui::app::App; +use improvise_command::ui::effect::{self, ViewTarget}; +use improvise_command::ui::host::Host; +use improvise_command::ui::project::{self, Viewport}; +use improvise_command::ui::reduce::caught_up; +use improvise_io::persistence::{format_md, parse_md, save_to}; +use improvise_io::storage::MemoryFs; +use improvise_protocol::{ + ClientFrame, KeyCode, KeyEvent, KeyModifiers, ModelCommand, RecordHash, render::RecordCell, +}; + +fn lcg(s: &mut u64, n: usize) -> usize { + *s = s + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + (*s >> 33) as usize % n +} + +struct Cfg { + k: usize, + n: usize, + cats: usize, + prune: bool, + mode: String, + groups: bool, + ops: String, +} + +fn arg(args: &HashMap, k: &str, d: &str) -> String { + args.get(k).cloned().unwrap_or_else(|| d.to_string()) +} + +fn document(c: &Cfg) -> (Workbook, Vec) { + let mut wb = Workbook::new("bench"); + for ci in 0..c.cats { + let cat = format!("C{ci}"); + wb.add_category(cat.clone()).expect("under the limit"); + for i in 0..c.k { + if c.groups && ci == 0 { + wb.add_item_in_group(&cat, format!("i{i}"), format!("G{}", i / 10)) + .expect("added"); + } else { + wb.add_item(&cat, format!("i{i}")).expect("added"); + } + } + } + for m in ["Cost", "Rev"] { + wb.add_item("_Measure", m).expect("virtual"); + } + let space = (c.k as f64).powi(c.cats as i32); + let wanted = c.n.min((space / 2.0).max(1.0) as usize); + let (mut s, mut seen, mut recs) = (0x9E37_79B9_7F4A_7C15u64, HashSet::new(), Vec::new()); + while seen.len() < wanted { + let d: Vec = (0..c.cats).map(|_| lcg(&mut s, c.k)).collect(); + if !seen.insert(d.clone()) { + continue; + } + let coords: Vec<(String, String)> = d + .iter() + .enumerate() + .map(|(ci, x)| (format!("C{ci}"), format!("i{x}"))) + .collect(); + recs.push(CellKey::new(coords.clone())); + for (m, v) in [("Cost", 3.0), ("Rev", 5.0)] { + let mut key = coords.clone(); + key.push(("_Measure".to_string(), m.to_string())); + wb.model.set_cell(CellKey::new(key), CellValue::Number(v)); + } + } + let view = wb.active_view_mut(); + for ci in 0..c.cats { + let axis = if c.mode == "records" || c.mode == "none" { Axis::None } else { Axis::Row }; + view.set_axis(&format!("C{ci}"), axis); + } + if c.mode == "records" { + view.set_axis("_Index", Axis::Row); + view.set_axis("_Dim", Axis::Column); + } else { + view.set_axis("_Measure", Axis::Column); + } + view.prune_empty = c.prune; + (wb, recs) +} + +fn maxrss_mb() -> f64 { + let mut u: libc::rusage = unsafe { std::mem::zeroed() }; + unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut u) }; + u.ru_maxrss as f64 / 1048576.0 +} + +fn rss_mb() -> f64 { + let out = std::process::Command::new("ps") + .args(["-o", "rss=", "-p", &std::process::id().to_string()]) + .output() + .expect("ps"); + String::from_utf8_lossy(&out.stdout).trim().parse::().unwrap_or(0.0) / 1024.0 +} + +/// Sample `f` up to `reps` times, stopping once `budget_ms` is spent (one sample minimum). +fn sample(name: &str, reps: usize, budget_ms: f64, mut f: impl FnMut(usize)) { + let mut v = Vec::new(); + let start = Instant::now(); + for r in 0..reps { + let t = Instant::now(); + f(r); + v.push(t.elapsed().as_secs_f64() * 1000.0); + if start.elapsed().as_secs_f64() * 1000.0 > budget_ms { + break; + } + } + v.sort_by(|a, b| a.partial_cmp(b).expect("finite")); + println!( + "{name} min={:.3} med={:.3} max={:.3} reps={}", + v[0], + v[v.len() / 2], + v[v.len() - 1], + v.len() + ); + std::io::stdout().flush().ok(); +} + +fn once(name: &str, f: impl FnOnce() -> T) -> T { + let t = Instant::now(); + let v = f(); + println!("{name} min={0:.3} med={0:.3} max={0:.3} reps=1", t.elapsed().as_secs_f64() * 1000.0); + std::io::stdout().flush().ok(); + v +} + +fn key(code: KeyCode) -> KeyEvent { + KeyEvent::new(code, KeyModifiers::NONE) +} + +fn main() { + let args: HashMap = std::env::args() + .skip(1) + .filter_map(|a| a.split_once('=').map(|(k, v)| (k.to_string(), v.to_string()))) + .collect(); + let c = Cfg { + k: arg(&args, "k", "1000").parse().expect("k"), + n: arg(&args, "n", "100000").parse().expect("n"), + cats: arg(&args, "cats", "3").parse().expect("cats"), + prune: arg(&args, "prune", "1") == "1", + mode: arg(&args, "mode", "grid"), + groups: arg(&args, "groups", "0") == "1", + ops: arg(&args, "ops", "open,key,frame,window,setcell,host,reshape,io"), + }; + let rows_cap: f64 = arg(&args, "rows_cap", "30000000").parse().expect("cap"); + let product = (c.k as f64).powi(c.cats as i32); + if c.mode == "grid" && product > rows_cap { + println!("REFUSED product_rows={product:.3e} (> {rows_cap:.0}: layout would materialize it)"); + return; + } + let want = |o: &str| c.ops.split(',').any(|x| x == o); + let (wb, recs) = document(&c); + println!("cells={} records={} base_rss_mb={:.0}", wb.model.data.iter_cells().count(), recs.len(), rss_mb()); + let mut app = once("open", || App::new(wb, None)); + println!("rows={} cols={} rss_after_open_mb={:.0}", app.layout.row_count(), app.layout.col_count(), rss_mb()); + app.term_width = 200; + app.term_height = 50; + let name = app.view_name().to_string(); + + if want("key") { + sample("keypress_down", 8, 6000.0, |_| { + app.dispatch_key(key(KeyCode::Down)); + }); + } + if want("frame") { + let whole = |app: &App| Viewport { row_offset: 0, col_offset: 0, rows: app.layout.row_count(), cols: app.layout.col_count() }; + sample("tui_frame_grid_of_whole", 3, 8000.0, |_| { + std::hint::black_box(project::grid_of(&app, whole(&app))); + }); + let cache = project::grid_of(&app, whole(&app)); + let area = ratatui::layout::Rect::new(0, 0, 200, 50); + sample("tui_render_gridwidget_buffer", 5, 4000.0, |_| { + let mut buf = ratatui::buffer::Buffer::empty(area); + let w = improvise::ui::grid::GridWidget::new( + &cache, app.cursor(), &app.view_state.active_view, &app.layout, + &app.view_state.mode, &app.view_state.search_query, + &app.view_state.buffers, app.drill_state.as_ref(), + ); + ratatui::widgets::Widget::render(w, area, &mut buf); + std::hint::black_box(buf); + }); + } + if want("window") { + let mid = app.layout.row_count() / 2; + for (label, off) in [("top", 0), ("mid", mid)] { + let vp = Viewport { row_offset: off, col_offset: 0, rows: 45, cols: 12 }; + sample(&format!("browser_grid_of_{label}"), 5, 6000.0, |_| { + std::hint::black_box(project::grid_of(&app, vp)); + }); + } + } + if want("setcell") && !recs.is_empty() { + let existing = recs[recs.len() / 2].clone().with("_Measure", "Rev"); + sample("setcell_existing", 5, 6000.0, |r| { + app.apply_effects(vec![Box::new(effect::SetCell(existing.clone(), CellValue::Number(10.0 + r as f64)))]); + }); + let mut s = 99u64; + sample("setcell_new_record", 5, 6000.0, |_| { + let mut coords: Vec<(String, String)> = (0..c.cats).map(|ci| (format!("C{ci}"), format!("i{}", lcg(&mut s, c.k)))).collect(); + coords.push(("_Measure".into(), "Rev".into())); + app.apply_effects(vec![Box::new(effect::SetCell(CellKey::new(coords), CellValue::Number(1.0)))]); + }); + } + if want("reshape") && c.mode == "grid" { + let v = || ViewTarget::Active; + sample("reshape_hide_item", 3, 6000.0, |r| { + app.apply_effects(vec![Box::new(effect::HideItem { view: v(), category: "C1".into(), item: format!("i{r}") })]); + }); + if c.groups { + sample("reshape_toggle_group", 3, 6000.0, |_| { + app.apply_effects(vec![Box::new(effect::ToggleGroup { view: v(), category: "C0".into(), group: "G0".into() })]); + }); + } + let last = format!("C{}", c.cats - 1); + sample("reshape_set_axis", 4, 6000.0, |r| { + let axis = if r % 2 == 0 { Axis::Page } else { Axis::Row }; + app.apply_effects(vec![Box::new(effect::SetAxis { view: v(), category: last.clone(), axis })]); + }); + sample("reshape_toggle_prune", 2, 6000.0, |_| { + app.apply_effects(vec![Box::new(effect::TogglePruneEmpty(v()))]); + }); + } + if want("io") { + let wbr = &app.model_state.workbook; + let text = once("format_md", || format_md(wbr)); + println!("improv_mb={:.2}", text.len() as f64 / 1048576.0); + once("parse_md", || parse_md(&text).expect("parses")); + let fs = MemoryFs::new(); + sample("persist_per_command_save_to", 3, 6000.0, |_| { + save_to(&fs, wbr, std::path::Path::new("d.improv")).expect("write"); + }); + } + if want("host") && !recs.is_empty() { + let mut host = Host::new(app); + let (id, snap) = once("host_connect_snapshot", || host.connect(45, 12)); + sample("json_encode_snapshot", 3, 4000.0, |_| { + std::hint::black_box(serde_json::to_vec(&snap.msg).expect("json")); + }); + println!("snapshot_kb={:.1}", serde_json::to_vec(&snap.msg).expect("json").len() as f64 / 1024.0); + let rec = recs[recs.len() / 2].clone(); + let cell = RecordCell { record: RecordHash::of(&rec), measure: Some("Rev".into()) }; + let mut enc: Vec<(f64, usize)> = Vec::new(); + sample("host_handle_setcell_and_project", 4, 8000.0, |r| { + let wire = caught_up(host.app(), ModelCommand::SetCell { view: name.clone(), cell: cell.clone(), value: CellValue::Number(50.0 + r as f64) }); + let out = host.handle(id, ClientFrame::command(wire)); + let t = Instant::now(); + let bytes: usize = out.iter().map(|a| serde_json::to_vec(&a.msg).expect("json").len()).sum(); + enc.push((t.elapsed().as_secs_f64() * 1000.0, bytes)); + }); + println!("json_encode_setcell_updates ms={:?}", enc); + } + println!("maxrss_mb={:.0}", maxrss_mb()); +} diff --git a/context/test-evidence/scale-bisect.md b/context/test-evidence/scale-bisect.md new file mode 100644 index 00000000..7bf797d6 --- /dev/null +++ b/context/test-evidence/scale-bisect.md @@ -0,0 +1,218 @@ +# Scale bisect: how big a document does improvise handle today (improvise-felo.10) + +Measured on `gh/main` @ 8220a065, **Apple M4 Max, 64 GB** (`sysctl`: `hw.memsize 68719476736`), macOS, +release build (`lto`, `opt-level 3`), native only (no wasm, no real OPFS, no terminal). +Harness: `benches/scale_bisect.rs` (one configuration per process) driven by `bench/scale_bisect.py`, +`bench/sweep.py`, `bench/edge.py`, `bench/full.py`. Reproduce: +`nix develop --command cargo build --release --bench scale_bisect`, copy the binary from +`target/release/deps/scale_bisect-*` to `/tmp/sb`, then `python3 bench/edge.py`. + +Budgets (user ruling 2026-10-03): **8 ms is the headline** (120 FPS, hot path or defer to background); +16 ms, 100 ms (noticeable) and 1 s (broken) are reference crossings. + +## Method, and what is real + +Every number goes through the production path: `App::new` (open: formula recompute + +`GridLayout::with_frozen_records`), `App::dispatch_key(Down)` (**keypress/scroll**: it calls +`rebuild_layout`, then `cmd_context`, which runs `compute_col_widths` over the whole grid, then applies effects), +`project::grid_of` (TUI frame: whole grid; browser: a 45x12 window), `App::apply_effects` with `effect::SetCell`, +`SetAxis`/`HideItem`/`ToggleGroup`/`TogglePruneEmpty` (reshape), `Host::connect` and `Host::handle` with a +`caught_up` `ModelCommand::SetCell` (browser path incl. projection), `format_md`/`parse_md`/`save_to(MemoryFs)`. +Document: C0..C{cats-1} with K items, N distinct random tuples ("records", capped at K^cats/2), two measures +each (2N cells), `_Measure` on Column. Timings are the **median** of up to 3-8 in-process repeats; boundary +rows are the median of **3 separate processes** run sequentially (spread shown). Earlier exploratory sweeps +ran 2-3 processes in parallel and agree with the sequential runs within ~5%. + +**`View::prune_empty` defaults to false** (`view/types.rs`), so "prune off" is what a user gets until they +toggle it; "prune on" is reported next to it. Rows below are the layout's `row_count`. + +**Dominant cost (reasoned, not profiled):** the release profile strips symbols, so `sample` could not +symbolicate. The reason is in the code and the timings agree with it: `GridLayout::cell_key`/`row_item` do +`data_items(&self.row_items).nth(row)` (`view/layout.rs` ~253/292/807/870), O(row) per lookup, and +`compute_col_widths` (every keypress, via `cmd_context`) and `grid_of` visit every cell, so both are +O(rows^2). Evidence: with prune off, rows 729 -> 5.4 ms, 4096 -> 74.6 ms, 19683 -> 1248 ms (x6.7 rows -> +x14 time -> x17 time). A 45x12 browser window costs **the same** as the whole-grid TUI frame (K=30: +2274 vs 2298 ms) because the widths pass is whole-grid whatever the viewport says, so viewport narrowing buys +nothing today. `GridLayout` build itself is cheap (~90 ns/row, ADR 008). + +## Attribution: state+projection vs render/serialize (K=10, 1000 rows, prune off; 3 runs) + +| step | ms | +|---|---| +| keypress (state: rebuild + widths + effect) | 9.5-9.7 | +| `grid_of` whole grid (TUI projection) | 11.7-12.0 | +| `GridWidget` render into a 200x50 ratatui `Buffer` | 0.12 | +| `grid_of` 45x12 window (browser projection) | 9.1-9.3 | +| JSON encode of one SetCell's projected updates | 0.001-0.005 (0.6-1.1 KB) | +| JSON encode of the connect Snapshot | 0.02 (21 KB) | + +Render and serialize are **<=0.5 ms in every row measured** (GridWidget 0.08-0.53 ms up to 14,500 records; +snapshot encode <=0.07 ms; records-mode SetCell updates 21 KB in 0.02 ms). The 8 ms is entirely state and +projection. Not measured: the rest of `draw()` (panels, status; `draw` is private to the binary), terminal I/O, +browser JSON *decode* and DOM paint. + +## Axis B: items per category K, 3 categories on Row, N = 100k (capped at K^3/2), prune OFF + +| K | rows | open | keypress | browser window | SetCell | maxrss MB | +|---|---|---|---|---|---|---| +| 5 | 125 | 0.1 | 0.5 | 0.5 | 0.2 | 8 | +| 9 | 729 | 0.2 | **5.4** (5.3-5.5) | 5.0 | 0.9 | 13 | +| 10 | 1,000 | 0.2 | **9.6** (9.5-9.7) | 9.2 | 1.2 | 14 | +| 11 | 1,331 | 0.3 | 12.7 | 12.7 | 1.6 | 10 | +| 12 | 1,728 | 0.3 | 20.0 | 19.3 | 2.1 | 11 | +| 15 | 3,375 | 0.5 | 65 | 64 | 4.0 | 14 | +| 16 | 4,096 | 0.6 | 74.6 (74.3-75.2) | 73.4 | - | - | +| 17 | 4,913 | 0.7 | **105** (105.0-105.5) | 103 | - | - | +| 20 | 8,000 | 1.1 | 302 | 301 | 9.4 | 21 | +| 25 | 15,625 | 1.9 | 773 | 771 | 18.8 | 33 | +| 26 | 17,576 | 2.2 | 967-977 | 962 | - | - | +| 27 | 19,683 | 2.4 | **1,248** (1,244-1,251) | 1,244 | - | - | +| 30 | 27,000 | 3.1 | 2,262 | 2,274 | 31 | 49 | +| 40 | 64,000 | 6.7 | 10,858 | 10,833 | 73 | 235 | +| 60 | 216,000 | 21.6 | timeout (>60 s) | - | - | - | +| 1000 | 10^9 | **refused** (see below) | | | | | + +Hot-path keypress/scroll and browser window: **largest inside 8 ms: K=9 (729 rows)**; inside 16 ms: K=11; +first over 100 ms: **K=17 (4,913 rows)**; first over 1 s: **K=27 (19,683 rows)**. SetCell alone +(no widths pass; layout rebuild only, ~linear in rows): inside 8 ms to K~19 (K=15 4.0, K=20 9.4), 16 ms ~K=24, +100 ms ~K=45 (K=40 = 73 ms), 1 s extrapolated ~K=95 (**extrapolated, not measured**). Open stays <25 ms to K=60. + +**3 x 1,000 on Row cannot be opened at any N**: the layout materializes the cross product (10^9 rows, ~150 B +each = ~150 GB, ADR 008). The harness refuses above 30M rows (`rows_cap`) rather than swap the machine, so +the 10^9 number is the ADR's estimate, not a measurement here. + +### Prune ON (same documents) + +| K | rows | open | keypress | browser window | SetCell | +|---|---|---|---|---|---| +| 7 | 171 | 2.0 | 4.5 (4.4-4.6) | 1.1 | 5.4 | +| 8 | 256 | 3.0 | **7.0** (7.0-7.1) | 1.6 | 8.5 | +| 9 | 364 | 5.3 | 12.8 | 2.7 | 15.3 | +| 11 | 665 | 12.2 | 29.6 | 6.2 | ~28 | +| 13 | 1,098 | 29.0 | 71 | 14.5 | ~70 | +| 14 | 1,372 | 48 | **119** | 24 | - | +| 20 | 4,000 | 300 | 747 | 145 | 903 | +| 21 | 4,630 | 401 | 968-992 | 190 | - | +| 25 | 7,812 | 767 | 1,899 | 366 | 2,328 | +| 30 | 13,500 | 2,255 | 5,555 | 1,047 | 6,804 | + +Prune on: keypress inside 8 ms to **K=8**, first over 100 ms **K=14**, ~1 s at **K~21-22** (K=22 measured +0.93-0.95 s, a different N cap; first clearly over: K=25). Pruning shrinks the grid, so the browser window is +cheaper than prune off at the same K (K=20: 145 ms against 302 ms), but **every keypress re-runs +the prune** (`dispatch_key` calls `rebuild_layout`), so keypress and SetCell are *worse* with prune on +(K=10: 23 vs 9.9 ms). Prune is also quadratic (`nth`). Conclusion: prune on/off moves the boundary by about +one K at 8 ms and makes 100 ms/1 s *earlier* (K=14/21 vs 17/27); it never rescues the 3x1000 case. + +## Axis A: records N at fixed K (K=30, 27,000 rows) + +| N | prune off keypress | prune on rows | prune on keypress | prune on window | +|---|---|---|---|---| +| 100 | 451 | 100 | 861 | 0.3 | +| 1,000 | 527 | 1,000 | 1,059 | 6.4 | +| 5,000 | 1,013 | 5,000 | 2,178 | 140 | +| 13,500 (cap) | 2,267 | 13,500 | 5,555 | 1,047 | + +Prune off: ~450 ms of widths over 27k empty rows + ~1.3 us per stored cell; **N never gets the keypress +under 100 ms at K=30**, and at K=1000 N is irrelevant because the layout cannot be built (rows = K^3). +The N-axis has no bisect point in the 3x1000 regime; N only matters once K is small enough that rows fit, and +there it is a second-order term. Open is flat (3.1-3.6 ms) in N with prune off. + +## Axis C: categories on Row (prune off) + +| K | cats | rows | open | keypress | note | +|---|---|---|---|---|---| +| 10 | 2 | 100 | 0.1 | 0.2 | | +| 10 | 3 | 1,000 | 0.2 | 9.9 | over 8 ms | +| 10 | 4 | 10,000 | 1.4 | **880** | ~1 s | +| 10 | 5 | 100,000 | 14 | timeout >100 s | | +| 3 | 6 | 729 | 0.3 | 27.9 | 5x worse than K=9 x 3 cats at the same rows | +| 3 | 8 | 6,561 | 2.0 | 2,798 | | +| 3 | 10-12 | 59k-531k | 18-210 | timeout | open alone passes 100 ms at 11 cats | +| 2 | 10 | 1,024 | 0.6 | 172 | | +| 2 | 12 | 4,096 | 2.2 | 3,190 | | +| 2 | 14 | 16,384 | 9 | 63,100 | | + +The cap of 24 is unreachable: rows = product of items, so a view with c Row categories of K items dies by +K^c ~ 10^4 rows (1 s) regardless of N, and with two items per category by 12-14 categories. Per-row cost +also grows with category count (each row_item is a Vec of c items). Not measured: Row categories with prune +on, other axes absorbing categories (the realistic mitigation: only 1-2 on Row). + +## Axis D: records mode over N records (`with_frozen_records`, the improvise-sk32 quadratic) + +| N | open | keypress | browser window | SetCell (apply) | host SetCell+project | maxrss MB | +|---|---|---|---|---|---|---| +| 300 | 1.5 | 0.7 | 0.1 | 1.6 | - | 10 | +| 1,000 | 2.9 | 5.6 (5.5-5.7) | 0.3 | 10 | - | 12 | +| 1,200 | 3.9 | **7.9** (7.7-8.0) | 0.3 | - | - | - | +| 1,300 | 4.5 | 9.3 | 0.3 | 15.5 | 17 | 26 | +| 1,800 | 8.4 | 16.5 | 0.4 | - | - | - | +| 3,000 | 22.9 | 47 | 0.7 | 74 | - | 20 | +| 4,400-4,600 | 47-51 | **95-102** | 0.9 | 149 | 158 | 55 | +| 10,000 | 248 | 492 | 1.8 | 734 | - | 46 | +| 14,000-14,500 | 472-500 | **940-1,002** | 2.6-2.8 | 1,435 | 1,459 | 135-148 | +| 30,000 | 2,198 | 4,314 | 5.6 | 6,467 | - | 125 | +| 100,000 | **25,420** | **51,234** | 19.3 | not reached (killed at 100 s) | - | not reached | + +Records mode, keypress/scroll: inside 8 ms **to N~1,200**; 16 ms N~1,800; 100 ms N~4,500 (first over: 4,600 = +102 ms); 1 s N~14,500. Open crosses 8 ms ~1,750, 16 ms ~2,300, 100 ms ~6,500, 1 s ~20,000. Time ~ N^1.95. +SetCell is ~1.5x keypress (it rebuilds and re-groups). The *browser window* is cheap (<20 ms even at 100k) +because records mode has two columns and the width walk is short, so the 51 s is `rebuild_layout`, not +projection. Prune on/off does not change records mode (prune applies to pivoted axes; the sweep ran off). +Records per KB: ~4 KB/record process-wide at 14k (maxrss includes the document generator, so read as an +upper bound); the 100k memory was not obtained. + +## Axis E: save / load and per-command persistence + +Mode `none` (all categories off-axis, so layout is trivial and only the store scales), 3 x 1,000, N records += 2N cells: + +| cells | format_md | text MB | parse_md | persist per accepted command (`save_to` MemoryFs) | maxrss MB | +|---|---|---|---|---|---| +| 2,000 | 13.8 | 0.1 | 4.5 | 12.4 | 18 | +| 20,000 | 156 | 0.86 | 37 | 161 | 80 | +| 60,000 | 527 | 2.5 | 119 | 539 | 236 | +| **200,000 (the target)** | **1,974** | 8.4 | **401** | **1,984** | 663 | +| 600,000 | 6,470 | 25 | 1,320 | 6,460 | 1,894 | +| 2,000,000 | 23,120 | 84 | 4,668 | 23,088 | 5,611 | + +Linear: ~10 us/cell to format, ~2 us/cell to parse. Crossings, format_md: 8 ms ~700 cells, 16 ms ~1.5k, +100 ms ~11k, 1 s ~100k cells. parse_md: 8 ms ~4k cells, 100 ms ~50k, 1 s ~450k. **At the target (200k cells) +every accepted command in the browser worker costs ~2 s of format alone** before OPFS write time (not +measured: MemoryFs only). Persist per command also appears in smaller documents inside the grid axes +(K=10 grid: 5.5 ms; records N=4,400: 63 ms). + +## Other operations (3 runs each; "grouped" fixtures start with C0 groups collapsed, so rows are not K^3) + +Reshape (prune off, K=10, 1000 rows): hide item 0.25 ms, set axis 0.19 ms, **toggle prune 12.7 ms**. +K=26 grouped, 2,028 rows: hide 0.5, toggle group 1.8, set axis 1.8, **toggle prune 727-750 ms**. +Prune on K=13 grouped, 321 rows: hide 9.4, toggle group 37, set axis 37, toggle prune 37-39 ms. +`Host::connect` snapshot: K=10 6.5 ms, K=26 375 ms, records N=14,000 2.9 ms. Host SetCell+project: K=10 7.7 ms, +K=26 ~400 ms, records N=4,400 158 ms (equal to apply + a whole-grid width walk). + +## Classification and feasibility of yielding + +| operation | class | today's first size over 8 ms | dominated by | chunked/yielding form | +|---|---|---|---|---| +| keypress / scroll | **hot (<=8 ms)** | 1,000 rows (K=10) / N=1,300 records | `rebuild_layout` + whole-grid `compute_col_widths` (O(rows^2) `nth`), recomputed though nothing changed | not a slicing problem: cache widths/layout across keys, O(1) row lookup (ADR 008/009). Slicing would only hide it | +| browser window projection | **hot** | 1,000 rows | same width walk, whole grid regardless of window | needs the same fix; then window cost is O(window) | +| SetCell apply + project | **hot** | K~19 (prune off), 1,300 records | layout rebuild O(rows) + widths + projection | incremental update of one row/total, not a rebuild | +| column totals | hot | inside `grid_of` (not separable here) | whole-grid sum in the projection | maintained total (ADR 009) | +| open / layout build | deferrable | 40k rows (prune off); N~1,750 (records) | build is ~90 ns/row (sliceable by outer index); the prune/records grouping is the quadratic part | build is feasible in slices once the quadratic is removed; the quadratic itself is not sliceable | +| reshape (hide/collapse/axis) | deferrable | cheap prune-off (<2 ms to ~2k rows); 37 ms prune-on at 321 rows | layout rebuild + prune | one sort/remap at 100k occupied tuples was 2.3 ms in ADR 008's prototype, so a reshape can fit inline; prune toggle needs the quadratic fixed first | +| save (format_md) | deferrable | ~700 cells | per-cell formatting, linear | the Data section is a flat per-cell line stream, so chunking by cell range with a yield every ~400 cells looks feasible, but needs a consistent snapshot (clone or journal); not profiled by function | +| load (parse_md) | deferrable | ~4k cells | pest parse + tree walk, linear | tree walk is chunkable by line; the pest parse of one text is not, unless the Data section is parsed line by line (grammar is line oriented); not profiled by function | +| persistence per command | deferrable (debounce) | ~700 cells | the whole-document `format_md` | coalesce/debounce or append-journal: formatting 2 s per command at target is unusable inline at any slicing | + +## What I could not measure + +* wasm: all numbers are native; the browser worker is typically slower. Real OPFS I/O, JSON *decode* and + DOM paint, the full `draw()` frame, and Safari are not covered. +* Records mode at 100k: SetCell, host and RSS (killed after open and keypress, ~76 s). +* Per-function profile: symbols are stripped in `[profile.release]`; attribution is reasoned from the + code and the timing exponents, not a flame graph. +* SetCell 100 ms/1 s for prune off past K=40 are extrapolations. +* Memory (`maxrss`) includes the generator's hash set and `recs` vector; it is an upper bound on the + document. Layout cost ~150 B/row per ADR 008, consistent with K=40 (64k rows, 235 MB peak with prune off + including the generator and a second layout during rebuild). +* Noise: boundary rows are 3 sequential processes, spread <=3% (e.g. K=17: 105.0-105.5 ms). Non-boundary + rows are single runs, some taken with 2-3 concurrent processes on a 16+ core machine; treat them as +-5-10%.