Merge gh/main into fix/31oe-cmdlang; renumber cli.md §13/§14 to §14/§15
#321 added its own cli.md section 13, so this PR's sections become 14 (pipe quoting, surplus arguments) and 15 (names the document lacks), with every citation renumbered. The stale descriptions of the deleted unknown_* helpers in cli.md, command.md and the test evidence now describe RequireKnown, and section 15 says a refused name stops the rest of a `:` line but not a headless one. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
This commit is contained in:
Claude (cmdlang)
@@ -31,6 +31,11 @@ name = "reshape_observation"
|
||||
harness = false
|
||||
test = false
|
||||
|
||||
[[bench]]
|
||||
name = "scale_bisect"
|
||||
harness = false
|
||||
test = false
|
||||
|
||||
[dependencies]
|
||||
gp-tree = { path = "crates/gp-tree" }
|
||||
improvise-core = { path = "crates/improvise-core" }
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
import sys; sys.path.insert(0,'bench')
|
||||
from scale_bisect import run
|
||||
cfgs=[dict(k=k,prune=0) for k in (16,17,18,26,27)]+[dict(k=k,prune=1) for k in (7,11,12,13,14,21,22)]+[dict(k=1000,n=n,mode='records') for n in (1000,1200,1500,1800,2000,4600,14500)]
|
||||
for c in cfgs:
|
||||
c.setdefault('n',100000); c['ops']='open,key,window'
|
||||
v=[run('/tmp/sb',c,timeout=200) for _ in range(3)]
|
||||
f=lambda k:'/'.join(f"{r[k][1]:.1f}" if k in r else '-' for r in v)
|
||||
print({k:c[k] for k in c if k!='ops'},'rows',v[0].get('rows'),'open',f('open'),'key',f('keypress_down'),'win',f('browser_grid_of_top'),'tuiframe',f('tui_frame_grid_of_whole'),flush=True)
|
||||
@@ -0,0 +1,10 @@
|
||||
import sys, json; sys.path.insert(0,'bench')
|
||||
from scale_bisect import run
|
||||
cfgs=[dict(k=9,prune=0),dict(k=10,prune=0),dict(k=17,prune=0,groups=1),dict(k=26,prune=0,groups=1),
|
||||
dict(k=8,prune=1),dict(k=13,prune=1,groups=1),
|
||||
dict(k=1000,n=1300,mode='records'),dict(k=1000,n=4400,mode='records'),dict(k=1000,n=14000,mode='records')]
|
||||
for c in cfgs:
|
||||
c.setdefault('n',100000)
|
||||
for rep in range(3):
|
||||
r=run('/tmp/sb',c,timeout=300)
|
||||
print(json.dumps({'cfg':c,'rep':rep,**r}),flush=True)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Drive benches/scale_bisect one configuration per process; parse its lines.
|
||||
usage: scale_bisect.py BIN k=.. n=.. [cats=..] [prune=..] [mode=..] [ops=..] [--timeout S]
|
||||
"""
|
||||
import subprocess, sys, re, json
|
||||
|
||||
def run(binary, cfg, timeout=90):
|
||||
args = [binary] + [f"{k}={v}" for k, v in cfg.items()]
|
||||
try:
|
||||
p = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
||||
out, status = p.stdout, "ok"
|
||||
except subprocess.TimeoutExpired as e:
|
||||
out, status = (e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or ""), "timeout"
|
||||
res = {"status": status}
|
||||
for line in out.splitlines():
|
||||
if line.startswith("REFUSED"):
|
||||
res["status"] = "refused"
|
||||
m = re.match(r"(\w+) min=([\d.]+) med=([\d.]+) max=([\d.]+)", line)
|
||||
if m:
|
||||
res[m[1]] = (float(m[2]), float(m[3]), float(m[4]))
|
||||
for kv in re.findall(r"(\w+)=([\d.]+)", line):
|
||||
if kv[0] in ("rows", "cols", "maxrss_mb", "improv_mb", "snapshot_kb", "rss_after_open_mb", "cells"):
|
||||
res[kv[0]] = float(kv[1])
|
||||
m = re.match(r"json_encode_setcell_updates ms=(.*)", line)
|
||||
if m:
|
||||
res["json_setcell"] = m[1]
|
||||
return res
|
||||
|
||||
if __name__ == "__main__":
|
||||
binary = sys.argv[1]
|
||||
cfg, timeout = {}, 90
|
||||
rest = sys.argv[2:]
|
||||
for a in rest:
|
||||
if a.startswith("--timeout"):
|
||||
timeout = int(a.split("=")[1])
|
||||
else:
|
||||
k, v = a.split("=", 1)
|
||||
cfg[k] = v
|
||||
print(json.dumps(run(binary, cfg, timeout)))
|
||||
@@ -0,0 +1,10 @@
|
||||
import sys; sys.path.insert(0,'bench')
|
||||
from scale_bisect import run
|
||||
import itertools
|
||||
OPS=sys.argv[1];
|
||||
cfgs=eval(sys.argv[2])
|
||||
cols=sys.argv[3].split(',')
|
||||
for c in cfgs:
|
||||
c=dict(c); c['ops']=OPS
|
||||
r=run('/tmp/sb',c,timeout=int(sys.argv[4]) if len(sys.argv)>4 else 90)
|
||||
print(c, r['status'], 'rows',r.get('rows'), *[f"{k}={r[k][1]:.1f}" if k in r else f"{k}=-" for k in cols], f"rss={r.get('maxrss_mb')}", flush=True)
|
||||
@@ -0,0 +1,347 @@
|
||||
//! One configuration per process: build a sparse document, time the real paths.
|
||||
//! `scale_bisect k=1000 n=100000 cats=3 prune=1 mode=grid`; see context/test-evidence/scale-bisect.md.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Write;
|
||||
use std::time::Instant;
|
||||
|
||||
use improvise::model::cell::{CellKey, CellValue};
|
||||
use improvise::view::Axis;
|
||||
use improvise::workbook::Workbook;
|
||||
use improvise_command::ui::app::App;
|
||||
use improvise_command::ui::effect::{self, ViewTarget};
|
||||
use improvise_command::ui::host::Host;
|
||||
use improvise_command::ui::project::{self, Viewport};
|
||||
use improvise_command::ui::reduce::caught_up;
|
||||
use improvise_io::persistence::{format_md, parse_md, save_to};
|
||||
use improvise_io::storage::MemoryFs;
|
||||
use improvise_protocol::{
|
||||
ClientFrame, KeyCode, KeyEvent, KeyModifiers, ModelCommand, RecordHash, render::RecordCell,
|
||||
};
|
||||
|
||||
fn lcg(s: &mut u64, n: usize) -> usize {
|
||||
*s = s
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
(*s >> 33) as usize % n
|
||||
}
|
||||
|
||||
struct Cfg {
|
||||
k: usize,
|
||||
n: usize,
|
||||
cats: usize,
|
||||
prune: bool,
|
||||
mode: String,
|
||||
groups: bool,
|
||||
ops: String,
|
||||
}
|
||||
|
||||
fn arg(args: &HashMap<String, String>, k: &str, d: &str) -> String {
|
||||
args.get(k).cloned().unwrap_or_else(|| d.to_string())
|
||||
}
|
||||
|
||||
fn document(c: &Cfg) -> (Workbook, Vec<CellKey>) {
|
||||
let mut wb = Workbook::new("bench");
|
||||
for ci in 0..c.cats {
|
||||
let cat = format!("C{ci}");
|
||||
wb.add_category(cat.clone()).expect("under the limit");
|
||||
for i in 0..c.k {
|
||||
if c.groups && ci == 0 {
|
||||
wb.add_item_in_group(&cat, format!("i{i}"), format!("G{}", i / 10))
|
||||
.expect("added");
|
||||
} else {
|
||||
wb.add_item(&cat, format!("i{i}")).expect("added");
|
||||
}
|
||||
}
|
||||
}
|
||||
for m in ["Cost", "Rev"] {
|
||||
wb.add_item("_Measure", m).expect("virtual");
|
||||
}
|
||||
let space = (c.k as f64).powi(c.cats as i32);
|
||||
let wanted = c.n.min((space / 2.0).max(1.0) as usize);
|
||||
let (mut s, mut seen, mut recs) = (0x9E37_79B9_7F4A_7C15u64, HashSet::new(), Vec::new());
|
||||
while seen.len() < wanted {
|
||||
let d: Vec<usize> = (0..c.cats).map(|_| lcg(&mut s, c.k)).collect();
|
||||
if !seen.insert(d.clone()) {
|
||||
continue;
|
||||
}
|
||||
let coords: Vec<(String, String)> = d
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(ci, x)| (format!("C{ci}"), format!("i{x}")))
|
||||
.collect();
|
||||
recs.push(CellKey::new(coords.clone()));
|
||||
for (m, v) in [("Cost", 3.0), ("Rev", 5.0)] {
|
||||
let mut key = coords.clone();
|
||||
key.push(("_Measure".to_string(), m.to_string()));
|
||||
wb.model.set_cell(CellKey::new(key), CellValue::Number(v));
|
||||
}
|
||||
}
|
||||
let view = wb.active_view_mut();
|
||||
for ci in 0..c.cats {
|
||||
let axis = if c.mode == "records" || c.mode == "none" {
|
||||
Axis::None
|
||||
} else {
|
||||
Axis::Row
|
||||
};
|
||||
view.set_axis(&format!("C{ci}"), axis);
|
||||
}
|
||||
if c.mode == "records" {
|
||||
view.set_axis("_Index", Axis::Row);
|
||||
view.set_axis("_Dim", Axis::Column);
|
||||
} else {
|
||||
view.set_axis("_Measure", Axis::Column);
|
||||
}
|
||||
view.prune_empty = c.prune;
|
||||
(wb, recs)
|
||||
}
|
||||
|
||||
fn maxrss_mb() -> f64 {
|
||||
let mut u: libc::rusage = unsafe { std::mem::zeroed() };
|
||||
unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut u) };
|
||||
u.ru_maxrss as f64 / 1048576.0
|
||||
}
|
||||
|
||||
fn rss_mb() -> f64 {
|
||||
let out = std::process::Command::new("ps")
|
||||
.args(["-o", "rss=", "-p", &std::process::id().to_string()])
|
||||
.output()
|
||||
.expect("ps");
|
||||
String::from_utf8_lossy(&out.stdout)
|
||||
.trim()
|
||||
.parse::<f64>()
|
||||
.unwrap_or(0.0)
|
||||
/ 1024.0
|
||||
}
|
||||
|
||||
/// Sample `f` up to `reps` times, stopping once `budget_ms` is spent (one sample minimum).
|
||||
fn sample(name: &str, reps: usize, budget_ms: f64, mut f: impl FnMut(usize)) {
|
||||
let mut v = Vec::new();
|
||||
let start = Instant::now();
|
||||
for r in 0..reps {
|
||||
let t = Instant::now();
|
||||
f(r);
|
||||
v.push(t.elapsed().as_secs_f64() * 1000.0);
|
||||
if start.elapsed().as_secs_f64() * 1000.0 > budget_ms {
|
||||
break;
|
||||
}
|
||||
}
|
||||
v.sort_by(|a, b| a.partial_cmp(b).expect("finite"));
|
||||
println!(
|
||||
"{name} min={:.3} med={:.3} max={:.3} reps={}",
|
||||
v[0],
|
||||
v[v.len() / 2],
|
||||
v[v.len() - 1],
|
||||
v.len()
|
||||
);
|
||||
std::io::stdout().flush().ok();
|
||||
}
|
||||
|
||||
fn once<T>(name: &str, f: impl FnOnce() -> T) -> T {
|
||||
let t = Instant::now();
|
||||
let v = f();
|
||||
println!(
|
||||
"{name} min={0:.3} med={0:.3} max={0:.3} reps=1",
|
||||
t.elapsed().as_secs_f64() * 1000.0
|
||||
);
|
||||
std::io::stdout().flush().ok();
|
||||
v
|
||||
}
|
||||
|
||||
fn key(code: KeyCode) -> KeyEvent {
|
||||
KeyEvent::new(code, KeyModifiers::NONE)
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args: HashMap<String, String> = std::env::args()
|
||||
.skip(1)
|
||||
.filter_map(|a| {
|
||||
a.split_once('=')
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
})
|
||||
.collect();
|
||||
let c = Cfg {
|
||||
k: arg(&args, "k", "1000").parse().expect("k"),
|
||||
n: arg(&args, "n", "100000").parse().expect("n"),
|
||||
cats: arg(&args, "cats", "3").parse().expect("cats"),
|
||||
prune: arg(&args, "prune", "1") == "1",
|
||||
mode: arg(&args, "mode", "grid"),
|
||||
groups: arg(&args, "groups", "0") == "1",
|
||||
ops: arg(
|
||||
&args,
|
||||
"ops",
|
||||
"open,key,frame,window,setcell,host,reshape,io",
|
||||
),
|
||||
};
|
||||
let rows_cap: f64 = arg(&args, "rows_cap", "30000000").parse().expect("cap");
|
||||
let product = (c.k as f64).powi(c.cats as i32);
|
||||
if c.mode == "grid" && product > rows_cap {
|
||||
println!(
|
||||
"REFUSED product_rows={product:.3e} (> {rows_cap:.0}: layout would materialize it)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let want = |o: &str| c.ops.split(',').any(|x| x == o);
|
||||
let (wb, recs) = document(&c);
|
||||
println!(
|
||||
"cells={} records={} base_rss_mb={:.0}",
|
||||
wb.model.data.iter_cells().count(),
|
||||
recs.len(),
|
||||
rss_mb()
|
||||
);
|
||||
let mut app = once("open", || App::new(wb, None));
|
||||
println!(
|
||||
"rows={} cols={} rss_after_open_mb={:.0}",
|
||||
app.layout.row_count(),
|
||||
app.layout.col_count(),
|
||||
rss_mb()
|
||||
);
|
||||
app.term_width = 200;
|
||||
app.term_height = 50;
|
||||
let name = app.view_name().to_string();
|
||||
|
||||
if want("key") {
|
||||
sample("keypress_down", 8, 6000.0, |_| {
|
||||
app.dispatch_key(key(KeyCode::Down));
|
||||
});
|
||||
}
|
||||
if want("frame") {
|
||||
let whole = |app: &App| Viewport {
|
||||
row_offset: 0,
|
||||
col_offset: 0,
|
||||
rows: app.layout.row_count(),
|
||||
cols: app.layout.col_count(),
|
||||
};
|
||||
sample("tui_frame_grid_of_whole", 3, 8000.0, |_| {
|
||||
std::hint::black_box(project::grid_of(&app, whole(&app)));
|
||||
});
|
||||
let cache = project::grid_of(&app, whole(&app));
|
||||
let area = ratatui::layout::Rect::new(0, 0, 200, 50);
|
||||
sample("tui_render_gridwidget_buffer", 5, 4000.0, |_| {
|
||||
let mut buf = ratatui::buffer::Buffer::empty(area);
|
||||
let w = improvise::ui::grid::GridWidget::new(
|
||||
&cache,
|
||||
app.cursor(),
|
||||
&app.view_state.active_view,
|
||||
&app.layout,
|
||||
&app.view_state.mode,
|
||||
&app.view_state.search_query,
|
||||
&app.view_state.buffers,
|
||||
app.drill_state.as_ref(),
|
||||
);
|
||||
ratatui::widgets::Widget::render(w, area, &mut buf);
|
||||
std::hint::black_box(buf);
|
||||
});
|
||||
}
|
||||
if want("window") {
|
||||
let mid = app.layout.row_count() / 2;
|
||||
for (label, off) in [("top", 0), ("mid", mid)] {
|
||||
let vp = Viewport {
|
||||
row_offset: off,
|
||||
col_offset: 0,
|
||||
rows: 45,
|
||||
cols: 12,
|
||||
};
|
||||
sample(&format!("browser_grid_of_{label}"), 5, 6000.0, |_| {
|
||||
std::hint::black_box(project::grid_of(&app, vp));
|
||||
});
|
||||
}
|
||||
}
|
||||
if want("setcell") && !recs.is_empty() {
|
||||
let existing = recs[recs.len() / 2].clone().with("_Measure", "Rev");
|
||||
sample("setcell_existing", 5, 6000.0, |r| {
|
||||
app.apply_effects(vec![Box::new(effect::SetCell(
|
||||
existing.clone(),
|
||||
CellValue::Number(10.0 + r as f64),
|
||||
))]);
|
||||
});
|
||||
let mut s = 99u64;
|
||||
sample("setcell_new_record", 5, 6000.0, |_| {
|
||||
let mut coords: Vec<(String, String)> = (0..c.cats)
|
||||
.map(|ci| (format!("C{ci}"), format!("i{}", lcg(&mut s, c.k))))
|
||||
.collect();
|
||||
coords.push(("_Measure".into(), "Rev".into()));
|
||||
app.apply_effects(vec![Box::new(effect::SetCell(
|
||||
CellKey::new(coords),
|
||||
CellValue::Number(1.0),
|
||||
))]);
|
||||
});
|
||||
}
|
||||
if want("reshape") && c.mode == "grid" {
|
||||
let v = || ViewTarget::Active;
|
||||
sample("reshape_hide_item", 3, 6000.0, |r| {
|
||||
app.apply_effects(vec![Box::new(effect::HideItem {
|
||||
view: v(),
|
||||
category: "C1".into(),
|
||||
item: format!("i{r}"),
|
||||
})]);
|
||||
});
|
||||
if c.groups {
|
||||
sample("reshape_toggle_group", 3, 6000.0, |_| {
|
||||
app.apply_effects(vec![Box::new(effect::ToggleGroup {
|
||||
view: v(),
|
||||
category: "C0".into(),
|
||||
group: "G0".into(),
|
||||
})]);
|
||||
});
|
||||
}
|
||||
let last = format!("C{}", c.cats - 1);
|
||||
sample("reshape_set_axis", 4, 6000.0, |r| {
|
||||
let axis = if r % 2 == 0 { Axis::Page } else { Axis::Row };
|
||||
app.apply_effects(vec![Box::new(effect::SetAxis {
|
||||
view: v(),
|
||||
category: last.clone(),
|
||||
axis,
|
||||
})]);
|
||||
});
|
||||
sample("reshape_toggle_prune", 2, 6000.0, |_| {
|
||||
app.apply_effects(vec![Box::new(effect::TogglePruneEmpty(v()))]);
|
||||
});
|
||||
}
|
||||
if want("io") {
|
||||
let wbr = &app.model_state.workbook;
|
||||
let text = once("format_md", || format_md(wbr));
|
||||
println!("improv_mb={:.2}", text.len() as f64 / 1048576.0);
|
||||
once("parse_md", || parse_md(&text).expect("parses"));
|
||||
let fs = MemoryFs::new();
|
||||
sample("persist_per_command_save_to", 3, 6000.0, |_| {
|
||||
save_to(&fs, wbr, std::path::Path::new("d.improv")).expect("write");
|
||||
});
|
||||
}
|
||||
if want("host") && !recs.is_empty() {
|
||||
let mut host = Host::new(app);
|
||||
let (id, snap) = once("host_connect_snapshot", || host.connect(45, 12));
|
||||
sample("json_encode_snapshot", 3, 4000.0, |_| {
|
||||
std::hint::black_box(serde_json::to_vec(&snap.msg).expect("json"));
|
||||
});
|
||||
println!(
|
||||
"snapshot_kb={:.1}",
|
||||
serde_json::to_vec(&snap.msg).expect("json").len() as f64 / 1024.0
|
||||
);
|
||||
let rec = recs[recs.len() / 2].clone();
|
||||
let cell = RecordCell {
|
||||
record: RecordHash::of(&rec),
|
||||
measure: Some("Rev".into()),
|
||||
};
|
||||
let mut enc: Vec<(f64, usize)> = Vec::new();
|
||||
sample("host_handle_setcell_and_project", 4, 8000.0, |r| {
|
||||
let wire = caught_up(
|
||||
host.app(),
|
||||
ModelCommand::SetCell {
|
||||
view: name.clone(),
|
||||
cell: cell.clone(),
|
||||
value: CellValue::Number(50.0 + r as f64),
|
||||
},
|
||||
);
|
||||
let out = host.handle(id, ClientFrame::command(wire));
|
||||
let t = Instant::now();
|
||||
let bytes: usize = out
|
||||
.iter()
|
||||
.map(|a| serde_json::to_vec(&a.msg).expect("json").len())
|
||||
.sum();
|
||||
enc.push((t.elapsed().as_secs_f64() * 1000.0, bytes));
|
||||
});
|
||||
println!("json_encode_setcell_updates ms={:?}", enc);
|
||||
}
|
||||
println!("maxrss_mb={:.0}", maxrss_mb());
|
||||
}
|
||||
+51
-10
@@ -393,7 +393,40 @@ The headless path matters most, because the code-maat pipeline
|
||||
pins the views, the sparse Data lines and the byte-identical second run, with
|
||||
the first run's output sitting in the folder it reads.
|
||||
|
||||
## 13 — the command language reads `|pipe quoting|`, and a surplus argument is refused (improvise-31oe.3)
|
||||
## 13 — a headless import refuses rather than saving less than was asked (improvise-8jk6.4, 8jk6.5, 8jk6.6)
|
||||
|
||||
The user treats the headless command and model layer as a product of its own,
|
||||
and the scripted `import --no-wizard` is where it meets a pipeline. The
|
||||
repo-stats pipeline (improvise-1xfb.5.1) found three ways that import saved
|
||||
something other than what the flags asked for, and still exited 0. The
|
||||
reasoning for each is in `context/code/formula-and-io.md`, "A CSV field is
|
||||
text, and the field's role reads it" and "An import refuses what it cannot
|
||||
keep". The CLI's contract:
|
||||
|
||||
- **`--category` keeps the CSV text.** A forced category's items are the
|
||||
file's text, verbatim: `00565796` stays `00565796`, and `220944e2` stays
|
||||
`220944e2`. Measures still read as numbers. A JSON number has no source
|
||||
text, so it is named as JSON writes it (`1e3` is `1000.0`).
|
||||
- **An unparseable `--formula` is a refusal.** It exits non-zero and writes
|
||||
nothing. Every bad formula is named on stderr, each with its parse error.
|
||||
This is §10's rule: the request was not met, so the exit code says so. The
|
||||
wizard refuses the same text at the formula step, with the same wording.
|
||||
- **An import that kept no record is a refusal.** It exits non-zero, writes
|
||||
nothing, and says why: every record was skipped as not a JSON object, or
|
||||
the source had none.
|
||||
- **An import that skipped *some* records exits 0 by default.** It writes the
|
||||
model and warns with a count. With `--strict` (the user's ruling on PR #321)
|
||||
any skipped record is a refusal instead: exit 1, the same count named, and
|
||||
nothing written. The argument is in `formula-and-io.md`.
|
||||
|
||||
Two refusals bail in `run_headless_import` before the build: the `--strict`
|
||||
skip check (`src/main.rs:305`) and the `--on-duplicate refuse` collision
|
||||
check. The rest come from `build_model_with_warnings` as an `Err`. All of
|
||||
them reach `main` as exit 1 before `persistence::save`. So "writes nothing"
|
||||
holds by statement order, not by where the refusal is raised. The headless
|
||||
tests check it by asserting that the output path does not exist.
|
||||
|
||||
## 14 — the command language reads `|pipe quoting|`, and a surplus argument is refused (improvise-31oe.3)
|
||||
|
||||
`create-view |By area|` used to make a view named `|By` and drop `area|` without
|
||||
a word. Two separate defects, fixed separately.
|
||||
@@ -454,7 +487,7 @@ Total = Widget + Gadget` still says.
|
||||
items or coordinates. `no_command_silently_drops_an_argument_it_was_given` names
|
||||
that list, so widening another command to `Any` has to add it there.
|
||||
|
||||
## 14 — a command that names something the document lacks is refused (improvise-31oe.2)
|
||||
## 15 — a command that names something the document lacks is refused (improvise-31oe.2)
|
||||
|
||||
`set-axis Regoin row`, `set-page Region Nowhere` and `set-axis Region sideways`
|
||||
all exited 0 and said nothing. §10's rule applies: nothing changed, so the
|
||||
@@ -464,14 +497,15 @@ request was not met.
|
||||
with `parse_axis` when it is parsed, so a script is refused before any of it
|
||||
runs. `execute` still refuses an axis it cannot read rather than returning
|
||||
nothing. That arm cannot be reached through the registry.
|
||||
- **An unknown category, item, group or view is a refusal**, through
|
||||
`effect::refuse` in `execute`, because only the context knows what the
|
||||
document holds. `unknown_category` asks the active view (`try_axis_of`). That
|
||||
is exactly the condition under which `View::set_axis` was a silent no-op.
|
||||
`unknown_item` asks `effective_item_names`, so a formula target is a known
|
||||
`_Measure` item. `unknown_group` asks both an item's own group and the
|
||||
category's `groups`. `add_item_in_group` registers no `Group`, and a parent
|
||||
group has no items, so either one alone misses a real group.
|
||||
- **An unknown category, item, group or view is a refusal**, from `RequireKnown`
|
||||
(below), because only the document knows what it holds. `effect::unknown_name`
|
||||
asks the active view (`try_axis_of`) for a category. That is exactly the
|
||||
condition under which `View::set_axis` was a silent no-op. It asks
|
||||
`effective_item_names` for an item, so a formula target is a known `_Measure`
|
||||
item. It asks both an item's own group and the category's `groups` for a
|
||||
group. `add_item_in_group` registers no `Group`, and a parent group has no
|
||||
items, so either one alone misses a real group. A view is asked of
|
||||
`workbook.views`.
|
||||
|
||||
The siblings fixed with it: `hide-item` and `show-item` (hiding an unknown item
|
||||
wrote it into `hidden_items`), `toggle-group`, and `switch-view` (the effect
|
||||
@@ -493,6 +527,13 @@ stay non-refusing, since a refusing effect there becomes a frame rejection
|
||||
a browser composes these only from names it was sent. It is left for a ticket
|
||||
of its own.
|
||||
|
||||
A refused name stops the rest of a `:` line, because the abort ends the one batch
|
||||
the whole line runs as. Headless runs each segment as its own batch, so an abort
|
||||
never crosses a `.` there: `:set-page Type Nowhere . add-item Type X` adds
|
||||
nothing, while `improvise cmd` on the same line exits 1 and still adds `X`.
|
||||
`load`/`Save` already behaved this way. The two front ends' chain semantics are
|
||||
left for a ticket of their own.
|
||||
|
||||
The interactive cursor fallbacks of `clear-cell`, `yank`, `paste` and
|
||||
`drill-into-cell` answer only for coordinates that were OMITTED
|
||||
(`only_when_omitted`). Coordinates that were given and are malformed are
|
||||
|
||||
@@ -2999,7 +2999,7 @@ and the re-export keeps `crate::command::key::…` paths working.
|
||||
line separated by `.`, coordinate pairs written `Category/Item`, verbatim quoted
|
||||
strings (`"Profit = Revenue - Cost"`) and `|pipe-quoted names|` read by
|
||||
`improvise_formula::quote`. One lexer serves every entry point, so the separator
|
||||
and the quoting cannot disagree (improvise-31oe.3, `context/code/cli.md` §13).
|
||||
and the quoting cannot disagree (improvise-31oe.3, `context/code/cli.md` §14).
|
||||
|
||||
### `cmd/core.rs`
|
||||
|
||||
@@ -3020,7 +3020,7 @@ Every registration also declares its `MaxArgs`, and the registry refuses surplus
|
||||
arguments on both arms after the command is built. `register_nullary` declares
|
||||
`AtMost(0)` for itself, and an `effect_cmd!` declares its limit in the macro
|
||||
call as `MAX_ARGS`. So the arity rule is written once and is total over the
|
||||
registry (improvise-31oe.3, `context/code/cli.md` §13).
|
||||
registry (improvise-31oe.3, `context/code/cli.md` §14).
|
||||
|
||||
`write_buffer` is the effect that writes a named buffer — the mirror of
|
||||
`read_buffer`, and the only place that knows "search" is not an entry in the buffers
|
||||
@@ -3380,13 +3380,13 @@ The parseable model-mutation commands are thin `Cmd` wrappers around effects,
|
||||
constructible from string args, sharing the same execution path as
|
||||
keymap-dispatched commands (gotcha 16).
|
||||
|
||||
A wrapper that names a category, item or group checks it in `execute` through
|
||||
`unknown_category`, `unknown_item` or `unknown_group`, `switch-view` checks
|
||||
`workbook.views`, and each emits
|
||||
`effect::refuse` in place of its effect when the document lacks it
|
||||
(improvise-31oe.2). The effects stay silent about unknown names, because they
|
||||
are also the wire's mutation path. Why the check sits here and not there:
|
||||
`context/code/cli.md` §14.
|
||||
A wrapper that names a category, item, group or view returns
|
||||
`effect::RequireKnown(Needs)` followed by its real effect (improvise-31oe.2).
|
||||
`RequireKnown` asks `unknown_name` against the live app when the batch applies
|
||||
and refuses and aborts if the document lacks the name, so a `:` line sees what
|
||||
its earlier segments made. The effects themselves stay silent about unknown
|
||||
names, because they are also the wire's mutation path. Why the check is a
|
||||
separate effect: `context/code/cli.md` §15.
|
||||
|
||||
### `cmd/wizard.rs` (improvise-2wm, improvise-907)
|
||||
|
||||
|
||||
@@ -658,6 +658,118 @@ one proves nothing.
|
||||
backend rather than only a file (improvise-fuj); the path-taking wrappers remain
|
||||
for the native CLI.
|
||||
|
||||
### A CSV field is text, and the field's role reads it (improvise-8jk6.4)
|
||||
|
||||
The parser used to type every field as it read it: text that parsed as an
|
||||
`i64` or a finite `f64` became a `Value::Number`. That is a lossy step, and it
|
||||
ran before anyone knew what the column was for. A column the user then forced
|
||||
to Category (`--category`, or the wizard's `c`) took its item names from the
|
||||
number's display, not from the file. So commit `00565796` became item `565796`,
|
||||
and `220944e2` was read as scientific notation and became `22094400.0`. The
|
||||
repo-stats pipeline (improvise-1xfb.5.1) found it.
|
||||
|
||||
CSV has no types. Whether `00565796` is a number depends on whether its column
|
||||
is a measure, and the parser cannot know that. So the parser now yields every
|
||||
non-empty field as a `Value::String` of its text (blank is still `Null`), and
|
||||
two readers apply the old numeric rule, `csv_parser::number_in`, only where a
|
||||
number is wanted:
|
||||
|
||||
- **Analysis** proposes a kind from a typed copy of the records
|
||||
(`Scalars::typed_record`), so a numeric CSV column is still proposed as a
|
||||
Measure, exactly as before.
|
||||
- **A measure** takes its value through `Scalars::number`.
|
||||
|
||||
A category or a label is named from the text itself, so it is verbatim.
|
||||
|
||||
`ImportPipeline` carries the reading as a private `scalars: Scalars`:
|
||||
`Typed` for JSON and `Text` for CSV. JSON cannot share the text rule, because
|
||||
a JSON string is text by the file's own say. `"02139"` in JSON is a zip code,
|
||||
and reading it as a number would propose it as a Measure. A JSON number
|
||||
forced to Category has no source text to keep, because serde_json does not
|
||||
keep one. It is named by `Value`'s own display, which is what writing it back
|
||||
as JSON would say: `42`, `1.5`, and `1000.0` for `1e3`.
|
||||
|
||||
**The parser returns `CsvRecords`, not `Vec<Value>`.** It derefs to
|
||||
`[Value]`, so a caller can read it. It converts to a `Vec` only inside the
|
||||
crate, and the only public way into a pipeline is `ImportPipeline::from_csv`.
|
||||
This is not caution for its own sake. When the parser first changed, eleven
|
||||
tests in four files failed. Every one handed parser output to
|
||||
`ImportPipeline::new`, the JSON constructor, which would read every column as
|
||||
text and propose `Revenue` as a Category. Production had the same shape at all
|
||||
three CSV entry points: `main.rs`'s single file, `ImportPipeline::from_merge`
|
||||
(the merge path, from PR #314), and `ImportJsonHeadless`. `from_merge` merged
|
||||
in with `Self::new` and now calls `Self::from_csv`. Now that mistake does not
|
||||
compile. Two holes remain, and
|
||||
both take a deliberate step. `ImportPipeline::records` is still a public
|
||||
`Vec<Value>`, and a test helper once copied it out of a CSV pipeline into a
|
||||
JSON one. And `CsvRecords` derefs to a slice that `.to_vec()` will copy.
|
||||
|
||||
### An import refuses what it cannot keep (improvise-8jk6.5, improvise-8jk6.6)
|
||||
|
||||
Two failures used to end in an import that saved and reported success.
|
||||
|
||||
**A formula that does not parse.** `build_model_with_warnings` ran
|
||||
`if let Ok(formula) = parse_formula(..)`, so a bad `--formula` vanished. The
|
||||
headless run then saved and exited 0, and the wizard kept the bad text in its
|
||||
list and dropped it the same way at build. Both now go through one
|
||||
function, `import_formula`, whose error names the formula and carries pest's
|
||||
parse error: `Formula error in 'Y = x +': …`. The prefix matches the TUI
|
||||
formula minibuffer's `Formula error:`.
|
||||
|
||||
- `ImportPipeline::parsed_formulas` parses them all. If any fails, it refuses
|
||||
and names **every** formula that failed, one per line (a message the user
|
||||
needs is never truncated). The build calls it before creating anything, so a
|
||||
refusal writes nothing.
|
||||
- The wizard checks at `confirm_formula`, the one place the user can still fix
|
||||
the text (gotcha 22). A formula that does not parse stays in the buffer with
|
||||
the error as the step's message, and never enters the list.
|
||||
|
||||
**Nothing kept.** The one skip left after improvise-8jk6.2 is a record that
|
||||
is not a JSON object. When every record is skipped, there are no fields, so
|
||||
there are no proposals and no categories. The build did refuse that case, but
|
||||
by accident, with "At least one category must be accepted". That tells the
|
||||
user to change a field's kind when there are no fields. The ticket's own repro
|
||||
(`[1, 2, "three"]`) already exited 1 on the lpzu base, so the headline did not
|
||||
reproduce there. What did reproduce was the reason. `nothing_to_import` now
|
||||
states the rule and runs before the category check: `Nothing to import: all 3
|
||||
records were skipped as not JSON objects`, or `the source has no records`. The
|
||||
wizard asks it first on `ReviewProposals`, where another array path can still
|
||||
be chosen.
|
||||
|
||||
One case meets PR #314's "a header-only file is still a file". A folder whose
|
||||
CSVs are **all** header-only has no records, only sources. Built from
|
||||
`from_merge`'s seeded proposals alone, it would save `File` items, empty
|
||||
columns and one empty records view per file. It is refused here, because it
|
||||
kept no record. A header-only file *beside* files with rows is unaffected,
|
||||
since some record was kept. The user confirmed this on PR #321: "Refuse (as
|
||||
#321 does)".
|
||||
|
||||
**A partial skip stays a warning.** Some records kept and some skipped still
|
||||
builds, exits 0, and warns with a count: `Skipped 1 of 3 records that were not
|
||||
JSON objects`. The argument: a non-object in an array of records is not a
|
||||
record that failed to import. It is structurally not a record, such as a
|
||||
trailing `null` or a stray scalar in a hand-edited file. Every record that
|
||||
*is* a record was kept, and the count is on stderr for a script to read.
|
||||
Making it fatal would fail every import that carries one stray element.
|
||||
Making it a distinct non-zero exit that still writes gives a contract where
|
||||
`import … && next` stops even though the file was written.
|
||||
|
||||
The user ruled on PR #321: "Add a --strict flag". The default stays exit 0
|
||||
with the counted warning, pinned by
|
||||
`an_import_that_skips_some_records_warns_with_a_count_and_still_writes`.
|
||||
`import --no-wizard --strict` turns any skipped record into a refusal:
|
||||
`--strict: Skipped 1 of 3 records that were not JSON objects`, exit 1.
|
||||
|
||||
**A strict refusal writes nothing.** A refusal that still wrote the file would
|
||||
be the half-met contract argued against above: `&&` stops while the output
|
||||
sits there, and a script that checks only the file gets a model that `--strict`
|
||||
existed to prevent. So the check runs in `run_headless_import` before the
|
||||
build. It reads `ImportPipeline::skip_warning`, the same function the build's
|
||||
warning comes from, so the two cannot disagree about what was skipped.
|
||||
"Skipped" here means a record that is not an object, the only skip left after
|
||||
8jk6.2. An unparsed date or a short CSV row is a warning about a record that
|
||||
was kept, and `--strict` does not refuse it.
|
||||
|
||||
### A folder of CSVs is one sparse model, with a view per file (improvise-lpzu)
|
||||
|
||||
`improvise import <folder>` exists for improvise-1xfb.5.1. code-maat writes one
|
||||
|
||||
@@ -448,6 +448,57 @@ Rivals R2, R7 and R9 each fail it. Since the review of `4e68372c` the fixture
|
||||
also holds a header-only `soc.csv`. Its view must carry `File` at `soc`, be a
|
||||
records view, and draw zero rows. Rivals F7 and F8 fail that half.
|
||||
|
||||
### a_csv_column_forced_to_category_keeps_its_text_verbatim
|
||||
|
||||
improvise-8jk6.4. Before the fix, `commit,lines` with `00565796` and
|
||||
`220944e2` forced to Category imported as items `565796` and `22094400.0`.
|
||||
The CSV parser had typed both as numbers, and the category was named from the
|
||||
number. On the base this failed with the items line `- |565796|, |22094400.0|`.
|
||||
The test also asserts both Data lines, which proves the measure `lines` still
|
||||
reads 3 and 5 through the same records.
|
||||
|
||||
- **R1**, a category named from the typed value, fails it.
|
||||
- **R2**, a CSV measure read as a JSON value (`as_f64` on the text), fails it
|
||||
on the Data lines.
|
||||
|
||||
### a_json_number_forced_to_category_is_named_as_json_writes_it
|
||||
|
||||
improvise-8jk6.4, the JSON half. This is not a bug demonstration: it passed
|
||||
on the base, and it pins that the fix left JSON alone. JSON keeps no source
|
||||
text for a number, so `42`, `1.5` and `1e3` name items `42`, `1.5` and
|
||||
`1000.0`. `1e3` is the discriminating value, because its source text, its f64
|
||||
display (`1000`) and its JSON display (`1000.0`) all differ. **R10**, naming a
|
||||
number by its `f64` display, fails it on `1000.0`.
|
||||
|
||||
### an_import_formula_that_does_not_parse_refuses_names_it_and_writes_nothing
|
||||
|
||||
improvise-8jk6.5. Before the fix, `--formula 'Y = x +'` was dropped: the run
|
||||
printed `Saved to …` and exited 0. That is the base failure (`a dropped
|
||||
formula exited 0`). The test passes a good formula too (`Z = x`) and asserts
|
||||
it is not named, so a refusal that lists every formula cannot pass.
|
||||
`expected` comes from pest's error, which proves the parse error is carried
|
||||
and not only the formula text. **R4**, a build that drops what does not
|
||||
parse, fails it. Since improvise-8jk6.17 it passes two bad formulas, so
|
||||
**R5**, naming only the first, fails it here too. At `efaa5050` only the io
|
||||
unit test saw R5.
|
||||
|
||||
### an_import_that_keeps_no_record_refuses_and_says_why
|
||||
|
||||
improvise-8jk6.6. On the lpzu base `[1, 2, "three"]` already exited 1, so the
|
||||
exit-code half was not the bug. The reason was: `At least one category must
|
||||
be accepted`, which is true only because no fields exist, and which points
|
||||
the user at a fix that cannot help. The base failure was on the message.
|
||||
**R7**, removing `nothing_to_import` from the build, fails it the same way.
|
||||
|
||||
### an_import_that_skips_some_records_warns_with_a_count_and_still_writes
|
||||
|
||||
improvise-8jk6.6, the partial case. It passed on the base. It pins the
|
||||
default the user kept on PR #321 ("Add a --strict flag"), so that changing
|
||||
the default is a decision and not a drift. It asserts the warning's count (`1 of 3`), exit 0,
|
||||
and a Data line from the record *after* the skipped one. That last check
|
||||
shows the skip did not end the import early. **R9**, making any skip fatal,
|
||||
fails it.
|
||||
|
||||
### a_folder_of_one_csv_still_gets_a_file_category_and_its_view_but_a_lone_file_does_not
|
||||
|
||||
improvise-lpzu, after the user's ruling on PR #314 ("we'd want file identity").
|
||||
@@ -510,11 +561,14 @@ column`, with an empty status. Each row asserts the refusal flag, the missing
|
||||
name in the message, and that the document's `.improv` text and the active view
|
||||
are unchanged. The last two catch a refusal written after the write. The
|
||||
control is `a_command_naming_what_the_document_has_is_not_refused`, whose group
|
||||
row is the one that caught `unknown_group` reading only `Category::groups`:
|
||||
`add_item_in_group` registers no `Group`. Rivals, each run: `unknown_item`
|
||||
answering `None` fails at `set-page ["Type", "Nowhere"]`. `unknown_category`
|
||||
answering `None` fails at the first row. The `switch-view` check disabled fails
|
||||
at its row.
|
||||
row is the one that caught a group check reading only `Category::groups`:
|
||||
`add_item_in_group` registers no `Group`. The checks are `effect::unknown_name`,
|
||||
run by `RequireKnown` when the batch applies. Rivals, each run: the
|
||||
`Needs::Item` arm answering `None` fails at `set-page ["Type", "Nowhere"]`. The
|
||||
category test (`try_axis_of(..).is_none()`) never firing fails at the first
|
||||
row. The `Needs::View` arm answering `None` fails at the `switch-view` row.
|
||||
`RequireKnown` placed after the write, or refusing without `refuse_and_stop`'s
|
||||
abort, fails the document-unchanged assertion.
|
||||
|
||||
### a_colon_line_naming_something_missing_refuses_on_the_status_line
|
||||
|
||||
@@ -565,3 +619,42 @@ name checks in `execute` refused `create-view Second . switch-view Second` and
|
||||
`add-item Type Tools . set-page Type Tools` after the first half landed. The
|
||||
checks now run when the effect applies. Rival: the checks back in `execute`
|
||||
fails both lines with `No view named "Second"` and `Unknown item 'Tools'`.
|
||||
|
||||
### a_strict_import_that_skips_a_record_refuses_and_writes_nothing
|
||||
|
||||
improvise-8jk6.6, the user's ruling on PR #321. Written before `--strict`
|
||||
existed, it failed on clap's unknown-argument error. It asserts the refusal
|
||||
names the flag and the same count as the default warning, and that the output
|
||||
path does not exist.
|
||||
|
||||
- **S1**, the flag parsed but ignored, fails it on the exit status.
|
||||
- **S3**, refusing only after `persistence::save`, fails it on the output
|
||||
path. That is the "a strict refusal still writes" contract this PR decided
|
||||
against.
|
||||
|
||||
### a_strict_import_that_skips_nothing_still_writes
|
||||
|
||||
The control for the entry above. Without it, a `--strict` that refuses
|
||||
**every** import passes that test. **S2** is that rival, and it fails here.
|
||||
It failed before the flag existed too, but only on clap's error, which is not
|
||||
the discriminating failure. The discriminating one is S2.
|
||||
|
||||
### a_csv_decimal_measure_imports_as_its_number
|
||||
|
||||
improvise-8jk6.17 (F3), from the adversarial review of #321. The fixture has
|
||||
decimal CSV measures (`1000.50`, `-2.25`). With `number_in`'s `f64` branch
|
||||
disabled (integers only), no headless test failed at `efaa5050`, and a decimal
|
||||
measure was silently dropped. **F3** now fails this test and the next one.
|
||||
This is coverage, not a bug: it passed when written.
|
||||
|
||||
### import_json_on_a_csv_reads_its_numeric_column_as_a_measure
|
||||
|
||||
improvise-8jk6.17 (F1). `improvise cmd 'import-json f.csv'` reaches
|
||||
`ImportJsonHeadless`, a third CSV entry point besides `import`. **F1**
|
||||
restores its CSV branch to the JSON constructor via
|
||||
`ImportPipeline::new(Value::Array(recs.to_vec()))`. That still compiles,
|
||||
because `CsvRecords` derefs to a slice. `Revenue` then becomes a category of
|
||||
text items, and this test fails on its Data lines. At `efaa5050` only
|
||||
`import_headless_refuses_records_that_collide` failed under F1, and only by
|
||||
accident. The `## Category: Revenue` assertion names the symptom directly.
|
||||
This is coverage: it passed when written.
|
||||
|
||||
@@ -882,6 +882,11 @@ The control: a **lone** `\r` is itself a record terminator to this reader, so
|
||||
it is not part of a CRLF pair, and it is why "two records, clean values" is a
|
||||
real observation rather than a coincidence.
|
||||
|
||||
Since improvise-8jk6.4 the parser yields text, so the assertion is that
|
||||
`Revenue` is the text `1`. A kept `\r` would make it `1\r`, which fails that
|
||||
assertion, and which `number_in` would not read as a number where a measure
|
||||
wanted one.
|
||||
|
||||
### an_empty_file_is_no_records_and_no_warnings
|
||||
|
||||
An empty file is no records and no complaint. The wizard's own preconditions
|
||||
@@ -919,12 +924,17 @@ The consequence downstream is deliberate and worth stating: a column carrying
|
||||
Label rather than a Measure. That is the honest answer — the column is not
|
||||
numeric — and it is visible in the wizard, where a fabricated zero was not.
|
||||
|
||||
Since improvise-8jk6.4 the numeric rule is `csv_parser::number_in`, applied
|
||||
through `typed` only where a number is wanted, and the parser itself yields
|
||||
text. So this test asserts that `typed` leaves each token as text. The
|
||||
`unwrap_or` fault would be injected into `number_in` now.
|
||||
|
||||
### an_ordinary_float_is_still_a_number
|
||||
|
||||
The control for the entry above. Guarding non-finiteness by deleting the float
|
||||
branch entirely would also pass that test, so this one pins that `1000.50` is
|
||||
still a `Number` and that `0` — the value the bug invented — is still reachable
|
||||
when the file really says so.
|
||||
when the file really says so. Since improvise-8jk6.4 both go through `typed`.
|
||||
|
||||
### a_repeated_header_is_renamed_rather_than_overwriting_its_twin
|
||||
|
||||
@@ -1818,6 +1828,68 @@ document still has the three views in order, `File: filter` at the file's item,
|
||||
and the same drawn values. This is the first persisted, non-virtual use of
|
||||
`Axis::Filter` (gotcha 27). R7 and R9 fail it.
|
||||
|
||||
### a_csv_column_read_as_a_category_keeps_its_text_and_as_a_measure_its_number
|
||||
|
||||
improvise-8jk6.4, at the pipeline. It checks three things in one fixture,
|
||||
one per reader of a CSV value:
|
||||
|
||||
- analysis still proposes the numeric-looking `commit` as a Measure (typed);
|
||||
- forced to Category, its items are `00565796` and `220944e2` (text);
|
||||
- `lines` is stored as the number 5 (typed).
|
||||
|
||||
The rivals, each run:
|
||||
|
||||
- **R1**, a category named from the typed value, fails it on the items.
|
||||
- **R2**, a CSV measure read with `as_f64` on its text, fails it on the cell.
|
||||
- **R3**, CSV analysed without typing, fails it on the proposal.
|
||||
|
||||
R2 and R3 also fail the five lpzu merge tests in this module and five
|
||||
headless imports. That is how much of the suite leans on CSV measures reading
|
||||
as numbers.
|
||||
|
||||
This test uses `from_csv` and so cannot compile on the base. The base failure
|
||||
for 8jk6.4 is the headless test
|
||||
`a_csv_column_forced_to_category_keeps_its_text_verbatim`.
|
||||
|
||||
### a_formula_that_does_not_parse_stops_the_build_and_is_named
|
||||
|
||||
improvise-8jk6.5. On the base the build returned `Ok`, keeping `Good` and
|
||||
silently dropping `Bad`. Two formulas fail to parse here, so the test proves
|
||||
every failure is named. **R5**, naming only the first, fails it on `Worse`,
|
||||
and since improvise-8jk6.17 the headless
|
||||
`an_import_formula_that_does_not_parse_refuses_names_it_and_writes_nothing`
|
||||
sees R5 too (it passes two bad formulas). It also asserts `Good` is not named, so a
|
||||
message that lists every formula cannot pass. **R4** (drop, as the base did)
|
||||
fails it.
|
||||
|
||||
### confirming_a_formula_that_does_not_parse_keeps_it_out_and_the_text_in
|
||||
|
||||
improvise-8jk6.5, the wizard half. On the base `confirm_formula` pushed
|
||||
`Bad = revenue +` into the list, which failed on `pipeline.formulas`. The test
|
||||
asserts the text is still in the buffer and the editor still open. A fix that
|
||||
refused but cleared the buffer would throw away what the user typed, the same
|
||||
hazard improvise-v4hg fixed in the TUI formula minibuffer. **R6**, confirming
|
||||
regardless, fails it. The headless suite cannot see R6.
|
||||
|
||||
### a_build_that_keeps_no_record_refuses_and_says_why
|
||||
|
||||
improvise-8jk6.6. On the base both halves got `At least one category must be
|
||||
accepted`, a refusal right only by accident. It asserts the exact message for
|
||||
the two cases `nothing_to_import` distinguishes: all skipped, and no records
|
||||
at all.
|
||||
|
||||
- **R7**, the check removed from the build, fails it.
|
||||
- **R9**, `any` swapped for `all`, also fails it, on the empty source: `all`
|
||||
is vacuously true over no records.
|
||||
|
||||
### the_review_step_says_when_no_record_can_be_imported
|
||||
|
||||
improvise-8jk6.6, the wizard half (gotcha 22: the check lives on the step
|
||||
where the user can still choose another array path). On the base the step
|
||||
refused with `No categories: … Press c to cycle a field's kind`, which offers
|
||||
a key that has no field to act on. **R8**, the step without the check, fails
|
||||
it. The build test does not see R8.
|
||||
|
||||
### a_header_only_file_keeps_its_file_item_its_columns_and_its_view
|
||||
|
||||
The ChatGPT Codex P2 on PR #314 at `4e68372c`, confirmed by the adversarial
|
||||
@@ -1866,3 +1938,14 @@ improvise-8jk6.12 (4). A `region` of `" "` is kept as a record with no
|
||||
predates this PR (improvise-ych), but the sparse-record rule now rests on it:
|
||||
without it the value becomes an item named `" "`. Rival F4 (the reviewer's
|
||||
fault) fails it.
|
||||
|
||||
### a_folder_of_only_header_only_files_keeps_no_record_and_refuses
|
||||
|
||||
improvise-8jk6.6 meeting PR #314's rule that a header-only file is still a
|
||||
file. Written after merging `main` at `c91cc055`, where the two rules first
|
||||
met. The file seeds give the pipeline a `File` Category proposal even with no
|
||||
records. So without `nothing_to_import` the build succeeds and saves `File`
|
||||
items and empty views: the category check alone cannot refuse. The `setup`
|
||||
assertion proves that seeded proposal exists, so the test is not passing only
|
||||
because there was no category. **R7**, the check removed from the build, fails
|
||||
it. The user confirmed the refusal on PR #321: "Refuse (as #321 does)".
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
# Layout and keypress cost (improvise-felo.13)
|
||||
|
||||
Measured on an **Apple M4 Max, 64 GB**, macOS, release build, with
|
||||
`benches/scale_bisect.rs` (merged from `bench/scale-bisect`, `13918bb9`), one
|
||||
configuration per process, 3 categories of K items on Row, N = 100,000 capped at
|
||||
K^3/2 records, prune off unless stated. Baseline is this branch with only the
|
||||
tests added; "after" is the branch head. `key` is `App::dispatch_key(Down)`;
|
||||
"cold" is the first key after open or after an edit, which pays the one
|
||||
whole-grid measurement; "warm" is every key after it.
|
||||
|
||||
## Before and after
|
||||
|
||||
| rows | before key | after key warm | after key cold | before browser window | after browser window | SetCell before / after |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 729 (K=9) | 5.5 | 0.00 | 1.4 | 5.0 | 0.13 | 0.9 / 0.9 |
|
||||
| 4,913 (K=17) | 108 | 0.00 | 6.0 | 105 | 0.12 | 5.8 / 5.9 |
|
||||
| 5,832 (K=18) | | 0.00 | 7.7 | | 0.12 | 6.8 |
|
||||
| 6,859 (K=19) | | 0.00 | 8.8 | | 0.12 | 8.6 |
|
||||
| 19,683 (K=27) | 1,271 | 0.00 | 24.5 | 1,259 | 0.14 | 23.0 / 22.7 |
|
||||
| 64,000 (K=40) | 10,771 | 0.00 | 84 | 10,746 | 0.20 | 79 / 80 |
|
||||
|
||||
Prune on: K=17 (2,456 rows) key 264 ms -> 0.00 warm, 3.6 cold; prune toggle
|
||||
172 -> 8.8 ms; K=27 prune toggle 2,189 -> 41.5 ms. Records mode (3 x 1000, 14,500
|
||||
records): key 1,004 ms -> 0.00 warm, 2.8 cold; **SetCell 1,511 ms -> 1,512 ms,
|
||||
unchanged** (not this ticket's cost).
|
||||
|
||||
Boundaries, warm key and browser window: inside 8 ms at every size measured (to
|
||||
216,000 rows, 303 ms cold). Cold key: inside 8 ms to K=18 (5,832 rows), over 8 ms at
|
||||
K=19. Before: inside 8 ms to 729 rows, over 100 ms at 4,913, over 1 s at 19,683.
|
||||
The remaining cold cost is linear (1.2-1.4 us per row) and is the whole-grid width
|
||||
and total pass; the warm path does not repeat it.
|
||||
|
||||
## What the ticket's reading got right and what it did not predict
|
||||
|
||||
The O(rows^2) in `cell_key` was real and is one of three causes. Two more sat
|
||||
under it. First, `compute_col_widths` ran on **every** keypress, through
|
||||
`cmd_context`, and `dispatch_key` and `apply_effects` each rebuilt the layout
|
||||
whether or not anything had moved. Second, once the index was O(1) the width pass
|
||||
was still O(K^2) per cell: with `_Index`/`_Dim` on `Axis::None` the aggregation is
|
||||
not `Exact`, so each cell went through `DataStore::matching_values`, which
|
||||
narrows by the smallest index set (about N/K candidates) and probes the rest. A
|
||||
partial as wide as the widest stored cell can match only one cell, so
|
||||
`exact_positions` probes it directly. That second cause is why K=40's cold key
|
||||
stayed at 8 s after the first two fixes.
|
||||
|
||||
### a_cursor_key_rebuilds_neither_the_layout_nor_the_widths
|
||||
|
||||
`ui::app::tests`. Two keys after a warm-up leave `layout_builds` and
|
||||
`width_passes` where they were. Failed before the fix on the layout count (a
|
||||
cursor key rebuilt it). Rival 1 (`refresh_layout` always rebuilding): fails on
|
||||
`layout rebuilt on a cursor key`. The test seats the cursor at row 2, so "the
|
||||
keys moved the cursor" is asserted and a key that does nothing cannot pass.
|
||||
|
||||
### an_edit_rebuilds_and_remeasures_so_the_widths_follow_the_value
|
||||
|
||||
The control for the test above, and the guard on invalidation. A `SetCell` of a
|
||||
long text must rebuild the layout and remeasure, and the cached widths must equal
|
||||
a fresh `compute_col_widths`. Rival 2 (`rebuild_layout` not dropping the cached
|
||||
geometry): fails on `an edit must remeasure`. It also asserts the widths changed,
|
||||
so a fixture where the edit does not widen a column cannot pass.
|
||||
|
||||
### switching_view_rebuilds_though_the_effect_is_view_only
|
||||
|
||||
`SwitchView` is `ViewOnly` (gotcha 3), so skipping the rebuild by `EffectKind`
|
||||
alone would leave the old view's layout. The layout records the view it drew and
|
||||
the drill snapshot it froze, and rebuilds when either differs.
|
||||
|
||||
### an_axis_lookup_does_work_independent_of_the_row_index
|
||||
|
||||
`view::layout::lookup_cost_tests`. Counts the items `data_items` yields during
|
||||
`cell_key`, `row_tuple`, `row_label` and `column_key`; row 399 of 400 must cost no
|
||||
more than row 0 plus two. Failed before the fix: `cell_key: row 399 cost 402 steps
|
||||
against 3 at row 0`. Rival (`cell_key` going back to `data_items(..).nth(row)`):
|
||||
`cost 400 steps against 1 at row 0`. Limit: the counter lives in `data_items`, so
|
||||
a different linear walk written without it is not seen; the next test is what
|
||||
holds the index to the walk's answers.
|
||||
|
||||
### the_position_index_answers_what_the_walk_answered
|
||||
|
||||
Compares `row_tuple`, `data_row_to_visual` and `row_folds_a_group` against the
|
||||
walk over a grouped, collapsed and pruned axis, including two indices past the
|
||||
end. Rival (`data_index_to_visual` reading `data_idx + 1`): fails at `visual 0`.
|
||||
|
||||
### a_full_width_partial_probes_one_cell_instead_of_narrowing
|
||||
|
||||
`model::cell::exact_path_tests`. A full-width partial must not reach set
|
||||
narrowing, and an absent cell and an unseen item must match nothing. Rival
|
||||
(`exact_positions` returning `None`): the narrowing count is 1, not 0. Rival
|
||||
(`widest <= partial.len() + 1`): fails `a_wider_cell_turns_the_shortcut_off_until_it_is_removed`,
|
||||
which stores a cell one coordinate wider and expects it matched.
|
||||
`a_narrower_partial_still_narrows_and_matches_every_superset` is the control: the
|
||||
shortcut is not a blanket exact lookup. `an_unsorted_full_width_partial_is_still_answered`
|
||||
holds the canonical-order condition: a partial not in a stored key's order falls
|
||||
back.
|
||||
|
||||
### a_window_over_a_held_summary_equals_the_one_walk_grid_body
|
||||
|
||||
`ui::viewport::tests`. The host keeps one `GridSummary` (widths, total formats,
|
||||
totals) per layout and reads a window over it with `grid_window`; it must equal
|
||||
what `grid_body` builds for the same window, over the whole grid, an interior
|
||||
window, the last row and the empty window. Rival (`grid_window` reading column
|
||||
`ci + 1`): fails at `window 0..2 x 0..10`. Widths stay the **whole-grid** maximum,
|
||||
because the summary is the same walk `grid_body` did; nothing the user sees moves.
|
||||
|
||||
### a_frozen_records_snapshot_replaces_the_live_row_index
|
||||
|
||||
`view::layout::lookup_cost_tests`. Three records are frozen, two are then cleared,
|
||||
and `with_frozen_records` over the snapshot must draw 3 rows (the control: with no
|
||||
snapshot it draws 1). A drill held open while another client removes records is the
|
||||
production analogue. Rival (deleting `layout.reindex()` after the snapshot swap):
|
||||
fails `left: 1, right: 3`; the whole workspace passed it before this test.
|
||||
|
||||
### a_drill_snapshot_replaced_alone_rebuilds_the_layout
|
||||
|
||||
`ui::app::tests`. `StartDrill` is `ViewOnly`, so a batch of it alone has no `Model`
|
||||
effect to mark the layout stale; the drill pointer in `LayoutBasis` is the only
|
||||
thing that rebuilds it. The test starts one snapshot, then a second: the first
|
||||
step alone passes even with a constant pointer (drill absent to present changes the
|
||||
basis), which is why the second step is the test. Rival (pointer replaced by a
|
||||
constant): fails `no rebuild for a second drill snapshot`.
|
||||
|
||||
### Observation: the TUI frame
|
||||
|
||||
`tui_frame_grid_of_whole` is still 47-53 ms at 19,683 rows after this change: the
|
||||
draw path projects the whole grid, linear and out of scope for this ticket.
|
||||
@@ -0,0 +1,218 @@
|
||||
# Scale bisect: how big a document does improvise handle today (improvise-felo.10)
|
||||
|
||||
Measured on `gh/main` @ 8220a065, **Apple M4 Max, 64 GB** (`sysctl`: `hw.memsize 68719476736`), macOS,
|
||||
release build (`lto`, `opt-level 3`), native only (no wasm, no real OPFS, no terminal).
|
||||
Harness: `benches/scale_bisect.rs` (one configuration per process) driven by `bench/scale_bisect.py`,
|
||||
`bench/sweep.py`, `bench/edge.py`, `bench/full.py`. Reproduce:
|
||||
`nix develop --command cargo build --release --bench scale_bisect`, copy the binary from
|
||||
`target/release/deps/scale_bisect-*` to `/tmp/sb`, then `python3 bench/edge.py`.
|
||||
|
||||
Budgets (user ruling 2026-10-03): **8 ms is the headline** (120 FPS, hot path or defer to background);
|
||||
16 ms, 100 ms (noticeable) and 1 s (broken) are reference crossings.
|
||||
|
||||
## Method, and what is real
|
||||
|
||||
Every number goes through the production path: `App::new` (open: formula recompute +
|
||||
`GridLayout::with_frozen_records`), `App::dispatch_key(Down)` (**keypress/scroll**: it calls
|
||||
`rebuild_layout`, then `cmd_context`, which runs `compute_col_widths` over the whole grid, then applies effects),
|
||||
`project::grid_of` (TUI frame: whole grid; browser: a 45x12 window), `App::apply_effects` with `effect::SetCell`,
|
||||
`SetAxis`/`HideItem`/`ToggleGroup`/`TogglePruneEmpty` (reshape), `Host::connect` and `Host::handle` with a
|
||||
`caught_up` `ModelCommand::SetCell` (browser path incl. projection), `format_md`/`parse_md`/`save_to(MemoryFs)`.
|
||||
Document: C0..C{cats-1} with K items, N distinct random tuples ("records", capped at K^cats/2), two measures
|
||||
each (2N cells), `_Measure` on Column. Timings are the **median** of up to 3-8 in-process repeats; boundary
|
||||
rows are the median of **3 separate processes** run sequentially (spread shown). Earlier exploratory sweeps
|
||||
ran 2-3 processes in parallel and agree with the sequential runs within ~5%.
|
||||
|
||||
**`View::prune_empty` defaults to false** (`view/types.rs`), so "prune off" is what a user gets until they
|
||||
toggle it; "prune on" is reported next to it. Rows below are the layout's `row_count`.
|
||||
|
||||
**Dominant cost (reasoned, not profiled):** the release profile strips symbols, so `sample` could not
|
||||
symbolicate. The reason is in the code and the timings agree with it: `GridLayout::cell_key`/`row_item` do
|
||||
`data_items(&self.row_items).nth(row)` (`view/layout.rs` ~253/292/807/870), O(row) per lookup, and
|
||||
`compute_col_widths` (every keypress, via `cmd_context`) and `grid_of` visit every cell, so both are
|
||||
O(rows^2). Evidence: with prune off, rows 729 -> 5.4 ms, 4096 -> 74.6 ms, 19683 -> 1248 ms (x6.7 rows ->
|
||||
x14 time -> x17 time). A 45x12 browser window costs **the same** as the whole-grid TUI frame (K=30:
|
||||
2274 vs 2298 ms) because the widths pass is whole-grid whatever the viewport says, so viewport narrowing buys
|
||||
nothing today. `GridLayout` build itself is cheap (~90 ns/row, ADR 008).
|
||||
|
||||
## Attribution: state+projection vs render/serialize (K=10, 1000 rows, prune off; 3 runs)
|
||||
|
||||
| step | ms |
|
||||
|---|---|
|
||||
| keypress (state: rebuild + widths + effect) | 9.5-9.7 |
|
||||
| `grid_of` whole grid (TUI projection) | 11.7-12.0 |
|
||||
| `GridWidget` render into a 200x50 ratatui `Buffer` | 0.12 |
|
||||
| `grid_of` 45x12 window (browser projection) | 9.1-9.3 |
|
||||
| JSON encode of one SetCell's projected updates | 0.001-0.005 (0.6-1.1 KB) |
|
||||
| JSON encode of the connect Snapshot | 0.02 (21 KB) |
|
||||
|
||||
Render and serialize are **<=0.5 ms in every row measured** (GridWidget 0.08-0.53 ms up to 14,500 records;
|
||||
snapshot encode <=0.07 ms; records-mode SetCell updates 21 KB in 0.02 ms). The 8 ms is entirely state and
|
||||
projection. Not measured: the rest of `draw()` (panels, status; `draw` is private to the binary), terminal I/O,
|
||||
browser JSON *decode* and DOM paint.
|
||||
|
||||
## Axis B: items per category K, 3 categories on Row, N = 100k (capped at K^3/2), prune OFF
|
||||
|
||||
| K | rows | open | keypress | browser window | SetCell | maxrss MB |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 5 | 125 | 0.1 | 0.5 | 0.5 | 0.2 | 8 |
|
||||
| 9 | 729 | 0.2 | **5.4** (5.3-5.5) | 5.0 | 0.9 | 13 |
|
||||
| 10 | 1,000 | 0.2 | **9.6** (9.5-9.7) | 9.2 | 1.2 | 14 |
|
||||
| 11 | 1,331 | 0.3 | 12.7 | 12.7 | 1.6 | 10 |
|
||||
| 12 | 1,728 | 0.3 | 20.0 | 19.3 | 2.1 | 11 |
|
||||
| 15 | 3,375 | 0.5 | 65 | 64 | 4.0 | 14 |
|
||||
| 16 | 4,096 | 0.6 | 74.6 (74.3-75.2) | 73.4 | - | - |
|
||||
| 17 | 4,913 | 0.7 | **105** (105.0-105.5) | 103 | - | - |
|
||||
| 20 | 8,000 | 1.1 | 302 | 301 | 9.4 | 21 |
|
||||
| 25 | 15,625 | 1.9 | 773 | 771 | 18.8 | 33 |
|
||||
| 26 | 17,576 | 2.2 | 967-977 | 962 | - | - |
|
||||
| 27 | 19,683 | 2.4 | **1,248** (1,244-1,251) | 1,244 | - | - |
|
||||
| 30 | 27,000 | 3.1 | 2,262 | 2,274 | 31 | 49 |
|
||||
| 40 | 64,000 | 6.7 | 10,858 | 10,833 | 73 | 235 |
|
||||
| 60 | 216,000 | 21.6 | timeout (>60 s) | - | - | - |
|
||||
| 1000 | 10^9 | **refused** (see below) | | | | |
|
||||
|
||||
Hot-path keypress/scroll and browser window: **largest inside 8 ms: K=9 (729 rows)**; inside 16 ms: K=11;
|
||||
first over 100 ms: **K=17 (4,913 rows)**; first over 1 s: **K=27 (19,683 rows)**. SetCell alone
|
||||
(no widths pass; layout rebuild only, ~linear in rows): inside 8 ms to K~19 (K=15 4.0, K=20 9.4), 16 ms ~K=24,
|
||||
100 ms ~K=45 (K=40 = 73 ms), 1 s extrapolated ~K=95 (**extrapolated, not measured**). Open stays <25 ms to K=60.
|
||||
|
||||
**3 x 1,000 on Row cannot be opened at any N**: the layout materializes the cross product (10^9 rows, ~150 B
|
||||
each = ~150 GB, ADR 008). The harness refuses above 30M rows (`rows_cap`) rather than swap the machine, so
|
||||
the 10^9 number is the ADR's estimate, not a measurement here.
|
||||
|
||||
### Prune ON (same documents)
|
||||
|
||||
| K | rows | open | keypress | browser window | SetCell |
|
||||
|---|---|---|---|---|---|
|
||||
| 7 | 171 | 2.0 | 4.5 (4.4-4.6) | 1.1 | 5.4 |
|
||||
| 8 | 256 | 3.0 | **7.0** (7.0-7.1) | 1.6 | 8.5 |
|
||||
| 9 | 364 | 5.3 | 12.8 | 2.7 | 15.3 |
|
||||
| 11 | 665 | 12.2 | 29.6 | 6.2 | ~28 |
|
||||
| 13 | 1,098 | 29.0 | 71 | 14.5 | ~70 |
|
||||
| 14 | 1,372 | 48 | **119** | 24 | - |
|
||||
| 20 | 4,000 | 300 | 747 | 145 | 903 |
|
||||
| 21 | 4,630 | 401 | 968-992 | 190 | - |
|
||||
| 25 | 7,812 | 767 | 1,899 | 366 | 2,328 |
|
||||
| 30 | 13,500 | 2,255 | 5,555 | 1,047 | 6,804 |
|
||||
|
||||
Prune on: keypress inside 8 ms to **K=8**, first over 100 ms **K=14**, ~1 s at **K~21-22** (K=22 measured
|
||||
0.93-0.95 s, a different N cap; first clearly over: K=25). Pruning shrinks the grid, so the browser window is
|
||||
cheaper than prune off at the same K (K=20: 145 ms against 302 ms), but **every keypress re-runs
|
||||
the prune** (`dispatch_key` calls `rebuild_layout`), so keypress and SetCell are *worse* with prune on
|
||||
(K=10: 23 vs 9.9 ms). Prune is also quadratic (`nth`). Conclusion: prune on/off moves the boundary by about
|
||||
one K at 8 ms and makes 100 ms/1 s *earlier* (K=14/21 vs 17/27); it never rescues the 3x1000 case.
|
||||
|
||||
## Axis A: records N at fixed K (K=30, 27,000 rows)
|
||||
|
||||
| N | prune off keypress | prune on rows | prune on keypress | prune on window |
|
||||
|---|---|---|---|---|
|
||||
| 100 | 451 | 100 | 861 | 0.3 |
|
||||
| 1,000 | 527 | 1,000 | 1,059 | 6.4 |
|
||||
| 5,000 | 1,013 | 5,000 | 2,178 | 140 |
|
||||
| 13,500 (cap) | 2,267 | 13,500 | 5,555 | 1,047 |
|
||||
|
||||
Prune off: ~450 ms of widths over 27k empty rows + ~1.3 us per stored cell; **N never gets the keypress
|
||||
under 100 ms at K=30**, and at K=1000 N is irrelevant because the layout cannot be built (rows = K^3).
|
||||
The N-axis has no bisect point in the 3x1000 regime; N only matters once K is small enough that rows fit, and
|
||||
there it is a second-order term. Open is flat (3.1-3.6 ms) in N with prune off.
|
||||
|
||||
## Axis C: categories on Row (prune off)
|
||||
|
||||
| K | cats | rows | open | keypress | note |
|
||||
|---|---|---|---|---|---|
|
||||
| 10 | 2 | 100 | 0.1 | 0.2 | |
|
||||
| 10 | 3 | 1,000 | 0.2 | 9.9 | over 8 ms |
|
||||
| 10 | 4 | 10,000 | 1.4 | **880** | ~1 s |
|
||||
| 10 | 5 | 100,000 | 14 | timeout >100 s | |
|
||||
| 3 | 6 | 729 | 0.3 | 27.9 | 5x worse than K=9 x 3 cats at the same rows |
|
||||
| 3 | 8 | 6,561 | 2.0 | 2,798 | |
|
||||
| 3 | 10-12 | 59k-531k | 18-210 | timeout | open alone passes 100 ms at 11 cats |
|
||||
| 2 | 10 | 1,024 | 0.6 | 172 | |
|
||||
| 2 | 12 | 4,096 | 2.2 | 3,190 | |
|
||||
| 2 | 14 | 16,384 | 9 | 63,100 | |
|
||||
|
||||
The cap of 24 is unreachable: rows = product of items, so a view with c Row categories of K items dies by
|
||||
K^c ~ 10^4 rows (1 s) regardless of N, and with two items per category by 12-14 categories. Per-row cost
|
||||
also grows with category count (each row_item is a Vec of c items). Not measured: Row categories with prune
|
||||
on, other axes absorbing categories (the realistic mitigation: only 1-2 on Row).
|
||||
|
||||
## Axis D: records mode over N records (`with_frozen_records`, the improvise-sk32 quadratic)
|
||||
|
||||
| N | open | keypress | browser window | SetCell (apply) | host SetCell+project | maxrss MB |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 300 | 1.5 | 0.7 | 0.1 | 1.6 | - | 10 |
|
||||
| 1,000 | 2.9 | 5.6 (5.5-5.7) | 0.3 | 10 | - | 12 |
|
||||
| 1,200 | 3.9 | **7.9** (7.7-8.0) | 0.3 | - | - | - |
|
||||
| 1,300 | 4.5 | 9.3 | 0.3 | 15.5 | 17 | 26 |
|
||||
| 1,800 | 8.4 | 16.5 | 0.4 | - | - | - |
|
||||
| 3,000 | 22.9 | 47 | 0.7 | 74 | - | 20 |
|
||||
| 4,400-4,600 | 47-51 | **95-102** | 0.9 | 149 | 158 | 55 |
|
||||
| 10,000 | 248 | 492 | 1.8 | 734 | - | 46 |
|
||||
| 14,000-14,500 | 472-500 | **940-1,002** | 2.6-2.8 | 1,435 | 1,459 | 135-148 |
|
||||
| 30,000 | 2,198 | 4,314 | 5.6 | 6,467 | - | 125 |
|
||||
| 100,000 | **25,420** | **51,234** | 19.3 | not reached (killed at 100 s) | - | not reached |
|
||||
|
||||
Records mode, keypress/scroll: inside 8 ms **to N~1,200**; 16 ms N~1,800; 100 ms N~4,500 (first over: 4,600 =
|
||||
102 ms); 1 s N~14,500. Open crosses 8 ms ~1,750, 16 ms ~2,300, 100 ms ~6,500, 1 s ~20,000. Time ~ N^1.95.
|
||||
SetCell is ~1.5x keypress (it rebuilds and re-groups). The *browser window* is cheap (<20 ms even at 100k)
|
||||
because records mode has two columns and the width walk is short, so the 51 s is `rebuild_layout`, not
|
||||
projection. Prune on/off does not change records mode (prune applies to pivoted axes; the sweep ran off).
|
||||
Records per KB: ~4 KB/record process-wide at 14k (maxrss includes the document generator, so read as an
|
||||
upper bound); the 100k memory was not obtained.
|
||||
|
||||
## Axis E: save / load and per-command persistence
|
||||
|
||||
Mode `none` (all categories off-axis, so layout is trivial and only the store scales), 3 x 1,000, N records
|
||||
= 2N cells:
|
||||
|
||||
| cells | format_md | text MB | parse_md | persist per accepted command (`save_to` MemoryFs) | maxrss MB |
|
||||
|---|---|---|---|---|---|
|
||||
| 2,000 | 13.8 | 0.1 | 4.5 | 12.4 | 18 |
|
||||
| 20,000 | 156 | 0.86 | 37 | 161 | 80 |
|
||||
| 60,000 | 527 | 2.5 | 119 | 539 | 236 |
|
||||
| **200,000 (the target)** | **1,974** | 8.4 | **401** | **1,984** | 663 |
|
||||
| 600,000 | 6,470 | 25 | 1,320 | 6,460 | 1,894 |
|
||||
| 2,000,000 | 23,120 | 84 | 4,668 | 23,088 | 5,611 |
|
||||
|
||||
Linear: ~10 us/cell to format, ~2 us/cell to parse. Crossings, format_md: 8 ms ~700 cells, 16 ms ~1.5k,
|
||||
100 ms ~11k, 1 s ~100k cells. parse_md: 8 ms ~4k cells, 100 ms ~50k, 1 s ~450k. **At the target (200k cells)
|
||||
every accepted command in the browser worker costs ~2 s of format alone** before OPFS write time (not
|
||||
measured: MemoryFs only). Persist per command also appears in smaller documents inside the grid axes
|
||||
(K=10 grid: 5.5 ms; records N=4,400: 63 ms).
|
||||
|
||||
## Other operations (3 runs each; "grouped" fixtures start with C0 groups collapsed, so rows are not K^3)
|
||||
|
||||
Reshape (prune off, K=10, 1000 rows): hide item 0.25 ms, set axis 0.19 ms, **toggle prune 12.7 ms**.
|
||||
K=26 grouped, 2,028 rows: hide 0.5, toggle group 1.8, set axis 1.8, **toggle prune 727-750 ms**.
|
||||
Prune on K=13 grouped, 321 rows: hide 9.4, toggle group 37, set axis 37, toggle prune 37-39 ms.
|
||||
`Host::connect` snapshot: K=10 6.5 ms, K=26 375 ms, records N=14,000 2.9 ms. Host SetCell+project: K=10 7.7 ms,
|
||||
K=26 ~400 ms, records N=4,400 158 ms (equal to apply + a whole-grid width walk).
|
||||
|
||||
## Classification and feasibility of yielding
|
||||
|
||||
| operation | class | today's first size over 8 ms | dominated by | chunked/yielding form |
|
||||
|---|---|---|---|---|
|
||||
| keypress / scroll | **hot (<=8 ms)** | 1,000 rows (K=10) / N=1,300 records | `rebuild_layout` + whole-grid `compute_col_widths` (O(rows^2) `nth`), recomputed though nothing changed | not a slicing problem: cache widths/layout across keys, O(1) row lookup (ADR 008/009). Slicing would only hide it |
|
||||
| browser window projection | **hot** | 1,000 rows | same width walk, whole grid regardless of window | needs the same fix; then window cost is O(window) |
|
||||
| SetCell apply + project | **hot** | K~19 (prune off), 1,300 records | layout rebuild O(rows) + widths + projection | incremental update of one row/total, not a rebuild |
|
||||
| column totals | hot | inside `grid_of` (not separable here) | whole-grid sum in the projection | maintained total (ADR 009) |
|
||||
| open / layout build | deferrable | 40k rows (prune off); N~1,750 (records) | build is ~90 ns/row (sliceable by outer index); the prune/records grouping is the quadratic part | build is feasible in slices once the quadratic is removed; the quadratic itself is not sliceable |
|
||||
| reshape (hide/collapse/axis) | deferrable | cheap prune-off (<2 ms to ~2k rows); 37 ms prune-on at 321 rows | layout rebuild + prune | one sort/remap at 100k occupied tuples was 2.3 ms in ADR 008's prototype, so a reshape can fit inline; prune toggle needs the quadratic fixed first |
|
||||
| save (format_md) | deferrable | ~700 cells | per-cell formatting, linear | the Data section is a flat per-cell line stream, so chunking by cell range with a yield every ~400 cells looks feasible, but needs a consistent snapshot (clone or journal); not profiled by function |
|
||||
| load (parse_md) | deferrable | ~4k cells | pest parse + tree walk, linear | tree walk is chunkable by line; the pest parse of one text is not, unless the Data section is parsed line by line (grammar is line oriented); not profiled by function |
|
||||
| persistence per command | deferrable (debounce) | ~700 cells | the whole-document `format_md` | coalesce/debounce or append-journal: formatting 2 s per command at target is unusable inline at any slicing |
|
||||
|
||||
## What I could not measure
|
||||
|
||||
* wasm: all numbers are native; the browser worker is typically slower. Real OPFS I/O, JSON *decode* and
|
||||
DOM paint, the full `draw()` frame, and Safari are not covered.
|
||||
* Records mode at 100k: SetCell, host and RSS (killed after open and keypress, ~76 s).
|
||||
* Per-function profile: symbols are stripped in `[profile.release]`; attribution is reasoned from the
|
||||
code and the timing exponents, not a flame graph.
|
||||
* SetCell 100 ms/1 s for prune off past K=40 are extrapolations.
|
||||
* Memory (`maxrss`) includes the generator's hash set and `recs` vector; it is an upper bound on the
|
||||
document. Layout cost ~150 B/row per ADR 008, consistent with K=40 (64k rows, 235 MB peak with prune off
|
||||
including the generator and a second layout during rebuild).
|
||||
* Noise: boundary rows are 3 sequential processes, spread <=3% (e.g. K=17: 105.0-105.5 ms). Non-boundary
|
||||
rows are single runs, some taken with 2-3 concurrent processes on a 16+ core machine; treat them as +-5-10%.
|
||||
@@ -251,7 +251,7 @@ pub trait AtCursor: Cmd {
|
||||
}
|
||||
|
||||
/// How many arguments a command takes. The registry refuses any beyond it, for
|
||||
/// every command in one place — `context/code/cli.md` §13.
|
||||
/// every command in one place — `context/code/cli.md` §14.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MaxArgs {
|
||||
AtMost(usize),
|
||||
@@ -435,7 +435,7 @@ pub(super) fn require_args(args: &[String], n: usize) -> Result<(), String> {
|
||||
}
|
||||
|
||||
/// Parse Cat/Item coordinate args into a CellKey, refusing any token that is not
|
||||
/// one rather than dropping it — `context/code/cli.md` §13.
|
||||
/// one rather than dropping it — `context/code/cli.md` §14.
|
||||
pub(super) fn parse_cell_key_from_args(
|
||||
args: &[String],
|
||||
) -> Result<crate::model::cell::CellKey, String> {
|
||||
@@ -452,7 +452,7 @@ pub(super) fn parse_cell_key_from_args(
|
||||
}
|
||||
|
||||
/// A cursor fallback answers only for coordinates that were OMITTED; given and
|
||||
/// wrong, they are refused, never the cursor's cell — `context/code/cli.md` §13.
|
||||
/// wrong, they are refused, never the cursor's cell — `context/code/cli.md` §14.
|
||||
pub(super) fn only_when_omitted(args: &[String]) -> Result<(), String> {
|
||||
if args.is_empty() {
|
||||
return Ok(());
|
||||
|
||||
@@ -252,7 +252,7 @@ macro_rules! effect_cmd {
|
||||
}
|
||||
|
||||
/// `effect`, preceded by the check that what it names exists, run at APPLY time:
|
||||
/// a `:` line is built before any of it runs — `context/code/cli.md` §14.
|
||||
/// a `:` line is built before any of it runs — `context/code/cli.md` §15.
|
||||
fn unless_unknown(needs: effect::Needs, effect: impl Effect + 'static) -> Vec<Box<dyn Effect>> {
|
||||
vec![Box::new(effect::RequireKnown(needs)), Box::new(effect)]
|
||||
}
|
||||
|
||||
@@ -82,7 +82,7 @@ fn groups(line: &str) -> Result<Vec<Vec<Word>>, String> {
|
||||
Ok(groups)
|
||||
}
|
||||
|
||||
/// Split `line` into words. The quoting rules: `context/code/cli.md` §13.
|
||||
/// Split `line` into words. The quoting rules: `context/code/cli.md` §14.
|
||||
fn lex(line: &str) -> Result<Vec<Word>, String> {
|
||||
let mut words = Vec::new();
|
||||
let mut chars = line.char_indices().peekable();
|
||||
@@ -433,7 +433,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// `context/code/cli.md` §13: only a BARE `.` separates commands. A quoted one
|
||||
/// `context/code/cli.md` §14: only a BARE `.` separates commands. A quoted one
|
||||
/// is an argument, so `|.|` and `"."` name a thing called `.`.
|
||||
#[test]
|
||||
fn a_quoted_dot_is_an_argument_not_a_separator() {
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::persistence;
|
||||
use crate::ui::effect::{EffectKind, ModelChange, ViewTarget};
|
||||
use crate::ui::project::Dirty;
|
||||
use crate::ui::viewport::{
|
||||
compute_col_widths, compute_row_header_width, compute_visible_cols, grid_width,
|
||||
GridSummary, compute_row_header_width, compute_visible_cols, grid_summary, grid_width,
|
||||
};
|
||||
use crate::view::{GridLayout, View};
|
||||
use crate::workbook::Workbook;
|
||||
@@ -434,6 +434,20 @@ impl Default for ModelState {
|
||||
}
|
||||
}
|
||||
|
||||
/// What a layout was built from other than the model.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
struct LayoutBasis {
|
||||
view: String,
|
||||
/// The address of the frozen records, never dereferenced.
|
||||
drill: Option<usize>,
|
||||
}
|
||||
|
||||
/// What the keypress path measures of a built layout.
|
||||
pub(crate) struct GridGeometry {
|
||||
pub summary: GridSummary,
|
||||
pub row_header_width: u16,
|
||||
}
|
||||
|
||||
/// Top-level application state: `model_state`, `view_state`, and a residue of
|
||||
/// fields tagged below. The five residue tags are in `context/code/command.md`.
|
||||
pub struct App {
|
||||
@@ -453,8 +467,21 @@ pub struct App {
|
||||
pub term_width: u16,
|
||||
pub term_height: u16,
|
||||
/// **Derived cache**: a pure function of the slices, rebuilt by
|
||||
/// `rebuild_layout()`. Owned by `App` because the rebuild is the host's job.
|
||||
/// `rebuild_layout()`; a writer that bypasses effects must call it (felo.13).
|
||||
pub layout: GridLayout,
|
||||
/// How many times `rebuild_layout` has run: what a keypress that changes
|
||||
/// neither the model nor the view must leave alone (improvise-felo.13).
|
||||
layout_builds: usize,
|
||||
/// How many times the grid's column widths were measured, for the same reason.
|
||||
width_passes: std::cell::Cell<usize>,
|
||||
/// The widths of the layout above, measured on first use and dropped with it.
|
||||
geometry: std::cell::OnceCell<GridGeometry>,
|
||||
/// What the layout above was built from besides the model: the view it drew
|
||||
/// and the drill snapshot it froze. `layout_current` compares it.
|
||||
layout_basis: LayoutBasis,
|
||||
/// The model changed since the layout was built. Set by the one writer that
|
||||
/// does not rebuild at once.
|
||||
layout_stale: bool,
|
||||
/// **Transient**: set during `apply_effects` to skip the rest of the batch,
|
||||
/// reset at the start of every call. Reach it through `AbortChain`.
|
||||
pub abort_effects: bool,
|
||||
@@ -553,6 +580,11 @@ impl App {
|
||||
term_width: 80,
|
||||
term_height: 24,
|
||||
layout,
|
||||
layout_builds: 0,
|
||||
width_passes: std::cell::Cell::new(0),
|
||||
geometry: std::cell::OnceCell::new(),
|
||||
layout_basis: LayoutBasis::default(),
|
||||
layout_stale: false,
|
||||
abort_effects: false,
|
||||
batch_aborted: false,
|
||||
batch_refused: false,
|
||||
@@ -566,6 +598,7 @@ impl App {
|
||||
keymap_set: KeymapSet::default_keymaps(),
|
||||
storage,
|
||||
};
|
||||
app.layout_basis = app.current_layout_basis();
|
||||
app.settle_normal_mode();
|
||||
app
|
||||
}
|
||||
@@ -610,6 +643,7 @@ impl App {
|
||||
self.view_state.view_back_stack.clear();
|
||||
self.view_state.view_forward_stack.clear();
|
||||
self.model_state.workbook = workbook;
|
||||
self.layout_stale = true;
|
||||
// Every client's mirror is scoped to the outgoing document too, and an
|
||||
// effect cannot reach the session list. The generation is how it hears.
|
||||
self.document_generation += 1;
|
||||
@@ -900,6 +934,42 @@ impl App {
|
||||
let frozen = self.drill_state.as_ref().map(|s| Rc::clone(&s.records));
|
||||
self.layout =
|
||||
GridLayout::with_frozen_records(&self.model_state.workbook.model, view, frozen);
|
||||
self.layout_builds += 1;
|
||||
self.layout_stale = false;
|
||||
self.layout_basis = self.current_layout_basis();
|
||||
self.geometry = std::cell::OnceCell::new();
|
||||
}
|
||||
|
||||
/// Rebuild only if the model, the view drawn or the drill snapshot moved; a
|
||||
/// cursor key moves none of them (layout-cost.md).
|
||||
fn refresh_layout(&mut self) {
|
||||
if self.layout_stale || self.layout_basis != self.current_layout_basis() {
|
||||
self.rebuild_layout();
|
||||
}
|
||||
}
|
||||
|
||||
fn current_layout_basis(&self) -> LayoutBasis {
|
||||
LayoutBasis {
|
||||
view: self.view_name().to_string(),
|
||||
drill: self
|
||||
.drill_state
|
||||
.as_ref()
|
||||
.map(|d| Rc::as_ptr(&d.records) as usize),
|
||||
}
|
||||
}
|
||||
|
||||
/// The measurements a keypress needs of the grid: column widths and the row
|
||||
/// header. Counted, so a test can see whether a keypress paid for them.
|
||||
pub(crate) fn grid_geometry(&self) -> &GridGeometry {
|
||||
self.geometry.get_or_init(|| {
|
||||
self.width_passes.set(self.width_passes.get() + 1);
|
||||
let model = &self.model_state.workbook.model;
|
||||
let plan = FormatPlan::of(model, self.view());
|
||||
GridGeometry {
|
||||
summary: grid_summary(model, &self.layout, &plan),
|
||||
row_header_width: compute_row_header_width(&self.layout),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Editable text for the cell under the cursor: staged drill edit, else the
|
||||
@@ -980,12 +1050,10 @@ impl App {
|
||||
display_value: self.display_value_at_cursor(),
|
||||
visible_rows: (self.term_height as usize).saturating_sub(8),
|
||||
visible_cols: {
|
||||
let model = &self.model_state.workbook.model;
|
||||
let col_widths = compute_col_widths(model, layout, &FormatPlan::of(model, view));
|
||||
let row_header_width = compute_row_header_width(layout);
|
||||
let geometry = self.grid_geometry();
|
||||
compute_visible_cols(
|
||||
&col_widths,
|
||||
row_header_width,
|
||||
&geometry.summary.col_widths,
|
||||
geometry.row_header_width,
|
||||
self.grid_width(),
|
||||
cursor.col_offset,
|
||||
)
|
||||
@@ -1116,7 +1184,8 @@ impl App {
|
||||
// is read only behind the epoch gate — `context/code/command.md`.
|
||||
let reshaped = self.last_reshaped_views.clone();
|
||||
self.last_reframed_views.extend(reshaped);
|
||||
self.rebuild_layout();
|
||||
self.layout_stale |= touches_model;
|
||||
self.refresh_layout();
|
||||
// After the rebuild, or a batch that crosses into or out of records mode
|
||||
// settles against the grid it started on — `context/code/command.md`.
|
||||
self.settle_normal_mode();
|
||||
@@ -1196,7 +1265,7 @@ impl App {
|
||||
/// Resolve one keypress through the keymap and apply what it produces. The
|
||||
/// projection half is factored out; `ui::reduce` is the named door (improvise-gxi).
|
||||
pub fn dispatch_key(&mut self, key: KeyEvent) -> Dirty {
|
||||
self.rebuild_layout();
|
||||
self.refresh_layout();
|
||||
|
||||
// An armed prefix key takes priority. Resolve it back to its
|
||||
// sub-keymap through the keymap set (improvise-jc8).
|
||||
@@ -1594,6 +1663,32 @@ mod tests {
|
||||
app
|
||||
}
|
||||
|
||||
/// improvise-felo.13: `StartDrill` is `ViewOnly`, so a batch of it alone moves
|
||||
/// the layout with no `Model` effect to mark it stale. Evidence: layout-cost.md.
|
||||
#[test]
|
||||
fn a_drill_snapshot_replaced_alone_rebuilds_the_layout() {
|
||||
let mut app = records_app_for_drill();
|
||||
assert_eq!(app.layout.row_count(), 3);
|
||||
let rows: Vec<_> = app.layout.records.as_ref().unwrap().to_vec();
|
||||
let start = |app: &mut App, n: usize| {
|
||||
let snapshot = rows.iter().take(n).cloned().collect();
|
||||
app.apply_effects(vec![Box::new(crate::ui::effect::StartDrill(snapshot))]);
|
||||
};
|
||||
start(&mut app, 1);
|
||||
assert_eq!(
|
||||
app.layout.row_count(),
|
||||
1,
|
||||
"the layout draws the frozen snapshot"
|
||||
);
|
||||
let builds = app.layout_builds;
|
||||
start(&mut app, 2);
|
||||
assert!(
|
||||
app.layout_builds > builds,
|
||||
"no rebuild for a second drill snapshot"
|
||||
);
|
||||
assert_eq!(app.layout.row_count(), 2);
|
||||
}
|
||||
|
||||
/// The opening basis may be read off `App::layout` only when no drill froze
|
||||
/// it. Evidence: `context/test-evidence/command.md`.
|
||||
#[test]
|
||||
@@ -1898,6 +1993,71 @@ mod tests {
|
||||
App::new(wb, None)
|
||||
}
|
||||
|
||||
/// improvise-felo.13: context/test-evidence/layout-cost.md.
|
||||
#[test]
|
||||
fn a_cursor_key_rebuilds_neither_the_layout_nor_the_widths() {
|
||||
let mut app = two_col_model();
|
||||
let down = || KeyEvent::new(KeyCode::Down, KeyModifiers::NONE);
|
||||
app.dispatch_key(down());
|
||||
let (builds, widths) = (app.layout_builds, app.width_passes.get());
|
||||
app.dispatch_key(down());
|
||||
app.dispatch_key(down());
|
||||
assert_eq!(app.cursor().selected.0, 2, "the keys moved the cursor");
|
||||
assert_eq!(app.layout_builds, builds, "layout rebuilt on a cursor key");
|
||||
assert_eq!(app.width_passes.get(), widths, "widths remeasured");
|
||||
}
|
||||
|
||||
/// improvise-felo.13: the control for the test above, and the guard on
|
||||
/// invalidation: an edit must rebuild, and the widths must follow the value.
|
||||
#[test]
|
||||
fn an_edit_rebuilds_and_remeasures_so_the_widths_follow_the_value() {
|
||||
use crate::ui::effect::SetCell;
|
||||
let mut app = two_col_model();
|
||||
let down = || KeyEvent::new(KeyCode::Down, KeyModifiers::NONE);
|
||||
app.dispatch_key(down());
|
||||
let narrow = app.grid_geometry().summary.col_widths.clone();
|
||||
let (builds, widths) = (app.layout_builds, app.width_passes.get());
|
||||
let key = CellKey::new(vec![("Row".into(), "A".into()), ("Col".into(), "X".into())]);
|
||||
let long = CellValue::Text("a value far wider than a column".into());
|
||||
app.apply_effects(vec![Box::new(SetCell(key, long))]);
|
||||
assert!(
|
||||
app.layout_builds > builds,
|
||||
"an edit must rebuild the layout"
|
||||
);
|
||||
app.dispatch_key(down());
|
||||
assert!(app.width_passes.get() > widths, "an edit must remeasure");
|
||||
let model = &app.model_state.workbook.model;
|
||||
let fresh = crate::ui::viewport::compute_col_widths(
|
||||
model,
|
||||
&app.layout,
|
||||
&FormatPlan::of(model, app.view()),
|
||||
);
|
||||
assert_eq!(
|
||||
app.grid_geometry().summary.col_widths,
|
||||
fresh,
|
||||
"stale widths"
|
||||
);
|
||||
assert_ne!(
|
||||
fresh, narrow,
|
||||
"the edit widened a column, or this proves nothing"
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-felo.13: a `ViewOnly` effect that changes which view is drawn
|
||||
/// must still rebuild, so skipping the rebuild is not keyed on `EffectKind` alone.
|
||||
#[test]
|
||||
fn switching_view_rebuilds_though_the_effect_is_view_only() {
|
||||
let mut app = two_col_model();
|
||||
app.model_state.workbook.create_view("Second");
|
||||
app.rebuild_layout();
|
||||
let builds = app.layout_builds;
|
||||
app.apply_effects(vec![Box::new(crate::ui::effect::SwitchView(
|
||||
"Second".into(),
|
||||
))]);
|
||||
assert_eq!(app.view_state.active_view.as_str(), "Second");
|
||||
assert!(app.layout_builds > builds, "the layout still draws Default");
|
||||
}
|
||||
|
||||
/// improvise-htl6. Evidence: `context/test-evidence/command.md`.
|
||||
#[test]
|
||||
fn the_view_history_walks_both_ways_from_the_keyboard() {
|
||||
@@ -2134,6 +2294,7 @@ mod tests {
|
||||
.unwrap()
|
||||
.add_item(format!("R{i}"));
|
||||
}
|
||||
app.rebuild_layout();
|
||||
app.term_height = 28; // ~20 visible rows → delta = 15
|
||||
app.cursor_mut().selected = (0, 0);
|
||||
app.handle_key(KeyEvent::new(KeyCode::PageDown, KeyModifiers::NONE))
|
||||
@@ -2158,6 +2319,7 @@ mod tests {
|
||||
.unwrap()
|
||||
.add_item(format!("R{i}"));
|
||||
}
|
||||
app.rebuild_layout();
|
||||
app.term_height = 28;
|
||||
app.cursor_mut().selected = (20, 0);
|
||||
app.handle_key(KeyEvent::new(KeyCode::PageUp, KeyModifiers::NONE))
|
||||
@@ -2177,6 +2339,7 @@ mod tests {
|
||||
.unwrap()
|
||||
.add_item(format!("R{i}"));
|
||||
}
|
||||
app.rebuild_layout();
|
||||
app.term_height = 13; // ~5 visible rows
|
||||
app.cursor_mut().selected = (0, 0);
|
||||
// G jumps to last row (row 12)
|
||||
@@ -2204,6 +2367,7 @@ mod tests {
|
||||
.unwrap()
|
||||
.add_item(format!("R{i}"));
|
||||
}
|
||||
app.rebuild_layout();
|
||||
app.term_height = 13; // ~5 visible rows
|
||||
app.cursor_mut().selected = (0, 0);
|
||||
// Ctrl+d scrolls by 5 rows
|
||||
|
||||
@@ -1245,7 +1245,7 @@ pub struct ImportJsonHeadless {
|
||||
}
|
||||
impl Effect for ImportJsonHeadless {
|
||||
fn apply(&self, app: &mut App) {
|
||||
use crate::import::analyzer::{FieldKind, analyze_records};
|
||||
use crate::import::analyzer::FieldKind;
|
||||
use crate::import::wizard::{DuplicatePolicy, ImportPipeline};
|
||||
|
||||
let is_csv = self
|
||||
@@ -1253,10 +1253,10 @@ impl Effect for ImportJsonHeadless {
|
||||
.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("csv"));
|
||||
|
||||
// Parse the file exactly once: `records` feeds analysis, `raw` is reused
|
||||
// as the pipeline's value (improvise-oaq). CSV warnings go to the status.
|
||||
// Parse the file exactly once, into the pipeline that reads it
|
||||
// (improvise-oaq, improvise-8jk6.4). CSV warnings go to the status.
|
||||
let mut warnings: Vec<String> = vec![];
|
||||
let (records, raw) = if is_csv {
|
||||
let mut pipeline = if is_csv {
|
||||
let bytes = match app.storage.read(&self.path) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
@@ -1267,8 +1267,7 @@ impl Effect for ImportJsonHeadless {
|
||||
match crate::import::csv_parser::parse_csv_bytes_with_warnings(&bytes) {
|
||||
Ok((recs, w)) => {
|
||||
warnings = w;
|
||||
let raw = serde_json::Value::Array(recs.clone());
|
||||
(recs, raw)
|
||||
ImportPipeline::from_csv(recs)
|
||||
}
|
||||
Err(e) => {
|
||||
refuse_and_stop(app, format!("CSV error: {e}"));
|
||||
@@ -1291,38 +1290,28 @@ impl Effect for ImportJsonHeadless {
|
||||
}
|
||||
};
|
||||
match json_import_records(&value, self.array_path.as_deref()) {
|
||||
Ok(recs) => (recs, value),
|
||||
Ok(recs) => {
|
||||
let mut pipeline = ImportPipeline::new(value);
|
||||
pipeline.selected_path = self.array_path.clone().unwrap_or_default();
|
||||
pipeline.records = recs;
|
||||
pipeline.analyze();
|
||||
pipeline
|
||||
}
|
||||
Err(msg) => {
|
||||
refuse_and_stop(app, msg);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let proposals = analyze_records(&records);
|
||||
|
||||
let pipeline = ImportPipeline {
|
||||
raw,
|
||||
array_paths: vec![],
|
||||
selected_path: self.array_path.as_deref().unwrap_or("").to_string(),
|
||||
records,
|
||||
proposals: proposals
|
||||
.into_iter()
|
||||
.map(|mut p| {
|
||||
p.accepted = p.kind != FieldKind::Label;
|
||||
p
|
||||
})
|
||||
.collect(),
|
||||
model_name: self
|
||||
.model_name
|
||||
.as_deref()
|
||||
.unwrap_or("Imported Model")
|
||||
.to_string(),
|
||||
formulas: vec![],
|
||||
// Headless: nobody to ask, so a collision refuses (improvise-cgqt).
|
||||
on_duplicate: DuplicatePolicy::Refuse,
|
||||
sources: vec![],
|
||||
};
|
||||
pipeline.model_name = self
|
||||
.model_name
|
||||
.clone()
|
||||
.unwrap_or_else(|| "Imported Model".to_string());
|
||||
// Headless: nobody to ask, so a collision refuses (improvise-cgqt).
|
||||
pipeline.on_duplicate = DuplicatePolicy::Refuse;
|
||||
for p in &mut pipeline.proposals {
|
||||
p.accepted = p.kind != FieldKind::Label;
|
||||
}
|
||||
|
||||
match pipeline.build_model_with_warnings() {
|
||||
Ok((new_workbook, build_warnings)) => {
|
||||
@@ -1422,7 +1411,7 @@ fn unknown_name(app: &App, needs: &Needs) -> Option<String> {
|
||||
}
|
||||
|
||||
/// Refuse and stop unless the document has what the next effects name, asked at
|
||||
/// apply time so a `:` line sees its earlier segments (`cli.md` §14).
|
||||
/// apply time so a `:` line sees its earlier segments (`cli.md` §15).
|
||||
#[derive(Debug)]
|
||||
pub struct RequireKnown(pub Needs);
|
||||
impl Effect for RequireKnown {
|
||||
|
||||
@@ -25,7 +25,7 @@ use improvise_protocol::{
|
||||
use crate::ui::app::App;
|
||||
use crate::ui::effect::{EffectKind, ModelChange};
|
||||
use crate::ui::reduce::{Checked, apply_command};
|
||||
use crate::ui::viewport::{GridBody, grid_body};
|
||||
use crate::ui::viewport::{GridBody, grid_body, grid_window};
|
||||
|
||||
/// Which widgets a batch of effects could have changed. A gate, not a description:
|
||||
/// over-reporting costs work, under-reporting is a correctness bug (gotcha 35).
|
||||
@@ -142,9 +142,19 @@ pub fn grid_in(app: &App, view_name: &str, viewport: Viewport) -> GridCache {
|
||||
let plan = FormatPlan::of(model, view);
|
||||
let rows = viewport.rows_in(layout.row_count());
|
||||
let cols = viewport.cols_in(layout.col_count());
|
||||
// ONE walk: the widths, the totals and this window's values all come off the
|
||||
// same evaluation — `context/code/command.md` (improvise-jmjz).
|
||||
let body = grid_body(model, layout, &plan, rows.clone(), cols.clone());
|
||||
// One walk gives widths, totals and window (improvise-jmjz); the host's layout
|
||||
// keeps the first two until rebuilt, so its window costs its own cells.
|
||||
let hosted = elsewhere.is_none() && derived.is_none();
|
||||
let body = match hosted {
|
||||
true => grid_window(
|
||||
model,
|
||||
layout,
|
||||
&app.grid_geometry().summary,
|
||||
rows.clone(),
|
||||
cols.clone(),
|
||||
),
|
||||
false => grid_body(model, layout, &plan, rows.clone(), cols.clone()),
|
||||
};
|
||||
|
||||
GridCache {
|
||||
row_frame: axis_frame(&layout.row_cats, &layout.row_items, rows.clone()),
|
||||
|
||||
@@ -65,7 +65,7 @@ const NOT_ON_THE_WIRE: &[(&str, Verdict)] = &[
|
||||
),
|
||||
(
|
||||
"RequireKnown",
|
||||
Verdict::Off("a command-side name check that writes no state (gotcha 21, cli.md §14)"),
|
||||
Verdict::Off("a command-side name check that writes no state (gotcha 21, cli.md §15)"),
|
||||
),
|
||||
(
|
||||
"AbortChain",
|
||||
|
||||
@@ -101,6 +101,50 @@ pub fn grid_body(
|
||||
}
|
||||
}
|
||||
|
||||
/// What of a grid build no window changes: the widths, each column's total
|
||||
/// format and the totals. Measured over every cell, so a host keeps one per layout.
|
||||
pub struct GridSummary {
|
||||
pub col_widths: Vec<u16>,
|
||||
pub col_formats: Vec<NumberFormat>,
|
||||
pub totals: Vec<Option<CellValue>>,
|
||||
}
|
||||
|
||||
/// The whole-grid walk, answering only what a window cannot.
|
||||
pub fn grid_summary(model: &Model, layout: &GridLayout, plan: &FormatPlan) -> GridSummary {
|
||||
let body = grid_body(model, layout, plan, 0..0, 0..0);
|
||||
GridSummary {
|
||||
col_widths: body.col_widths,
|
||||
col_formats: body.col_formats,
|
||||
totals: body.totals,
|
||||
}
|
||||
}
|
||||
|
||||
/// [`grid_body`] for a window, given the `summary` of the same layout: evaluates
|
||||
/// only the window's cells. Equal to `grid_body` where `summary` is current.
|
||||
pub fn grid_window(
|
||||
model: &Model,
|
||||
layout: &GridLayout,
|
||||
summary: &GridSummary,
|
||||
rows: std::ops::Range<usize>,
|
||||
cols: std::ops::Range<usize>,
|
||||
) -> GridBody {
|
||||
let eval = layout.evaluator(model);
|
||||
let records = layout.is_records_mode();
|
||||
let window = rows
|
||||
.map(|ri| {
|
||||
cols.clone()
|
||||
.map(|ci| cell_value(layout, &eval, records, ri, ci).1)
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
GridBody {
|
||||
col_widths: summary.col_widths.clone(),
|
||||
col_formats: summary.col_formats.clone(),
|
||||
totals: summary.totals.clone(),
|
||||
window,
|
||||
}
|
||||
}
|
||||
|
||||
/// The one evaluation. Records mode reads its wide row; pivot mode aggregates.
|
||||
fn cell_value(
|
||||
layout: &GridLayout,
|
||||
@@ -146,7 +190,7 @@ fn widen(widths: &mut [u16], col: usize, width: usize) {
|
||||
/// Adaptive column widths alone, for the keypress path, which needs no cells.
|
||||
/// It is [`grid_body`] with an empty window so there is one width rule.
|
||||
pub fn compute_col_widths(model: &Model, layout: &GridLayout, plan: &FormatPlan) -> Vec<u16> {
|
||||
grid_body(model, layout, plan, 0..0, 0..0).col_widths
|
||||
grid_summary(model, layout, plan).col_widths
|
||||
}
|
||||
|
||||
/// The total row-header width for a layout. The rule is the protocol's; this is the
|
||||
@@ -272,4 +316,33 @@ mod tests {
|
||||
"F0..F8 once for the Q1 row, once for the totals"
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-felo.13: a window read off a held summary is the window `grid_body`
|
||||
/// builds. Evidence: `context/test-evidence/layout-cost.md`.
|
||||
#[test]
|
||||
fn a_window_over_a_held_summary_equals_the_one_walk_grid_body() {
|
||||
let wb = folded_chain();
|
||||
let view = wb.active_view();
|
||||
let layout = GridLayout::new(&wb.model, view);
|
||||
let plan = FormatPlan::of(&wb.model, view);
|
||||
let summary = grid_summary(&wb.model, &layout, &plan);
|
||||
let (rows, cols) = (layout.row_count(), layout.col_count());
|
||||
assert!(
|
||||
rows >= 2 && cols >= 3,
|
||||
"a window strictly inside is possible"
|
||||
);
|
||||
for (r, c) in [
|
||||
(0..rows, 0..cols),
|
||||
(1..2, 1..3),
|
||||
(rows - 1..rows, 0..1),
|
||||
(0..0, 0..0),
|
||||
] {
|
||||
let walked = grid_body(&wb.model, &layout, &plan, r.clone(), c.clone());
|
||||
let held = grid_window(&wb.model, &layout, &summary, r.clone(), c.clone());
|
||||
assert_eq!(held.window, walked.window, "window {r:?} x {c:?}");
|
||||
assert_eq!(held.col_widths, walked.col_widths);
|
||||
assert_eq!(held.totals, walked.totals);
|
||||
assert_eq!(held.col_formats, walked.col_formats);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,6 +194,12 @@ impl RecordEntry {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
/// How many lookups fell through to set narrowing, for the exact-path test.
|
||||
static NARROWED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct DataStore {
|
||||
/// Primary storage: interned keys, insertion-ordered so records mode can
|
||||
@@ -206,6 +212,9 @@ pub struct DataStore {
|
||||
/// Reverse index for the client-facing surrogate (ADR 002), plus each
|
||||
/// record's write counter. Why it lives here: core.md §8.
|
||||
records: HashMap<RecordHash, RecordEntry>,
|
||||
/// How many stored cells have each number of coordinates; its largest key
|
||||
/// says when a partial can match only one exact cell.
|
||||
arities: std::collections::BTreeMap<usize, usize>,
|
||||
}
|
||||
|
||||
/// Resolved through the symbol table and sorted wherever the storage is a
|
||||
@@ -217,6 +226,7 @@ impl std::fmt::Debug for DataStore {
|
||||
symbols,
|
||||
index,
|
||||
records,
|
||||
arities: _,
|
||||
} = self;
|
||||
let cells: Vec<_> = cells
|
||||
.iter()
|
||||
@@ -291,7 +301,11 @@ impl DataStore {
|
||||
for pair in &ikey.0 {
|
||||
self.index.entry(*pair).or_default().insert(ikey.clone());
|
||||
}
|
||||
let width = ikey.0.len();
|
||||
let is_new_cell = self.cells.insert(ikey, value).is_none();
|
||||
if is_new_cell {
|
||||
*self.arities.entry(width).or_default() += 1;
|
||||
}
|
||||
let entry = self.records.entry(hash).or_insert(RecordEntry {
|
||||
record,
|
||||
cells: 0,
|
||||
@@ -327,6 +341,12 @@ impl DataStore {
|
||||
return;
|
||||
};
|
||||
if self.cells.shift_remove(&ikey).is_some() {
|
||||
if let Some(n) = self.arities.get_mut(&ikey.0.len()) {
|
||||
*n -= 1;
|
||||
if *n == 0 {
|
||||
self.arities.remove(&ikey.0.len());
|
||||
}
|
||||
}
|
||||
for pair in &ikey.0 {
|
||||
if let Some(set) = self.index.get_mut(pair) {
|
||||
set.remove(&ikey);
|
||||
@@ -378,11 +398,28 @@ impl DataStore {
|
||||
/// Positions in `cells` of every cell matching `partial`, **in insertion
|
||||
/// order**. The index narrows; the primary map orders (§8).
|
||||
fn matching_positions(&self, partial: &[(String, String)]) -> Vec<usize> {
|
||||
if let Some(exact) = self.exact_positions(partial) {
|
||||
return exact;
|
||||
}
|
||||
self.intern_partial(partial)
|
||||
.map(|pairs| self.positions_of(&pairs))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The answer without narrowing, when `partial` is as wide as the widest cell
|
||||
/// stored: only an equal cell can match (layout-cost.md). `None` otherwise.
|
||||
fn exact_positions(&self, partial: &[(String, String)]) -> Option<Vec<usize>> {
|
||||
let widest = *self.arities.keys().next_back()?;
|
||||
// Strictly sorted by category, the order a stored key is in, so one probe is the lookup.
|
||||
let canonical = partial.windows(2).all(|w| w[0].0 < w[1].0);
|
||||
(!partial.is_empty() && widest <= partial.len() && canonical).then(|| {
|
||||
let found = self
|
||||
.intern_partial(partial)
|
||||
.and_then(|pairs| self.cells.get_index_of(&InternedKey(pairs)));
|
||||
found.into_iter().collect()
|
||||
})
|
||||
}
|
||||
|
||||
/// `partial` as symbols, or `None` when a name was never interned, which
|
||||
/// no stored cell can then match.
|
||||
fn intern_partial(&self, partial: &[(String, String)]) -> Option<Vec<(Symbol, Symbol)>> {
|
||||
@@ -412,6 +449,8 @@ impl DataStore {
|
||||
}
|
||||
|
||||
sets.sort_by_key(|s| s.len());
|
||||
#[cfg(test)]
|
||||
NARROWED.with(|n| n.set(n.get() + 1));
|
||||
|
||||
let mut positions: Vec<usize> = match sets.split_first() {
|
||||
Some((first, rest)) => first
|
||||
@@ -1356,3 +1395,105 @@ mod record_index {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod exact_path_tests {
|
||||
use super::{CellKey, CellValue, DataStore, NARROWED};
|
||||
|
||||
fn key(a: &str, b: &str, c: &str) -> CellKey {
|
||||
CellKey::new(vec![
|
||||
("A".into(), a.into()),
|
||||
("B".into(), b.into()),
|
||||
("C".into(), c.into()),
|
||||
])
|
||||
}
|
||||
|
||||
fn store() -> DataStore {
|
||||
let mut d = DataStore::new();
|
||||
for i in 0..6 {
|
||||
for j in 0..6 {
|
||||
d.set(
|
||||
key(&format!("a{i}"), &format!("b{j}"), "c"),
|
||||
CellValue::Number((i * 6 + j) as f64),
|
||||
);
|
||||
}
|
||||
}
|
||||
d
|
||||
}
|
||||
|
||||
fn narrowings(f: impl FnOnce()) -> usize {
|
||||
NARROWED.with(|n| n.set(0));
|
||||
f();
|
||||
NARROWED.with(|n| n.get())
|
||||
}
|
||||
|
||||
/// improvise-felo.13: context/test-evidence/layout-cost.md.
|
||||
#[test]
|
||||
fn a_full_width_partial_probes_one_cell_instead_of_narrowing() {
|
||||
let d = store();
|
||||
let full = key("a2", "b3", "c").0;
|
||||
let mut got = Vec::new();
|
||||
assert_eq!(
|
||||
narrowings(|| got = d.matching_values(&full).into_iter().cloned().collect()),
|
||||
0
|
||||
);
|
||||
assert_eq!(got, vec![CellValue::Number(15.0)]);
|
||||
let missing = key("a2", "b9", "c").0;
|
||||
assert!(
|
||||
d.matching_values(&missing).is_empty(),
|
||||
"an absent cell matches nothing"
|
||||
);
|
||||
let unknown = key("a2", "b3", "zzz").0;
|
||||
assert!(
|
||||
d.matching_values(&unknown).is_empty(),
|
||||
"an unseen item matches nothing"
|
||||
);
|
||||
}
|
||||
|
||||
/// The control: a narrower partial still sums its supersets, so the shortcut
|
||||
/// is not a blanket exact lookup.
|
||||
#[test]
|
||||
fn a_narrower_partial_still_narrows_and_matches_every_superset() {
|
||||
let d = store();
|
||||
let narrow = vec![
|
||||
("A".to_string(), "a2".to_string()),
|
||||
("C".to_string(), "c".to_string()),
|
||||
];
|
||||
let mut n = 0;
|
||||
assert_eq!(narrowings(|| n = d.matching_values(&narrow).len()), 1);
|
||||
assert_eq!(n, 6);
|
||||
}
|
||||
|
||||
/// A wider cell appearing, or leaving, moves the line the shortcut is allowed at.
|
||||
#[test]
|
||||
fn a_wider_cell_turns_the_shortcut_off_until_it_is_removed() {
|
||||
let mut d = store();
|
||||
let wide = CellKey::new(vec![
|
||||
("A".into(), "a2".into()),
|
||||
("B".into(), "b3".into()),
|
||||
("C".into(), "c".into()),
|
||||
("D".into(), "d".into()),
|
||||
]);
|
||||
d.set(wide.clone(), CellValue::Number(100.0));
|
||||
let full = key("a2", "b3", "c").0;
|
||||
assert_eq!(
|
||||
d.matching_values(&full).len(),
|
||||
2,
|
||||
"the exact cell and the wider cell under it"
|
||||
);
|
||||
d.remove(&wide);
|
||||
assert_eq!(
|
||||
narrowings(|| assert_eq!(d.matching_values(&full).len(), 1)),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
/// An unsorted partial is not a stored key's order, so it must not be probed.
|
||||
#[test]
|
||||
fn an_unsorted_full_width_partial_is_still_answered() {
|
||||
let d = store();
|
||||
let mut full = key("a2", "b3", "c").0;
|
||||
full.reverse();
|
||||
assert_eq!(d.matching_values(&full).len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,23 +235,47 @@ impl AxisEntry {
|
||||
|
||||
/// The data-item tuples on an axis, skipping group headers. Every public
|
||||
/// `row`/`col` argument indexes this sequence, not the visual entry list.
|
||||
#[cfg(test)]
|
||||
fn data_items(items: &[AxisEntry]) -> impl Iterator<Item = &Vec<String>> {
|
||||
items.iter().filter_map(|e| match e {
|
||||
AxisEntry::DataItem { tuple, .. } => Some(tuple),
|
||||
AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the data item at `idx` folds a collapsed group.
|
||||
fn folds(items: &[AxisEntry], idx: usize) -> bool {
|
||||
items
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
AxisEntry::DataItem { collapsed, .. } => Some(*collapsed),
|
||||
AxisEntry::DataItem { tuple, .. } => Some(tuple),
|
||||
AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None,
|
||||
})
|
||||
.nth(idx)
|
||||
.unwrap_or(false)
|
||||
.inspect(|_| DATA_ITEM_STEPS.with(|n| n.set(n.get() + 1)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
/// Items `data_items` has yielded on this thread: the work a lookup did.
|
||||
static DATA_ITEM_STEPS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
|
||||
}
|
||||
|
||||
/// Entry `positions(items)[n]` is data item `n`: a lookup by data index is O(1),
|
||||
/// not a walk past every header before it (layout-cost.md).
|
||||
fn positions(items: &[AxisEntry]) -> Vec<usize> {
|
||||
items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(at, e)| matches!(e, AxisEntry::DataItem { .. }).then_some(at))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The data item at `idx`, by the axis's position index.
|
||||
fn data_item<'a>(items: &'a [AxisEntry], pos: &[usize], idx: usize) -> Option<&'a Vec<String>> {
|
||||
match items.get(*pos.get(idx)?)? {
|
||||
AxisEntry::DataItem { tuple, .. } => Some(tuple),
|
||||
AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the data item at `idx` folds a collapsed group.
|
||||
fn folds(items: &[AxisEntry], pos: &[usize], idx: usize) -> bool {
|
||||
match pos.get(idx).and_then(|at| items.get(*at)) {
|
||||
Some(AxisEntry::DataItem { collapsed, .. }) => *collapsed,
|
||||
Some(AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. }) | None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The data index of the first item in the run `cat`/`group` opens under
|
||||
@@ -287,21 +311,15 @@ fn run_start(
|
||||
}
|
||||
|
||||
/// The display label of the data item at `idx`, or empty when out of bounds.
|
||||
fn axis_label(items: &[AxisEntry], idx: usize) -> String {
|
||||
data_items(items)
|
||||
.nth(idx)
|
||||
fn axis_label(items: &[AxisEntry], pos: &[usize], idx: usize) -> String {
|
||||
data_item(items, pos, idx)
|
||||
.map(|v| v.join("/"))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Visual index of the nth data entry (skipping group headers).
|
||||
fn data_index_to_visual(items: &[AxisEntry], data_idx: usize) -> Option<usize> {
|
||||
items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, e)| matches!(e, AxisEntry::DataItem { .. }))
|
||||
.map(|(vi, _)| vi)
|
||||
.nth(data_idx)
|
||||
fn data_index_to_visual(pos: &[usize], data_idx: usize) -> Option<usize> {
|
||||
pos.get(data_idx).copied()
|
||||
}
|
||||
|
||||
/// Where a run sits: `(level, depth)`, ordered, so "inside" is "greater than".
|
||||
@@ -468,6 +486,10 @@ pub struct GridLayout {
|
||||
pub page_coords: Vec<PageAxis>,
|
||||
pub row_items: Vec<AxisEntry>,
|
||||
pub col_items: Vec<AxisEntry>,
|
||||
/// `positions(row_items)` / `positions(col_items)`: derived, so every writer of
|
||||
/// the items calls `reindex`, and nothing outside this module writes them.
|
||||
row_pos: Vec<usize>,
|
||||
col_pos: Vec<usize>,
|
||||
/// Categories on `Axis::None` — hidden, implicitly aggregated.
|
||||
pub none_cats: Vec<String>,
|
||||
/// Records mode: one entry per row, grouping the cells that share every
|
||||
@@ -497,6 +519,7 @@ impl GridLayout {
|
||||
.collect();
|
||||
layout.row_items = row_items;
|
||||
layout.records = Some(records);
|
||||
layout.reindex();
|
||||
}
|
||||
if view.prune_empty {
|
||||
layout.prune_empty(model);
|
||||
@@ -556,6 +579,8 @@ impl GridLayout {
|
||||
row_cats,
|
||||
col_cats,
|
||||
page_coords,
|
||||
row_pos: positions(&row_items),
|
||||
col_pos: positions(&col_items),
|
||||
row_items,
|
||||
col_items,
|
||||
none_cats,
|
||||
@@ -590,6 +615,8 @@ impl GridLayout {
|
||||
row_cats: vec![INDEX_CATEGORY.to_string()],
|
||||
col_cats: vec![DIM_CATEGORY.to_string()],
|
||||
page_coords,
|
||||
row_pos: positions(&row_items),
|
||||
col_pos: positions(&col_items),
|
||||
row_items,
|
||||
col_items,
|
||||
none_cats,
|
||||
@@ -728,6 +755,13 @@ impl GridLayout {
|
||||
prune_axis_items(&mut self.col_items, |ci| {
|
||||
(0..rc).any(|ri| has_value[ri][ci])
|
||||
});
|
||||
self.reindex();
|
||||
}
|
||||
|
||||
/// Rebuild the position indexes after the items change.
|
||||
fn reindex(&mut self) {
|
||||
self.row_pos = positions(&self.row_items);
|
||||
self.col_pos = positions(&self.col_items);
|
||||
}
|
||||
|
||||
/// Whether this layout is in records mode.
|
||||
@@ -737,20 +771,20 @@ impl GridLayout {
|
||||
|
||||
/// Number of data rows (group headers excluded).
|
||||
pub fn row_count(&self) -> usize {
|
||||
data_items(&self.row_items).count()
|
||||
self.row_pos.len()
|
||||
}
|
||||
|
||||
/// Number of data columns (group headers excluded).
|
||||
pub fn col_count(&self) -> usize {
|
||||
data_items(&self.col_items).count()
|
||||
self.col_pos.len()
|
||||
}
|
||||
|
||||
pub fn row_label(&self, row: usize) -> String {
|
||||
axis_label(&self.row_items, row)
|
||||
axis_label(&self.row_items, &self.row_pos, row)
|
||||
}
|
||||
|
||||
pub fn col_label(&self, col: usize) -> String {
|
||||
axis_label(&self.col_items, col)
|
||||
axis_label(&self.col_items, &self.col_pos, col)
|
||||
}
|
||||
|
||||
/// The value of a pivot key as THIS grid reads it: aggregated over its None
|
||||
@@ -804,8 +838,8 @@ impl GridLayout {
|
||||
(DIM_CATEGORY.to_string(), col.to_string()),
|
||||
]));
|
||||
}
|
||||
let row_item = data_items(&self.row_items).nth(row)?;
|
||||
let col_item = data_items(&self.col_items).nth(col)?;
|
||||
let row_item = data_item(&self.row_items, &self.row_pos, row)?;
|
||||
let col_item = data_item(&self.col_items, &self.col_pos, col)?;
|
||||
let mut coords = page_constraints(&self.page_coords);
|
||||
for (cat, item) in self.row_cats.iter().zip(row_item.iter()) {
|
||||
coords.push((cat.clone(), item.clone()));
|
||||
@@ -819,7 +853,7 @@ impl GridLayout {
|
||||
/// The coordinates a COLUMN answers for once its rows are aggregated away:
|
||||
/// the page/filter constraints and the column's own tuple (core.md §6).
|
||||
pub fn column_key(&self, col: usize) -> Option<CellKey> {
|
||||
let col_item = data_items(&self.col_items).nth(col)?;
|
||||
let col_item = data_item(&self.col_items, &self.col_pos, col)?;
|
||||
let mut coords = page_constraints(&self.page_coords);
|
||||
for (cat, item) in self.col_cats.iter().zip(col_item.iter()) {
|
||||
coords.push((cat.clone(), item.clone()));
|
||||
@@ -829,12 +863,12 @@ impl GridLayout {
|
||||
|
||||
/// Visual index of the nth data row (skipping group headers).
|
||||
pub fn data_row_to_visual(&self, data_row: usize) -> Option<usize> {
|
||||
data_index_to_visual(&self.row_items, data_row)
|
||||
data_index_to_visual(&self.row_pos, data_row)
|
||||
}
|
||||
|
||||
/// Visual index of the nth data column (skipping group headers).
|
||||
pub fn data_col_to_visual(&self, data_col: usize) -> Option<usize> {
|
||||
data_index_to_visual(&self.col_items, data_col)
|
||||
data_index_to_visual(&self.col_pos, data_col)
|
||||
}
|
||||
|
||||
/// The `(cat_name, group_name)` of the innermost group the Nth data row
|
||||
@@ -846,12 +880,12 @@ impl GridLayout {
|
||||
/// Whether the cell at `(row, col)` folds a collapsed group: an aggregate
|
||||
/// with no record under it. The one home of that answer (§14).
|
||||
pub fn folds_a_group(&self, row: usize, col: usize) -> bool {
|
||||
self.row_folds_a_group(row) || folds(&self.col_items, col)
|
||||
self.row_folds_a_group(row) || folds(&self.col_items, &self.col_pos, col)
|
||||
}
|
||||
|
||||
/// The row half of [`Self::folds_a_group`], for a command about the row alone.
|
||||
pub fn row_folds_a_group(&self, row: usize) -> bool {
|
||||
folds(&self.row_items, row)
|
||||
folds(&self.row_items, &self.row_pos, row)
|
||||
}
|
||||
|
||||
/// The first data row of `cat`/`group`'s run under `prefix` — where a toggle
|
||||
@@ -867,12 +901,12 @@ impl GridLayout {
|
||||
|
||||
/// The coordinate tuple of the Nth data row.
|
||||
pub fn row_tuple(&self, row: usize) -> Option<&[String]> {
|
||||
data_items(&self.row_items).nth(row).map(Vec::as_slice)
|
||||
data_item(&self.row_items, &self.row_pos, row).map(Vec::as_slice)
|
||||
}
|
||||
|
||||
/// The coordinate tuple of the Nth data column.
|
||||
pub fn col_tuple(&self, col: usize) -> Option<&[String]> {
|
||||
data_items(&self.col_items).nth(col).map(Vec::as_slice)
|
||||
data_item(&self.col_items, &self.col_pos, col).map(Vec::as_slice)
|
||||
}
|
||||
|
||||
/// The `(cat_name, group_name)` of the innermost group the Nth data
|
||||
@@ -2970,3 +3004,171 @@ mod hierarchy {
|
||||
assert_eq!(l.row_items[0], header(0, "2024", false));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod lookup_cost_tests {
|
||||
use super::{DATA_ITEM_STEPS, GridLayout};
|
||||
use crate::view::Axis;
|
||||
use crate::workbook::Workbook;
|
||||
|
||||
fn tall(rows: usize) -> GridLayout {
|
||||
let mut wb = Workbook::new("T");
|
||||
wb.add_category("Row").unwrap();
|
||||
wb.add_category("Col").unwrap();
|
||||
for i in 0..rows {
|
||||
wb.add_item("Row", format!("r{i}")).unwrap();
|
||||
}
|
||||
wb.add_item("Col", "c0").unwrap();
|
||||
wb.add_item("Col", "c1").unwrap();
|
||||
wb.active_view_mut().set_axis("Row", Axis::Row);
|
||||
wb.active_view_mut().set_axis("Col", Axis::Column);
|
||||
GridLayout::new(&wb.model, wb.active_view())
|
||||
}
|
||||
|
||||
fn steps(f: impl FnOnce()) -> usize {
|
||||
DATA_ITEM_STEPS.with(|n| n.set(0));
|
||||
f();
|
||||
DATA_ITEM_STEPS.with(|n| n.get())
|
||||
}
|
||||
|
||||
/// improvise-felo.13: context/test-evidence/layout-cost.md.
|
||||
#[test]
|
||||
fn an_axis_lookup_does_work_independent_of_the_row_index() {
|
||||
let layout = tall(400);
|
||||
let last = layout.row_count() - 1;
|
||||
assert_eq!(last, 399, "the fixture is as tall as the test needs");
|
||||
let lookups: [(&str, &dyn Fn(usize)); 4] = [
|
||||
("cell_key", &|r| {
|
||||
let _ = layout.cell_key(r, 1);
|
||||
}),
|
||||
("row_tuple", &|r| {
|
||||
let _ = layout.row_tuple(r);
|
||||
}),
|
||||
("row_label", &|r| {
|
||||
let _ = layout.row_label(r);
|
||||
}),
|
||||
("column_key", &|r| {
|
||||
let _ = layout.column_key(r % 2);
|
||||
}),
|
||||
];
|
||||
for (name, lookup) in lookups {
|
||||
let first = steps(|| lookup(0));
|
||||
let deep = steps(|| lookup(last));
|
||||
assert!(
|
||||
deep <= first + 2,
|
||||
"{name}: row {last} cost {deep} steps against {first} at row 0"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// improvise-felo.13: the index answers what the walk it replaced did, through
|
||||
/// groups, a collapsed fold and a pruned axis. Evidence: layout-cost.md.
|
||||
#[test]
|
||||
fn the_position_index_answers_what_the_walk_answered() {
|
||||
use super::{AxisEntry, data_items};
|
||||
use crate::model::cell::{CellKey, CellValue};
|
||||
let mut wb = Workbook::new("T");
|
||||
for cat in ["Row", "Col"] {
|
||||
wb.add_category(cat).unwrap();
|
||||
}
|
||||
for (i, g) in ["Q1", "Q1", "Q2", "Q2", "Q3"].iter().enumerate() {
|
||||
wb.add_item_in_group("Row", format!("r{i}"), *g).unwrap();
|
||||
}
|
||||
wb.add_item("Row", "ghost").unwrap();
|
||||
wb.add_item("Col", "c0").unwrap();
|
||||
wb.active_view_mut().set_axis("Row", Axis::Row);
|
||||
wb.active_view_mut().set_axis("Col", Axis::Column);
|
||||
wb.active_view_mut().set_expand_depth("Row", 1);
|
||||
wb.active_view_mut().toggle_group_collapse("Row", &["Q2"]);
|
||||
wb.active_view_mut().prune_empty = true;
|
||||
for i in 0..5 {
|
||||
let key = CellKey::new(vec![
|
||||
("Row".into(), format!("r{i}")),
|
||||
("Col".into(), "c0".into()),
|
||||
]);
|
||||
wb.model.set_cell(key, CellValue::Number(1.0));
|
||||
}
|
||||
let layout = GridLayout::with_frozen_records(&wb.model, wb.active_view(), None);
|
||||
assert!(
|
||||
layout
|
||||
.row_items
|
||||
.iter()
|
||||
.any(|e| !matches!(e, AxisEntry::DataItem { .. }))
|
||||
);
|
||||
assert!(layout.row_items.iter().any(|e| matches!(
|
||||
e,
|
||||
AxisEntry::DataItem {
|
||||
collapsed: true,
|
||||
..
|
||||
}
|
||||
)));
|
||||
let walked: Vec<&Vec<String>> = data_items(&layout.row_items).collect();
|
||||
assert_eq!(layout.row_count(), walked.len());
|
||||
let visual: Vec<usize> = layout
|
||||
.row_items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, e)| matches!(e, AxisEntry::DataItem { .. }))
|
||||
.map(|(at, _)| at)
|
||||
.collect();
|
||||
for i in 0..walked.len() + 2 {
|
||||
assert_eq!(
|
||||
layout.row_tuple(i),
|
||||
walked.get(i).map(|t| t.as_slice()),
|
||||
"row {i}"
|
||||
);
|
||||
assert_eq!(
|
||||
layout.data_row_to_visual(i),
|
||||
visual.get(i).copied(),
|
||||
"visual {i}"
|
||||
);
|
||||
let folded = matches!(
|
||||
data_items_entry(&layout.row_items, i),
|
||||
Some(AxisEntry::DataItem {
|
||||
collapsed: true,
|
||||
..
|
||||
})
|
||||
);
|
||||
assert_eq!(layout.row_folds_a_group(i), folded, "fold {i}");
|
||||
}
|
||||
}
|
||||
|
||||
/// improvise-felo.13: a drill held open while records are removed elsewhere.
|
||||
/// Evidence: layout-cost.md.
|
||||
#[test]
|
||||
fn a_frozen_records_snapshot_replaces_the_live_row_index() {
|
||||
use crate::model::cell::{CellKey, CellValue};
|
||||
let mut wb = Workbook::new("T");
|
||||
wb.add_category("Region").unwrap();
|
||||
wb.add_item("_Measure", "Revenue").unwrap();
|
||||
let cell = |i: usize| {
|
||||
CellKey::new(vec![
|
||||
("Region".into(), format!("r{i}")),
|
||||
("_Measure".into(), "Revenue".into()),
|
||||
])
|
||||
};
|
||||
for i in 0..3 {
|
||||
wb.add_item("Region", format!("r{i}")).unwrap();
|
||||
wb.model.set_cell(cell(i), CellValue::Number(1.0));
|
||||
}
|
||||
let v = wb.active_view_mut();
|
||||
v.set_axis("_Index", Axis::Row);
|
||||
v.set_axis("_Dim", Axis::Column);
|
||||
let frozen = GridLayout::new(&wb.model, wb.active_view()).records;
|
||||
wb.model.clear_cell(&cell(1));
|
||||
wb.model.clear_cell(&cell(2));
|
||||
let live = GridLayout::with_frozen_records(&wb.model, wb.active_view(), None);
|
||||
assert_eq!(live.row_count(), 1, "the control: one record is live");
|
||||
let held = GridLayout::with_frozen_records(&wb.model, wb.active_view(), frozen);
|
||||
assert_eq!(held.row_count(), 3);
|
||||
assert_eq!(held.row_tuple(2), Some(&["2".to_string()][..]));
|
||||
assert!(held.cell_key(2, 0).is_some());
|
||||
}
|
||||
|
||||
fn data_items_entry(items: &[super::AxisEntry], n: usize) -> Option<&super::AxisEntry> {
|
||||
items
|
||||
.iter()
|
||||
.filter(|e| matches!(e, super::AxisEntry::DataItem { .. }))
|
||||
.nth(n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,14 +7,32 @@ use serde_json::Value;
|
||||
|
||||
use crate::storage::{PhysicalFs, Storage};
|
||||
|
||||
/// Parsed CSV records. Every value is its source text, so they reach an import
|
||||
/// only through `ImportPipeline::from_csv` (improvise-8jk6.4).
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct CsvRecords(Vec<Value>);
|
||||
|
||||
impl std::ops::Deref for CsvRecords {
|
||||
type Target = [Value];
|
||||
fn deref(&self) -> &[Value] {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl CsvRecords {
|
||||
pub(crate) fn into_values(self) -> Vec<Value> {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
pub fn csv_path_p(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("csv"))
|
||||
}
|
||||
|
||||
/// Parse a CSV file into a `serde_json::Value` array. Short-row warnings go to
|
||||
/// stderr; [`parse_csv_with_warnings`] captures them instead.
|
||||
pub fn parse_csv(path: &Path) -> Result<Vec<Value>> {
|
||||
/// Parse a CSV file into records. Short-row warnings go to stderr;
|
||||
/// [`parse_csv_with_warnings`] captures them instead.
|
||||
pub fn parse_csv(path: &Path) -> Result<CsvRecords> {
|
||||
let (records, warnings) = parse_csv_with_warnings(path)?;
|
||||
for warning in &warnings {
|
||||
eprintln!("warning: {warning}");
|
||||
@@ -24,7 +42,7 @@ pub fn parse_csv(path: &Path) -> Result<Vec<Value>> {
|
||||
|
||||
/// Parse a CSV file, returning records plus warnings. A short row keeps every
|
||||
/// header column, missing ones `Null`, each reported with a row count.
|
||||
pub fn parse_csv_with_warnings(path: &Path) -> Result<(Vec<Value>, Vec<String>)> {
|
||||
pub fn parse_csv_with_warnings(path: &Path) -> Result<(CsvRecords, Vec<String>)> {
|
||||
let bytes = std::fs::read(path)
|
||||
.with_context(|| format!("Failed to open CSV file: {}", path.display()))?;
|
||||
parse_csv_bytes_with_warnings(&bytes)
|
||||
@@ -32,9 +50,9 @@ pub fn parse_csv_with_warnings(path: &Path) -> Result<(Vec<Value>, Vec<String>)>
|
||||
|
||||
/// As [`parse_csv_with_warnings`] but over bytes, so import can come from any
|
||||
/// [`Storage`](crate::storage::Storage) backend, not only a file (improvise-fuj).
|
||||
pub fn parse_csv_bytes_with_warnings(bytes: &[u8]) -> Result<(Vec<Value>, Vec<String>)> {
|
||||
pub fn parse_csv_bytes_with_warnings(bytes: &[u8]) -> Result<(CsvRecords, Vec<String>)> {
|
||||
let (_, records, warnings) = parse_csv_bytes(bytes)?;
|
||||
Ok((records, warnings))
|
||||
Ok((CsvRecords(records), warnings))
|
||||
}
|
||||
|
||||
/// The header (after renaming), the records and the warnings.
|
||||
@@ -152,7 +170,7 @@ fn disambiguate_headers(raw: Vec<String>) -> (Vec<String>, Vec<String>) {
|
||||
|
||||
/// Parse multiple CSV files and merge into a single JSON array.
|
||||
/// Each record gets a "File" field set to the filename stem (e.g., "sales" from "sales.csv").
|
||||
pub fn merge_csvs(paths: &[impl AsRef<Path>]) -> Result<Vec<Value>> {
|
||||
pub fn merge_csvs(paths: &[impl AsRef<Path>]) -> Result<CsvRecords> {
|
||||
let merged = merge_csvs_from(&PhysicalFs, paths)?;
|
||||
for warning in &merged.warnings {
|
||||
eprintln!("warning: {warning}");
|
||||
@@ -173,7 +191,7 @@ pub struct SourceFile {
|
||||
/// Several CSV files as one record set, with each file's own columns kept.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Merged {
|
||||
pub records: Vec<Value>,
|
||||
pub records: CsvRecords,
|
||||
/// One per distinct `File` item, in the order the files were read.
|
||||
pub sources: Vec<SourceFile>,
|
||||
/// Each file's parse warnings, prefixed with its path.
|
||||
@@ -184,7 +202,7 @@ pub struct Merged {
|
||||
/// and two files with one stem share its item and its columns (improvise-lpzu).
|
||||
pub fn merge_csvs_from(fs: &dyn Storage, paths: &[impl AsRef<Path>]) -> Result<Merged> {
|
||||
let mut merged = Merged {
|
||||
records: Vec::new(),
|
||||
records: CsvRecords::default(),
|
||||
sources: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
};
|
||||
@@ -212,7 +230,7 @@ pub fn merge_csvs_from(fs: &dyn Storage, paths: &[impl AsRef<Path>]) -> Result<M
|
||||
if let Value::Object(ref mut map) = record {
|
||||
map.insert(FILE_CATEGORY.to_string(), Value::String(stem.clone()));
|
||||
}
|
||||
merged.records.push(record);
|
||||
merged.records.0.push(record);
|
||||
}
|
||||
}
|
||||
Ok(merged)
|
||||
@@ -266,26 +284,33 @@ fn csv_documents_in(fs: &dyn Storage, dir: &Path) -> Result<Vec<PathBuf>> {
|
||||
Ok(csvs)
|
||||
}
|
||||
|
||||
/// A field is its source text: which ones are numbers is decided by the role
|
||||
/// the import gives the column (improvise-8jk6.4, `context/code/formula-and-io.md`).
|
||||
fn parse_csv_field(field: &str) -> Value {
|
||||
if field.is_empty() {
|
||||
return Value::Null;
|
||||
match field.is_empty() {
|
||||
true => Value::Null,
|
||||
false => Value::String(field.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
// Try to parse as number (integer or float)
|
||||
if let Ok(num) = field.parse::<i64>() {
|
||||
return Value::Number(serde_json::Number::from(num));
|
||||
/// The number a CSV field's text reads as, where a number is wanted.
|
||||
pub fn number_in(text: &str) -> Option<serde_json::Number> {
|
||||
if let Ok(num) = text.parse::<i64>() {
|
||||
return Some(serde_json::Number::from(num));
|
||||
}
|
||||
|
||||
// NaN and the infinities have no JSON number. A fabricated 0 is an invented
|
||||
// measurement (improvise-jc95), so the token falls through and stays text.
|
||||
if let Ok(num) = field.parse::<f64>()
|
||||
&& let Some(number) = serde_json::Number::from_f64(num)
|
||||
{
|
||||
return Value::Number(number);
|
||||
}
|
||||
// measurement (improvise-jc95), so the token stays text.
|
||||
text.parse::<f64>()
|
||||
.ok()
|
||||
.and_then(serde_json::Number::from_f64)
|
||||
}
|
||||
|
||||
// Otherwise treat as string
|
||||
Value::String(field.to_string())
|
||||
/// A CSV value as a typed JSON value: numeric text becomes its number.
|
||||
pub fn typed(value: &Value) -> Value {
|
||||
match value.as_str().and_then(number_in) {
|
||||
Some(number) => Value::Number(number),
|
||||
None => value.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -310,8 +335,9 @@ mod tests {
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
|
||||
assert_eq!(records[0]["Product"], Value::String("Shirts".to_string()));
|
||||
assert_eq!(records[0]["Revenue"], Value::String("1000".to_string()));
|
||||
assert_eq!(
|
||||
records[0]["Revenue"],
|
||||
typed(&records[0]["Revenue"]),
|
||||
Value::Number(serde_json::Number::from(1000))
|
||||
);
|
||||
}
|
||||
@@ -323,9 +349,10 @@ mod tests {
|
||||
let records = parse_csv(&path).unwrap();
|
||||
|
||||
assert_eq!(records.len(), 2);
|
||||
assert!(records[0]["Revenue"].is_f64());
|
||||
assert_eq!(records[0]["Revenue"], Value::String("1000.50".to_string()));
|
||||
assert!(typed(&records[0]["Revenue"]).is_f64());
|
||||
assert_eq!(
|
||||
records[0]["Revenue"],
|
||||
typed(&records[0]["Revenue"]),
|
||||
Value::Number(serde_json::Number::from_f64(1000.50).unwrap())
|
||||
);
|
||||
}
|
||||
@@ -363,10 +390,10 @@ mod tests {
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0]["Name"], Value::String("Widget".to_string()));
|
||||
assert_eq!(
|
||||
records[0]["Count"],
|
||||
typed(&records[0]["Count"]),
|
||||
Value::Number(serde_json::Number::from(5))
|
||||
);
|
||||
assert!(records[0]["Price"].is_f64());
|
||||
assert!(typed(&records[0]["Price"]).is_f64());
|
||||
assert_eq!(records[0]["Active"], Value::String("true".to_string()));
|
||||
}
|
||||
|
||||
@@ -386,10 +413,7 @@ mod tests {
|
||||
assert_eq!(records[3]["File"], Value::String("expenses".to_string()));
|
||||
// Original fields preserved
|
||||
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
|
||||
assert_eq!(
|
||||
records[2]["Revenue"],
|
||||
Value::Number(serde_json::Number::from(50))
|
||||
);
|
||||
assert_eq!(records[2]["Revenue"], Value::String("50".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -492,10 +516,10 @@ mod tests {
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
merged.records,
|
||||
*merged.records,
|
||||
[
|
||||
serde_json::json!({"entity": "src/a.rs", "coupled": "src/b.rs", "degree": 80, "File": "coupling"}),
|
||||
serde_json::json!({"entity": "src/a.rs", "age-months": 2, "File": "age"}),
|
||||
serde_json::json!({"entity": "src/a.rs", "coupled": "src/b.rs", "degree": "80", "File": "coupling"}),
|
||||
serde_json::json!({"entity": "src/a.rs", "age-months": "2", "File": "age"}),
|
||||
]
|
||||
);
|
||||
}
|
||||
@@ -656,10 +680,7 @@ mod tests {
|
||||
assert!(warnings.is_empty());
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
|
||||
assert_eq!(
|
||||
records[0]["Revenue"],
|
||||
Value::Number(serde_json::Number::from(1))
|
||||
);
|
||||
assert_eq!(records[0]["Revenue"], Value::String("1".to_string()));
|
||||
assert_eq!(records[1]["Region"], Value::String("West".to_string()));
|
||||
|
||||
let (control, _) = parse_csv_bytes_with_warnings(b"Region,Revenue\nEa\rst,1\n").unwrap();
|
||||
@@ -714,7 +735,8 @@ mod tests {
|
||||
token.parse::<f64>().is_ok_and(|n| !n.is_finite()),
|
||||
"precondition: {token} parses as a non-finite f64"
|
||||
);
|
||||
assert_eq!(parse_csv_field(token), Value::String(token.to_string()));
|
||||
let text = Value::String(token.to_string());
|
||||
assert_eq!(typed(&text), text);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -723,10 +745,10 @@ mod tests {
|
||||
#[test]
|
||||
fn an_ordinary_float_is_still_a_number() {
|
||||
assert_eq!(
|
||||
parse_csv_field("1000.50"),
|
||||
typed(&Value::String("1000.50".into())),
|
||||
Value::Number(serde_json::Number::from_f64(1000.50).unwrap())
|
||||
);
|
||||
assert_eq!(parse_csv_field("0"), Value::Number(0.into()));
|
||||
assert_eq!(typed(&Value::String("0".into())), Value::Number(0.into()));
|
||||
}
|
||||
|
||||
/// improvise-5hfh: two equal header names are one JSON key, so the later
|
||||
@@ -735,9 +757,9 @@ mod tests {
|
||||
fn a_repeated_header_is_renamed_rather_than_overwriting_its_twin() {
|
||||
let (records, warnings) = parse_csv_bytes_with_warnings(b"A,B,A\n1,2,3\n").unwrap();
|
||||
let map = records[0].as_object().unwrap();
|
||||
assert_eq!(map["A"], Value::Number(1.into()));
|
||||
assert_eq!(map["B"], Value::Number(2.into()));
|
||||
assert_eq!(map["A_2"], Value::Number(3.into()));
|
||||
assert_eq!(map["A"], Value::String("1".into()));
|
||||
assert_eq!(map["B"], Value::String("2".into()));
|
||||
assert_eq!(map["A_2"], Value::String("3".into()));
|
||||
assert_eq!(warnings.len(), 1);
|
||||
assert!(warnings[0].contains("'A'") && warnings[0].contains("'A_2'"));
|
||||
assert!(warnings[0].contains("column 3"), "{}", warnings[0]);
|
||||
@@ -750,8 +772,8 @@ mod tests {
|
||||
let (records, _) = parse_csv_bytes_with_warnings(b"A,A_2,A\n1,2,3\n").unwrap();
|
||||
let map = records[0].as_object().unwrap();
|
||||
assert_eq!(map.len(), 3, "{map:?}");
|
||||
assert_eq!(map["A_2"], Value::Number(2.into()));
|
||||
assert_eq!(map["A_3"], Value::Number(3.into()));
|
||||
assert_eq!(map["A_2"], Value::String("2".into()));
|
||||
assert_eq!(map["A_3"], Value::String("3".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -786,8 +808,9 @@ mod tests {
|
||||
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0]["Date"], Value::String("03/31/2026".to_string()));
|
||||
assert_eq!(records[0]["Amount"], Value::String("-50.00".to_string()));
|
||||
assert_eq!(
|
||||
records[0]["Amount"],
|
||||
typed(&records[0]["Amount"]),
|
||||
Value::Number(serde_json::Number::from_f64(-50.00).unwrap())
|
||||
);
|
||||
assert_eq!(records[0]["Flag"], Value::String("*".to_string()));
|
||||
@@ -797,7 +820,7 @@ mod tests {
|
||||
Value::String("VENMO PAYMENT 260331".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
records[1]["Amount"],
|
||||
typed(&records[1]["Amount"]),
|
||||
Value::Number(serde_json::Number::from_f64(-240.00).unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@ use super::analyzer::{
|
||||
DateComponent, FieldKind, FieldProposal, analyze_records, extract_array_at_path,
|
||||
extract_date_component, find_array_paths,
|
||||
};
|
||||
use super::csv_parser::{FILE_CATEGORY, Merged, SourceFile};
|
||||
use crate::formula::parse_formula;
|
||||
use super::csv_parser::{self, CsvRecords, FILE_CATEGORY, Merged, SourceFile};
|
||||
use crate::formula::{Formula, parse_formula};
|
||||
use crate::model::MAX_CATEGORIES;
|
||||
use crate::model::cell::{CellKey, CellValue};
|
||||
use crate::view::{Axis, VIRTUAL_VIEW_NAMES, View, ViewStep, records_steps};
|
||||
@@ -33,6 +33,42 @@ pub struct ImportPipeline {
|
||||
/// The files a CSV merge read, each of which gets its own view. Empty for
|
||||
/// a single file or a JSON import (improvise-lpzu).
|
||||
pub sources: Vec<SourceFile>,
|
||||
/// What the records' string values are, fixed by the constructor (8jk6.4).
|
||||
scalars: Scalars,
|
||||
}
|
||||
|
||||
/// How a record's values read. Why CSV is text: `context/code/formula-and-io.md`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Scalars {
|
||||
/// JSON: a string is text and a number is a number.
|
||||
Typed,
|
||||
/// CSV: every value is source text, read as a number only where one is wanted.
|
||||
Text,
|
||||
}
|
||||
|
||||
impl Scalars {
|
||||
/// The record as analysis sees it: typed, whatever its source.
|
||||
fn typed_record(self, record: &Value) -> Value {
|
||||
match (self, record) {
|
||||
(Scalars::Text, Value::Object(map)) => Value::Object(
|
||||
map.iter()
|
||||
.map(|(k, v)| (k.clone(), csv_parser::typed(v)))
|
||||
.collect(),
|
||||
),
|
||||
(Scalars::Text, other) | (Scalars::Typed, other) => other.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The value a measure takes from `value`, if it is a number.
|
||||
fn number(self, value: &Value) -> Option<f64> {
|
||||
match self {
|
||||
Scalars::Typed => value.as_f64(),
|
||||
Scalars::Text => value
|
||||
.as_str()
|
||||
.and_then(csv_parser::number_in)
|
||||
.and_then(|n| n.as_f64()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Several source records that land on one coordinate tuple (improvise-cgqt).
|
||||
@@ -57,7 +93,17 @@ pub enum DuplicatePolicy {
|
||||
}
|
||||
|
||||
impl ImportPipeline {
|
||||
/// A pipeline over a JSON value.
|
||||
pub fn new(raw: Value) -> Self {
|
||||
Self::reading(raw, Scalars::Typed)
|
||||
}
|
||||
|
||||
/// A pipeline over CSV records, whose values are their source text.
|
||||
pub fn from_csv(records: CsvRecords) -> Self {
|
||||
Self::reading(Value::Array(records.into_values()), Scalars::Text)
|
||||
}
|
||||
|
||||
fn reading(raw: Value, scalars: Scalars) -> Self {
|
||||
let array_paths = find_array_paths(&raw);
|
||||
let mut pipeline = Self {
|
||||
raw: raw.clone(),
|
||||
@@ -69,6 +115,7 @@ impl ImportPipeline {
|
||||
formulas: vec![],
|
||||
on_duplicate: DuplicatePolicy::Refuse,
|
||||
sources: vec![],
|
||||
scalars,
|
||||
};
|
||||
|
||||
// Auto-select if root is an array or there is exactly one candidate path.
|
||||
@@ -84,7 +131,7 @@ impl ImportPipeline {
|
||||
|
||||
/// A pipeline over a CSV merge, carrying its sources for the per-file views.
|
||||
pub fn from_merge(merged: Merged) -> Self {
|
||||
let mut pipeline = Self::new(Value::Array(merged.records));
|
||||
let mut pipeline = Self::from_csv(merged.records);
|
||||
pipeline.seed_sources(merged.sources);
|
||||
pipeline
|
||||
}
|
||||
@@ -112,7 +159,52 @@ impl ImportPipeline {
|
||||
self.selected_path = path.to_string();
|
||||
if let Some(arr) = extract_array_at_path(&self.raw, path) {
|
||||
self.records = arr.clone();
|
||||
self.proposals = analyze_records(&self.records);
|
||||
self.analyze();
|
||||
}
|
||||
}
|
||||
|
||||
/// Propose a kind for every field, reading the records as `scalars` says.
|
||||
pub fn analyze(&mut self) {
|
||||
let typed: Vec<Value> = self
|
||||
.records
|
||||
.iter()
|
||||
.map(|r| self.scalars.typed_record(r))
|
||||
.collect();
|
||||
self.proposals = analyze_records(&typed);
|
||||
}
|
||||
|
||||
/// The warning for the records the build will skip, if it skips any.
|
||||
pub fn skip_warning(&self) -> Option<String> {
|
||||
let skipped = self.records.iter().filter(|r| !r.is_object()).count();
|
||||
skip_warnings(self.records.len(), skipped)
|
||||
}
|
||||
|
||||
/// Why there is nothing to build, when no record is an object to place.
|
||||
pub fn nothing_to_import(&self) -> Option<String> {
|
||||
let total = self.records.len();
|
||||
match (total, self.records.iter().any(Value::is_object)) {
|
||||
(_, true) => None,
|
||||
(0, false) => Some("Nothing to import: the source has no records".into()),
|
||||
(_, false) => Some(format!(
|
||||
"Nothing to import: all {total} records were skipped as not JSON objects"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The formulas, parsed. Every one that does not parse is named with its
|
||||
/// error, never dropped (improvise-8jk6.5).
|
||||
pub fn parsed_formulas(&self) -> Result<Vec<Formula>> {
|
||||
let mut parsed = vec![];
|
||||
let mut refused: Vec<String> = vec![];
|
||||
for raw in &self.formulas {
|
||||
match import_formula(raw) {
|
||||
Ok(formula) => parsed.push(formula),
|
||||
Err(e) => refused.push(e.to_string()),
|
||||
}
|
||||
}
|
||||
match refused.is_empty() {
|
||||
true => Ok(parsed),
|
||||
false => Err(anyhow!(refused.join("\n"))),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,6 +319,7 @@ impl ImportPipeline {
|
||||
measures: accepted(&[FieldKind::Measure]),
|
||||
labels: accepted(&[FieldKind::Label]),
|
||||
date_extractions,
|
||||
scalars: self.scalars,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,9 +330,13 @@ impl ImportPipeline {
|
||||
let categories = &plan.categories;
|
||||
let date_extractions = &plan.date_extractions;
|
||||
|
||||
if let Some(why) = self.nothing_to_import() {
|
||||
return Err(anyhow!(why));
|
||||
}
|
||||
if categories.is_empty() {
|
||||
return Err(anyhow!("At least one category must be accepted"));
|
||||
}
|
||||
let formulas = self.parsed_formulas()?;
|
||||
|
||||
let placements: Vec<Placement> = self.records.iter().map(|r| plan.place(r)).collect();
|
||||
let collisions = collisions_among(&placements);
|
||||
@@ -288,7 +385,6 @@ impl ImportPipeline {
|
||||
wb.add_item("_Measure", &m.field)?;
|
||||
}
|
||||
|
||||
let mut skipped_non_objects = 0usize;
|
||||
// Per-extraction tally of values the detected format could not parse. A
|
||||
// derived coordinate that is merely absent is improvise-yss5.
|
||||
let mut unparsed_dates = vec![0usize; date_extractions.len()];
|
||||
@@ -304,7 +400,7 @@ impl ImportPipeline {
|
||||
|
||||
for (at, placement) in placements.into_iter().enumerate() {
|
||||
match placement {
|
||||
Placement::NotObject => skipped_non_objects += 1,
|
||||
Placement::NotObject => {}
|
||||
Placement::Placed {
|
||||
mut coords,
|
||||
measures,
|
||||
@@ -352,22 +448,15 @@ impl ImportPipeline {
|
||||
wb.model.set_cell(cell.key, CellValue::Number(cell.total));
|
||||
}
|
||||
|
||||
// Parse and add formulas
|
||||
// Formulas target the "_Measure" category by default.
|
||||
let formula_cat: String = "_Measure".to_string();
|
||||
for raw in &self.formulas {
|
||||
if let Ok(formula) = parse_formula(raw, &formula_cat) {
|
||||
wb.model.add_formula(formula);
|
||||
}
|
||||
for formula in formulas {
|
||||
wb.model.add_formula(formula);
|
||||
}
|
||||
|
||||
for view in source_views(&wb, &self.sources, &plan) {
|
||||
wb.views.insert(view.name.clone(), view);
|
||||
}
|
||||
|
||||
let mut warnings: Vec<String> = skip_warnings(self.records.len(), skipped_non_objects)
|
||||
.into_iter()
|
||||
.collect();
|
||||
let mut warnings: Vec<String> = self.skip_warning().into_iter().collect();
|
||||
warnings.extend(unparsed_date_warnings(date_extractions, &unparsed_dates));
|
||||
warnings.extend(collision_warning(&summed));
|
||||
warnings.extend(occurrence.map(|name| separated_note(name, &collisions)));
|
||||
@@ -463,6 +552,7 @@ struct Plan<'a> {
|
||||
measures: Vec<&'a FieldProposal>,
|
||||
labels: Vec<&'a FieldProposal>,
|
||||
date_extractions: Vec<(&'a str, &'a str, DateComponent, String)>,
|
||||
scalars: Scalars,
|
||||
}
|
||||
|
||||
/// Where one source record lands. Worked out once, for the build and for the
|
||||
@@ -519,7 +609,7 @@ impl Plan<'_> {
|
||||
let measures = self
|
||||
.measures
|
||||
.iter()
|
||||
.filter_map(|m| Some((m.field.clone(), map.get(&m.field)?.as_f64()?)))
|
||||
.filter_map(|m| Some((m.field.clone(), self.scalars.number(map.get(&m.field)?)?)))
|
||||
.collect();
|
||||
Placement::Placed {
|
||||
coords,
|
||||
@@ -635,6 +725,11 @@ fn unparsed_date_warnings(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// An import formula, targeting `_Measure`, or the error naming it (improvise-8jk6.5).
|
||||
fn import_formula(raw: &str) -> Result<Formula> {
|
||||
parse_formula(raw, "_Measure").map_err(|e| anyhow!("Formula error in '{raw}': {e}"))
|
||||
}
|
||||
|
||||
/// The warning for records that could not be placed in the grid. A record
|
||||
/// missing a category value is placed, so only a non-object is skipped.
|
||||
fn skip_warnings(total: usize, skipped_non_objects: usize) -> Option<String> {
|
||||
@@ -822,7 +917,9 @@ impl ImportWizard {
|
||||
fn validation_error(&self) -> Option<String> {
|
||||
match self.step {
|
||||
WizardStep::ReviewProposals => self
|
||||
.no_category_message()
|
||||
.pipeline
|
||||
.nothing_to_import()
|
||||
.or_else(|| self.no_category_message())
|
||||
.or_else(|| self.category_limit_message()),
|
||||
WizardStep::ConfigureDates => self.category_limit_message(),
|
||||
WizardStep::ResolveDuplicates => match &self.pipeline.on_duplicate {
|
||||
@@ -1117,14 +1214,24 @@ impl ImportWizard {
|
||||
self.formula_buffer.pop();
|
||||
}
|
||||
|
||||
/// Commit the current formula buffer to the pipeline's formula list.
|
||||
/// Commit the current formula buffer to the pipeline's formula list. One
|
||||
/// that does not parse stays in the buffer, with the error (improvise-8jk6.5).
|
||||
pub fn confirm_formula(&mut self) {
|
||||
let text = self.formula_buffer.trim().to_string();
|
||||
if !text.is_empty() {
|
||||
self.pipeline.formulas.push(text);
|
||||
match (text.is_empty(), import_formula(&text)) {
|
||||
(false, Err(e)) => self.message = Some(e.to_string()),
|
||||
(false, Ok(_)) => {
|
||||
self.pipeline.formulas.push(text);
|
||||
self.close_formula_edit();
|
||||
}
|
||||
(true, _) => self.close_formula_edit(),
|
||||
}
|
||||
}
|
||||
|
||||
fn close_formula_edit(&mut self) {
|
||||
self.formula_buffer.clear();
|
||||
self.formula_editing = false;
|
||||
self.message = None;
|
||||
self.cursor = self.pipeline.formulas.len().saturating_sub(1);
|
||||
}
|
||||
|
||||
@@ -2244,8 +2351,11 @@ mod tests {
|
||||
),
|
||||
];
|
||||
|
||||
fn code_maat_records() -> Value {
|
||||
Value::Array(merged_pipeline(&CODE_MAAT).records)
|
||||
/// The code-maat merge without its sources, so no per-file views.
|
||||
fn code_maat_without_views() -> ImportPipeline {
|
||||
let mut p = merged_pipeline(&CODE_MAAT);
|
||||
p.sources.clear();
|
||||
p
|
||||
}
|
||||
|
||||
/// Every non-empty cell a view's grid draws, row by row.
|
||||
@@ -2519,7 +2629,7 @@ mod tests {
|
||||
/// 0, "" or a skip. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn files_with_different_headers_import_as_one_sparse_model() {
|
||||
let p = ImportPipeline::new(code_maat_records());
|
||||
let p = code_maat_without_views();
|
||||
let kinds: Vec<(&str, FieldKind)> = p
|
||||
.proposals
|
||||
.iter()
|
||||
@@ -2561,7 +2671,7 @@ mod tests {
|
||||
/// "". Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_label_column_a_record_lacks_is_no_coordinate_not_an_empty_item() {
|
||||
let mut p = ImportPipeline::new(code_maat_records());
|
||||
let mut p = code_maat_without_views();
|
||||
p.proposals
|
||||
.iter_mut()
|
||||
.find(|f| f.field == "coupled")
|
||||
@@ -3164,4 +3274,108 @@ mod tests {
|
||||
w.message
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.5. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_formula_that_does_not_parse_stops_the_build_and_is_named() {
|
||||
let mut p = ImportPipeline::new(json!([{"region": "East", "revenue": 1.0}]));
|
||||
p.formulas = vec![
|
||||
"Good = revenue".into(),
|
||||
"Bad = revenue +".into(),
|
||||
"Worse = * 2".into(),
|
||||
];
|
||||
let err = p.build_model().expect_err("an unparseable formula refuses");
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("Formula error in 'Bad = revenue +'"), "{msg}");
|
||||
assert!(msg.contains("Formula error in 'Worse = * 2'"), "{msg}");
|
||||
assert!(!msg.contains("Good"), "{msg}");
|
||||
}
|
||||
|
||||
/// improvise-8jk6.5: the wizard refuses where the text can still be
|
||||
/// fixed. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn confirming_a_formula_that_does_not_parse_keeps_it_out_and_the_text_in() {
|
||||
let mut w = sample_wizard();
|
||||
w.start_formula_edit();
|
||||
for c in "Bad = revenue +".chars() {
|
||||
w.push_formula_char(c);
|
||||
}
|
||||
w.confirm_formula();
|
||||
assert!(w.pipeline.formulas.is_empty(), "{:?}", w.pipeline.formulas);
|
||||
assert_eq!(w.formula_buffer, "Bad = revenue +");
|
||||
assert!(w.formula_editing);
|
||||
let message = w.message.clone().unwrap_or_default();
|
||||
assert!(
|
||||
message.starts_with("Formula error in 'Bad = revenue +'"),
|
||||
"{message}"
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.4: the role reads the text. Evidence:
|
||||
/// `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_csv_column_read_as_a_category_keeps_its_text_and_as_a_measure_its_number() {
|
||||
use crate::import::csv_parser::parse_csv_bytes_with_warnings;
|
||||
use crate::model::cell::{CellKey, CellValue};
|
||||
let csv = b"commit,lines\n00565796,3\n220944e2,5\n";
|
||||
let (records, _) = parse_csv_bytes_with_warnings(csv).unwrap();
|
||||
let mut p = ImportPipeline::from_csv(records);
|
||||
let kind = |p: &ImportPipeline, f: &str| {
|
||||
p.proposals
|
||||
.iter()
|
||||
.find(|x| x.field == f)
|
||||
.unwrap()
|
||||
.kind
|
||||
.clone()
|
||||
};
|
||||
assert_eq!(kind(&p, "commit"), FieldKind::Measure, "analysis types");
|
||||
let commit = p.proposals.iter_mut().find(|x| x.field == "commit");
|
||||
commit.unwrap().kind = FieldKind::Category;
|
||||
let wb = p.build_model().unwrap();
|
||||
let items: Vec<&String> = wb.model.category("commit").unwrap().items.keys().collect();
|
||||
assert_eq!(items, ["00565796", "220944e2"]);
|
||||
let key = CellKey::new(vec![
|
||||
("_Measure".into(), "lines".into()),
|
||||
("commit".into(), "220944e2".into()),
|
||||
]);
|
||||
assert_eq!(wb.model.get_cell(&key), Some(&CellValue::Number(5.0)));
|
||||
}
|
||||
|
||||
/// improvise-8jk6.6 meets lpzu's header-only rule: a folder of nothing but
|
||||
/// headers kept no record. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_folder_of_only_header_only_files_keeps_no_record_and_refuses() {
|
||||
let p = merged_pipeline(&[("m/a.csv", "entity,x\n"), ("m/b.csv", "entity,y\n")]);
|
||||
assert!(p.proposals.iter().any(|f| f.field == "File"), "setup");
|
||||
let msg = p.build_model().expect_err("refuses").to_string();
|
||||
assert_eq!(msg, "Nothing to import: the source has no records");
|
||||
}
|
||||
|
||||
/// improvise-8jk6.6: the wizard asks on the review step, where another
|
||||
/// path can still be chosen. Evidence: the test-evidence doc.
|
||||
#[test]
|
||||
fn the_review_step_says_when_no_record_can_be_imported() {
|
||||
let mut w = ImportWizard::new(json!([1, 2]));
|
||||
assert_eq!(w.step, WizardStep::ReviewProposals, "setup");
|
||||
w.advance();
|
||||
assert_eq!(w.step, WizardStep::ReviewProposals);
|
||||
assert_eq!(
|
||||
w.message.as_deref(),
|
||||
Some("Nothing to import: all 2 records were skipped as not JSON objects")
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.6. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_build_that_keeps_no_record_refuses_and_says_why() {
|
||||
let skipped = ImportPipeline::new(json!([1, "two", null]));
|
||||
let msg = skipped.build_model().expect_err("refuses").to_string();
|
||||
assert_eq!(
|
||||
msg,
|
||||
"Nothing to import: all 3 records were skipped as not JSON objects"
|
||||
);
|
||||
let empty = ImportPipeline::new(json!([]));
|
||||
let msg = empty.build_model().expect_err("refuses").to_string();
|
||||
assert_eq!(msg, "Nothing to import: the source has no records");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1031,7 +1031,7 @@ fn a_csv_header_with_an_embedded_newline_survives_a_save() {
|
||||
let csv = "\"Region\nName\",Amount\nEast,1\nWest,2\n";
|
||||
let (records, _) =
|
||||
crate::import::csv_parser::parse_csv_bytes_with_warnings(csv.as_bytes()).unwrap();
|
||||
let wb = crate::import::wizard::ImportPipeline::new(serde_json::Value::Array(records))
|
||||
let wb = crate::import::wizard::ImportPipeline::from_csv(records)
|
||||
.build_model()
|
||||
.unwrap();
|
||||
assert!(
|
||||
|
||||
@@ -81,6 +81,8 @@ const SECTIONS = [
|
||||
["context/test-evidence/cli.md", "test-evidence/cli", "The headless CLI"],
|
||||
["context/test-evidence/gp.md", "test-evidence/gp", "gp-tree and improvise-gp"],
|
||||
["context/test-evidence/tooling.md", "test-evidence/tooling", "The repository's own gates"],
|
||||
["context/test-evidence/layout-cost.md", "test-evidence/layout-cost", "Layout and keypress cost"],
|
||||
["context/test-evidence/scale-bisect.md", "test-evidence/scale-bisect", "Scale bisect"],
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
+11
-1
@@ -97,6 +97,10 @@ struct ImportArgs {
|
||||
#[arg(long, value_enum, default_value_t = OnDuplicate::Refuse)]
|
||||
on_duplicate: OnDuplicate,
|
||||
|
||||
/// With --no-wizard, refuse (and write nothing) if any record is skipped
|
||||
#[arg(long)]
|
||||
strict: bool,
|
||||
|
||||
/// Save to file instead of opening TUI
|
||||
#[arg(short, long)]
|
||||
output: Option<PathBuf>,
|
||||
@@ -157,6 +161,7 @@ impl Runnable for ImportArgs {
|
||||
formulas: self.formula,
|
||||
name: self.name,
|
||||
on_duplicate: self.on_duplicate,
|
||||
strict: self.strict,
|
||||
};
|
||||
|
||||
if self.no_wizard {
|
||||
@@ -191,6 +196,7 @@ struct ImportConfig {
|
||||
formulas: Vec<String>,
|
||||
name: Option<String>,
|
||||
on_duplicate: OnDuplicate,
|
||||
strict: bool,
|
||||
}
|
||||
|
||||
fn parse_colon_pairs(args: &[String]) -> Vec<(String, String)> {
|
||||
@@ -295,6 +301,10 @@ fn run_headless_import(
|
||||
model_file: Option<PathBuf>,
|
||||
) -> Result<()> {
|
||||
apply_config_to_pipeline(&mut pipeline, config);
|
||||
// A strict skip refuses before the build, so nothing is written (8jk6.6).
|
||||
if let (true, Some(skipped)) = (config.strict, pipeline.skip_warning()) {
|
||||
anyhow::bail!("--strict: {skipped}");
|
||||
}
|
||||
if pipeline.on_duplicate == DuplicatePolicy::Refuse {
|
||||
let collisions = pipeline.collisions();
|
||||
if !collisions.is_empty() {
|
||||
@@ -381,7 +391,7 @@ fn get_import_data(args: &[PathBuf]) -> Option<ImportPipeline> {
|
||||
Ok(content) => {
|
||||
if csv_path_p(path) {
|
||||
match crate::import::csv_parser::parse_csv(path) {
|
||||
Ok(records) => Some(ImportPipeline::new(Value::Array(records))),
|
||||
Ok(records) => Some(ImportPipeline::from_csv(records)),
|
||||
Err(e) => {
|
||||
eprintln!("CSV parse error: {e}");
|
||||
None
|
||||
|
||||
@@ -64,8 +64,8 @@ fn duplicate_csv_headers_not_silent() {
|
||||
2,
|
||||
"both columns must survive as separate fields: {fields:?}"
|
||||
);
|
||||
assert_eq!(fields["A"].as_f64(), Some(1.0), "{fields:?}");
|
||||
assert_eq!(fields["A_2"].as_f64(), Some(2.0), "{fields:?}");
|
||||
assert_eq!(fields["A"], "1", "{fields:?}");
|
||||
assert_eq!(fields["A_2"], "2", "{fields:?}");
|
||||
assert!(
|
||||
warnings
|
||||
.iter()
|
||||
@@ -78,7 +78,7 @@ fn duplicate_csv_headers_not_silent() {
|
||||
fn blank_measure_still_infers_numeric() {
|
||||
use improvise_io::import::{analyzer::FieldKind, wizard::ImportPipeline};
|
||||
let (records, _) = parse_csv_bytes_with_warnings(b"Region,Revenue\nEast,10\nWest,\n").unwrap();
|
||||
let p = ImportPipeline::new(serde_json::Value::Array(records));
|
||||
let p = ImportPipeline::from_csv(records);
|
||||
let proposal = p.proposals.iter().find(|p| p.field == "Revenue").unwrap();
|
||||
assert_eq!(proposal.kind, FieldKind::Measure);
|
||||
}
|
||||
@@ -87,7 +87,7 @@ fn repeated_import_coordinates_not_silently_overwritten() {
|
||||
use improvise_io::import::wizard::ImportPipeline;
|
||||
let (records, _) =
|
||||
parse_csv_bytes_with_warnings(b"Region,Revenue\nEast,10\nEast,20\n").unwrap();
|
||||
let mut pipeline = ImportPipeline::new(serde_json::Value::Array(records));
|
||||
let mut pipeline = ImportPipeline::from_csv(records);
|
||||
pipeline.on_duplicate = improvise_io::import::wizard::DuplicatePolicy::Sum;
|
||||
let (wb, warnings) = pipeline.build_model_with_warnings().unwrap();
|
||||
let values: Vec<f64> = wb
|
||||
@@ -400,7 +400,7 @@ fn ordinary_controls() {
|
||||
let (records, warnings) =
|
||||
parse_csv_bytes_with_warnings(b"Region,Revenue\nEast,10\nWest,20\n").unwrap();
|
||||
assert!(warnings.is_empty());
|
||||
let p = ImportPipeline::new(serde_json::Value::Array(records));
|
||||
let p = ImportPipeline::from_csv(records);
|
||||
assert_eq!(
|
||||
p.proposals
|
||||
.iter()
|
||||
|
||||
@@ -761,6 +761,192 @@ fn a_headless_import_takes_files_and_folders_together_and_refuses_an_empty_folde
|
||||
assert!(!scratch.path("never.improv").exists());
|
||||
}
|
||||
|
||||
/// Runs `import <source> --no-wizard <extra> -o out.improv` on a source it writes.
|
||||
fn import_of(scratch: &Scratch, name: &str, body: &str, extra: &[&str]) -> (Output, PathBuf) {
|
||||
let source = scratch.path(name);
|
||||
std::fs::write(&source, body).unwrap();
|
||||
let out_path = scratch.path("out.improv");
|
||||
(run_import(&source, &out_path, extra), out_path)
|
||||
}
|
||||
|
||||
/// improvise-8jk6.4: a forced category's values are item names, so they keep
|
||||
/// their source text. Evidence: `context/test-evidence/cli.md`.
|
||||
#[test]
|
||||
fn a_csv_column_forced_to_category_keeps_its_text_verbatim() {
|
||||
let scratch = Scratch::new("forced_category_text");
|
||||
let (out, out_path) = import_of(
|
||||
&scratch,
|
||||
"commits.csv",
|
||||
"commit,lines\n00565796,3\n220944e2,5\n",
|
||||
&["--category", "commit", "--measure", "lines"],
|
||||
);
|
||||
assert!(out.status.success(), "{}", stderr_of(&out));
|
||||
let doc = std::fs::read_to_string(&out_path).unwrap();
|
||||
assert!(
|
||||
doc.lines().any(|l| l == "- |00565796|, |220944e2|"),
|
||||
"the items are the source text:\n{doc}"
|
||||
);
|
||||
for line in [
|
||||
"_Measure=lines, commit=|00565796| = 3",
|
||||
"_Measure=lines, commit=|220944e2| = 5",
|
||||
] {
|
||||
assert!(doc.lines().any(|l| l == line), "{line} missing from\n{doc}");
|
||||
}
|
||||
}
|
||||
|
||||
/// improvise-8jk6.4: JSON has no source text for a number, so a forced
|
||||
/// category names it as serde_json writes it. Evidence file.
|
||||
#[test]
|
||||
fn a_json_number_forced_to_category_is_named_as_json_writes_it() {
|
||||
let scratch = Scratch::new("forced_category_json");
|
||||
let (out, out_path) = import_of(
|
||||
&scratch,
|
||||
"commits.json",
|
||||
r#"[{"commit": 42, "n": 1}, {"commit": 1.5, "n": 2}, {"commit": 1e3, "n": 3}]"#,
|
||||
&["--category", "commit", "--measure", "n"],
|
||||
);
|
||||
assert!(out.status.success(), "{}", stderr_of(&out));
|
||||
let doc = std::fs::read_to_string(&out_path).unwrap();
|
||||
assert!(
|
||||
doc.lines().any(|l| l == "- |42|, |1.5|, |1000.0|"),
|
||||
"the items are the numbers' JSON text:\n{doc}"
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.5: a formula the user named and that does not parse is a
|
||||
/// refusal, not a silent drop. Evidence: `context/test-evidence/cli.md`.
|
||||
#[test]
|
||||
fn an_import_formula_that_does_not_parse_refuses_names_it_and_writes_nothing() {
|
||||
let scratch = Scratch::new("bad_formula");
|
||||
let (out, out_path) = import_of(
|
||||
&scratch,
|
||||
"f.csv",
|
||||
"a,x\nk,1\n",
|
||||
&[
|
||||
"--measure",
|
||||
"x",
|
||||
"--formula",
|
||||
"Y = x +",
|
||||
"--formula",
|
||||
"Z = x",
|
||||
"--formula",
|
||||
"W = * 2",
|
||||
],
|
||||
);
|
||||
let err = stderr_of(&out);
|
||||
assert!(!out.status.success(), "a dropped formula exited 0: {err}");
|
||||
assert!(err.contains("'Y = x +'"), "names the formula: {err}");
|
||||
assert!(err.contains("'W = * 2'"), "names every bad one: {err}");
|
||||
assert!(err.contains("expected"), "carries the parse error: {err}");
|
||||
assert!(!err.contains("'Z = x'"), "names only the bad one: {err}");
|
||||
assert!(!out_path.exists(), "nothing is written");
|
||||
}
|
||||
|
||||
/// improvise-8jk6.6: an import that kept no record says why, and exits
|
||||
/// non-zero writing nothing. Evidence: `context/test-evidence/cli.md`.
|
||||
#[test]
|
||||
fn an_import_that_keeps_no_record_refuses_and_says_why() {
|
||||
let scratch = Scratch::new("kept_none");
|
||||
let (out, out_path) = import_of(&scratch, "r.json", r#"[1, 2, "three"]"#, &[]);
|
||||
let err = stderr_of(&out);
|
||||
assert!(!out.status.success(), "{err}");
|
||||
assert!(
|
||||
err.contains("all 3 records were skipped as not JSON objects"),
|
||||
"says why: {err}"
|
||||
);
|
||||
assert!(!out_path.exists(), "nothing is written");
|
||||
}
|
||||
|
||||
/// improvise-8jk6.6: a partial skip warns with a count and exits 0, a choice
|
||||
/// put to the user. Evidence: `context/test-evidence/cli.md`.
|
||||
#[test]
|
||||
fn an_import_that_skips_some_records_warns_with_a_count_and_still_writes() {
|
||||
let scratch = Scratch::new("kept_some");
|
||||
let (out, out_path) = import_of(
|
||||
&scratch,
|
||||
"p.json",
|
||||
r#"[{"a": "k", "x": 1}, 2, {"a": "m", "x": 4}]"#,
|
||||
&[],
|
||||
);
|
||||
let err = stderr_of(&out);
|
||||
assert!(out.status.success(), "{err}");
|
||||
assert!(
|
||||
err.contains("warning: Skipped 1 of 3 records that were not JSON objects"),
|
||||
"{err}"
|
||||
);
|
||||
let doc = std::fs::read_to_string(&out_path).unwrap();
|
||||
assert!(doc.contains("_Measure=x, a=m = 4"), "{doc}");
|
||||
}
|
||||
|
||||
/// A JSON array with one stray scalar among its records.
|
||||
const ONE_STRAY: &str = r#"[{"a": "k", "x": 1}, 2, {"a": "m", "x": 4}]"#;
|
||||
|
||||
/// improvise-8jk6.6, the user's ruling: `--strict` makes any skip a refusal
|
||||
/// that writes nothing. Evidence: `context/test-evidence/cli.md`.
|
||||
#[test]
|
||||
fn a_strict_import_that_skips_a_record_refuses_and_writes_nothing() {
|
||||
let scratch = Scratch::new("strict_skip");
|
||||
let (out, out_path) = import_of(&scratch, "p.json", ONE_STRAY, &["--strict"]);
|
||||
let err = stderr_of(&out);
|
||||
assert!(!out.status.success(), "{err}");
|
||||
assert!(
|
||||
err.contains("--strict: Skipped 1 of 3 records that were not JSON objects"),
|
||||
"{err}"
|
||||
);
|
||||
assert!(!out_path.exists(), "a strict refusal writes nothing");
|
||||
}
|
||||
|
||||
/// The control: `--strict` refuses a skip, not every import.
|
||||
#[test]
|
||||
fn a_strict_import_that_skips_nothing_still_writes() {
|
||||
let scratch = Scratch::new("strict_clean");
|
||||
let body = r#"[{"a": "k", "x": 1}, {"a": "m", "x": 4}]"#;
|
||||
let (out, out_path) = import_of(&scratch, "p.json", body, &["--strict"]);
|
||||
assert!(out.status.success(), "{}", stderr_of(&out));
|
||||
let doc = std::fs::read_to_string(&out_path).unwrap();
|
||||
assert!(doc.contains("_Measure=x, a=m = 4"), "{doc}");
|
||||
}
|
||||
|
||||
/// improvise-8jk6.17 (F3): a CSV decimal measure is its number, not dropped
|
||||
/// text. Evidence: `context/test-evidence/cli.md`.
|
||||
#[test]
|
||||
fn a_csv_decimal_measure_imports_as_its_number() {
|
||||
let scratch = Scratch::new("csv_decimal");
|
||||
let body = "Region,Revenue\nEast,1000.50\nWest,-2.25\n";
|
||||
let (out, out_path) = import_of(&scratch, "f.csv", body, &[]);
|
||||
assert!(out.status.success(), "{}", stderr_of(&out));
|
||||
let doc = std::fs::read_to_string(&out_path).unwrap();
|
||||
for line in [
|
||||
"Region=East, _Measure=Revenue = 1000.5",
|
||||
"Region=West, _Measure=Revenue = -2.25",
|
||||
] {
|
||||
assert!(doc.lines().any(|l| l == line), "{line} missing from\n{doc}");
|
||||
}
|
||||
}
|
||||
|
||||
/// improvise-8jk6.17 (F1): `import-json` reads a CSV as CSV, so its numeric
|
||||
/// column is a measure. Evidence: `context/test-evidence/cli.md`.
|
||||
#[test]
|
||||
fn import_json_on_a_csv_reads_its_numeric_column_as_a_measure() {
|
||||
let scratch = Scratch::new("import_json_csv");
|
||||
let csv = scratch.path("f.csv");
|
||||
std::fs::write(&csv, "Region,Revenue\nEast,1000.50\nWest,7\n").unwrap();
|
||||
let doc = scratch.path("m.improv");
|
||||
let out = run_cmd(&[&format!("import-json {}", csv.display())], &doc);
|
||||
assert!(out.status.success(), "{}", stderr_of(&out));
|
||||
let text = std::fs::read_to_string(&doc).unwrap();
|
||||
assert!(!text.contains("## Category: Revenue"), "{text}");
|
||||
for line in [
|
||||
"Region=East, _Measure=Revenue = 1000.5",
|
||||
"Region=West, _Measure=Revenue = 7",
|
||||
] {
|
||||
assert!(
|
||||
text.lines().any(|l| l == line),
|
||||
"{line} missing from\n{text}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// improvise-lpzu, the user's ruling: a folder always merges, so even one CSV
|
||||
/// keeps its file identity. Evidence: `context/test-evidence/cli.md`.
|
||||
#[test]
|
||||
|
||||
@@ -18,7 +18,7 @@ fn imported_model_shows_one_row_per_record_with_measure_columns() {
|
||||
.expect("write csv");
|
||||
|
||||
let records = improvise::import::csv_parser::parse_csv(&csv).expect("csv parses");
|
||||
let pipeline = ImportPipeline::new(serde_json::Value::Array(records));
|
||||
let pipeline = ImportPipeline::from_csv(records);
|
||||
let workbook = pipeline.build_model().expect("model builds");
|
||||
|
||||
let saved = dir.path().join("w.improv");
|
||||
|
||||
Reference in New Issue
Block a user