fiddlerwoaroof/improvise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge gh/main into fix/31oe-cmdlang; renumber cli.md §13/§14 to §14/§15

Browse Source 
#321 added its own cli.md section 13, so this PR's sections become 14 (pipe
quoting, surplus arguments) and 15 (names the document lacks), with every
citation renumbered. The stale descriptions of the deleted unknown_* helpers in
cli.md, command.md and the test evidence now describe RequireKnown, and section
15 says a refused name stops the rest of a `:` line but not a headless one.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
This commit is contained in:
Claude (cmdlang)
2026-10-02 22:00:38 -07:00
committed by Edward Langley
parent c99353777b 8da5a3101b
commit 8fa504fbdc
31 changed files with 2300 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/improvise-command/src/command/cmd/core.rs
+3 -3
View File
@@ -251,7 +251,7 @@ pub trait AtCursor: Cmd {
}
/// How many arguments a command takes. The registry refuses any beyond it, for
/// every command in one place — `context/code/cli.md` §13.
/// every command in one place — `context/code/cli.md` §14.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MaxArgs {
AtMost(usize),
@@ -435,7 +435,7 @@ pub(super) fn require_args(args: &[String], n: usize) -> Result<(), String> {
}
/// Parse Cat/Item coordinate args into a CellKey, refusing any token that is not
/// one rather than dropping it — `context/code/cli.md` §13.
/// one rather than dropping it — `context/code/cli.md` §14.
pub(super) fn parse_cell_key_from_args(
args: &[String],
) -> Result<crate::model::cell::CellKey, String> {
@@ -452,7 +452,7 @@ pub(super) fn parse_cell_key_from_args(
}
/// A cursor fallback answers only for coordinates that were OMITTED; given and
/// wrong, they are refused, never the cursor's cell — `context/code/cli.md` §13.
/// wrong, they are refused, never the cursor's cell — `context/code/cli.md` §14.
pub(super) fn only_when_omitted(args: &[String]) -> Result<(), String> {
if args.is_empty() {
return Ok(());
crates/improvise-command/src/command/cmd/effect_cmds.rs
+1 -1
View File
@@ -252,7 +252,7 @@ macro_rules! effect_cmd {
}
/// `effect`, preceded by the check that what it names exists, run at APPLY time:
/// a `:` line is built before any of it runs — `context/code/cli.md` §14.
/// a `:` line is built before any of it runs — `context/code/cli.md` §15.
fn unless_unknown(needs: effect::Needs, effect: impl Effect + 'static) -> Vec<Box<dyn Effect>> {
vec![Box::new(effect::RequireKnown(needs)), Box::new(effect)]
}
crates/improvise-command/src/command/parse.rs
+2 -2
View File
@@ -82,7 +82,7 @@ fn groups(line: &str) -> Result<Vec<Vec<Word>>, String> {
Ok(groups)
}
/// Split `line` into words. The quoting rules: `context/code/cli.md` §13.
/// Split `line` into words. The quoting rules: `context/code/cli.md` §14.
fn lex(line: &str) -> Result<Vec<Word>, String> {
let mut words = Vec::new();
let mut chars = line.char_indices().peekable();
@@ -433,7 +433,7 @@ mod tests {
);
}
/// `context/code/cli.md` §13: only a BARE `.` separates commands. A quoted one
/// `context/code/cli.md` §14: only a BARE `.` separates commands. A quoted one
/// is an argument, so `|.|` and `"."` name a thing called `.`.
#[test]
fn a_quoted_dot_is_an_argument_not_a_separator() {
crates/improvise-command/src/ui/app.rs
+173 -9
View File
@@ -24,7 +24,7 @@ use crate::persistence;
use crate::ui::effect::{EffectKind, ModelChange, ViewTarget};
use crate::ui::project::Dirty;
use crate::ui::viewport::{
compute_col_widths, compute_row_header_width, compute_visible_cols, grid_width,
GridSummary, compute_row_header_width, compute_visible_cols, grid_summary, grid_width,
};
use crate::view::{GridLayout, View};
use crate::workbook::Workbook;
@@ -434,6 +434,20 @@ impl Default for ModelState {
}
}
/// What a layout was built from other than the model.
#[derive(Debug, Default, PartialEq, Eq)]
struct LayoutBasis {
view: String,
/// The address of the frozen records, never dereferenced.
drill: Option<usize>,
}
/// What the keypress path measures of a built layout.
pub(crate) struct GridGeometry {
pub summary: GridSummary,
pub row_header_width: u16,
}
/// Top-level application state: `model_state`, `view_state`, and a residue of
/// fields tagged below. The five residue tags are in `context/code/command.md`.
pub struct App {
@@ -453,8 +467,21 @@ pub struct App {
pub term_width: u16,
pub term_height: u16,
/// **Derived cache**: a pure function of the slices, rebuilt by
/// `rebuild_layout()`. Owned by `App` because the rebuild is the host's job.
/// `rebuild_layout()`; a writer that bypasses effects must call it (felo.13).
pub layout: GridLayout,
/// How many times `rebuild_layout` has run: what a keypress that changes
/// neither the model nor the view must leave alone (improvise-felo.13).
layout_builds: usize,
/// How many times the grid's column widths were measured, for the same reason.
width_passes: std::cell::Cell<usize>,
/// The widths of the layout above, measured on first use and dropped with it.
geometry: std::cell::OnceCell<GridGeometry>,
/// What the layout above was built from besides the model: the view it drew
/// and the drill snapshot it froze. `layout_current` compares it.
layout_basis: LayoutBasis,
/// The model changed since the layout was built. Set by the one writer that
/// does not rebuild at once.
layout_stale: bool,
/// **Transient**: set during `apply_effects` to skip the rest of the batch,
/// reset at the start of every call. Reach it through `AbortChain`.
pub abort_effects: bool,
@@ -553,6 +580,11 @@ impl App {
term_width: 80,
term_height: 24,
layout,
layout_builds: 0,
width_passes: std::cell::Cell::new(0),
geometry: std::cell::OnceCell::new(),
layout_basis: LayoutBasis::default(),
layout_stale: false,
abort_effects: false,
batch_aborted: false,
batch_refused: false,
@@ -566,6 +598,7 @@ impl App {
keymap_set: KeymapSet::default_keymaps(),
storage,
};
app.layout_basis = app.current_layout_basis();
app.settle_normal_mode();
app
}
@@ -610,6 +643,7 @@ impl App {
self.view_state.view_back_stack.clear();
self.view_state.view_forward_stack.clear();
self.model_state.workbook = workbook;
self.layout_stale = true;
// Every client's mirror is scoped to the outgoing document too, and an
// effect cannot reach the session list. The generation is how it hears.
self.document_generation += 1;
@@ -900,6 +934,42 @@ impl App {
let frozen = self.drill_state.as_ref().map(|s| Rc::clone(&s.records));
self.layout =
GridLayout::with_frozen_records(&self.model_state.workbook.model, view, frozen);
self.layout_builds += 1;
self.layout_stale = false;
self.layout_basis = self.current_layout_basis();
self.geometry = std::cell::OnceCell::new();
}
/// Rebuild only if the model, the view drawn or the drill snapshot moved; a
/// cursor key moves none of them (layout-cost.md).
fn refresh_layout(&mut self) {
if self.layout_stale || self.layout_basis != self.current_layout_basis() {
self.rebuild_layout();
}
}
fn current_layout_basis(&self) -> LayoutBasis {
LayoutBasis {
view: self.view_name().to_string(),
drill: self
.drill_state
.as_ref()
.map(|d| Rc::as_ptr(&d.records) as usize),
}
}
/// The measurements a keypress needs of the grid: column widths and the row
/// header. Counted, so a test can see whether a keypress paid for them.
pub(crate) fn grid_geometry(&self) -> &GridGeometry {
self.geometry.get_or_init(|| {
self.width_passes.set(self.width_passes.get() + 1);
let model = &self.model_state.workbook.model;
let plan = FormatPlan::of(model, self.view());
GridGeometry {
summary: grid_summary(model, &self.layout, &plan),
row_header_width: compute_row_header_width(&self.layout),
}
})
}
/// Editable text for the cell under the cursor: staged drill edit, else the
@@ -980,12 +1050,10 @@ impl App {
display_value: self.display_value_at_cursor(),
visible_rows: (self.term_height as usize).saturating_sub(8),
visible_cols: {
let model = &self.model_state.workbook.model;
let col_widths = compute_col_widths(model, layout, &FormatPlan::of(model, view));
let row_header_width = compute_row_header_width(layout);
let geometry = self.grid_geometry();
compute_visible_cols(
&col_widths,
row_header_width,
&geometry.summary.col_widths,
geometry.row_header_width,
self.grid_width(),
cursor.col_offset,
)
@@ -1116,7 +1184,8 @@ impl App {
// is read only behind the epoch gate — `context/code/command.md`.
let reshaped = self.last_reshaped_views.clone();
self.last_reframed_views.extend(reshaped);
self.rebuild_layout();
self.layout_stale |= touches_model;
self.refresh_layout();
// After the rebuild, or a batch that crosses into or out of records mode
// settles against the grid it started on — `context/code/command.md`.
self.settle_normal_mode();
@@ -1196,7 +1265,7 @@ impl App {
/// Resolve one keypress through the keymap and apply what it produces. The
/// projection half is factored out; `ui::reduce` is the named door (improvise-gxi).
pub fn dispatch_key(&mut self, key: KeyEvent) -> Dirty {
self.rebuild_layout();
self.refresh_layout();
// An armed prefix key takes priority. Resolve it back to its
// sub-keymap through the keymap set (improvise-jc8).
@@ -1594,6 +1663,32 @@ mod tests {
app
}
/// improvise-felo.13: `StartDrill` is `ViewOnly`, so a batch of it alone moves
/// the layout with no `Model` effect to mark it stale. Evidence: layout-cost.md.
#[test]
fn a_drill_snapshot_replaced_alone_rebuilds_the_layout() {
let mut app = records_app_for_drill();
assert_eq!(app.layout.row_count(), 3);
let rows: Vec<_> = app.layout.records.as_ref().unwrap().to_vec();
let start = |app: &mut App, n: usize| {
let snapshot = rows.iter().take(n).cloned().collect();
app.apply_effects(vec![Box::new(crate::ui::effect::StartDrill(snapshot))]);
};
start(&mut app, 1);
assert_eq!(
app.layout.row_count(),
1,
"the layout draws the frozen snapshot"
);
let builds = app.layout_builds;
start(&mut app, 2);
assert!(
app.layout_builds > builds,
"no rebuild for a second drill snapshot"
);
assert_eq!(app.layout.row_count(), 2);
}
/// The opening basis may be read off `App::layout` only when no drill froze
/// it. Evidence: `context/test-evidence/command.md`.
#[test]
@@ -1898,6 +1993,71 @@ mod tests {
App::new(wb, None)
}
/// improvise-felo.13: context/test-evidence/layout-cost.md.
#[test]
fn a_cursor_key_rebuilds_neither_the_layout_nor_the_widths() {
let mut app = two_col_model();
let down = || KeyEvent::new(KeyCode::Down, KeyModifiers::NONE);
app.dispatch_key(down());
let (builds, widths) = (app.layout_builds, app.width_passes.get());
app.dispatch_key(down());
app.dispatch_key(down());
assert_eq!(app.cursor().selected.0, 2, "the keys moved the cursor");
assert_eq!(app.layout_builds, builds, "layout rebuilt on a cursor key");
assert_eq!(app.width_passes.get(), widths, "widths remeasured");
}
/// improvise-felo.13: the control for the test above, and the guard on
/// invalidation: an edit must rebuild, and the widths must follow the value.
#[test]
fn an_edit_rebuilds_and_remeasures_so_the_widths_follow_the_value() {
use crate::ui::effect::SetCell;
let mut app = two_col_model();
let down = || KeyEvent::new(KeyCode::Down, KeyModifiers::NONE);
app.dispatch_key(down());
let narrow = app.grid_geometry().summary.col_widths.clone();
let (builds, widths) = (app.layout_builds, app.width_passes.get());
let key = CellKey::new(vec![("Row".into(), "A".into()), ("Col".into(), "X".into())]);
let long = CellValue::Text("a value far wider than a column".into());
app.apply_effects(vec![Box::new(SetCell(key, long))]);
assert!(
app.layout_builds > builds,
"an edit must rebuild the layout"
);
app.dispatch_key(down());
assert!(app.width_passes.get() > widths, "an edit must remeasure");
let model = &app.model_state.workbook.model;
let fresh = crate::ui::viewport::compute_col_widths(
model,
&app.layout,
&FormatPlan::of(model, app.view()),
);
assert_eq!(
app.grid_geometry().summary.col_widths,
fresh,
"stale widths"
);
assert_ne!(
fresh, narrow,
"the edit widened a column, or this proves nothing"
);
}
/// improvise-felo.13: a `ViewOnly` effect that changes which view is drawn
/// must still rebuild, so skipping the rebuild is not keyed on `EffectKind` alone.
#[test]
fn switching_view_rebuilds_though_the_effect_is_view_only() {
let mut app = two_col_model();
app.model_state.workbook.create_view("Second");
app.rebuild_layout();
let builds = app.layout_builds;
app.apply_effects(vec![Box::new(crate::ui::effect::SwitchView(
"Second".into(),
))]);
assert_eq!(app.view_state.active_view.as_str(), "Second");
assert!(app.layout_builds > builds, "the layout still draws Default");
}
/// improvise-htl6. Evidence: `context/test-evidence/command.md`.
#[test]
fn the_view_history_walks_both_ways_from_the_keyboard() {
@@ -2134,6 +2294,7 @@ mod tests {
.unwrap()
.add_item(format!("R{i}"));
}
app.rebuild_layout();
app.term_height = 28; // ~20 visible rows → delta = 15
app.cursor_mut().selected = (0, 0);
app.handle_key(KeyEvent::new(KeyCode::PageDown, KeyModifiers::NONE))
@@ -2158,6 +2319,7 @@ mod tests {
.unwrap()
.add_item(format!("R{i}"));
}
app.rebuild_layout();
app.term_height = 28;
app.cursor_mut().selected = (20, 0);
app.handle_key(KeyEvent::new(KeyCode::PageUp, KeyModifiers::NONE))
@@ -2177,6 +2339,7 @@ mod tests {
.unwrap()
.add_item(format!("R{i}"));
}
app.rebuild_layout();
app.term_height = 13; // ~5 visible rows
app.cursor_mut().selected = (0, 0);
// G jumps to last row (row 12)
@@ -2204,6 +2367,7 @@ mod tests {
.unwrap()
.add_item(format!("R{i}"));
}
app.rebuild_layout();
app.term_height = 13; // ~5 visible rows
app.cursor_mut().selected = (0, 0);
// Ctrl+d scrolls by 5 rows
crates/improvise-command/src/ui/effect.rs
+22 -33
View File
@@ -1245,7 +1245,7 @@ pub struct ImportJsonHeadless {
}
impl Effect for ImportJsonHeadless {
fn apply(&self, app: &mut App) {
use crate::import::analyzer::{FieldKind, analyze_records};
use crate::import::analyzer::FieldKind;
use crate::import::wizard::{DuplicatePolicy, ImportPipeline};
let is_csv = self
@@ -1253,10 +1253,10 @@ impl Effect for ImportJsonHeadless {
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("csv"));
// Parse the file exactly once: `records` feeds analysis, `raw` is reused
// as the pipeline's value (improvise-oaq). CSV warnings go to the status.
// Parse the file exactly once, into the pipeline that reads it
// (improvise-oaq, improvise-8jk6.4). CSV warnings go to the status.
let mut warnings: Vec<String> = vec![];
let (records, raw) = if is_csv {
let mut pipeline = if is_csv {
let bytes = match app.storage.read(&self.path) {
Ok(b) => b,
Err(e) => {
@@ -1267,8 +1267,7 @@ impl Effect for ImportJsonHeadless {
match crate::import::csv_parser::parse_csv_bytes_with_warnings(&bytes) {
Ok((recs, w)) => {
warnings = w;
let raw = serde_json::Value::Array(recs.clone());
(recs, raw)
ImportPipeline::from_csv(recs)
}
Err(e) => {
refuse_and_stop(app, format!("CSV error: {e}"));
@@ -1291,38 +1290,28 @@ impl Effect for ImportJsonHeadless {
}
};
match json_import_records(&value, self.array_path.as_deref()) {
Ok(recs) => (recs, value),
Ok(recs) => {
let mut pipeline = ImportPipeline::new(value);
pipeline.selected_path = self.array_path.clone().unwrap_or_default();
pipeline.records = recs;
pipeline.analyze();
pipeline
}
Err(msg) => {
refuse_and_stop(app, msg);
return;
}
}
};
let proposals = analyze_records(&records);
let pipeline = ImportPipeline {
raw,
array_paths: vec![],
selected_path: self.array_path.as_deref().unwrap_or("").to_string(),
records,
proposals: proposals
.into_iter()
.map(|mut p| {
p.accepted = p.kind != FieldKind::Label;
p
})
.collect(),
model_name: self
.model_name
.as_deref()
.unwrap_or("Imported Model")
.to_string(),
formulas: vec![],
// Headless: nobody to ask, so a collision refuses (improvise-cgqt).
on_duplicate: DuplicatePolicy::Refuse,
sources: vec![],
};
pipeline.model_name = self
.model_name
.clone()
.unwrap_or_else(|| "Imported Model".to_string());
// Headless: nobody to ask, so a collision refuses (improvise-cgqt).
pipeline.on_duplicate = DuplicatePolicy::Refuse;
for p in &mut pipeline.proposals {
p.accepted = p.kind != FieldKind::Label;
}
match pipeline.build_model_with_warnings() {
Ok((new_workbook, build_warnings)) => {
@@ -1422,7 +1411,7 @@ fn unknown_name(app: &App, needs: &Needs) -> Option<String> {
}
/// Refuse and stop unless the document has what the next effects name, asked at
/// apply time so a `:` line sees its earlier segments (`cli.md` §14).
/// apply time so a `:` line sees its earlier segments (`cli.md` §15).
#[derive(Debug)]
pub struct RequireKnown(pub Needs);
impl Effect for RequireKnown {
crates/improvise-command/src/ui/project.rs
+14 -4
View File
@@ -25,7 +25,7 @@ use improvise_protocol::{
use crate::ui::app::App;
use crate::ui::effect::{EffectKind, ModelChange};
use crate::ui::reduce::{Checked, apply_command};
use crate::ui::viewport::{GridBody, grid_body};
use crate::ui::viewport::{GridBody, grid_body, grid_window};
/// Which widgets a batch of effects could have changed. A gate, not a description:
/// over-reporting costs work, under-reporting is a correctness bug (gotcha 35).
@@ -142,9 +142,19 @@ pub fn grid_in(app: &App, view_name: &str, viewport: Viewport) -> GridCache {
let plan = FormatPlan::of(model, view);
let rows = viewport.rows_in(layout.row_count());
let cols = viewport.cols_in(layout.col_count());
// ONE walk: the widths, the totals and this window's values all come off the
// same evaluation — `context/code/command.md` (improvise-jmjz).
let body = grid_body(model, layout, &plan, rows.clone(), cols.clone());
// One walk gives widths, totals and window (improvise-jmjz); the host's layout
// keeps the first two until rebuilt, so its window costs its own cells.
let hosted = elsewhere.is_none() && derived.is_none();
let body = match hosted {
true => grid_window(
model,
layout,
&app.grid_geometry().summary,
rows.clone(),
cols.clone(),
),
false => grid_body(model, layout, &plan, rows.clone(), cols.clone()),
};
GridCache {
row_frame: axis_frame(&layout.row_cats, &layout.row_items, rows.clone()),
crates/improvise-command/src/ui/reduce/vocabulary_is_complete.rs
+1 -1
View File
@@ -65,7 +65,7 @@ const NOT_ON_THE_WIRE: &[(&str, Verdict)] = &[
),
(
"RequireKnown",
Verdict::Off("a command-side name check that writes no state (gotcha 21, cli.md §14)"),
Verdict::Off("a command-side name check that writes no state (gotcha 21, cli.md §15)"),
),
(
"AbortChain",
crates/improvise-command/src/ui/viewport.rs
+74 -1
View File
@@ -101,6 +101,50 @@ pub fn grid_body(
}
}
/// What of a grid build no window changes: the widths, each column's total
/// format and the totals. Measured over every cell, so a host keeps one per layout.
pub struct GridSummary {
pub col_widths: Vec<u16>,
pub col_formats: Vec<NumberFormat>,
pub totals: Vec<Option<CellValue>>,
}
/// The whole-grid walk, answering only what a window cannot.
pub fn grid_summary(model: &Model, layout: &GridLayout, plan: &FormatPlan) -> GridSummary {
let body = grid_body(model, layout, plan, 0..0, 0..0);
GridSummary {
col_widths: body.col_widths,
col_formats: body.col_formats,
totals: body.totals,
}
}
/// [`grid_body`] for a window, given the `summary` of the same layout: evaluates
/// only the window's cells. Equal to `grid_body` where `summary` is current.
pub fn grid_window(
model: &Model,
layout: &GridLayout,
summary: &GridSummary,
rows: std::ops::Range<usize>,
cols: std::ops::Range<usize>,
) -> GridBody {
let eval = layout.evaluator(model);
let records = layout.is_records_mode();
let window = rows
.map(|ri| {
cols.clone()
.map(|ci| cell_value(layout, &eval, records, ri, ci).1)
.collect()
})
.collect();
GridBody {
col_widths: summary.col_widths.clone(),
col_formats: summary.col_formats.clone(),
totals: summary.totals.clone(),
window,
}
}
/// The one evaluation. Records mode reads its wide row; pivot mode aggregates.
fn cell_value(
layout: &GridLayout,
@@ -146,7 +190,7 @@ fn widen(widths: &mut [u16], col: usize, width: usize) {
/// Adaptive column widths alone, for the keypress path, which needs no cells.
/// It is [`grid_body`] with an empty window so there is one width rule.
pub fn compute_col_widths(model: &Model, layout: &GridLayout, plan: &FormatPlan) -> Vec<u16> {
grid_body(model, layout, plan, 0..0, 0..0).col_widths
grid_summary(model, layout, plan).col_widths
}
/// The total row-header width for a layout. The rule is the protocol's; this is the
@@ -272,4 +316,33 @@ mod tests {
"F0..F8 once for the Q1 row, once for the totals"
);
}
/// improvise-felo.13: a window read off a held summary is the window `grid_body`
/// builds. Evidence: `context/test-evidence/layout-cost.md`.
#[test]
fn a_window_over_a_held_summary_equals_the_one_walk_grid_body() {
let wb = folded_chain();
let view = wb.active_view();
let layout = GridLayout::new(&wb.model, view);
let plan = FormatPlan::of(&wb.model, view);
let summary = grid_summary(&wb.model, &layout, &plan);
let (rows, cols) = (layout.row_count(), layout.col_count());
assert!(
rows >= 2 && cols >= 3,
"a window strictly inside is possible"
);
for (r, c) in [
(0..rows, 0..cols),
(1..2, 1..3),
(rows - 1..rows, 0..1),
(0..0, 0..0),
] {
let walked = grid_body(&wb.model, &layout, &plan, r.clone(), c.clone());
let held = grid_window(&wb.model, &layout, &summary, r.clone(), c.clone());
assert_eq!(held.window, walked.window, "window {r:?} x {c:?}");
assert_eq!(held.col_widths, walked.col_widths);
assert_eq!(held.totals, walked.totals);
assert_eq!(held.col_formats, walked.col_formats);
}
}
}
crates/improvise-core/src/model/cell.rs
+141
View File
@@ -194,6 +194,12 @@ impl RecordEntry {
}
}
#[cfg(test)]
thread_local! {
/// How many lookups fell through to set narrowing, for the exact-path test.
static NARROWED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
#[derive(Clone, Default)]
pub struct DataStore {
/// Primary storage: interned keys, insertion-ordered so records mode can
@@ -206,6 +212,9 @@ pub struct DataStore {
/// Reverse index for the client-facing surrogate (ADR 002), plus each
/// record's write counter. Why it lives here: core.md §8.
records: HashMap<RecordHash, RecordEntry>,
/// How many stored cells have each number of coordinates; its largest key
/// says when a partial can match only one exact cell.
arities: std::collections::BTreeMap<usize, usize>,
}
/// Resolved through the symbol table and sorted wherever the storage is a
@@ -217,6 +226,7 @@ impl std::fmt::Debug for DataStore {
symbols,
index,
records,
arities: _,
} = self;
let cells: Vec<_> = cells
.iter()
@@ -291,7 +301,11 @@ impl DataStore {
for pair in &ikey.0 {
self.index.entry(*pair).or_default().insert(ikey.clone());
}
let width = ikey.0.len();
let is_new_cell = self.cells.insert(ikey, value).is_none();
if is_new_cell {
*self.arities.entry(width).or_default() += 1;
}
let entry = self.records.entry(hash).or_insert(RecordEntry {
record,
cells: 0,
@@ -327,6 +341,12 @@ impl DataStore {
return;
};
if self.cells.shift_remove(&ikey).is_some() {
if let Some(n) = self.arities.get_mut(&ikey.0.len()) {
*n -= 1;
if *n == 0 {
self.arities.remove(&ikey.0.len());
}
}
for pair in &ikey.0 {
if let Some(set) = self.index.get_mut(pair) {
set.remove(&ikey);
@@ -378,11 +398,28 @@ impl DataStore {
/// Positions in `cells` of every cell matching `partial`, **in insertion
/// order**. The index narrows; the primary map orders (§8).
fn matching_positions(&self, partial: &[(String, String)]) -> Vec<usize> {
if let Some(exact) = self.exact_positions(partial) {
return exact;
}
self.intern_partial(partial)
.map(|pairs| self.positions_of(&pairs))
.unwrap_or_default()
}
/// The answer without narrowing, when `partial` is as wide as the widest cell
/// stored: only an equal cell can match (layout-cost.md). `None` otherwise.
fn exact_positions(&self, partial: &[(String, String)]) -> Option<Vec<usize>> {
let widest = *self.arities.keys().next_back()?;
// Strictly sorted by category, the order a stored key is in, so one probe is the lookup.
let canonical = partial.windows(2).all(|w| w[0].0 < w[1].0);
(!partial.is_empty() && widest <= partial.len() && canonical).then(|| {
let found = self
.intern_partial(partial)
.and_then(|pairs| self.cells.get_index_of(&InternedKey(pairs)));
found.into_iter().collect()
})
}
/// `partial` as symbols, or `None` when a name was never interned, which
/// no stored cell can then match.
fn intern_partial(&self, partial: &[(String, String)]) -> Option<Vec<(Symbol, Symbol)>> {
@@ -412,6 +449,8 @@ impl DataStore {
}
sets.sort_by_key(|s| s.len());
#[cfg(test)]
NARROWED.with(|n| n.set(n.get() + 1));
let mut positions: Vec<usize> = match sets.split_first() {
Some((first, rest)) => first
@@ -1356,3 +1395,105 @@ mod record_index {
}
}
}
#[cfg(test)]
mod exact_path_tests {
use super::{CellKey, CellValue, DataStore, NARROWED};
fn key(a: &str, b: &str, c: &str) -> CellKey {
CellKey::new(vec![
("A".into(), a.into()),
("B".into(), b.into()),
("C".into(), c.into()),
])
}
fn store() -> DataStore {
let mut d = DataStore::new();
for i in 0..6 {
for j in 0..6 {
d.set(
key(&format!("a{i}"), &format!("b{j}"), "c"),
CellValue::Number((i * 6 + j) as f64),
);
}
}
d
}
fn narrowings(f: impl FnOnce()) -> usize {
NARROWED.with(|n| n.set(0));
f();
NARROWED.with(|n| n.get())
}
/// improvise-felo.13: context/test-evidence/layout-cost.md.
#[test]
fn a_full_width_partial_probes_one_cell_instead_of_narrowing() {
let d = store();
let full = key("a2", "b3", "c").0;
let mut got = Vec::new();
assert_eq!(
narrowings(|| got = d.matching_values(&full).into_iter().cloned().collect()),
0
);
assert_eq!(got, vec![CellValue::Number(15.0)]);
let missing = key("a2", "b9", "c").0;
assert!(
d.matching_values(&missing).is_empty(),
"an absent cell matches nothing"
);
let unknown = key("a2", "b3", "zzz").0;
assert!(
d.matching_values(&unknown).is_empty(),
"an unseen item matches nothing"
);
}
/// The control: a narrower partial still sums its supersets, so the shortcut
/// is not a blanket exact lookup.
#[test]
fn a_narrower_partial_still_narrows_and_matches_every_superset() {
let d = store();
let narrow = vec![
("A".to_string(), "a2".to_string()),
("C".to_string(), "c".to_string()),
];
let mut n = 0;
assert_eq!(narrowings(|| n = d.matching_values(&narrow).len()), 1);
assert_eq!(n, 6);
}
/// A wider cell appearing, or leaving, moves the line the shortcut is allowed at.
#[test]
fn a_wider_cell_turns_the_shortcut_off_until_it_is_removed() {
let mut d = store();
let wide = CellKey::new(vec![
("A".into(), "a2".into()),
("B".into(), "b3".into()),
("C".into(), "c".into()),
("D".into(), "d".into()),
]);
d.set(wide.clone(), CellValue::Number(100.0));
let full = key("a2", "b3", "c").0;
assert_eq!(
d.matching_values(&full).len(),
2,
"the exact cell and the wider cell under it"
);
d.remove(&wide);
assert_eq!(
narrowings(|| assert_eq!(d.matching_values(&full).len(), 1)),
0
);
}
/// An unsorted partial is not a stored key's order, so it must not be probed.
#[test]
fn an_unsorted_full_width_partial_is_still_answered() {
let d = store();
let mut full = key("a2", "b3", "c").0;
full.reverse();
assert_eq!(d.matching_values(&full).len(), 1);
}
}
crates/improvise-core/src/view/layout.rs
+236 -34
View File
@@ -235,23 +235,47 @@ impl AxisEntry {
/// The data-item tuples on an axis, skipping group headers. Every public
/// `row`/`col` argument indexes this sequence, not the visual entry list.
#[cfg(test)]
fn data_items(items: &[AxisEntry]) -> impl Iterator<Item = &Vec<String>> {
items.iter().filter_map(|e| match e {
AxisEntry::DataItem { tuple, .. } => Some(tuple),
AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None,
})
}
/// Whether the data item at `idx` folds a collapsed group.
fn folds(items: &[AxisEntry], idx: usize) -> bool {
items
.iter()
.filter_map(|e| match e {
AxisEntry::DataItem { collapsed, .. } => Some(*collapsed),
AxisEntry::DataItem { tuple, .. } => Some(tuple),
AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None,
})
.nth(idx)
.unwrap_or(false)
.inspect(|_| DATA_ITEM_STEPS.with(|n| n.set(n.get() + 1)))
}
#[cfg(test)]
thread_local! {
/// Items `data_items` has yielded on this thread: the work a lookup did.
static DATA_ITEM_STEPS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
/// Entry `positions(items)[n]` is data item `n`: a lookup by data index is O(1),
/// not a walk past every header before it (layout-cost.md).
fn positions(items: &[AxisEntry]) -> Vec<usize> {
items
.iter()
.enumerate()
.filter_map(|(at, e)| matches!(e, AxisEntry::DataItem { .. }).then_some(at))
.collect()
}
/// The data item at `idx`, by the axis's position index.
fn data_item<'a>(items: &'a [AxisEntry], pos: &[usize], idx: usize) -> Option<&'a Vec<String>> {
match items.get(*pos.get(idx)?)? {
AxisEntry::DataItem { tuple, .. } => Some(tuple),
AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. } => None,
}
}
/// Whether the data item at `idx` folds a collapsed group.
fn folds(items: &[AxisEntry], pos: &[usize], idx: usize) -> bool {
match pos.get(idx).and_then(|at| items.get(*at)) {
Some(AxisEntry::DataItem { collapsed, .. }) => *collapsed,
Some(AxisEntry::GroupHeader { .. } | AxisEntry::GroupEnd { .. }) | None => false,
}
}
/// The data index of the first item in the run `cat`/`group` opens under
@@ -287,21 +311,15 @@ fn run_start(
}
/// The display label of the data item at `idx`, or empty when out of bounds.
fn axis_label(items: &[AxisEntry], idx: usize) -> String {
data_items(items)
.nth(idx)
fn axis_label(items: &[AxisEntry], pos: &[usize], idx: usize) -> String {
data_item(items, pos, idx)
.map(|v| v.join("/"))
.unwrap_or_default()
}
/// Visual index of the nth data entry (skipping group headers).
fn data_index_to_visual(items: &[AxisEntry], data_idx: usize) -> Option<usize> {
items
.iter()
.enumerate()
.filter(|(_, e)| matches!(e, AxisEntry::DataItem { .. }))
.map(|(vi, _)| vi)
.nth(data_idx)
fn data_index_to_visual(pos: &[usize], data_idx: usize) -> Option<usize> {
pos.get(data_idx).copied()
}
/// Where a run sits: `(level, depth)`, ordered, so "inside" is "greater than".
@@ -468,6 +486,10 @@ pub struct GridLayout {
pub page_coords: Vec<PageAxis>,
pub row_items: Vec<AxisEntry>,
pub col_items: Vec<AxisEntry>,
/// `positions(row_items)` / `positions(col_items)`: derived, so every writer of
/// the items calls `reindex`, and nothing outside this module writes them.
row_pos: Vec<usize>,
col_pos: Vec<usize>,
/// Categories on `Axis::None` — hidden, implicitly aggregated.
pub none_cats: Vec<String>,
/// Records mode: one entry per row, grouping the cells that share every
@@ -497,6 +519,7 @@ impl GridLayout {
.collect();
layout.row_items = row_items;
layout.records = Some(records);
layout.reindex();
}
if view.prune_empty {
layout.prune_empty(model);
@@ -556,6 +579,8 @@ impl GridLayout {
row_cats,
col_cats,
page_coords,
row_pos: positions(&row_items),
col_pos: positions(&col_items),
row_items,
col_items,
none_cats,
@@ -590,6 +615,8 @@ impl GridLayout {
row_cats: vec![INDEX_CATEGORY.to_string()],
col_cats: vec![DIM_CATEGORY.to_string()],
page_coords,
row_pos: positions(&row_items),
col_pos: positions(&col_items),
row_items,
col_items,
none_cats,
@@ -728,6 +755,13 @@ impl GridLayout {
prune_axis_items(&mut self.col_items, |ci| {
(0..rc).any(|ri| has_value[ri][ci])
});
self.reindex();
}
/// Rebuild the position indexes after the items change.
fn reindex(&mut self) {
self.row_pos = positions(&self.row_items);
self.col_pos = positions(&self.col_items);
}
/// Whether this layout is in records mode.
@@ -737,20 +771,20 @@ impl GridLayout {
/// Number of data rows (group headers excluded).
pub fn row_count(&self) -> usize {
data_items(&self.row_items).count()
self.row_pos.len()
}
/// Number of data columns (group headers excluded).
pub fn col_count(&self) -> usize {
data_items(&self.col_items).count()
self.col_pos.len()
}
pub fn row_label(&self, row: usize) -> String {
axis_label(&self.row_items, row)
axis_label(&self.row_items, &self.row_pos, row)
}
pub fn col_label(&self, col: usize) -> String {
axis_label(&self.col_items, col)
axis_label(&self.col_items, &self.col_pos, col)
}
/// The value of a pivot key as THIS grid reads it: aggregated over its None
@@ -804,8 +838,8 @@ impl GridLayout {
(DIM_CATEGORY.to_string(), col.to_string()),
]));
}
let row_item = data_items(&self.row_items).nth(row)?;
let col_item = data_items(&self.col_items).nth(col)?;
let row_item = data_item(&self.row_items, &self.row_pos, row)?;
let col_item = data_item(&self.col_items, &self.col_pos, col)?;
let mut coords = page_constraints(&self.page_coords);
for (cat, item) in self.row_cats.iter().zip(row_item.iter()) {
coords.push((cat.clone(), item.clone()));
@@ -819,7 +853,7 @@ impl GridLayout {
/// The coordinates a COLUMN answers for once its rows are aggregated away:
/// the page/filter constraints and the column's own tuple (core.md §6).
pub fn column_key(&self, col: usize) -> Option<CellKey> {
let col_item = data_items(&self.col_items).nth(col)?;
let col_item = data_item(&self.col_items, &self.col_pos, col)?;
let mut coords = page_constraints(&self.page_coords);
for (cat, item) in self.col_cats.iter().zip(col_item.iter()) {
coords.push((cat.clone(), item.clone()));
@@ -829,12 +863,12 @@ impl GridLayout {
/// Visual index of the nth data row (skipping group headers).
pub fn data_row_to_visual(&self, data_row: usize) -> Option<usize> {
data_index_to_visual(&self.row_items, data_row)
data_index_to_visual(&self.row_pos, data_row)
}
/// Visual index of the nth data column (skipping group headers).
pub fn data_col_to_visual(&self, data_col: usize) -> Option<usize> {
data_index_to_visual(&self.col_items, data_col)
data_index_to_visual(&self.col_pos, data_col)
}
/// The `(cat_name, group_name)` of the innermost group the Nth data row
@@ -846,12 +880,12 @@ impl GridLayout {
/// Whether the cell at `(row, col)` folds a collapsed group: an aggregate
/// with no record under it. The one home of that answer (§14).
pub fn folds_a_group(&self, row: usize, col: usize) -> bool {
self.row_folds_a_group(row) || folds(&self.col_items, col)
self.row_folds_a_group(row) || folds(&self.col_items, &self.col_pos, col)
}
/// The row half of [`Self::folds_a_group`], for a command about the row alone.
pub fn row_folds_a_group(&self, row: usize) -> bool {
folds(&self.row_items, row)
folds(&self.row_items, &self.row_pos, row)
}
/// The first data row of `cat`/`group`'s run under `prefix` — where a toggle
@@ -867,12 +901,12 @@ impl GridLayout {
/// The coordinate tuple of the Nth data row.
pub fn row_tuple(&self, row: usize) -> Option<&[String]> {
data_items(&self.row_items).nth(row).map(Vec::as_slice)
data_item(&self.row_items, &self.row_pos, row).map(Vec::as_slice)
}
/// The coordinate tuple of the Nth data column.
pub fn col_tuple(&self, col: usize) -> Option<&[String]> {
data_items(&self.col_items).nth(col).map(Vec::as_slice)
data_item(&self.col_items, &self.col_pos, col).map(Vec::as_slice)
}
/// The `(cat_name, group_name)` of the innermost group the Nth data
@@ -2970,3 +3004,171 @@ mod hierarchy {
assert_eq!(l.row_items[0], header(0, "2024", false));
}
}
#[cfg(test)]
mod lookup_cost_tests {
use super::{DATA_ITEM_STEPS, GridLayout};
use crate::view::Axis;
use crate::workbook::Workbook;
fn tall(rows: usize) -> GridLayout {
let mut wb = Workbook::new("T");
wb.add_category("Row").unwrap();
wb.add_category("Col").unwrap();
for i in 0..rows {
wb.add_item("Row", format!("r{i}")).unwrap();
}
wb.add_item("Col", "c0").unwrap();
wb.add_item("Col", "c1").unwrap();
wb.active_view_mut().set_axis("Row", Axis::Row);
wb.active_view_mut().set_axis("Col", Axis::Column);
GridLayout::new(&wb.model, wb.active_view())
}
fn steps(f: impl FnOnce()) -> usize {
DATA_ITEM_STEPS.with(|n| n.set(0));
f();
DATA_ITEM_STEPS.with(|n| n.get())
}
/// improvise-felo.13: context/test-evidence/layout-cost.md.
#[test]
fn an_axis_lookup_does_work_independent_of_the_row_index() {
let layout = tall(400);
let last = layout.row_count() - 1;
assert_eq!(last, 399, "the fixture is as tall as the test needs");
let lookups: [(&str, &dyn Fn(usize)); 4] = [
("cell_key", &|r| {
let _ = layout.cell_key(r, 1);
}),
("row_tuple", &|r| {
let _ = layout.row_tuple(r);
}),
("row_label", &|r| {
let _ = layout.row_label(r);
}),
("column_key", &|r| {
let _ = layout.column_key(r % 2);
}),
];
for (name, lookup) in lookups {
let first = steps(|| lookup(0));
let deep = steps(|| lookup(last));
assert!(
deep <= first + 2,
"{name}: row {last} cost {deep} steps against {first} at row 0"
);
}
}
/// improvise-felo.13: the index answers what the walk it replaced did, through
/// groups, a collapsed fold and a pruned axis. Evidence: layout-cost.md.
#[test]
fn the_position_index_answers_what_the_walk_answered() {
use super::{AxisEntry, data_items};
use crate::model::cell::{CellKey, CellValue};
let mut wb = Workbook::new("T");
for cat in ["Row", "Col"] {
wb.add_category(cat).unwrap();
}
for (i, g) in ["Q1", "Q1", "Q2", "Q2", "Q3"].iter().enumerate() {
wb.add_item_in_group("Row", format!("r{i}"), *g).unwrap();
}
wb.add_item("Row", "ghost").unwrap();
wb.add_item("Col", "c0").unwrap();
wb.active_view_mut().set_axis("Row", Axis::Row);
wb.active_view_mut().set_axis("Col", Axis::Column);
wb.active_view_mut().set_expand_depth("Row", 1);
wb.active_view_mut().toggle_group_collapse("Row", &["Q2"]);
wb.active_view_mut().prune_empty = true;
for i in 0..5 {
let key = CellKey::new(vec![
("Row".into(), format!("r{i}")),
("Col".into(), "c0".into()),
]);
wb.model.set_cell(key, CellValue::Number(1.0));
}
let layout = GridLayout::with_frozen_records(&wb.model, wb.active_view(), None);
assert!(
layout
.row_items
.iter()
.any(|e| !matches!(e, AxisEntry::DataItem { .. }))
);
assert!(layout.row_items.iter().any(|e| matches!(
e,
AxisEntry::DataItem {
collapsed: true,
..
}
)));
let walked: Vec<&Vec<String>> = data_items(&layout.row_items).collect();
assert_eq!(layout.row_count(), walked.len());
let visual: Vec<usize> = layout
.row_items
.iter()
.enumerate()
.filter(|(_, e)| matches!(e, AxisEntry::DataItem { .. }))
.map(|(at, _)| at)
.collect();
for i in 0..walked.len() + 2 {
assert_eq!(
layout.row_tuple(i),
walked.get(i).map(|t| t.as_slice()),
"row {i}"
);
assert_eq!(
layout.data_row_to_visual(i),
visual.get(i).copied(),
"visual {i}"
);
let folded = matches!(
data_items_entry(&layout.row_items, i),
Some(AxisEntry::DataItem {
collapsed: true,
..
})
);
assert_eq!(layout.row_folds_a_group(i), folded, "fold {i}");
}
}
/// improvise-felo.13: a drill held open while records are removed elsewhere.
/// Evidence: layout-cost.md.
#[test]
fn a_frozen_records_snapshot_replaces_the_live_row_index() {
use crate::model::cell::{CellKey, CellValue};
let mut wb = Workbook::new("T");
wb.add_category("Region").unwrap();
wb.add_item("_Measure", "Revenue").unwrap();
let cell = |i: usize| {
CellKey::new(vec![
("Region".into(), format!("r{i}")),
("_Measure".into(), "Revenue".into()),
])
};
for i in 0..3 {
wb.add_item("Region", format!("r{i}")).unwrap();
wb.model.set_cell(cell(i), CellValue::Number(1.0));
}
let v = wb.active_view_mut();
v.set_axis("_Index", Axis::Row);
v.set_axis("_Dim", Axis::Column);
let frozen = GridLayout::new(&wb.model, wb.active_view()).records;
wb.model.clear_cell(&cell(1));
wb.model.clear_cell(&cell(2));
let live = GridLayout::with_frozen_records(&wb.model, wb.active_view(), None);
assert_eq!(live.row_count(), 1, "the control: one record is live");
let held = GridLayout::with_frozen_records(&wb.model, wb.active_view(), frozen);
assert_eq!(held.row_count(), 3);
assert_eq!(held.row_tuple(2), Some(&["2".to_string()][..]));
assert!(held.cell_key(2, 0).is_some());
}
fn data_items_entry(items: &[super::AxisEntry], n: usize) -> Option<&super::AxisEntry> {
items
.iter()
.filter(|e| matches!(e, super::AxisEntry::DataItem { .. }))
.nth(n)
}
}
crates/improvise-io/src/import/csv_parser.rs
+73 -50
View File
@@ -7,14 +7,32 @@ use serde_json::Value;
use crate::storage::{PhysicalFs, Storage};
/// Parsed CSV records. Every value is its source text, so they reach an import
/// only through `ImportPipeline::from_csv` (improvise-8jk6.4).
#[derive(Debug, Clone, Default, PartialEq)]
pub struct CsvRecords(Vec<Value>);
impl std::ops::Deref for CsvRecords {
type Target = [Value];
fn deref(&self) -> &[Value] {
&self.0
}
}
impl CsvRecords {
pub(crate) fn into_values(self) -> Vec<Value> {
self.0
}
}
pub fn csv_path_p(path: &Path) -> bool {
path.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("csv"))
}
/// Parse a CSV file into a `serde_json::Value` array. Short-row warnings go to
/// stderr; [`parse_csv_with_warnings`] captures them instead.
pub fn parse_csv(path: &Path) -> Result<Vec<Value>> {
/// Parse a CSV file into records. Short-row warnings go to stderr;
/// [`parse_csv_with_warnings`] captures them instead.
pub fn parse_csv(path: &Path) -> Result<CsvRecords> {
let (records, warnings) = parse_csv_with_warnings(path)?;
for warning in &warnings {
eprintln!("warning: {warning}");
@@ -24,7 +42,7 @@ pub fn parse_csv(path: &Path) -> Result<Vec<Value>> {
/// Parse a CSV file, returning records plus warnings. A short row keeps every
/// header column, missing ones `Null`, each reported with a row count.
pub fn parse_csv_with_warnings(path: &Path) -> Result<(Vec<Value>, Vec<String>)> {
pub fn parse_csv_with_warnings(path: &Path) -> Result<(CsvRecords, Vec<String>)> {
let bytes = std::fs::read(path)
.with_context(|| format!("Failed to open CSV file: {}", path.display()))?;
parse_csv_bytes_with_warnings(&bytes)
@@ -32,9 +50,9 @@ pub fn parse_csv_with_warnings(path: &Path) -> Result<(Vec<Value>, Vec<String>)>
/// As [`parse_csv_with_warnings`] but over bytes, so import can come from any
/// [`Storage`](crate::storage::Storage) backend, not only a file (improvise-fuj).
pub fn parse_csv_bytes_with_warnings(bytes: &[u8]) -> Result<(Vec<Value>, Vec<String>)> {
pub fn parse_csv_bytes_with_warnings(bytes: &[u8]) -> Result<(CsvRecords, Vec<String>)> {
let (_, records, warnings) = parse_csv_bytes(bytes)?;
Ok((records, warnings))
Ok((CsvRecords(records), warnings))
}
/// The header (after renaming), the records and the warnings.
@@ -152,7 +170,7 @@ fn disambiguate_headers(raw: Vec<String>) -> (Vec<String>, Vec<String>) {
/// Parse multiple CSV files and merge into a single JSON array.
/// Each record gets a "File" field set to the filename stem (e.g., "sales" from "sales.csv").
pub fn merge_csvs(paths: &[impl AsRef<Path>]) -> Result<Vec<Value>> {
pub fn merge_csvs(paths: &[impl AsRef<Path>]) -> Result<CsvRecords> {
let merged = merge_csvs_from(&PhysicalFs, paths)?;
for warning in &merged.warnings {
eprintln!("warning: {warning}");
@@ -173,7 +191,7 @@ pub struct SourceFile {
/// Several CSV files as one record set, with each file's own columns kept.
#[derive(Debug, Clone, PartialEq)]
pub struct Merged {
pub records: Vec<Value>,
pub records: CsvRecords,
/// One per distinct `File` item, in the order the files were read.
pub sources: Vec<SourceFile>,
/// Each file's parse warnings, prefixed with its path.
@@ -184,7 +202,7 @@ pub struct Merged {
/// and two files with one stem share its item and its columns (improvise-lpzu).
pub fn merge_csvs_from(fs: &dyn Storage, paths: &[impl AsRef<Path>]) -> Result<Merged> {
let mut merged = Merged {
records: Vec::new(),
records: CsvRecords::default(),
sources: Vec::new(),
warnings: Vec::new(),
};
@@ -212,7 +230,7 @@ pub fn merge_csvs_from(fs: &dyn Storage, paths: &[impl AsRef<Path>]) -> Result<M
if let Value::Object(ref mut map) = record {
map.insert(FILE_CATEGORY.to_string(), Value::String(stem.clone()));
}
merged.records.push(record);
merged.records.0.push(record);
}
}
Ok(merged)
@@ -266,26 +284,33 @@ fn csv_documents_in(fs: &dyn Storage, dir: &Path) -> Result<Vec<PathBuf>> {
Ok(csvs)
}
/// A field is its source text: which ones are numbers is decided by the role
/// the import gives the column (improvise-8jk6.4, `context/code/formula-and-io.md`).
fn parse_csv_field(field: &str) -> Value {
if field.is_empty() {
return Value::Null;
match field.is_empty() {
true => Value::Null,
false => Value::String(field.to_string()),
}
}
// Try to parse as number (integer or float)
if let Ok(num) = field.parse::<i64>() {
return Value::Number(serde_json::Number::from(num));
/// The number a CSV field's text reads as, where a number is wanted.
pub fn number_in(text: &str) -> Option<serde_json::Number> {
if let Ok(num) = text.parse::<i64>() {
return Some(serde_json::Number::from(num));
}
// NaN and the infinities have no JSON number. A fabricated 0 is an invented
// measurement (improvise-jc95), so the token falls through and stays text.
if let Ok(num) = field.parse::<f64>()
&& let Some(number) = serde_json::Number::from_f64(num)
{
return Value::Number(number);
}
// measurement (improvise-jc95), so the token stays text.
text.parse::<f64>()
.ok()
.and_then(serde_json::Number::from_f64)
}
// Otherwise treat as string
Value::String(field.to_string())
/// A CSV value as a typed JSON value: numeric text becomes its number.
pub fn typed(value: &Value) -> Value {
match value.as_str().and_then(number_in) {
Some(number) => Value::Number(number),
None => value.clone(),
}
}
#[cfg(test)]
@@ -310,8 +335,9 @@ mod tests {
assert_eq!(records.len(), 2);
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
assert_eq!(records[0]["Product"], Value::String("Shirts".to_string()));
assert_eq!(records[0]["Revenue"], Value::String("1000".to_string()));
assert_eq!(
records[0]["Revenue"],
typed(&records[0]["Revenue"]),
Value::Number(serde_json::Number::from(1000))
);
}
@@ -323,9 +349,10 @@ mod tests {
let records = parse_csv(&path).unwrap();
assert_eq!(records.len(), 2);
assert!(records[0]["Revenue"].is_f64());
assert_eq!(records[0]["Revenue"], Value::String("1000.50".to_string()));
assert!(typed(&records[0]["Revenue"]).is_f64());
assert_eq!(
records[0]["Revenue"],
typed(&records[0]["Revenue"]),
Value::Number(serde_json::Number::from_f64(1000.50).unwrap())
);
}
@@ -363,10 +390,10 @@ mod tests {
assert_eq!(records.len(), 2);
assert_eq!(records[0]["Name"], Value::String("Widget".to_string()));
assert_eq!(
records[0]["Count"],
typed(&records[0]["Count"]),
Value::Number(serde_json::Number::from(5))
);
assert!(records[0]["Price"].is_f64());
assert!(typed(&records[0]["Price"]).is_f64());
assert_eq!(records[0]["Active"], Value::String("true".to_string()));
}
@@ -386,10 +413,7 @@ mod tests {
assert_eq!(records[3]["File"], Value::String("expenses".to_string()));
// Original fields preserved
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
assert_eq!(
records[2]["Revenue"],
Value::Number(serde_json::Number::from(50))
);
assert_eq!(records[2]["Revenue"], Value::String("50".to_string()));
}
#[test]
@@ -492,10 +516,10 @@ mod tests {
]
);
assert_eq!(
merged.records,
*merged.records,
[
serde_json::json!({"entity": "src/a.rs", "coupled": "src/b.rs", "degree": 80, "File": "coupling"}),
serde_json::json!({"entity": "src/a.rs", "age-months": 2, "File": "age"}),
serde_json::json!({"entity": "src/a.rs", "coupled": "src/b.rs", "degree": "80", "File": "coupling"}),
serde_json::json!({"entity": "src/a.rs", "age-months": "2", "File": "age"}),
]
);
}
@@ -656,10 +680,7 @@ mod tests {
assert!(warnings.is_empty());
assert_eq!(records.len(), 2);
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
assert_eq!(
records[0]["Revenue"],
Value::Number(serde_json::Number::from(1))
);
assert_eq!(records[0]["Revenue"], Value::String("1".to_string()));
assert_eq!(records[1]["Region"], Value::String("West".to_string()));
let (control, _) = parse_csv_bytes_with_warnings(b"Region,Revenue\nEa\rst,1\n").unwrap();
@@ -714,7 +735,8 @@ mod tests {
token.parse::<f64>().is_ok_and(|n| !n.is_finite()),
"precondition: {token} parses as a non-finite f64"
);
assert_eq!(parse_csv_field(token), Value::String(token.to_string()));
let text = Value::String(token.to_string());
assert_eq!(typed(&text), text);
}
}
@@ -723,10 +745,10 @@ mod tests {
#[test]
fn an_ordinary_float_is_still_a_number() {
assert_eq!(
parse_csv_field("1000.50"),
typed(&Value::String("1000.50".into())),
Value::Number(serde_json::Number::from_f64(1000.50).unwrap())
);
assert_eq!(parse_csv_field("0"), Value::Number(0.into()));
assert_eq!(typed(&Value::String("0".into())), Value::Number(0.into()));
}
/// improvise-5hfh: two equal header names are one JSON key, so the later
@@ -735,9 +757,9 @@ mod tests {
fn a_repeated_header_is_renamed_rather_than_overwriting_its_twin() {
let (records, warnings) = parse_csv_bytes_with_warnings(b"A,B,A\n1,2,3\n").unwrap();
let map = records[0].as_object().unwrap();
assert_eq!(map["A"], Value::Number(1.into()));
assert_eq!(map["B"], Value::Number(2.into()));
assert_eq!(map["A_2"], Value::Number(3.into()));
assert_eq!(map["A"], Value::String("1".into()));
assert_eq!(map["B"], Value::String("2".into()));
assert_eq!(map["A_2"], Value::String("3".into()));
assert_eq!(warnings.len(), 1);
assert!(warnings[0].contains("'A'") && warnings[0].contains("'A_2'"));
assert!(warnings[0].contains("column 3"), "{}", warnings[0]);
@@ -750,8 +772,8 @@ mod tests {
let (records, _) = parse_csv_bytes_with_warnings(b"A,A_2,A\n1,2,3\n").unwrap();
let map = records[0].as_object().unwrap();
assert_eq!(map.len(), 3, "{map:?}");
assert_eq!(map["A_2"], Value::Number(2.into()));
assert_eq!(map["A_3"], Value::Number(3.into()));
assert_eq!(map["A_2"], Value::String("2".into()));
assert_eq!(map["A_3"], Value::String("3".into()));
}
#[test]
@@ -786,8 +808,9 @@ mod tests {
assert_eq!(records.len(), 2);
assert_eq!(records[0]["Date"], Value::String("03/31/2026".to_string()));
assert_eq!(records[0]["Amount"], Value::String("-50.00".to_string()));
assert_eq!(
records[0]["Amount"],
typed(&records[0]["Amount"]),
Value::Number(serde_json::Number::from_f64(-50.00).unwrap())
);
assert_eq!(records[0]["Flag"], Value::String("*".to_string()));
@@ -797,7 +820,7 @@ mod tests {
Value::String("VENMO PAYMENT 260331".to_string())
);
assert_eq!(
records[1]["Amount"],
typed(&records[1]["Amount"]),
Value::Number(serde_json::Number::from_f64(-240.00).unwrap())
);
}
crates/improvise-io/src/import/wizard.rs
+239 -25
View File
@@ -7,8 +7,8 @@ use super::analyzer::{
DateComponent, FieldKind, FieldProposal, analyze_records, extract_array_at_path,
extract_date_component, find_array_paths,
};
use super::csv_parser::{FILE_CATEGORY, Merged, SourceFile};
use crate::formula::parse_formula;
use super::csv_parser::{self, CsvRecords, FILE_CATEGORY, Merged, SourceFile};
use crate::formula::{Formula, parse_formula};
use crate::model::MAX_CATEGORIES;
use crate::model::cell::{CellKey, CellValue};
use crate::view::{Axis, VIRTUAL_VIEW_NAMES, View, ViewStep, records_steps};
@@ -33,6 +33,42 @@ pub struct ImportPipeline {
/// The files a CSV merge read, each of which gets its own view. Empty for
/// a single file or a JSON import (improvise-lpzu).
pub sources: Vec<SourceFile>,
/// What the records' string values are, fixed by the constructor (8jk6.4).
scalars: Scalars,
}
/// How a record's values read. Why CSV is text: `context/code/formula-and-io.md`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Scalars {
/// JSON: a string is text and a number is a number.
Typed,
/// CSV: every value is source text, read as a number only where one is wanted.
Text,
}
impl Scalars {
/// The record as analysis sees it: typed, whatever its source.
fn typed_record(self, record: &Value) -> Value {
match (self, record) {
(Scalars::Text, Value::Object(map)) => Value::Object(
map.iter()
.map(|(k, v)| (k.clone(), csv_parser::typed(v)))
.collect(),
),
(Scalars::Text, other) | (Scalars::Typed, other) => other.clone(),
}
}
/// The value a measure takes from `value`, if it is a number.
fn number(self, value: &Value) -> Option<f64> {
match self {
Scalars::Typed => value.as_f64(),
Scalars::Text => value
.as_str()
.and_then(csv_parser::number_in)
.and_then(|n| n.as_f64()),
}
}
}
/// Several source records that land on one coordinate tuple (improvise-cgqt).
@@ -57,7 +93,17 @@ pub enum DuplicatePolicy {
}
impl ImportPipeline {
/// A pipeline over a JSON value.
pub fn new(raw: Value) -> Self {
Self::reading(raw, Scalars::Typed)
}
/// A pipeline over CSV records, whose values are their source text.
pub fn from_csv(records: CsvRecords) -> Self {
Self::reading(Value::Array(records.into_values()), Scalars::Text)
}
fn reading(raw: Value, scalars: Scalars) -> Self {
let array_paths = find_array_paths(&raw);
let mut pipeline = Self {
raw: raw.clone(),
@@ -69,6 +115,7 @@ impl ImportPipeline {
formulas: vec![],
on_duplicate: DuplicatePolicy::Refuse,
sources: vec![],
scalars,
};
// Auto-select if root is an array or there is exactly one candidate path.
@@ -84,7 +131,7 @@ impl ImportPipeline {
/// A pipeline over a CSV merge, carrying its sources for the per-file views.
pub fn from_merge(merged: Merged) -> Self {
let mut pipeline = Self::new(Value::Array(merged.records));
let mut pipeline = Self::from_csv(merged.records);
pipeline.seed_sources(merged.sources);
pipeline
}
@@ -112,7 +159,52 @@ impl ImportPipeline {
self.selected_path = path.to_string();
if let Some(arr) = extract_array_at_path(&self.raw, path) {
self.records = arr.clone();
self.proposals = analyze_records(&self.records);
self.analyze();
}
}
/// Propose a kind for every field, reading the records as `scalars` says.
pub fn analyze(&mut self) {
let typed: Vec<Value> = self
.records
.iter()
.map(|r| self.scalars.typed_record(r))
.collect();
self.proposals = analyze_records(&typed);
}
/// The warning for the records the build will skip, if it skips any.
pub fn skip_warning(&self) -> Option<String> {
let skipped = self.records.iter().filter(|r| !r.is_object()).count();
skip_warnings(self.records.len(), skipped)
}
/// Why there is nothing to build, when no record is an object to place.
pub fn nothing_to_import(&self) -> Option<String> {
let total = self.records.len();
match (total, self.records.iter().any(Value::is_object)) {
(_, true) => None,
(0, false) => Some("Nothing to import: the source has no records".into()),
(_, false) => Some(format!(
"Nothing to import: all {total} records were skipped as not JSON objects"
)),
}
}
/// The formulas, parsed. Every one that does not parse is named with its
/// error, never dropped (improvise-8jk6.5).
pub fn parsed_formulas(&self) -> Result<Vec<Formula>> {
let mut parsed = vec![];
let mut refused: Vec<String> = vec![];
for raw in &self.formulas {
match import_formula(raw) {
Ok(formula) => parsed.push(formula),
Err(e) => refused.push(e.to_string()),
}
}
match refused.is_empty() {
true => Ok(parsed),
false => Err(anyhow!(refused.join("\n"))),
}
}
@@ -227,6 +319,7 @@ impl ImportPipeline {
measures: accepted(&[FieldKind::Measure]),
labels: accepted(&[FieldKind::Label]),
date_extractions,
scalars: self.scalars,
}
}
@@ -237,9 +330,13 @@ impl ImportPipeline {
let categories = &plan.categories;
let date_extractions = &plan.date_extractions;
if let Some(why) = self.nothing_to_import() {
return Err(anyhow!(why));
}
if categories.is_empty() {
return Err(anyhow!("At least one category must be accepted"));
}
let formulas = self.parsed_formulas()?;
let placements: Vec<Placement> = self.records.iter().map(|r| plan.place(r)).collect();
let collisions = collisions_among(&placements);
@@ -288,7 +385,6 @@ impl ImportPipeline {
wb.add_item("_Measure", &m.field)?;
}
let mut skipped_non_objects = 0usize;
// Per-extraction tally of values the detected format could not parse. A
// derived coordinate that is merely absent is improvise-yss5.
let mut unparsed_dates = vec![0usize; date_extractions.len()];
@@ -304,7 +400,7 @@ impl ImportPipeline {
for (at, placement) in placements.into_iter().enumerate() {
match placement {
Placement::NotObject => skipped_non_objects += 1,
Placement::NotObject => {}
Placement::Placed {
mut coords,
measures,
@@ -352,22 +448,15 @@ impl ImportPipeline {
wb.model.set_cell(cell.key, CellValue::Number(cell.total));
}
// Parse and add formulas
// Formulas target the "_Measure" category by default.
let formula_cat: String = "_Measure".to_string();
for raw in &self.formulas {
if let Ok(formula) = parse_formula(raw, &formula_cat) {
wb.model.add_formula(formula);
}
for formula in formulas {
wb.model.add_formula(formula);
}
for view in source_views(&wb, &self.sources, &plan) {
wb.views.insert(view.name.clone(), view);
}
let mut warnings: Vec<String> = skip_warnings(self.records.len(), skipped_non_objects)
.into_iter()
.collect();
let mut warnings: Vec<String> = self.skip_warning().into_iter().collect();
warnings.extend(unparsed_date_warnings(date_extractions, &unparsed_dates));
warnings.extend(collision_warning(&summed));
warnings.extend(occurrence.map(|name| separated_note(name, &collisions)));
@@ -463,6 +552,7 @@ struct Plan<'a> {
measures: Vec<&'a FieldProposal>,
labels: Vec<&'a FieldProposal>,
date_extractions: Vec<(&'a str, &'a str, DateComponent, String)>,
scalars: Scalars,
}
/// Where one source record lands. Worked out once, for the build and for the
@@ -519,7 +609,7 @@ impl Plan<'_> {
let measures = self
.measures
.iter()
.filter_map(|m| Some((m.field.clone(), map.get(&m.field)?.as_f64()?)))
.filter_map(|m| Some((m.field.clone(), self.scalars.number(map.get(&m.field)?)?)))
.collect();
Placement::Placed {
coords,
@@ -635,6 +725,11 @@ fn unparsed_date_warnings(
.collect()
}
/// An import formula, targeting `_Measure`, or the error naming it (improvise-8jk6.5).
fn import_formula(raw: &str) -> Result<Formula> {
parse_formula(raw, "_Measure").map_err(|e| anyhow!("Formula error in '{raw}': {e}"))
}
/// The warning for records that could not be placed in the grid. A record
/// missing a category value is placed, so only a non-object is skipped.
fn skip_warnings(total: usize, skipped_non_objects: usize) -> Option<String> {
@@ -822,7 +917,9 @@ impl ImportWizard {
fn validation_error(&self) -> Option<String> {
match self.step {
WizardStep::ReviewProposals => self
.no_category_message()
.pipeline
.nothing_to_import()
.or_else(|| self.no_category_message())
.or_else(|| self.category_limit_message()),
WizardStep::ConfigureDates => self.category_limit_message(),
WizardStep::ResolveDuplicates => match &self.pipeline.on_duplicate {
@@ -1117,14 +1214,24 @@ impl ImportWizard {
self.formula_buffer.pop();
}
/// Commit the current formula buffer to the pipeline's formula list.
/// Commit the current formula buffer to the pipeline's formula list. One
/// that does not parse stays in the buffer, with the error (improvise-8jk6.5).
pub fn confirm_formula(&mut self) {
let text = self.formula_buffer.trim().to_string();
if !text.is_empty() {
self.pipeline.formulas.push(text);
match (text.is_empty(), import_formula(&text)) {
(false, Err(e)) => self.message = Some(e.to_string()),
(false, Ok(_)) => {
self.pipeline.formulas.push(text);
self.close_formula_edit();
}
(true, _) => self.close_formula_edit(),
}
}
fn close_formula_edit(&mut self) {
self.formula_buffer.clear();
self.formula_editing = false;
self.message = None;
self.cursor = self.pipeline.formulas.len().saturating_sub(1);
}
@@ -2244,8 +2351,11 @@ mod tests {
),
];
fn code_maat_records() -> Value {
Value::Array(merged_pipeline(&CODE_MAAT).records)
/// The code-maat merge without its sources, so no per-file views.
fn code_maat_without_views() -> ImportPipeline {
let mut p = merged_pipeline(&CODE_MAAT);
p.sources.clear();
p
}
/// Every non-empty cell a view's grid draws, row by row.
@@ -2519,7 +2629,7 @@ mod tests {
/// 0, "" or a skip. Evidence: `context/test-evidence/formula-and-io.md`.
#[test]
fn files_with_different_headers_import_as_one_sparse_model() {
let p = ImportPipeline::new(code_maat_records());
let p = code_maat_without_views();
let kinds: Vec<(&str, FieldKind)> = p
.proposals
.iter()
@@ -2561,7 +2671,7 @@ mod tests {
/// "". Evidence: `context/test-evidence/formula-and-io.md`.
#[test]
fn a_label_column_a_record_lacks_is_no_coordinate_not_an_empty_item() {
let mut p = ImportPipeline::new(code_maat_records());
let mut p = code_maat_without_views();
p.proposals
.iter_mut()
.find(|f| f.field == "coupled")
@@ -3164,4 +3274,108 @@ mod tests {
w.message
);
}
/// improvise-8jk6.5. Evidence: `context/test-evidence/formula-and-io.md`.
#[test]
fn a_formula_that_does_not_parse_stops_the_build_and_is_named() {
let mut p = ImportPipeline::new(json!([{"region": "East", "revenue": 1.0}]));
p.formulas = vec![
"Good = revenue".into(),
"Bad = revenue +".into(),
"Worse = * 2".into(),
];
let err = p.build_model().expect_err("an unparseable formula refuses");
let msg = err.to_string();
assert!(msg.contains("Formula error in 'Bad = revenue +'"), "{msg}");
assert!(msg.contains("Formula error in 'Worse = * 2'"), "{msg}");
assert!(!msg.contains("Good"), "{msg}");
}
/// improvise-8jk6.5: the wizard refuses where the text can still be
/// fixed. Evidence: `context/test-evidence/formula-and-io.md`.
#[test]
fn confirming_a_formula_that_does_not_parse_keeps_it_out_and_the_text_in() {
let mut w = sample_wizard();
w.start_formula_edit();
for c in "Bad = revenue +".chars() {
w.push_formula_char(c);
}
w.confirm_formula();
assert!(w.pipeline.formulas.is_empty(), "{:?}", w.pipeline.formulas);
assert_eq!(w.formula_buffer, "Bad = revenue +");
assert!(w.formula_editing);
let message = w.message.clone().unwrap_or_default();
assert!(
message.starts_with("Formula error in 'Bad = revenue +'"),
"{message}"
);
}
/// improvise-8jk6.4: the role reads the text. Evidence:
/// `context/test-evidence/formula-and-io.md`.
#[test]
fn a_csv_column_read_as_a_category_keeps_its_text_and_as_a_measure_its_number() {
use crate::import::csv_parser::parse_csv_bytes_with_warnings;
use crate::model::cell::{CellKey, CellValue};
let csv = b"commit,lines\n00565796,3\n220944e2,5\n";
let (records, _) = parse_csv_bytes_with_warnings(csv).unwrap();
let mut p = ImportPipeline::from_csv(records);
let kind = |p: &ImportPipeline, f: &str| {
p.proposals
.iter()
.find(|x| x.field == f)
.unwrap()
.kind
.clone()
};
assert_eq!(kind(&p, "commit"), FieldKind::Measure, "analysis types");
let commit = p.proposals.iter_mut().find(|x| x.field == "commit");
commit.unwrap().kind = FieldKind::Category;
let wb = p.build_model().unwrap();
let items: Vec<&String> = wb.model.category("commit").unwrap().items.keys().collect();
assert_eq!(items, ["00565796", "220944e2"]);
let key = CellKey::new(vec![
("_Measure".into(), "lines".into()),
("commit".into(), "220944e2".into()),
]);
assert_eq!(wb.model.get_cell(&key), Some(&CellValue::Number(5.0)));
}
/// improvise-8jk6.6 meets lpzu's header-only rule: a folder of nothing but
/// headers kept no record. Evidence: `context/test-evidence/formula-and-io.md`.
#[test]
fn a_folder_of_only_header_only_files_keeps_no_record_and_refuses() {
let p = merged_pipeline(&[("m/a.csv", "entity,x\n"), ("m/b.csv", "entity,y\n")]);
assert!(p.proposals.iter().any(|f| f.field == "File"), "setup");
let msg = p.build_model().expect_err("refuses").to_string();
assert_eq!(msg, "Nothing to import: the source has no records");
}
/// improvise-8jk6.6: the wizard asks on the review step, where another
/// path can still be chosen. Evidence: the test-evidence doc.
#[test]
fn the_review_step_says_when_no_record_can_be_imported() {
let mut w = ImportWizard::new(json!([1, 2]));
assert_eq!(w.step, WizardStep::ReviewProposals, "setup");
w.advance();
assert_eq!(w.step, WizardStep::ReviewProposals);
assert_eq!(
w.message.as_deref(),
Some("Nothing to import: all 2 records were skipped as not JSON objects")
);
}
/// improvise-8jk6.6. Evidence: `context/test-evidence/formula-and-io.md`.
#[test]
fn a_build_that_keeps_no_record_refuses_and_says_why() {
let skipped = ImportPipeline::new(json!([1, "two", null]));
let msg = skipped.build_model().expect_err("refuses").to_string();
assert_eq!(
msg,
"Nothing to import: all 3 records were skipped as not JSON objects"
);
let empty = ImportPipeline::new(json!([]));
let msg = empty.build_model().expect_err("refuses").to_string();
assert_eq!(msg, "Nothing to import: the source has no records");
}
}
crates/improvise-io/src/persistence/roundtrip_tests.rs
+1 -1
View File
@@ -1031,7 +1031,7 @@ fn a_csv_header_with_an_embedded_newline_survives_a_save() {
let csv = "\"Region\nName\",Amount\nEast,1\nWest,2\n";
let (records, _) =
crate::import::csv_parser::parse_csv_bytes_with_warnings(csv.as_bytes()).unwrap();
let wb = crate::import::wizard::ImportPipeline::new(serde_json::Value::Array(records))
let wb = crate::import::wizard::ImportPipeline::from_csv(records)
.build_model()
.unwrap();
assert!(