fiddlerwoaroof/improvise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP improvise-eccl/f2oo: a fold evaluates its formula over the fold, and covers only drawn members

Browse Source 
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
This commit is contained in:
Edward Langley
2026-09-22 22:21:02 -07:00
parent b535245a21
commit e3aa2ccb4b
2 changed files with 275 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/improvise-core/src/model/types.rs
+156 -61
View File
@@ -51,17 +51,41 @@ fn member_records(model: &Model, partial: &CellKey) -> Vec<CellKey> {
seen.into_iter().collect()
}
/// A formula-level aggregate over the data matching `context` minus the
/// target. Deliberately *not* [`apply_agg`]'s emptiness rule — §2.
/// A formula-level aggregate over the data matching each of `contexts` minus the
/// target, pooled. Deliberately *not* [`apply_agg`]'s emptiness rule — §2.
fn eval_agg(
model: &Model,
context: &CellKey,
contexts: &[CellKey],
target_category: &str,
func: &AggFunc,
inner: &Expr,
agg_filter: Option<&Filter>,
eval_at: &dyn Fn(&Expr, &CellKey) -> Result<f64, String>,
) -> Result<f64, String> {
let values = contexts
.iter()
.map(|c| agg_values(model, c, target_category, inner, agg_filter, eval_at))
.collect::<Result<Vec<_>, String>>()?
.concat();
match func {
AggFunc::Sum => Ok(values.iter().sum()),
AggFunc::Count => Ok(values.len() as f64),
AggFunc::Avg | AggFunc::Min | AggFunc::Max => {
apply_agg(&values, func).ok_or_else(|| "empty".to_string())
}
}
}
/// The values an aggregate folds at one context: a stored ref's cells, or the
/// expression asked of each member record — §13.
fn agg_values(
model: &Model,
context: &CellKey,
target_category: &str,
inner: &Expr,
agg_filter: Option<&Filter>,
eval_at: &dyn Fn(&Expr, &CellKey) -> Result<f64, String>,
) -> Result<Vec<f64>, String> {
let mut partial = context.without(target_category);
// The fast path is the fold specialised to a ref that names a STORED
// cell; where it does not, the premise fails — §13, s08t, s2w3.
@@ -83,15 +107,15 @@ fn eval_agg(
if let Some(f) = agg_filter {
partial = partial.with(&f.category, &f.item);
}
let values: Vec<f64> = match select_raw {
match select_raw {
// A bare ref asks for the values that exist; anything else asks the
// expression of each member record and folds the answers — §13.
true => model
true => Ok(model
.data
.matching_values(&partial.0)
.into_iter()
.filter_map(|v| v.as_f64())
.collect(),
.collect()),
false => member_records(model, &partial)
.into_iter()
.map(|record| {
@@ -101,28 +125,34 @@ fn eval_agg(
.fold(partial.clone(), |k, (cat, item)| k.with(cat, item));
eval_at(inner, &at)
})
.collect::<Result<Vec<f64>, String>>()?,
};
match func {
AggFunc::Sum => Ok(values.iter().sum()),
AggFunc::Count => Ok(values.len() as f64),
AggFunc::Avg | AggFunc::Min | AggFunc::Max => {
apply_agg(&values, func).ok_or_else(|| "empty".to_string())
}
.collect(),
}
}
/// Which members of a collapsed group its fold covers. The model knows no
/// view, so the grid passes the members it draws (core.md §14).
pub type Covers<'a> = &'a dyn Fn(&str, &str) -> bool;
/// Where a formula's references are read: at one record, or across the members
/// a collapsed group's fold covers, as a None axis reads them (core.md §14).
#[derive(Clone, Copy)]
enum Reach<'a> {
Record,
Fold { agg: Aggregation, covers: Covers<'a> },
}
/// The **record-level** evaluator: no cache, because a member key is not a stem
/// the cache holds. Errors propagate immediately — §3, §13.
/// the cache holds. Errors propagate immediately — §3, §13. `reach`: §14.
fn eval_expr(
expr: &Expr,
context: &CellKey,
model: &Model,
target_category: &str,
depth: u8,
reach: Reach,
) -> Result<f64, String> {
use crate::formula::BinOp;
let eval = |e: &Expr| eval_expr(e, context, model, target_category, depth, reach);
match expr {
Expr::Number(n) => Ok(*n),
Expr::Ref(name) => {
@@ -130,15 +160,19 @@ fn eval_expr(
.find_item_category(name)
.ok_or_else(|| format!("ref:{name}"))?;
let new_key = context.clone().with(cat, name);
match model.evaluate_depth(&new_key, depth) {
let value = match reach {
Reach::Record => model.evaluate_depth(&new_key, depth),
Reach::Fold { agg, covers } => model.evaluate_fold(&new_key, agg, covers, depth),
};
match value {
Some(CellValue::Number(n)) => Ok(n),
Some(CellValue::Error(e)) => Err(e),
_ => Err(format!("ref:{name}")),
}
}
Expr::BinOp(op, l, r) => {
let lv = eval_expr(l, context, model, target_category, depth)?;
let rv = eval_expr(r, context, model, target_category, depth)?;
let lv = eval(l)?;
let rv = eval(r)?;
match op {
BinOp::Add => Ok(lv + rv),
BinOp::Sub => Ok(lv - rv),
@@ -153,22 +187,23 @@ fn eval_expr(
}
}
}
Expr::UnaryMinus(e) => Ok(-eval_expr(e, context, model, target_category, depth)?),
Expr::UnaryMinus(e) => Ok(-eval(e)?),
// A fold's aggregate pools every member record it covers, never the
// members' own aggregates (§14).
Expr::Agg(func, inner, agg_filter) => eval_agg(
model,
context,
&model.contexts_of(context, reach),
target_category,
func,
inner.as_ref(),
agg_filter.as_ref(),
&|e, at| eval_expr(e, at, model, target_category, depth),
&|e, at| eval_expr(e, at, model, target_category, depth, Reach::Record),
),
Expr::Is(filter) => Ok(eval_is(context, filter)),
Expr::If(cond, then, else_) => {
let cv = eval_bool(cond, context, model, target_category, depth)?;
match cv {
true => eval_expr(then, context, model, target_category, depth),
false => eval_expr(else_, context, model, target_category, depth),
match eval_bool(cond, context, model, target_category, depth, reach)? {
true => eval(then),
false => eval(else_),
}
}
}
@@ -182,12 +217,13 @@ fn eval_bool(
model: &Model,
target_category: &str,
depth: u8,
reach: Reach,
) -> Result<bool, String> {
use crate::formula::BinOp;
match expr {
Expr::BinOp(op, l, r) => {
let lv = eval_expr(l, context, model, target_category, depth)?;
let rv = eval_expr(r, context, model, target_category, depth)?;
let lv = eval_expr(l, context, model, target_category, depth, reach)?;
let rv = eval_expr(r, context, model, target_category, depth, reach)?;
match op {
BinOp::Eq => Ok((lv - rv).abs() < FLOAT_EQ_EPSILON),
BinOp::Ne => Ok((lv - rv).abs() >= FLOAT_EQ_EPSILON),
@@ -523,33 +559,64 @@ impl Model {
}
pub fn evaluate_aggregated(&self, key: &CellKey, agg: Aggregation) -> Option<CellValue> {
let members = self.group_expansion(key);
if !members.is_empty() {
return self.evaluate_group(key, &members, agg);
self.evaluate_covering(key, agg, &|_, _| true)
}
/// [`Self::evaluate_aggregated`], with a collapsed group's fold covering only
/// the members `covers` admits — the ones a grid draws (core.md §14).
pub fn evaluate_covering(
&self,
key: &CellKey,
agg: Aggregation,
covers: Covers,
) -> Option<CellValue> {
self.evaluate_fold(key, agg, covers, Self::MAX_EVAL_DEPTH)
}
/// A key naming a group folds; any other key reads as before. A formula is
/// evaluated OVER the fold, never folded itself — why: core.md §14.
fn evaluate_fold(
&self,
key: &CellKey,
agg: Aggregation,
covers: Covers,
depth: u8,
) -> Option<CellValue> {
let members = self.group_expansion(key, covers);
match (members.is_empty(), self.formula_for(key), depth) {
(true, _, _) => self.evaluate_unfolded(key, agg),
(false, Some(_), 0) => Some(CellValue::Error("circular".into())),
(false, Some(formula), _) => {
self.eval_formula_folded(formula, key, &members, agg, covers, depth - 1)
}
(false, None, _) => self.fold_stored(key, &members, agg),
}
}
/// A key that names no group: exact, the formula cache, or raw aggregation.
fn evaluate_unfolded(&self, key: &CellKey, agg: Aggregation) -> Option<CellValue> {
if agg == Aggregation::Exact {
return self.evaluate(key);
}
// Check the formula cache first.
if let Some(cached) = self.formula_cache.get(key) {
return Some(cached.clone());
}
// Aggregate raw data across all None-axis categories.
aggregate_cell_values(&self.data.matching_values(&key.0), self.agg_for(key))
}
/// One key per member where `key` names a collapsed GROUP rather than an item —
/// the cross product over every such coordinate. Empty when none does (§14).
fn group_expansion(&self, key: &CellKey) -> Vec<CellKey> {
/// One key per covered member where `key` names a collapsed GROUP rather than
/// an item — the cross product over every such coordinate. Empty when none does.
fn group_expansion(&self, key: &CellKey, covers: Covers) -> Vec<CellKey> {
let mut expanded = vec![key.clone()];
for (cat_name, name) in &key.0 {
let members = self
let members: Vec<&str> = self
.category(cat_name)
.filter(|cat| !cat.items.contains_key(name.as_str()))
.map(|cat| cat.items_in_group(name))
.unwrap_or_default();
.unwrap_or_default()
.into_iter()
.filter(|m| covers(cat_name, m))
.collect();
if members.is_empty() {
continue;
}
@@ -564,29 +631,53 @@ impl Model {
}
}
/// A group's cell: stored values fold ONCE over every record the group covers,
/// a derived one folds its members' results — why the two differ: core.md §14.
fn evaluate_group(
/// The contexts an aggregate inside a formula pools over at `context`.
fn contexts_of(&self, context: &CellKey, reach: Reach) -> Vec<CellKey> {
let members = match reach {
Reach::Record => Vec::new(),
Reach::Fold { covers, .. } => self.group_expansion(context, covers),
};
match members.is_empty() {
true => vec![context.clone()],
false => members,
}
}
/// A group's stored values fold ONCE over every record the group covers.
fn fold_stored(&self, key: &CellKey, members: &[CellKey], agg: Aggregation) -> Option<CellValue> {
let values: Vec<&CellValue> = members
.iter()
.flat_map(|k| match agg {
Aggregation::Exact => self.data.get(k).into_iter().collect(),
Aggregation::OverHidden => self.data.matching_values(&k.0),
})
.collect();
aggregate_cell_values(&values, self.agg_for(key))
}
/// A formula at a group key, its refs read through the fold — the value the
/// same formula has with the group's category on `Axis::None` (§14).
fn eval_formula_folded(
&self,
formula: &Formula,
key: &CellKey,
members: &[CellKey],
agg: Aggregation,
covers: Covers,
depth: u8,
) -> Option<CellValue> {
let values: Vec<CellValue> = match self.formula_for(key) {
Some(_) => members
.iter()
.filter_map(|k| self.evaluate_aggregated(k, agg))
.collect(),
None => members
.iter()
.flat_map(|k| match agg {
Aggregation::Exact => self.data.get(k).into_iter().collect(),
Aggregation::OverHidden => self.data.matching_values(&k.0),
})
.cloned()
.collect(),
};
aggregate_cell_values(&values.iter().collect::<Vec<_>>(), self.agg_for(key))
let applies = formula
.filter
.as_ref()
.is_none_or(|f| key.get(&f.category) == Some(f.item.as_str()));
if !applies {
return self.fold_stored(key, members, agg);
}
let reach = Reach::Fold { agg, covers };
match eval_expr(&formula.expr, key, self, &formula.target_category, depth, reach) {
Ok(n) => Some(CellValue::Number(n)),
Err(e) => Some(CellValue::Error(e)),
}
}
/// [`Self::evaluate_aggregated`] as `f64`, with `0.0` for an empty cell.
@@ -744,14 +835,17 @@ impl Model {
)?),
Expr::Agg(func, inner, agg_filter) => eval_agg(
model,
context,
std::slice::from_ref(context),
target_category,
func,
inner.as_ref(),
agg_filter.as_ref(),
// The fold evaluates at a MEMBER key, which is not a stem the
// cache holds — so the record-level evaluator, in both paths.
&|e, at| eval_expr(e, at, model, target_category, Model::MAX_EVAL_DEPTH),
&|e, at| {
let depth = Model::MAX_EVAL_DEPTH;
eval_expr(e, at, model, target_category, depth, Reach::Record)
},
),
Expr::Is(filter) => Ok(eval_is(context, filter)),
Expr::If(cond, then, else_) => {
@@ -841,6 +935,7 @@ impl Model {
self,
&formula.target_category,
depth,
Reach::Record,
) {
Ok(n) => Some(CellValue::Number(n)),
Err(e) => Some(CellValue::Error(e)),
@@ -891,7 +986,7 @@ mod agg_helper_tests {
let empty = Expr::Ref("Revenue".to_string());
let ctx = CellKey::new(vec![]);
let never = |_: &Expr, _: &CellKey| unreachable!("a bare Ref never folds per record");
let run = |func| eval_agg(&model, &ctx, "_Measure", &func, &empty, None, &never);
let run = |func| eval_agg(&model, &[ctx.clone()], "_Measure", &func, &empty, None, &never);
assert_eq!(run(AggFunc::Sum), Ok(0.0));
assert_eq!(run(AggFunc::Count), Ok(0.0));
crates/improvise-core/src/view/layout.rs
+119 -1
View File
@@ -1,3 +1,4 @@
use std::collections::{HashMap, HashSet};
use std::rc::Rc;
use crate::model::category::{DIM_CATEGORY, INDEX_CATEGORY, MEASURE_CATEGORY};
@@ -384,6 +385,8 @@ pub struct GridLayout {
/// In records mode: what each column holds, parallel to `col_items`.
/// Empty for normal pivot views.
record_cols: Vec<RecordCol>,
/// The view's hidden items: a fold covers only the members this grid draws.
hidden: HashMap<String, HashSet<String>>,
}
impl GridLayout {
@@ -467,6 +470,7 @@ impl GridLayout {
none_cats,
records: None,
record_cols: Vec::new(),
hidden: view.hidden_items.clone(),
}
}
}
@@ -500,6 +504,7 @@ impl GridLayout {
none_cats,
records: Some(Rc::new(records)),
record_cols,
hidden: HashMap::new(),
}
}
@@ -660,7 +665,8 @@ impl GridLayout {
/// The value of a pivot key as THIS grid reads it: aggregated over its None
/// axis, and a fold over the members it draws (core.md §14). One evaluation.
pub fn evaluate(&self, model: &Model, key: &CellKey) -> Option<CellValue> {
model.evaluate_aggregated(key, Aggregation::of(&self.none_cats))
let drawn = |cat: &str, item: &str| !self.hidden.get(cat).is_some_and(|h| h.contains(item));
model.evaluate_covering(key, Aggregation::of(&self.none_cats), &drawn)
}
pub fn display_text(
@@ -1747,4 +1753,116 @@ mod tests {
"Sep is in no group — Q2's GroupEnd closed Q1's re-opened run"
);
}
/// Q2 = {Apr}, Q1 = {Jan, Feb}, so the Q1 fold is row 1. Jan and Feb carry
/// ratios 0.5 and 0.9 whose sum, mean and pooled ratio all differ; Apr is empty.
fn margin_book(formulas: &[&str]) -> Workbook {
let mut wb = Workbook::new("T");
wb.add_category("Month").unwrap();
wb.add_item_in_group("Month", "Apr", "Q2").unwrap();
wb.add_item_in_group("Month", "Jan", "Q1").unwrap();
wb.add_item_in_group("Month", "Feb", "Q1").unwrap();
wb.add_item("_Measure", "Revenue").unwrap();
wb.add_item("_Measure", "Cost").unwrap();
for (month, revenue, cost) in [("Jan", 10.0, 5.0), ("Feb", 30.0, 3.0)] {
for (measure, n) in [("Revenue", revenue), ("Cost", cost)] {
wb.model.set_cell(
CellKey::new(vec![
("Month".into(), month.into()),
("_Measure".into(), measure.into()),
]),
CellValue::Number(n),
);
}
}
for f in formulas {
wb.model
.add_formula(crate::formula::parse_formula(f, "_Measure").unwrap());
}
wb.active_view_mut().set_axis("Month", Axis::Row);
wb.active_view_mut().set_axis("_Measure", Axis::Column);
wb
}
/// `measure` at the row labelled `row_label`, as the grid reads it.
fn grid_value(wb: &mut Workbook, row_label: &str, measure: &str) -> Option<f64> {
wb.model.recompute_formulas(&wb.active_view().none_cats());
let layout = GridLayout::new(&wb.model, wb.active_view());
let row = (0..layout.row_count()).find(|&r| layout.row_label(r) == row_label)?;
let col = (0..layout.col_count()).find(|&c| layout.col_label(c) == measure)?;
layout
.evaluate(&wb.model, &layout.cell_key(row, col)?)
.and_then(|v| v.as_f64())
}
/// improvise-eccl — see context/test-evidence/core.md.
#[test]
fn a_folds_ratio_formula_is_the_ratio_of_its_folded_inputs() {
for formulas in [
&["Margin = (Revenue - Cost) / Revenue"][..],
&["Profit = Revenue - Cost", "Margin = Profit / Revenue"][..],
] {
let mut folded = margin_book(formulas);
folded.active_view_mut().toggle_group_collapse("Month", "Q1");
let fold = grid_value(&mut folded, "Q1", "Margin").expect("a Q1 fold");
let mut none = margin_book(formulas);
none.active_view_mut().set_axis("Month", Axis::None);
let over_none = grid_value(&mut none, "", "Margin").expect("a None-axis cell");
assert!((fold - 0.8).abs() < 1e-9, "{formulas:?}: fold {fold}");
assert!((fold - over_none).abs() < 1e-9, "{formulas:?}: {fold} vs {over_none}");
}
}
/// improvise-eccl's guard on the other side: an aggregate INSIDE the formula
/// still pools every record the fold covers.
#[test]
fn a_folds_aggregate_formula_pools_its_members_records() {
let mut wb = margin_book(&["Top = MAX(Revenue)", "Total = SUM(Revenue)"]);
wb.active_view_mut().toggle_group_collapse("Month", "Q1");
assert_eq!(grid_value(&mut wb, "Q1", "Total"), Some(40.0));
assert_eq!(grid_value(&mut wb, "Q1", "Top"), Some(30.0));
}
/// Every row's value in column 0, summed: the totals strip's arithmetic.
fn column_total(wb: &Workbook) -> f64 {
let layout = GridLayout::new(&wb.model, wb.active_view());
(0..layout.row_count())
.filter_map(|r| layout.evaluate(&wb.model, &layout.cell_key(r, 0)?))
.filter_map(|v| v.as_f64())
.sum()
}
/// improvise-f2oo — see context/test-evidence/core.md.
#[test]
fn collapsing_a_group_with_a_hidden_member_leaves_the_total_where_it_was() {
let mut wb = Workbook::new("T");
wb.add_category("Month").unwrap();
wb.add_category("Type").unwrap();
wb.add_item("Type", "Food").unwrap();
for (month, quarter, n) in [("Jan", "Q1", 3.0), ("Feb", "Q1", 4.0), ("Apr", "Q2", 100.0)] {
wb.add_item_in_group("Month", month, quarter).unwrap();
wb.model.set_cell(
CellKey::new(vec![
("Month".into(), month.into()),
("Type".into(), "Food".into()),
]),
CellValue::Number(n),
);
}
wb.active_view_mut().set_axis("Month", Axis::Row);
wb.active_view_mut().set_axis("Type", Axis::Column);
wb.active_view_mut().hide_item("Month", "Feb");
assert_eq!(column_total(&wb), 103.0, "expanded: Jan and Apr are drawn");
wb.active_view_mut().toggle_group_collapse("Month", "Q1");
assert_eq!(column_total(&wb), 103.0, "z moved the total");
assert_eq!(grid_value(&mut wb, "Q1", "Food"), Some(3.0), "Feb is hidden");
// Every member hidden: no fold is drawn, and the total still agrees.
wb.active_view_mut().hide_item("Month", "Jan");
assert_eq!(grid_value(&mut wb, "Q1", "Food"), None, "no Q1 row");
assert_eq!(column_total(&wb), 100.0);
}
}