improvise/crates/improvise-incremental/tests/formulas.rs

//! `Incremental`'s formula cache against `Eager`'s, and what it costs.
//! Evidence: `context/test-evidence/improvise-incremental.md`.

mod support;

use improvise_core::formula::parse_formula;
use improvise_core::model::cell::{CellKey, CellValue};
use improvise_core::model::{Aggregation, Model};
use improvise_incremental::{Derived, Edit, Evaluation, Incremental, Maintained, differential_in};
use support::{
    FORMULA_POOL, bump, formula_edit, formula_workbook, key, none_of, same_model, unformula_edit,
};

#[test]
fn every_formula_in_the_pool_is_added_edited_around_and_removed_like_eager() {
    let wb = formula_workbook();
    let views: Vec<_> = wb.views.values().cloned().collect();
    let none = none_of(&wb);
    for (i, (raw, _)) in FORMULA_POOL.iter().enumerate() {
        let edits = [
            formula_edit(i),
            bump("East", "A", "Revenue", 99.0),
            bump("West", "B", "Units", 0.0),
            Edit::ClearCell(key("West", "A", "Cost")),
            unformula_edit(i),
        ];
        let ran = differential_in(
            &wb.model,
            &views,
            &none,
            Incremental::new(wb.model.clone(), none.clone()),
            &edits,
        );
        ran.unwrap_or_else(|d| panic!("{raw}: {d}"));
    }
}

#[test]
fn a_formula_that_keeps_moving_for_sixteen_passes_is_circular() {
    let wb = formula_workbook();
    let none = vec!["Region".to_string()];
    let seeds = [("Loop", 1.0), ("Pong", 1.0)];
    let mut edits: Vec<Edit> = vec![bump("West", "B", "Revenue", 1.0)];
    edits.extend([6, 7, 8].iter().map(|&i| formula_edit(i)));
    edits.extend(seeds.iter().map(|(m, v)| bump("East", "A", m, *v)));
    let mut backend = Incremental::new(wb.model.clone(), none);
    edits.iter().for_each(|e| backend.apply(e));
    let at = |m: &str| {
        CellKey::new(vec![
            ("Product".into(), "A".into()),
            ("_Measure".into(), m.into()),
        ])
    };
    let circular = Some(CellValue::Error("circular".into()));
    assert_eq!(
        backend.value(&at("Loop"), Aggregation::OverHidden),
        circular
    );
    assert_eq!(
        backend.value(&at("Ping"), Aggregation::OverHidden),
        circular
    );
    let unseeded = Some(CellValue::Error("ref:Loop".into()));
    let b = CellKey::new(vec![
        ("Product".into(), "B".into()),
        ("_Measure".into(), "Loop".into()),
    ]);
    assert_eq!(
        backend.value(&b, Aggregation::OverHidden),
        unseeded,
        "with no value to start from the cycle settles on its first error"
    );
}

fn chain(len: usize) -> Vec<Edit> {
    (0..len)
        .rev()
        .map(|i| {
            let raw = match i {
                0 => "C0 = Revenue".to_string(),
                _ => format!("C{i} = C{} + 1", i - 1),
            };
            Edit::AddFormula(parse_formula(&raw, "_Measure").unwrap())
        })
        .collect()
}

#[test]
fn a_chain_added_last_link_first_is_circular_exactly_where_eager_says() {
    let wb = formula_workbook();
    let none = vec!["Region".to_string()];
    for len in [14, 15, 16, 17, 18] {
        let mut edits = chain(len);
        edits.push(bump("East", "A", "Revenue", 7.0));
        for by in [Evaluation::Ordered, Evaluation::Plain] {
            same_model(&wb.model, &none, &edits, by).unwrap_or_else(|e| panic!("{len}: {e}"));
        }
    }
}

/// Twelve products by three measures per region, with Profit: stems number the products.
fn wide_table(products: usize) -> Model {
    let mut model = formula_workbook().model;
    for p in 0..products {
        let name = format!("P{p}");
        model.category_mut("Product").unwrap().add_item(&name);
        for (m, v) in [("Revenue", 5.0), ("Cost", 2.0), ("Units", 1.0)] {
            model.set_cell(key("East", &name, m), CellValue::Number(v + p as f64));
        }
    }
    model
}

#[test]
fn a_cell_edit_evaluates_the_one_stem_that_reads_it_whatever_the_table_size() {
    let none = vec!["Region".to_string()];
    for products in [12, 48] {
        let mut backend = Incremental::new(wide_table(products), none.clone());
        let before = backend.meter();
        backend.apply(&bump("East", "P3", "Revenue", 50.0));
        let spent = backend.meter();
        assert_eq!(spent.stems - before.stems, 1, "{products} products");
        assert_eq!(
            spent.formula_cells - before.formula_cells,
            2,
            "Profit and Margin"
        );
        assert_eq!(spent.rebuilds, before.rebuilds, "no stem index is rebuilt");
    }
}

#[test]
fn a_formula_edit_evaluates_every_stem_once_and_rebuilds_the_index_once() {
    let none = vec!["Region".to_string()];
    let mut backend = Incremental::new(wide_table(12), none);
    let before = backend.meter();
    backend.apply(&formula_edit(11));
    let spent = backend.meter();
    assert_eq!(spent.rebuilds - before.rebuilds, 1);
    assert_eq!(spent.stems - before.stems, 13, "twelve products and A");
}

#[test]
fn a_formula_replaced_in_place_is_evaluated_afresh() {
    let wb = formula_workbook();
    let none = vec!["Region".to_string()];
    let swapped = parse_formula("Profit = Revenue + Cost", "_Measure").unwrap();
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&wb.model, &none, &[Edit::AddFormula(swapped.clone())], by).unwrap();
    }
}

#[test]
fn overwriting_zero_with_negative_zero_changes_what_a_formula_reads() {
    let wb = formula_workbook();
    let none = vec!["Region".to_string()];
    let mut edits = vec![formula_edit(16)];
    edits.extend(["East", "West"].map(|r| bump(r, "A", "Units", 0.0)));
    edits.extend(["East", "West"].map(|r| bump(r, "A", "Units", -0.0)));
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&wb.model, &none, &edits, by).unwrap();
    }
}

#[test]
fn a_filter_on_a_category_the_stem_has_reads_the_cells_of_other_stems() {
    let wb = formula_workbook();
    let none = vec!["Region".to_string()];
    let edits = [
        bump("East", "B", "Revenue", 5.0),
        bump("West", "B", "Cost", 2.0),
        formula_edit(9),
        bump("East", "A", "Revenue", 100.0),
    ];
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&wb.model, &none, &edits, by).unwrap();
    }
}

#[test]
fn two_target_categories_read_each_others_stems_whatever_the_order_they_are_added_in() {
    let wb = formula_workbook();
    let none = none_of(&wb);
    let edits = [
        formula_edit(14),
        bump("East", "A", "Loop", 0.0),
        formula_edit(6),
    ];
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&wb.model, &none, &edits, by).unwrap();
    }
}

#[test]
fn the_whole_cache_recompute_of_a_wide_document_is_metered() {
    let wb = formula_workbook();
    let mut backend = Incremental::new(wb.model.clone(), none_of(&wb));
    let edit = bump("East", "A", "Revenue", 3.0);
    backend.apply(&edit);
    assert_eq!(
        backend.meter().recomputes,
        0,
        "one target category: by stem"
    );
    backend.apply(&formula_edit(14));
    let before = backend.meter();
    backend.apply(&edit);
    backend.apply(&bump("East", "A", "Revenue", 4.0));
    let spent = backend.meter();
    assert_eq!(
        spent.recomputes - before.recomputes,
        2,
        "Region and _Measure targets: wide"
    );
    assert_eq!(
        spent.stems, before.stems,
        "and no stem is evaluated on its own"
    );
}

#[test]
fn a_name_in_another_category_reads_another_stems_cell() {
    let mut wb = formula_workbook();
    wb.model.category_mut("Product").unwrap().add_item("B");
    let none = vec!["Region".to_string()];
    let pf = parse_formula("Pf = IF(Revenue > 1000, B, Revenue)", "_Measure").unwrap();
    let edits = [
        bump("East", "A", "Revenue", 5000.0),
        bump("East", "B", "Revenue", 7.0),
        Edit::AddFormula(pf),
        bump("East", "B", "Revenue", 9.0),
    ];
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&wb.model, &none, &edits, by).unwrap();
    }
}

#[test]
fn a_formula_cell_holding_nan_is_nan_and_not_circular() {
    let wb = formula_workbook();
    let none = vec!["Region".to_string()];
    let nan = bump("East", "A", "Revenue", f64::NAN);
    let mut backend = Incremental::new(wb.model.clone(), none.clone());
    backend.apply(&nan);
    let at = |m: &str| {
        CellKey::new(vec![
            ("Product".into(), "A".into()),
            ("_Measure".into(), m.into()),
        ])
    };
    for measure in ["Profit", "Margin"] {
        let got = backend.value(&at(measure), Aggregation::OverHidden);
        assert!(
            matches!(got, Some(CellValue::Number(n)) if n.is_nan()),
            "{measure}: {got:?}"
        );
    }
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&wb.model, &none, std::slice::from_ref(&nan), by).unwrap();
    }
}

#[test]
fn a_new_aggregation_for_the_measure_a_filtered_formula_falls_back_to_changes_it() {
    let wb = formula_workbook();
    let none = vec!["Region".to_string()];
    let edits = [
        formula_edit(4),
        bump("East", "A", "Bonus", 3.0),
        bump("West", "A", "Bonus", 4.0),
        Edit::SetMeasureAgg {
            measure: "Bonus".into(),
            func: improvise_core::formula::ast::AggFunc::Count,
        },
    ];
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&wb.model, &none, &edits, by).unwrap();
    }
}

#[test]
fn a_new_record_adds_its_stem_and_a_cleared_one_takes_it_away() {
    let wb = formula_workbook();
    let none = vec!["Region".to_string()];
    let edits = [
        Edit::SetCell(key("East", "C", "Revenue"), CellValue::Number(4.0)),
        Edit::SetCell(key("East", "C", "Cost"), CellValue::Number(1.0)),
        Edit::ClearCell(key("East", "C", "Revenue")),
        Edit::ClearCell(key("East", "C", "Cost")),
    ];
    same_model(&wb.model, &none, &edits, Evaluation::Ordered).unwrap();
}

/// Region {East, West}, a Time category folded away, Revenue by quarter: `R0 = East + West`
/// targets Region, so its stems carry the measure but no formula names it.
fn region_target_model() -> Model {
    let mut model = Model::new("regions");
    for (cat, items) in [("Region", vec!["East", "West"]), ("Time", vec!["Q1", "Q2"])] {
        model.add_category(cat).unwrap();
        for item in items {
            model.category_mut(cat).unwrap().add_item(item);
        }
    }
    model.category_mut("_Measure").unwrap().add_item("Revenue");
    for (region, quarter, v) in [
        ("East", "Q1", 1.0),
        ("East", "Q2", 3.0),
        ("West", "Q1", 10.0),
    ] {
        let key = CellKey::new(vec![
            ("Region".into(), region.into()),
            ("Time".into(), quarter.into()),
            ("_Measure".into(), "Revenue".into()),
        ]);
        model.set_cell(key, CellValue::Number(v));
    }
    model.add_formula(parse_formula("R0 = East + West", "Region").unwrap());
    model
}

#[test]
fn a_new_aggregation_reaches_the_stems_of_a_formula_that_targets_another_category() {
    let model = region_target_model();
    let none = vec!["Time".to_string()];
    let edits = [Edit::SetMeasureAgg {
        measure: "Revenue".into(),
        func: improvise_core::formula::ast::AggFunc::Max,
    }];
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&model, &none, &edits, by).unwrap();
    }
    let mut backend = Incremental::new(model, none);
    backend.apply(&edits[0]);
    let at = CellKey::new(vec![
        ("Region".into(), "R0".into()),
        ("_Measure".into(), "Revenue".into()),
    ]);
    let want = Some(CellValue::Number(13.0));
    assert_eq!(
        backend.value(&at, Aggregation::OverHidden),
        want,
        "Max 3 + Max 10"
    );
}

#[test]
fn a_new_aggregation_reaches_a_filtered_sum_on_the_fallback_path() {
    let mut model = region_target_model();
    model.add_formula(parse_formula("P0 = SUM(Revenue) WHERE Region = East", "_Measure").unwrap());
    let none = vec!["Time".to_string()];
    let edits = [Edit::SetMeasureAgg {
        measure: "Revenue".into(),
        func: improvise_core::formula::ast::AggFunc::Max,
    }];
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&model, &none, &edits, by).unwrap();
    }
}

#[test]
fn ordered_evaluation_keeps_the_zero_eager_settles_on() {
    let mut model = Model::new("zeros");
    model.add_category("Region").unwrap();
    model.category_mut("Region").unwrap().add_item("East");
    let key = CellKey::new(vec![
        ("Region".into(), "East".into()),
        ("_Measure".into(), "F1".into()),
    ]);
    model.set_cell(key, CellValue::Number(0.0));
    let edits: Vec<Edit> = [
        ("F3 = F1", "_Measure"),
        ("F4 = F3 ^ -1", "_Measure"),
        ("F1 = -0", "_Measure"),
    ]
    .iter()
    .map(|(raw, cat)| Edit::AddFormula(parse_formula(raw, cat).unwrap()))
    .collect();
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&model, &[], &edits, by).unwrap_or_else(|e| panic!("{by:?}: {e}"));
    }
}
#[test]
fn a_new_aggregation_reaches_a_product_targeted_filtered_sum() {
    let mut model = Model::new("products");
    for (cat, items) in [("Region", vec!["East", "West"]), ("Product", vec!["F0"])] {
        model.add_category(cat).unwrap();
        for item in items {
            model.category_mut(cat).unwrap().add_item(item);
        }
    }
    model.category_mut("_Measure").unwrap().add_item("Cost");
    for (region, v) in [("East", 4.0), ("West", 9.0)] {
        let key = CellKey::new(vec![
            ("Region".into(), region.into()),
            ("Product".into(), "F0".into()),
            ("_Measure".into(), "Cost".into()),
        ]);
        model.set_cell(key, CellValue::Number(v));
    }
    model.add_formula(parse_formula("P0 = SUM(F0 WHERE Region = East)", "Product").unwrap());
    let none = vec![];
    let edits = [Edit::SetMeasureAgg {
        measure: "Cost".into(),
        func: improvise_core::formula::ast::AggFunc::Min,
    }];
    for by in [Evaluation::Ordered, Evaluation::Plain] {
        same_model(&model, &none, &edits, by).unwrap();
    }
}