improvise/crates/improvise-incremental/tests/pushdown.rs

//! `Pushdown` over a remote table against `Eager`: values, hidden items, groups, formulas.
//! Without a real SQL engine only the contract is proven: `context/code/improvise-incremental.md`.

mod support;

use improvise_core::formula::ast::AggFunc;
use improvise_core::formula::parse_formula;
use improvise_core::model::cell::{CellKey, CellValue};
use improvise_core::model::source::SharedSource;
use improvise_core::model::{Aggregation, Model};
use improvise_core::view::View;
use improvise_incremental::{Derived, Edit, Probe, Reading, differential_in};
use improvise_incremental::{Eager, Pushdown};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use support::remote::{
    ChangesAfterAFold, ChangesAfterStems, Hooked, IgnoresHidden, Pushed, Remote, SortsStems,
};
use support::{
    corpus, edits_for, grouped_model, hand_model, hidden_counted_workbook, hidden_workbook, key,
    none_of, ratio_workbook, wide_workbook,
};

fn run(model: &Model, views: &[View], none: &[String], edits: &[Edit]) -> usize {
    let candidate = Pushed::new(model, none.to_vec());
    match differential_in(model, views, none, candidate, edits) {
        Ok(report) => report.probes,
        Err(d) => panic!("{d}"),
    }
}

fn agg_edits(measure: &str) -> Vec<Edit> {
    let all = [
        AggFunc::Count,
        AggFunc::Min,
        AggFunc::Max,
        AggFunc::Avg,
        AggFunc::Sum,
    ];
    all.into_iter()
        .map(|func| Edit::SetMeasureAgg {
            measure: measure.into(),
            func,
        })
        .collect()
}

#[test]
fn a_pushdown_matches_eager_over_the_shipped_documents() {
    let (mut compared, mut files_done) = (0, Vec::<String>::new());
    for (name, model, none, views) in corpus() {
        let file = name.split(':').next().unwrap().to_string();
        if files_done.contains(&file) {
            continue;
        }
        files_done.push(file);
        let mut edits = edits_for(&model);
        if model.data.iter_cells().count() > 40 {
            let picked = [0, 2, 4].iter().filter_map(|&i| edits.get(i).cloned());
            edits = picked.collect();
        }
        compared += run(&model, &views, &none, &edits);
    }
    assert!(compared > 10_000, "only {compared} answers were compared");
}

#[test]
fn a_pushdown_matches_eager_on_hidden_items_and_collapsed_groups_under_every_aggregation() {
    for wb in [hidden_workbook(), hidden_counted_workbook()] {
        let views: Vec<View> = wb.views.values().cloned().collect();
        let none = none_of(&wb);
        let mut edits = edits_for(&wb.model);
        edits.extend(agg_edits("Revenue"));
        let hidden = key("East", "A", "Revenue");
        edits.push(Edit::SetCell(hidden.clone(), CellValue::Number(f64::NAN)));
        edits.push(Edit::SetCell(hidden.clone(), CellValue::Number(-0.0)));
        edits.push(Edit::SetCell(hidden, CellValue::Text("t".into())));
        // A cell stored under the group's own name is a cell its fold and its count both see.
        let named = key("Coastal", "B", "Revenue");
        edits.push(Edit::SetCell(named, CellValue::Number(4.0)));
        assert!(run(&wb.model, &views, &none, &edits) > 1_000);
    }
}

#[test]
fn a_pushdown_matches_eager_on_formulas_groups_and_wide_keys() {
    let ratio = ratio_workbook();
    let views: Vec<View> = ratio.views.values().cloned().collect();
    let mut edits = edits_for(&ratio.model);
    edits.extend(agg_edits("Revenue"));
    run(&ratio.model, &views, &none_of(&ratio), &edits);
    let hand = hand_model();
    let mut edits = agg_edits("Revenue");
    edits.extend(agg_edits("Note"));
    run(&hand, &[], &["Region".to_string()], &edits);
    run(&grouped_model(), &[], &[], &edits_for(&grouped_model()));
    let wide = wide_workbook();
    let views: Vec<View> = wide.views.values().cloned().collect();
    run(
        &wide.model,
        &views,
        &none_of(&wide),
        &edits_for(&wide.model),
    );
}

#[test]
fn a_fold_is_one_query_and_sends_no_row() {
    let wb = hidden_counted_workbook();
    let none = none_of(&wb);
    let pushed = Pushed::new(&wb.model, none);
    let at = |name: &str| key("Inland", name, "Revenue");
    let before = pushed.remote.asked();
    pushed.pd.raw_fold(&CellKey::new(vec![
        ("Product".into(), "A".into()),
        ("_Measure".into(), "Revenue".into()),
    ]));
    let reading = Reading::of_view(wb.active_view(), &none_of(&wb));
    let coastal = key("Coastal", "A", "Revenue");
    pushed.pd.covered(&coastal, &reading);
    let total = key("Inland", "A", "Revenue").without("Region");
    pushed.pd.total(&total, &["Region".to_string()], &reading);
    let spent = pushed.remote.asked();
    assert_eq!(spent.summaries - before.summaries, 3, "one query per fold");
    assert_eq!(spent.rows_sent, before.rows_sent, "and no row sent back");
    let one = pushed.pd.value(&at("A"), Aggregation::Exact);
    assert_eq!(one, Some(CellValue::Number(2.0)));
    assert_eq!(pushed.remote.asked().lookups - spent.lookups, 1);
}

#[test]
fn occupancy_is_one_count_whatever_the_groups_and_hidden_items() {
    let wb = hidden_workbook();
    let pushed = Pushed::new(&wb.model, none_of(&wb));
    let reading = Reading::of_view(wb.active_view(), &none_of(&wb));
    let before = pushed.remote.asked();
    let coastal = CellKey::new(vec![
        ("Region".into(), "Coastal".into()),
        ("Product".into(), "A".into()),
    ]);
    let seen = pushed.pd.occupancy(&coastal, &reading);
    let spent = pushed.remote.asked();
    assert_eq!(seen, 1, "East is hidden, West holds A");
    assert_eq!(spent.counts - before.counts, 1);
    assert_eq!(spent.rows_sent, before.rows_sent);
}

#[test]
fn formula_cells_are_computed_from_the_distinct_stems_not_from_every_cell() {
    let model = hand_model();
    let cells = model.data.iter_cells().count() as u64;
    let pushed = Pushed::new(&model, vec!["Region".to_string()]);
    let asked = pushed.remote.asked();
    assert_eq!(asked.stem_lists, 1, "one list for the one formula");
    assert_eq!(
        asked.rows_sent, 2,
        "Product A and Product B, of {cells} cells"
    );
    assert_eq!((asked.key_lists, asked.cell_lists), (0, 0));
    assert!(
        asked.summaries > 0,
        "and the folds the formula reads are queries"
    );
}

#[test]
fn a_stored_cell_is_current_and_a_formula_cell_waits_for_a_refresh() {
    let mut pushed = Pushed::new(&hand_model(), vec!["Region".to_string()]);
    let revenue = key("East", "A", "Revenue");
    let profit = CellKey::new(vec![
        ("Product".into(), "A".into()),
        ("_Measure".into(), "Profit".into()),
    ]);
    let fold = |p: &Pushed| p.pd.value(&profit, Aggregation::OverHidden);
    assert_eq!(fold(&pushed), Some(CellValue::Number(9.0)));
    pushed
        .remote
        .write(revenue.clone(), Some(CellValue::Number(50.0)));
    let now = pushed.pd.value(&revenue, Aggregation::Exact);
    assert_eq!(now, Some(CellValue::Number(50.0)), "a read is a query");
    assert_eq!(
        fold(&pushed),
        Some(CellValue::Number(9.0)),
        "the cache is stale"
    );
    pushed.pd.refresh();
    assert_eq!(fold(&pushed), Some(CellValue::Number(49.0)));
}

#[test]
fn a_source_that_forgets_the_hidden_items_is_caught_as_a_divergence() {
    let wb = hidden_workbook();
    let views: Vec<View> = wb.views.values().cloned().collect();
    let none = none_of(&wb);
    let faulty = Pushed::through(&wb.model, none.clone(), IgnoresHidden);
    let ran = differential_in(&wb.model, &views, &none, faulty, &[]);
    let divergence = ran.expect_err("a wrong hidden filter must not pass");
    assert!(
        matches!(*divergence.probe, Probe::Occupancy(..)),
        "{divergence}"
    );
    let sound = Pushed::new(&wb.model, none.clone());
    assert!(differential_in(&wb.model, &views, &none, sound, &[]).is_ok());
}

/// Eighteen products stored in chain order and named to sort the other way, and a formula that
/// reads the product before it: one pass along the chain, one per link against it.
fn chain() -> Model {
    let name = |k: usize| format!("n{}", 100 - k);
    let mut model = Model::new("chain");
    model.add_category("Product").unwrap();
    model.category_mut("_Measure").unwrap().add_item("Revenue");
    for k in 1..=18 {
        model.category_mut("Product").unwrap().add_item(name(k));
        let cell = CellKey::new(vec![
            ("Product".into(), name(k)),
            ("_Measure".into(), "Revenue".into()),
        ]);
        model.set_cell(cell, CellValue::Number(k as f64));
    }
    let mut text = "0".to_string();
    for k in (2..=18).rev() {
        text = format!(
            "IF((Product IS {}) > 0, {} + 1, {text})",
            name(k),
            name(k - 1)
        );
    }
    text = format!("IF((Product IS {}) > 0, 1, {text})", name(1));
    model.add_formula(parse_formula(&format!("Chain = {text}"), "_Measure").unwrap());
    model
}

#[test]
fn the_order_stems_are_listed_in_decides_a_result_today() {
    let model = chain();
    let last = CellKey::new(vec![
        ("Product".into(), "n82".into()),
        ("_Measure".into(), "Chain".into()),
    ]);
    let eager = Eager::new(model.clone(), vec![]);
    let want = eager.value(&last, Aggregation::OverHidden);
    assert_eq!(
        want,
        Some(CellValue::Number(18.0)),
        "along the chain it settles"
    );
    let sound = Pushed::new(&model, vec![]);
    assert_eq!(sound.pd.value(&last, Aggregation::OverHidden), want);
    let ran = differential_in(&model, &[], &[], sound, &[]);
    assert!(ran.is_ok(), "{}", ran.unwrap_err());
    let sorted = Pushed::through(&model, vec![], |remote| Hooked(remote, SortsStems));
    let got = sorted.pd.value(&last, Aggregation::OverHidden);
    assert_eq!(
        got,
        Some(CellValue::Error("circular".into())),
        "against it, the bound of 16 passes"
    );
    let ran = differential_in(&model, &[], &[], sorted, &[]);
    assert!(ran.is_err(), "a sorted stem list must be a divergence");
}

type Change = Box<dyn Fn(&Remote) + Send + Sync>;

/// Another client's writes: Product A's revenue and cost move, and Product C appears.
fn another_clients_writes() -> Change {
    Box::new(|remote| {
        remote.write(key("East", "A", "Revenue"), Some(CellValue::Number(50.0)));
        remote.write(key("East", "A", "Cost"), Some(CellValue::Number(0.0)));
        remote.write(key("East", "C", "Revenue"), Some(CellValue::Number(7.0)));
        remote.write(key("East", "C", "Cost"), Some(CellValue::Number(2.0)));
    })
}

fn profit_of(product: &str) -> CellKey {
    CellKey::new(vec![
        ("Product".into(), product.into()),
        ("_Measure".into(), "Profit".into()),
    ])
}

#[test]
fn writes_between_two_queries_of_a_refresh_are_healed_by_the_fixed_point() {
    let none = vec!["Region".to_string()];
    let armed = Arc::new(AtomicBool::new(false));
    let hook = ChangesAfterAFold {
        armed: armed.clone(),
        change: another_clients_writes(),
    };
    let none_cats = vec!["Region".to_string()];
    let mut pushed = Pushed::through(&hand_model(), none_cats, |r| Hooked(r, hook));
    assert_eq!(
        pushed.pd.value(&profit_of("A"), Aggregation::OverHidden),
        Some(CellValue::Number(9.0))
    );
    armed.store(true, Ordering::SeqCst);
    pushed.pd.refresh();
    assert!(
        !armed.load(Ordering::SeqCst),
        "the change happened mid-refresh"
    );
    let mut after = hand_model();
    after.set_cell(key("East", "A", "Revenue"), CellValue::Number(50.0));
    after.set_cell(key("East", "A", "Cost"), CellValue::Number(0.0));
    let truth = Eager::new(after, none);
    let want = truth.value(&profit_of("A"), Aggregation::OverHidden);
    assert_eq!(want, Some(CellValue::Number(53.0)));
    let got = pushed.pd.value(&profit_of("A"), Aggregation::OverHidden);
    assert_eq!(
        got, want,
        "the last pass read the new state: 55 - 2, and not 13"
    );
}

#[test]
fn a_stem_that_appears_between_the_stem_list_and_the_folds_waits_for_the_next_refresh() {
    let armed = Arc::new(AtomicBool::new(false));
    let hook = ChangesAfterStems {
        armed: armed.clone(),
        change: another_clients_writes(),
    };
    let none_cats = vec!["Region".to_string()];
    let mut pushed = Pushed::through(&hand_model(), none_cats, |r| Hooked(r, hook));
    armed.store(true, Ordering::SeqCst);
    pushed.pd.refresh();
    assert_eq!(
        pushed.pd.value(&profit_of("C"), Aggregation::OverHidden),
        None,
        "the stem list was taken before Product C had cells"
    );
    pushed.pd.refresh();
    assert_eq!(
        pushed.pd.value(&profit_of("C"), Aggregation::OverHidden),
        Some(CellValue::Number(5.0)),
        "and the next refresh has it"
    );
}

#[test]
#[should_panic(expected = "cell edits go through the source")]
fn a_cell_edit_on_a_pushdown_is_refused() {
    let mut pushed = Pushed::new(&hand_model(), vec![]);
    let edit = Edit::SetCell(key("East", "A", "Revenue"), CellValue::Number(1.0));
    pushed.pd.define(&edit);
}

#[test]
fn a_pushdown_can_be_built_over_the_exported_source_alias() {
    let model = hand_model();
    let rows = model.data.iter_cells().map(|(k, v)| (k, v.clone()));
    let source: SharedSource = Arc::new(Remote::of(rows));
    let pd = Pushdown::new(model, source, vec![]);
    let revenue = CellKey::new(vec![("_Measure".into(), "Revenue".into())]);
    assert_eq!(pd.raw_fold(&revenue), Some(CellValue::Number(23.0)));
}