improvise/crates/improvise-incremental/tests/pushdown.rs

//! `Pushdown` over a remote table against `Eager`: values, hidden items, groups, formulas.
//! Without a real SQL engine only the contract is proven: `context/code/improvise-incremental.md`.

mod support;

use improvise_core::formula::ast::AggFunc;
use improvise_core::model::cell::{CellKey, CellValue};
use improvise_core::model::{Aggregation, Model};
use improvise_core::view::View;
use improvise_incremental::{Derived, Edit, Probe, Reading, differential_in};
use support::remote::{IgnoresHidden, Pushed};
use support::{
    corpus, edits_for, grouped_model, hand_model, hidden_counted_workbook, hidden_workbook, key,
    none_of, ratio_workbook, wide_workbook,
};

fn run(model: &Model, views: &[View], none: &[String], edits: &[Edit]) -> usize {
    let candidate = Pushed::new(model, none.to_vec());
    match differential_in(model, views, none, candidate, edits) {
        Ok(report) => report.probes,
        Err(d) => panic!("{d}"),
    }
}

fn agg_edits(measure: &str) -> Vec<Edit> {
    let all = [
        AggFunc::Count,
        AggFunc::Min,
        AggFunc::Max,
        AggFunc::Avg,
        AggFunc::Sum,
    ];
    all.into_iter()
        .map(|func| Edit::SetMeasureAgg {
            measure: measure.into(),
            func,
        })
        .collect()
}

#[test]
fn a_pushdown_matches_eager_over_the_shipped_documents() {
    let (mut compared, mut files_done) = (0, Vec::<String>::new());
    for (name, model, none, views) in corpus() {
        let file = name.split(':').next().unwrap().to_string();
        if files_done.contains(&file) {
            continue;
        }
        files_done.push(file);
        let mut edits = edits_for(&model);
        if model.data.iter_cells().count() > 40 {
            let picked = [0, 2, 4].iter().filter_map(|&i| edits.get(i).cloned());
            edits = picked.collect();
        }
        compared += run(&model, &views, &none, &edits);
    }
    assert!(compared > 10_000, "only {compared} answers were compared");
}

#[test]
fn a_pushdown_matches_eager_on_hidden_items_and_collapsed_groups_under_every_aggregation() {
    for wb in [hidden_workbook(), hidden_counted_workbook()] {
        let views: Vec<View> = wb.views.values().cloned().collect();
        let none = none_of(&wb);
        let mut edits = edits_for(&wb.model);
        edits.extend(agg_edits("Revenue"));
        let hidden = key("East", "A", "Revenue");
        edits.push(Edit::SetCell(hidden.clone(), CellValue::Number(f64::NAN)));
        edits.push(Edit::SetCell(hidden.clone(), CellValue::Number(-0.0)));
        edits.push(Edit::SetCell(hidden, CellValue::Text("t".into())));
        // A cell stored under the group's own name is a cell its fold and its count both see.
        let named = key("Coastal", "B", "Revenue");
        edits.push(Edit::SetCell(named, CellValue::Number(4.0)));
        assert!(run(&wb.model, &views, &none, &edits) > 1_000);
    }
}

#[test]
fn a_pushdown_matches_eager_on_formulas_groups_and_wide_keys() {
    let ratio = ratio_workbook();
    let views: Vec<View> = ratio.views.values().cloned().collect();
    let mut edits = edits_for(&ratio.model);
    edits.extend(agg_edits("Revenue"));
    run(&ratio.model, &views, &none_of(&ratio), &edits);
    let hand = hand_model();
    let mut edits = agg_edits("Revenue");
    edits.extend(agg_edits("Note"));
    run(&hand, &[], &["Region".to_string()], &edits);
    run(&grouped_model(), &[], &[], &edits_for(&grouped_model()));
    let wide = wide_workbook();
    let views: Vec<View> = wide.views.values().cloned().collect();
    run(
        &wide.model,
        &views,
        &none_of(&wide),
        &edits_for(&wide.model),
    );
}

#[test]
fn a_fold_is_one_query_and_sends_no_row() {
    let wb = hidden_counted_workbook();
    let none = none_of(&wb);
    let pushed = Pushed::new(&wb.model, none);
    let at = |name: &str| key("Inland", name, "Revenue");
    let before = pushed.remote.asked();
    pushed.pd.raw_fold(&CellKey::new(vec![
        ("Product".into(), "A".into()),
        ("_Measure".into(), "Revenue".into()),
    ]));
    let reading = Reading::of_view(wb.active_view(), &none_of(&wb));
    let coastal = key("Coastal", "A", "Revenue");
    pushed.pd.covered(&coastal, &reading);
    let total = key("Inland", "A", "Revenue").without("Region");
    pushed.pd.total(&total, &["Region".to_string()], &reading);
    let spent = pushed.remote.asked();
    assert_eq!(spent.summaries - before.summaries, 3, "one query per fold");
    assert_eq!(spent.rows_sent, before.rows_sent, "and no row sent back");
    let one = pushed.pd.value(&at("A"), Aggregation::Exact);
    assert_eq!(one, Some(CellValue::Number(2.0)));
    assert_eq!(pushed.remote.asked().lookups - spent.lookups, 1);
}

#[test]
fn occupancy_is_one_count_whatever_the_groups_and_hidden_items() {
    let wb = hidden_workbook();
    let pushed = Pushed::new(&wb.model, none_of(&wb));
    let reading = Reading::of_view(wb.active_view(), &none_of(&wb));
    let before = pushed.remote.asked();
    let coastal = CellKey::new(vec![
        ("Region".into(), "Coastal".into()),
        ("Product".into(), "A".into()),
    ]);
    let seen = pushed.pd.occupancy(&coastal, &reading);
    let spent = pushed.remote.asked();
    assert_eq!(seen, 1, "East is hidden, West holds A");
    assert_eq!(spent.counts - before.counts, 1);
    assert_eq!(spent.rows_sent, before.rows_sent);
}

#[test]
fn formula_cells_are_computed_from_the_distinct_stems_not_from_every_cell() {
    let model = hand_model();
    let cells = model.data.iter_cells().count() as u64;
    let pushed = Pushed::new(&model, vec!["Region".to_string()]);
    let asked = pushed.remote.asked();
    assert_eq!(asked.stem_lists, 1, "one list for the one formula");
    assert_eq!(
        asked.rows_sent, 2,
        "Product A and Product B, of {cells} cells"
    );
    assert_eq!((asked.key_lists, asked.cell_lists), (0, 0));
    assert!(
        asked.summaries > 0,
        "and the folds the formula reads are queries"
    );
}

#[test]
fn a_stored_cell_is_current_and_a_formula_cell_waits_for_a_refresh() {
    let mut pushed = Pushed::new(&hand_model(), vec!["Region".to_string()]);
    let revenue = key("East", "A", "Revenue");
    let profit = CellKey::new(vec![
        ("Product".into(), "A".into()),
        ("_Measure".into(), "Profit".into()),
    ]);
    let fold = |p: &Pushed| p.pd.value(&profit, Aggregation::OverHidden);
    assert_eq!(fold(&pushed), Some(CellValue::Number(9.0)));
    pushed
        .remote
        .write(revenue.clone(), Some(CellValue::Number(50.0)));
    let now = pushed.pd.value(&revenue, Aggregation::Exact);
    assert_eq!(now, Some(CellValue::Number(50.0)), "a read is a query");
    assert_eq!(
        fold(&pushed),
        Some(CellValue::Number(9.0)),
        "the cache is stale"
    );
    pushed.pd.refresh();
    assert_eq!(fold(&pushed), Some(CellValue::Number(49.0)));
}

#[test]
fn a_source_that_forgets_the_hidden_items_is_caught_as_a_divergence() {
    let wb = hidden_workbook();
    let views: Vec<View> = wb.views.values().cloned().collect();
    let none = none_of(&wb);
    let faulty = Pushed::through(&wb.model, none.clone(), IgnoresHidden);
    let ran = differential_in(&wb.model, &views, &none, faulty, &[]);
    let divergence = ran.expect_err("a wrong hidden filter must not pass");
    assert!(
        matches!(*divergence.probe, Probe::Occupancy(..)),
        "{divergence}"
    );
    let sound = Pushed::new(&wb.model, none.clone());
    assert!(differential_in(&wb.model, &views, &none, sound, &[]).is_ok());
}