improvise/crates/improvise-incremental/tests/exact.rs

//! `Sum` and `Avg` in both backends: exact, order-free, and signed as IEEE signs the same addends
//! (ADR 009 clause 8). Evidence: `context/test-evidence/improvise-incremental.md`.

use improvise_core::formula::ast::AggFunc;
use improvise_core::model::Model;
use improvise_core::model::cell::{CellKey, CellValue};
use improvise_incremental::{Derived, Eager, Edit, Incremental, Maintained};
use numtower::ExactSum;

fn key(i: usize) -> CellKey {
    CellKey::new(vec![
        ("Region".into(), format!("r{i}")),
        ("_Measure".into(), "Revenue".into()),
    ])
}

fn the_measure() -> CellKey {
    CellKey::new(vec![("_Measure".into(), "Revenue".into())])
}

fn model_of(values: &[f64], func: AggFunc) -> Model {
    let mut model = Model::new("sums");
    model.add_category("Region").unwrap();
    model.category_mut("_Measure").unwrap().add_item("Revenue");
    model.measure_agg.insert("Revenue".into(), func);
    for (i, v) in values.iter().enumerate() {
        model
            .category_mut("Region")
            .unwrap()
            .add_item(format!("r{i}"));
        model.set_cell(key(i), CellValue::Number(*v));
    }
    model
}

fn exact(values: &[f64]) -> f64 {
    let mut sum = ExactSum::new();
    values.iter().for_each(|v| sum.add(*v, 1));
    sum.value()
}

/// Both backends' sum of `values` under `func`, as bits.
fn both(values: &[f64], func: AggFunc) -> (Option<u64>, Option<u64>) {
    let model = model_of(values, func);
    let bits = |v: Option<CellValue>| v.and_then(|v| v.as_f64()).map(f64::to_bits);
    let eager = Eager::new(model.clone(), vec![]);
    let incremental = Incremental::new(model, vec![]);
    let at = the_measure();
    (bits(eager.raw_fold(&at)), bits(incremental.raw_fold(&at)))
}

#[test]
fn sums_are_exact_whatever_the_order_the_cells_were_added_in() {
    let values = [1e16, 1.0, -1e16];
    let orders = [
        [0, 1, 2],
        [0, 2, 1],
        [1, 0, 2],
        [1, 2, 0],
        [2, 0, 1],
        [2, 1, 0],
    ];
    for order in orders {
        let added: Vec<f64> = order.iter().map(|&i| values[i]).collect();
        let want = Some(1.0f64.to_bits());
        assert_eq!(both(&added, AggFunc::Sum), (want, want), "{added:?}");
    }
}

#[test]
fn an_average_is_the_exact_total_over_the_count_rounded_once() {
    let values = [0.1, 0.2, 0.3, 1e16, -1e16];
    let mut sum = ExactSum::new();
    values.iter().for_each(|v| sum.add(*v, 1));
    let want = Some(sum.mean(values.len() as u64).to_bits());
    assert_eq!(both(&values, AggFunc::Avg), (want, want));
}

#[test]
fn an_average_does_not_overflow_with_its_sum_in_either_backend() {
    let want = Some(f64::MAX.to_bits());
    assert_eq!(both(&[f64::MAX, f64::MAX], AggFunc::Avg), (want, want));
}

#[test]
fn a_zero_total_has_the_sign_ieee_gives_the_same_addends() {
    let cases: [(&[f64], f64); 4] = [
        (&[-0.0, -0.0], -0.0),
        (&[-0.0, 0.0], 0.0),
        (&[1.0, -1.0], 0.0),
        (&[-0.0, 3.0, -3.0], 0.0),
    ];
    for (values, sign) in cases {
        let want = Some(sign.to_bits());
        assert_eq!(both(values, AggFunc::Sum), (want, want), "{values:?}");
    }
}

#[test]
fn overwriting_zero_with_negative_zero_moves_the_sign_of_a_sum() {
    let model = model_of(&[0.0, 0.0], AggFunc::Sum);
    let mut eager = Eager::new(model.clone(), vec![]);
    let mut incremental = Incremental::new(model, vec![]);
    let at = the_measure();
    let bits = |b: &dyn Derived| b.raw_fold(&at).and_then(|v| v.as_f64()).map(f64::to_bits);
    assert_eq!(bits(&incremental), Some(0.0f64.to_bits()));
    for i in 0..2 {
        let edit = Edit::SetCell(key(i), CellValue::Number(-0.0));
        eager.apply(&edit);
        incremental.apply(&edit);
    }
    let negative = Some((-0.0f64).to_bits());
    assert_eq!((bits(&eager), bits(&incremental)), (negative, negative));
    let back = Edit::SetCell(key(1), CellValue::Number(0.0));
    eager.apply(&back);
    incremental.apply(&back);
    let positive = Some(0.0f64.to_bits());
    assert_eq!((bits(&eager), bits(&incremental)), (positive, positive));
}

#[test]
fn twenty_thousand_overwrites_in_a_thousand_cell_group_do_not_drift() {
    let values: Vec<f64> = (0..1000).map(|i| 0.1 * f64::from(i % 7 + 1)).collect();
    let model = model_of(&values, AggFunc::Sum);
    let mut eager = Eager::new(model.clone(), vec![]);
    let mut incremental = Incremental::new(model, vec![]);
    let at = the_measure();
    let mut seed = 7u64;
    for step in 0..20_000 {
        seed = seed
            .wrapping_mul(6364136223846793005)
            .wrapping_add(1442695040888963407);
        let (i, wide) = ((seed >> 33) as usize % 1000, (seed >> 20) % 3);
        let v = [0.1, 1e12 + 0.3, -7.77e-3][wide as usize] * f64::from((step % 5) as u8 + 1);
        let edit = Edit::SetCell(key(i), CellValue::Number(v));
        eager.apply(&edit);
        incremental.apply(&edit);
    }
    let bits = |b: &dyn Derived| b.raw_fold(&at).and_then(|v| v.as_f64()).map(f64::to_bits);
    assert_eq!(bits(&eager), bits(&incremental));
    let fresh = Incremental::new(eager.model().clone(), vec![]);
    assert_eq!(
        bits(&fresh),
        bits(&incremental),
        "and equals a sum built afresh"
    );
}

#[test]
fn a_formula_sum_over_an_expression_is_exact_in_both_backends() {
    let formula = improvise_core::formula::parse_formula("Tot = SUM(Revenue * 1)", "_Measure");
    for order in [[0, 1, 2], [1, 0, 2], [2, 0, 1]] {
        let values = [1e16, 1.0, -1e16];
        let added: Vec<f64> = order.iter().map(|&i| values[i]).collect();
        let mut model = model_of(&added, AggFunc::Sum);
        model.add_formula(formula.as_ref().unwrap().clone());
        let none = vec!["Region".to_string()];
        let eager = Eager::new(model.clone(), none.clone());
        let incremental = Incremental::new(model, none);
        let at = CellKey::new(vec![("_Measure".into(), "Tot".into())]);
        let want = Some(CellValue::Number(1.0));
        let read = |b: &dyn Derived| b.value(&at, improvise_core::model::Aggregation::OverHidden);
        assert_eq!(
            (read(&eager), read(&incremental)),
            (want.clone(), want),
            "{added:?}"
        );
    }
}