improvise/crates/improvise-incremental/tests/sched_threads.rs

//! The scheduler under panics, idle workers and a full edge, with real threads behind a
//! watchdog. Evidence: `context/test-evidence/improvise-incremental.md`.

mod support;

use improvise_incremental::sched::{Executor, Pool, Turn, Unit};
use improvise_incremental::staged::{Cell, Staged, single_circuit};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use support::watchdog::within;
use zset::ZSet;

struct Boom;

impl Unit for Boom {
    fn turn(&mut self, _: usize) -> Turn {
        panic!("boom");
    }

    fn neighbours(&self) -> &[usize] {
        &[]
    }
}

/// A unit that counts its turns.
struct Count(Arc<AtomicUsize>);

impl Unit for Count {
    fn turn(&mut self, _: usize) -> Turn {
        self.0.fetch_add(1, Ordering::SeqCst);
        Turn {
            progress: false,
            more: false,
        }
    }

    fn neighbours(&self) -> &[usize] {
        &[]
    }
}

#[test]
fn a_panicking_turn_does_not_hang_finish() {
    within(5, "Pool::finish after a unit panicked", || {
        let exec = Arc::new(Executor::new(vec![Box::new(Boom)], 1));
        let pool = Pool::start(exec.clone(), 2);
        exec.kick(0);
        let _ = pool.finish();
    });
}

#[test]
fn a_panicking_unit_is_reported_and_its_neighbours_still_run() {
    let turns = Arc::new(AtomicUsize::new(0));
    let t = turns.clone();
    let err = within(5, "finish with one dead unit", move || {
        let units: Vec<Box<dyn Unit>> = vec![Box::new(Boom), Box::new(Count(t))];
        let exec = Arc::new(Executor::new(units, 1));
        let pool = Pool::start(exec.clone(), 2);
        exec.kick_all();
        let err = pool.finish().err();
        exec.kick(0);
        exec.kick(1);
        err
    });
    let err = err.expect("[panic] finish reports the panic");
    assert_eq!(err.unit, 0);
    assert!(
        err.message.contains("boom"),
        "[panic] the payload: {}",
        err.message
    );
    assert!(
        turns.load(Ordering::SeqCst) >= 1,
        "[panic] the live unit ran"
    );
}

#[test]
fn a_dead_unit_is_never_run_again() {
    let exec = Executor::new(vec![Box::new(Boom)], 1);
    exec.kick(0);
    let mut c = exec.claim(|_| 0).unwrap();
    let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| c.run(1)));
    assert!(r.is_err());
    exec.abandon(c, "boom".to_string());
    exec.kick(0);
    assert!(
        exec.claim(|_| 0).is_none(),
        "[dead] a kick does not revive it"
    );
    assert!(exec.idle());
}

#[test]
#[should_panic(expected = "no such unit")]
fn a_kick_of_a_unit_that_does_not_exist_panics_before_the_lock() {
    Executor::new(vec![Box::new(Boom)], 1).kick(5);
}

#[test]
fn an_executor_survives_a_bad_kick() {
    let exec = Executor::new(vec![Box::new(Boom)], 1);
    let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| exec.kick(5)));
    assert!(r.is_err());
    assert!(
        exec.idle(),
        "[lock] the lock was not poisoned by the bad index"
    );
}

#[test]
fn every_worker_stays_through_wakes_with_nothing_queued() {
    within(10, "the pool with idle wakes", || {
        let turns = Arc::new(AtomicUsize::new(0));
        let units: Vec<Box<dyn Unit>> =
            vec![Box::new(Count(turns.clone())), Box::new(Count(turns))];
        let exec = Arc::new(Executor::new(units, 1));
        let pool = Pool::start(exec.clone(), 4);
        assert_eq!(exec.workers(), 4);
        for round in 0..200 {
            exec.kick(round % 2);
            if round % 20 == 0 {
                let _ = exec.wait_idle();
            }
        }
        let _ = exec.wait_idle();
        assert_eq!(
            exec.workers(),
            4,
            "[workers] a wake with nothing queued sent a worker home"
        );
        let _ = pool.finish();
        assert_eq!(exec.workers(), 0, "[workers] finish stops them all");
    });
}

/// A unit whose turn is slow, so a drop that does not join returns before it ends.
struct Slow {
    started: Arc<AtomicUsize>,
    finished: Arc<AtomicUsize>,
}

impl Unit for Slow {
    fn turn(&mut self, _: usize) -> Turn {
        self.started.fetch_add(1, Ordering::SeqCst);
        std::thread::sleep(std::time::Duration::from_millis(300));
        self.finished.fetch_add(1, Ordering::SeqCst);
        Turn {
            progress: false,
            more: false,
        }
    }

    fn neighbours(&self) -> &[usize] {
        &[]
    }
}

#[test]
fn a_pool_that_is_dropped_waits_for_its_workers() {
    within(10, "dropping a pool", || {
        let (started, finished) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
        let unit = Slow {
            started: started.clone(),
            finished: finished.clone(),
        };
        let exec = Arc::new(Executor::new(vec![Box::new(unit)], 1));
        {
            let _pool = Pool::start(exec.clone(), 3);
            assert_eq!(exec.workers(), 3);
            exec.kick(0);
            while started.load(Ordering::SeqCst) == 0 {
                std::thread::yield_now();
            }
        }
        assert_eq!(
            finished.load(Ordering::SeqCst),
            1,
            "[drop] the turn in flight was joined"
        );
        assert_eq!(exec.workers(), 0, "[drop] no thread is left");
    });
}

fn cmd(r: u8, c: u8, v: i64) -> ZSet<Cell> {
    [((r, c, v), 1)].into_iter().collect()
}

#[test]
fn a_full_edge_does_not_make_its_sender_spin() {
    let staged = Staged::new(1, 1);
    let exec = staged.executor();
    // Hold the aggregate and the projection, so nothing drains the head's edge.
    let agg = exec.claim(|_| 1).unwrap();
    let proj = exec.claim(|_| 1).unwrap();
    assert_eq!((agg.id(), proj.id()), (1, 2));
    for n in 0..4 {
        staged.submit(cmd(0, 0, n));
        let mut head = exec.claim(|_| 0).unwrap();
        let t = head.run(exec.budget());
        exec.finish(head, t);
    }
    let mut turns = 0;
    while let Some(mut head) = exec.claim(|_| 0) {
        turns += 1;
        assert!(
            turns < 50,
            "[spin] the head keeps asking to run with its edge full"
        );
        let t = head.run(exec.budget());
        exec.finish(head, t);
    }
    let t = Turn {
        progress: false,
        more: false,
    };
    exec.finish(agg, t);
    exec.finish(proj, t);
}

#[test]
fn a_unit_whose_kicks_coalesced_still_drains_its_edge() {
    let staged = Staged::new(4, 1);
    let exec = staged.executor();
    // Settle the start-up requests.
    while let Some(mut u) = exec.claim(|_| 0) {
        let t = u.run(exec.budget());
        exec.finish(u, t);
    }
    // Two frames reach the projection's edge while the projection is never run: its kicks
    // coalesce into one queue entry, and a budget of 1 reads one frame per turn.
    let skip = Turn {
        progress: false,
        more: false,
    };
    for c in [cmd(0, 0, 1), cmd(1, 1, 2)] {
        staged.submit(c);
        while let Some(mut u) = exec.claim(|_| 0) {
            if u.id() == 2 {
                exec.finish(u, skip);
                continue;
            }
            let t = u.run(1);
            exec.finish(u, t);
        }
    }
    exec.kick(2);
    while let Some(mut u) = exec.claim(|_| 0) {
        let t = u.run(1);
        exec.finish(u, t);
    }
    let want = single_circuit(&[cmd(0, 0, 1), cmd(1, 1, 2)]);
    assert_eq!(
        staged.grid(),
        want,
        "[coalesced] the second frame was left unread"
    );
}