improvise/crates/improvise-incremental/tests/sched.rs

//! The scheduler: sub-circuits on an executor equal one circuit, whatever the number of workers
//! and the order they run in. Evidence: `context/test-evidence/improvise-incremental.md`.

mod support;

use improvise_incremental::sched::{Claimed, Executor, Pool, Turn, Unit};
use improvise_incremental::staged::{Cell, Staged, single_circuit};
use proptest::prelude::*;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use support::watchdog::within;
use zset::ZSet;

fn command() -> impl Strategy<Value = ZSet<Cell>> {
    prop::collection::vec(((0u8..3, 0u8..3, -4i64..=4), -2i64..=2), 0..4)
        .prop_map(|v| v.into_iter().collect())
}

/// Runs the executor with `workers` virtual workers, choosing each step from `choices`: either
/// claim a queued unit (running its turn at once) or hand a held one back. Returns the steps.
fn run_deterministic(
    exec: &Executor,
    workers: usize,
    choices: &mut impl Iterator<Item = u8>,
) -> usize {
    let mut held: Vec<(Claimed, Turn)> = Vec::new();
    let mut steps = 0;
    loop {
        let c = choices.next().unwrap_or(0) as usize;
        let can_claim = held.len() < workers;
        let mut started = false;
        if can_claim
            && (held.is_empty() || c.is_multiple_of(2))
            && let Some(mut claimed) = exec.claim(|n| c / 2 % n)
        {
            let turn = claimed.run(exec.budget());
            held.push((claimed, turn));
            started = true;
        }
        if !started {
            if held.is_empty() {
                return steps;
            }
            let (claimed, turn) = held.remove(c / 2 % held.len());
            exec.finish(claimed, turn);
        }
        steps += 1;
        assert!(steps < 100_000, "[sched] the executor never went quiet");
    }
}

proptest! {
    /// The placed graph, on 1 to 4 workers in a random order with commands arriving while it
    /// runs, ends where one circuit running the whole graph ends.
    #[test]
    fn placed_circuits_equal_one_circuit_for_any_workers_and_order(
        commands in prop::collection::vec(command(), 0..12),
        workers in 1usize..5,
        cap in 1usize..4,
        budget in 1usize..4,
        choices in prop::collection::vec(any::<u8>(), 0..400),
        gaps in prop::collection::vec(0usize..6, 0..12),
    ) {
        let staged = Staged::new(cap, budget);
        let mut it = choices.into_iter();
        for (i, d) in commands.iter().enumerate() {
            staged.submit(d.clone());
            // A few scheduler steps before the next command arrives.
            let mut few = it.by_ref().take(gaps.get(i).copied().unwrap_or(0));
            run_some(staged.executor(), workers, &mut few);
        }
        run_deterministic(staged.executor(), workers, &mut std::iter::repeat(0));
        prop_assert_eq!(staged.grid(), single_circuit(&commands), "[sched] the placed graph");
    }
}

/// A bounded number of claim-or-finish steps, leaving whatever is left in the queue.
fn run_some(exec: &Executor, workers: usize, choices: &mut impl Iterator<Item = u8>) {
    let mut held: Vec<(Claimed, Turn)> = Vec::new();
    for c in choices {
        let c = c as usize;
        if held.len() < workers
            && (held.is_empty() || c.is_multiple_of(2))
            && let Some(mut claimed) = exec.claim(|n| c / 2 % n)
        {
            let turn = claimed.run(exec.budget());
            held.push((claimed, turn));
            continue;
        }
        if !held.is_empty() {
            let (claimed, turn) = held.remove(c / 2 % held.len());
            exec.finish(claimed, turn);
        }
    }
    for (claimed, turn) in held {
        exec.finish(claimed, turn);
    }
}

#[test]
fn real_worker_threads_equal_one_circuit() {
    within(10, "real worker threads", || {
        let commands: Vec<ZSet<Cell>> = (0..400u32)
            .map(|n| {
                [(
                    ((n % 3) as u8, (n / 3 % 3) as u8, (n % 7) as i64 - 3),
                    1 - (n % 3) as i64,
                )]
                .into_iter()
                .collect()
            })
            .collect();
        for workers in 1..=4 {
            let staged = Staged::new(2, 3);
            let pool = Pool::start(staged.executor().clone(), workers);
            for d in &commands {
                staged.submit(d.clone());
            }
            pool.finish().unwrap();
            assert_eq!(
                staged.grid(),
                single_circuit(&commands),
                "[threads] {workers} workers"
            );
        }
    });
}

/// A unit that records which turns ran, and asks for `more` a fixed number of times.
struct Log {
    id: usize,
    more: usize,
    ran: Arc<std::sync::Mutex<Vec<usize>>>,
}

impl Unit for Log {
    fn turn(&mut self, _: usize) -> Turn {
        self.ran.lock().unwrap().push(self.id);
        let more = self.more > 0;
        self.more = self.more.saturating_sub(1);
        Turn {
            progress: false,
            more,
        }
    }

    fn neighbours(&self) -> &[usize] {
        &[]
    }
}

#[test]
fn a_unit_that_wants_more_goes_to_the_back_of_the_queue() {
    let ran = Arc::new(std::sync::Mutex::new(Vec::new()));
    let unit = |id, more| -> Box<dyn Unit> {
        Box::new(Log {
            id,
            more,
            ran: ran.clone(),
        })
    };
    let exec = Executor::new(vec![unit(0, 2), unit(1, 0), unit(2, 0)], 1);
    exec.kick_all();
    while let Some(mut c) = exec.claim(|_| 0) {
        let t = c.run(exec.budget());
        exec.finish(c, t);
    }
    assert_eq!(
        *ran.lock().unwrap(),
        vec![0, 1, 2, 0, 0],
        "[fair] the others run between its turns"
    );
    assert!(exec.idle());
}

#[test]
fn a_kick_while_a_unit_runs_queues_it_again_and_never_claims_it_twice() {
    let ran = Arc::new(std::sync::Mutex::new(Vec::new()));
    let exec = Executor::new(
        vec![Box::new(Log {
            id: 0,
            more: 0,
            ran: ran.clone(),
        })],
        1,
    );
    exec.kick(0);
    let mut claimed = exec.claim(|_| 0).unwrap();
    exec.kick(0);
    assert!(
        exec.claim(|_| 0).is_none(),
        "[own] a running unit is not claimable"
    );
    let t = claimed.run(1);
    exec.finish(claimed, t);
    assert!(
        !exec.idle(),
        "[again] the kick that arrived mid-turn is kept"
    );
    let mut again = exec.claim(|_| 0).unwrap();
    let t = again.run(1);
    exec.finish(again, t);
    assert!(exec.idle());
}

/// A unit that fails if two turns of it ever overlap.
struct Exclusive {
    inside: Arc<AtomicBool>,
    turns: Arc<AtomicUsize>,
}

impl Unit for Exclusive {
    fn turn(&mut self, _: usize) -> Turn {
        assert!(
            !self.inside.swap(true, Ordering::SeqCst),
            "[own] two workers ran one unit"
        );
        self.turns.fetch_add(1, Ordering::SeqCst);
        std::thread::yield_now();
        self.inside.store(false, Ordering::SeqCst);
        Turn {
            progress: false,
            more: false,
        }
    }

    fn neighbours(&self) -> &[usize] {
        &[]
    }
}

#[test]
fn at_most_one_worker_runs_a_unit_however_often_it_is_kicked() {
    within(10, "kicking threads", || {
        let (inside, turns) = (
            Arc::new(AtomicBool::new(false)),
            Arc::new(AtomicUsize::new(0)),
        );
        let exec = Arc::new(Executor::new(
            vec![Box::new(Exclusive {
                inside: inside.clone(),
                turns: turns.clone(),
            })],
            1,
        ));
        let pool = Pool::start(exec.clone(), 4);
        let kickers: Vec<_> = (0..4)
            .map(|_| {
                let exec = exec.clone();
                std::thread::spawn(move || {
                    for _ in 0..500 {
                        exec.kick(0);
                        std::thread::yield_now();
                    }
                })
            })
            .collect();
        for k in kickers {
            k.join().unwrap();
        }
        pool.finish().unwrap();
        assert!(turns.load(Ordering::SeqCst) > 0, "[own] it ran");
    });
}