#!/usr/bin/env bash # check_the_checker for scripts/pr-watch. Every case is a state this repo # actually hit; three of them are watchers that exited green while wrong. set -uo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WATCH="$here/pr-watch" work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT pass=0; fail=0 ok() { printf ' ok %s\n' "$1"; pass=$((pass+1)); } bad() { printf ' FAIL %s (exit %s, wanted %s)\n' "$1" "$2" "$3"; fail=$((fail+1)); } # $1 name, $2 expected exit, $3 fixture dir, rest: extra args run_case() { local name="$1" want="$2" dir="$3"; shift 3 PR_WATCH_FIXTURE_DIR="$dir" "$WATCH" 1 --interval 0 --timeout 1 "$@" >/dev/null 2>&1 local got=$? [ "$got" = "$want" ] && ok "$name" || bad "$name" "$got" "$want" } # Like run_case but captures stdout for content checks run_case_capture() { local name="$1" want="$2" dir="$3"; shift 3 local out="$work/output.txt" PR_WATCH_FIXTURE_DIR="$dir" "$WATCH" 1 --interval 0 --timeout 1 "$@" >"$out" 2>&1 local got=$? [ "$got" = "$want" ] && ok "$name" || bad "$name" "$got" "$want" cat "$out" } mk() { mkdir -p "$1"; cat > "$1/$2"; } # Create empty pr-reviews fixtures so pr-watch can call pr-reviews in fixture mode. # $1 dir, $2 head (must match the pr-watch fixture's headRefOid) mk_reviews_empty() { local dir="$1" head="$2" mkdir -p "$dir" printf '{"headRefOid":"%s","reviews":[]}\n' "$head" > "$dir/last.pr.json" echo '[]' > "$dir/last.inline.json" echo '[]' > "$dir/last.issue.json" } # Create pr-reviews fixtures with a review. $1 dir, $2 head, $3 review commit mk_reviews_with_review() { local dir="$1" head="$2" review_commit="$3" mkdir -p "$dir" cat > "$dir/last.pr.json" < "$dir/last.inline.json" echo '[]' > "$dir/last.issue.json" } # The state a fresh push is in: the PR exists, no check has registered yet. # A watcher keyed on "nothing pending" calls this finished and green. mk "$work/none" last.json <<'EOF' {"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[]} EOF run_case "no checks registered yet is not success" 3 "$work/none" # conclusion is a STRING, empty while running -- never null. A null test # matches nothing, so every check looks finished. mk "$work/running" last.json <<'EOF' {"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"UNSTABLE","statusCheckRollup":[ {"name":"build","status":"IN_PROGRESS","conclusion":""}, {"name":"test","status":"COMPLETED","conclusion":"SUCCESS"}]} EOF run_case "a running check with an empty conclusion is not finished" 3 "$work/running" mk "$work/green" last.json <<'EOF' {"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[ {"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}, {"name":"skipped","status":"COMPLETED","conclusion":"SKIPPED"}]} EOF mk_reviews_empty "$work/green" "aaaaaaaaaaaa" run_case "all completed and green" 0 "$work/green" # ── Reviews integration ──────────────────────────────────────────────────── # pr-watch now calls pr-reviews after checks are green. Test that it works. mk "$work/green-reviews" last.json <<'EOF' {"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[ {"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]} EOF mk_reviews_with_review "$work/green-reviews" "aaaaaaaaaaaa" "aaaaaaaaaaaa" out=$(run_case_capture "green with reviews shows review output" 0 "$work/green-reviews") if echo "$out" | grep -q "pr-reviews:"; then ok "green with reviews prints pr-reviews output" else bad "green with reviews prints pr-reviews output" "no pr-reviews output" "pr-reviews: in output" fi # --no-reviews skips the review printout out=$(run_case_capture "--no-reviews skips review output" 0 "$work/green-reviews" --no-reviews) if echo "$out" | grep -q "pr-reviews:"; then bad "--no-reviews skips review output" "pr-reviews in output" "no pr-reviews" else ok "--no-reviews actually skips review output" fi # pr-reviews failure: if pr-reviews fails, pr-watch exits 6 (not 0) # Create a green fixture with MALFORMED review fixtures so pr-reviews fails mk "$work/green-bad-reviews" last.json <<'EOF' {"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[ {"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]} EOF # Create malformed JSON that will cause jq to fail in pr-reviews echo 'NOT VALID JSON' > "$work/green-bad-reviews/last.pr.json" echo '[]' > "$work/green-bad-reviews/last.inline.json" echo '[]' > "$work/green-bad-reviews/last.issue.json" run_case "pr-reviews failure exits 6" 6 "$work/green-bad-reviews" mk "$work/red" last.json <<'EOF' {"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"UNSTABLE","statusCheckRollup":[ {"name":"build","status":"COMPLETED","conclusion":"FAILURE"}]} EOF run_case "a failed check fails" 1 "$work/red" mk "$work/conflict" last.json <<'EOF' {"headRefOid":"aaaaaaaaaaaa","mergeable":"CONFLICTING","mergeStateStatus":"DIRTY","statusCheckRollup":[ {"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]} EOF run_case "green checks on a conflicting PR still fail" 1 "$work/conflict" # The one that cost a real fix: a verdict about a tree that no longer exists # reads as a pass. PR #67 merged at a head that predated its correction. mkdir -p "$work/moved" cat > "$work/moved/1.json" <<'EOF' {"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"UNSTABLE","statusCheckRollup":[ {"name":"build","status":"IN_PROGRESS","conclusion":""}]} EOF cat > "$work/moved/last.json" <<'EOF' {"headRefOid":"bbbbbbbbbbbb","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[ {"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]} EOF run_case "a head that moves mid-wait is not a pass" 2 "$work/moved" --timeout 60 run_case "--expect refuses a different head" 2 "$work/green" --expect bbbbbbbb run_case "--expect accepts a prefix of the head" 0 "$work/green" --expect aaaa # improvise-edpl. GitHub computes mergeability lazily: for a while after the # base moves it reports UNKNOWN, and PRs #172 and #179 both exited 0 there. # This is the shape both had: every check green, mergeability unresolved. green_rollup='[{"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]' red_rollup='[{"name":"build","status":"COMPLETED","conclusion":"FAILURE"}]' # $1 dir, $2 file, $3 mergeable, $4 mergeStateStatus, $5 rollup (JSON values) state() { mkdir -p "$1" printf '{"headRefOid":"aaaaaaaaaaaa","mergeable":%s,"mergeStateStatus":%s,"statusCheckRollup":%s}\n' \ "$3" "$4" "$5" > "$1/$2" } state "$work/unk-conflict" 1.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup" state "$work/unk-conflict" last.json '"CONFLICTING"' '"DIRTY"' "$green_rollup" run_case "UNKNOWN that resolves to CONFLICTING fails" 1 "$work/unk-conflict" --timeout 60 state "$work/unk-clean" 1.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup" state "$work/unk-clean" 2.json '"MERGEABLE"' '"UNKNOWN"' "$green_rollup" state "$work/unk-clean" last.json '"MERGEABLE"' '"CLEAN"' "$green_rollup" run_case "UNKNOWN that resolves to MERGEABLE/CLEAN passes" 0 "$work/unk-clean" --timeout 60 state "$work/unk-forever" last.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup" run_case "UNKNOWN until the timeout is a timeout, not a pass" 3 "$work/unk-forever" # A failed check fails whatever mergeability says; waiting on it would turn a # known red into a timeout. state "$work/unk-red" last.json '"UNKNOWN"' '"UNKNOWN"' "$red_rollup" run_case "a failed check fails without waiting on mergeability" 1 "$work/unk-red" --timeout 0 # The whole table, enumerated rather than hand-picked, every check green. The # rule it states is in context/code/tooling.md, "pr-watch and mergeability". for m in MERGEABLE CONFLICTING UNKNOWN; do for s in BEHIND BLOCKED CLEAN DIRTY DRAFT HAS_HOOKS UNKNOWN UNSTABLE SOMETHING_NEW null; do if [ "$m" = CONFLICTING ]; then want=1 elif [ "$m" = UNKNOWN ] || [ "$s" = UNKNOWN ]; then want=3 else case "$s" in CLEAN|UNSTABLE|HAS_HOOKS) want=0 ;; BLOCKED) want=3 ;; *) want=1 ;; esac fi sv="\"$s\""; [ "$s" = null ] && sv=null state "$work/table-$m-$s" last.json "\"$m\"" "$sv" "$green_rollup" # Add review fixtures for cases that exit 0 (pr-watch calls pr-reviews) if [ "$want" = 0 ]; then mk_reviews_empty "$work/table-$m-$s" "aaaaaaaaaaaa" fi run_case "table: $m/$s" "$want" "$work/table-$m-$s" --timeout 0 done done # improvise-zlj2. The required "Browser harness" check comes from a job that # `needs: build`, so its check run does not exist until build finishes. In that # gap every registered check is green and GitHub says MERGEABLE/BLOCKED. The # rollup below is PR #189's, verbatim; the gap states drop or rewind entries. cat > "$work/rollup.jsonl" <<'EOF' {"__typename":"CheckRun","completedAt":"2026-09-23T04:41:53Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406461","name":"Build all bundles","startedAt":"2026-09-23T04:40:15Z","status":"COMPLETED","workflowName":"CI"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047406233","name":"plan","startedAt":"2026-09-23T04:40:14Z","status":"COMPLETED","workflowName":"Release"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047482378","name":"build-local-artifacts (${{ join(matrix.targets, ', ') }})","startedAt":"2026-09-23T04:40:32Z","status":"COMPLETED","workflowName":"Release"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:42:22Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406506","name":"Test and lint (host)","startedAt":"2026-09-23T04:40:16Z","status":"COMPLETED","workflowName":"CI"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047482733","name":"build-global-artifacts","startedAt":"2026-09-23T04:40:32Z","status":"COMPLETED","workflowName":"Release"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:40:40Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406409","name":"wasm32 check (browser bundle crates)","startedAt":"2026-09-23T04:40:15Z","status":"COMPLETED","workflowName":"CI"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047482801","name":"host","startedAt":"2026-09-23T04:40:33Z","status":"COMPLETED","workflowName":"Release"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:40:25Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406415","name":"Browser JS unit tests (node --test)","startedAt":"2026-09-23T04:40:14Z","status":"COMPLETED","workflowName":"CI"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047483313","name":"announce","startedAt":"2026-09-23T04:40:33Z","status":"COMPLETED","workflowName":"Release"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:41:17Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406264","name":"Knowledge base consistency","startedAt":"2026-09-23T04:40:14Z","status":"COMPLETED","workflowName":"CI"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:40:39Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406373","name":"Lint the harness crate","startedAt":"2026-09-23T04:40:14Z","status":"COMPLETED","workflowName":"CI"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:44:17Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047753280","name":"Browser harness (OPFS contract + shell)","startedAt":"2026-09-23T04:41:56Z","status":"COMPLETED","workflowName":"CI"} {"__typename":"CheckRun","completedAt":"2026-09-23T04:44:18Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107048259684","name":"Deploy to GitHub Pages","startedAt":"2026-09-23T04:44:18Z","status":"COMPLETED","workflowName":"CI"} EOF harness='Browser harness (OPFS contract + shell)' deploy='Deploy to GitHub Pages' # $1 dir, $2 file, $3 mergeable, $4 state, $5 jq filter over the rollup array gap_state() { state "$1" "$2" "\"$3\"" "\"$4\"" "$(jq -cs "$5" "$work/rollup.jsonl")" } build_running='map(if .name == "Build all bundles" then .status = "IN_PROGRESS" | .conclusion = "" else . end)' not_yet="map(select(.name != \"$harness\" and .name != \"$deploy\"))" harness_running="map(if .name == \"$harness\" then .status = \"IN_PROGRESS\" | .conclusion = \"\" else . end) | map(select(.name != \"$deploy\"))" gap_state "$work/gap" 1.json MERGEABLE BLOCKED "$not_yet | $build_running" gap_state "$work/gap" 2.json MERGEABLE BLOCKED "$not_yet" gap_state "$work/gap" 3.json MERGEABLE BLOCKED "$harness_running" gap_state "$work/gap" last.json MERGEABLE CLEAN '.' mk_reviews_empty "$work/gap" "aaaaaaaaaaaa" run_case "the needs-gap before the required check registers is not a failure" 0 "$work/gap" --timeout 60 gap_state "$work/gap-never" last.json MERGEABLE BLOCKED "$not_yet" run_case "a required check that never registers is a timeout" 3 "$work/gap-never" # improvise-9pfs. Here a failed REQUIRED check reports MERGEABLE/BLOCKED, not # UNSTABLE. The table above holds only green rollups, so this red was unpinned. gap_state "$work/blocked-red" last.json MERGEABLE BLOCKED \ 'map(if .name == "Test and lint (host)" then .conclusion = "FAILURE" else . end)' run_case "a failed required check under MERGEABLE/BLOCKED fails without waiting" 1 \ "$work/blocked-red" --timeout 0 # improvise-1qaa/29w2. Each ended fixture is a shape GitHub returned live (#189, # #74, #46); the last is the one a waiting loop would sit on. # $1 dir, $2 file, $3 head, $4 state, $5 mergeable, $6 mergeStateStatus ended() { mkdir -p "$1" printf '{"headRefOid":"%s","state":"%s","mergeable":"%s","mergeStateStatus":"%s","statusCheckRollup":%s}\n' \ "$3" "$4" "$5" "$6" "$green_rollup" > "$1/$2" } for shape in "MERGED UNKNOWN UNKNOWN" "CLOSED CONFLICTING DIRTY" "CLOSED MERGEABLE BLOCKED"; do set -- $shape d="$work/ended-$1-$3" ended "$d" last.json aaaaaaaaaaaa "$1" "$2" "$3" run_case "a $1 PR at $2/$3 ends the watch as no longer open" 5 "$d" --timeout 5 polls="$(cat "$d/.poll" 2>/dev/null || echo 0)" [ "$polls" = 1 ] && ok "a $1 PR at $2/$3 is read once, not waited on" \ || bad "a $1 PR at $2/$3 is read once, not waited on (polls)" "$polls" 1 done # improvise-24lz. A merge at a head the watcher never saw is the PR #67 hazard, # so a moved head outranks a closed PR, with or without --expect. state "$work/moved-merged" 1.json '"MERGEABLE"' '"UNSTABLE"' \ '[{"name":"build","status":"IN_PROGRESS","conclusion":""}]' ended "$work/moved-merged" last.json bbbbbbbbbbbb MERGED UNKNOWN UNKNOWN run_case "a head that moves and then merges is not reported as merged" 2 \ "$work/moved-merged" --timeout 60 ended "$work/merged-elsewhere" last.json bbbbbbbbbbbb MERGED UNKNOWN UNKNOWN run_case "a PR merged at a head other than --expect is a head mismatch" 2 \ "$work/merged-elsewhere" --expect aaaa --timeout 5 # improvise-h3ni. The head must stay pinned across the UNKNOWN wait, or a # verdict about b is reported as a verdict about a. state "$work/unk-moved" 1.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup" state "$work/unk-moved" 2.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup" state "$work/unk-moved" last.json '"MERGEABLE"' '"CLEAN"' "$green_rollup" for f in 2.json last.json; do sed 's/aaaaaaaaaaaa/bbbbbbbbbbbb/' "$work/unk-moved/$f" > "$work/unk-moved/$f.new" mv -f "$work/unk-moved/$f.new" "$work/unk-moved/$f" done run_case "a head that moves during the UNKNOWN wait is not a pass" 2 "$work/unk-moved" --timeout 60 printf '\npr-watch-test: %d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ]