Cursor Agent
190b330c4e
The run_case_capture helper was being called in command substitution
(out=$(run_case_capture ...)), which runs it in a subshell. This caused
the ok/bad updates to the pass/fail counters to be discarded.
Fix by having the helper write output to a caller-provided file parameter,
then reading from that file in the caller. This keeps the helper in the
parent shell where counter updates are preserved.
Addresses Codex P2 review on commit ec30c8a7.
Co-authored-by: Ed Langley <fiddlerwoaroof@users.noreply.github.com>
276 lines
17 KiB
Bash
Executable File
276 lines
17 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check_the_checker for scripts/pr-watch. Every case is a state this repo
|
|
# actually hit; three of them are watchers that exited green while wrong.
|
|
set -uo pipefail
|
|
|
|
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
WATCH="$here/pr-watch"
|
|
work="$(mktemp -d)"
|
|
trap 'rm -rf "$work"' EXIT
|
|
|
|
pass=0; fail=0
|
|
ok() { printf ' ok %s\n' "$1"; pass=$((pass+1)); }
|
|
bad() { printf ' FAIL %s (exit %s, wanted %s)\n' "$1" "$2" "$3"; fail=$((fail+1)); }
|
|
|
|
# $1 name, $2 expected exit, $3 fixture dir, rest: extra args
|
|
run_case() {
|
|
local name="$1" want="$2" dir="$3"; shift 3
|
|
PR_WATCH_FIXTURE_DIR="$dir" "$WATCH" 1 --interval 0 --timeout 1 "$@" >/dev/null 2>&1
|
|
local got=$?
|
|
[ "$got" = "$want" ] && ok "$name" || bad "$name" "$got" "$want"
|
|
}
|
|
|
|
# Like run_case but writes output to a caller-provided file.
|
|
# Caller must NOT use command substitution (subshell loses ok/bad counters).
|
|
# $1 name, $2 expected exit, $3 fixture dir, $4 output file, rest: extra args
|
|
run_case_capture() {
|
|
local name="$1" want="$2" dir="$3" outfile="$4"; shift 4
|
|
PR_WATCH_FIXTURE_DIR="$dir" "$WATCH" 1 --interval 0 --timeout 1 "$@" >"$outfile" 2>&1
|
|
local got=$?
|
|
[ "$got" = "$want" ] && ok "$name" || bad "$name" "$got" "$want"
|
|
}
|
|
|
|
mk() { mkdir -p "$1"; cat > "$1/$2"; }
|
|
|
|
# Create empty pr-reviews fixtures so pr-watch can call pr-reviews in fixture mode.
|
|
# $1 dir, $2 head (must match the pr-watch fixture's headRefOid)
|
|
mk_reviews_empty() {
|
|
local dir="$1" head="$2"
|
|
mkdir -p "$dir"
|
|
printf '{"headRefOid":"%s","reviews":[]}\n' "$head" > "$dir/last.pr.json"
|
|
echo '[]' > "$dir/last.inline.json"
|
|
echo '[]' > "$dir/last.issue.json"
|
|
}
|
|
|
|
# Create pr-reviews fixtures with a review. $1 dir, $2 head, $3 review commit
|
|
mk_reviews_with_review() {
|
|
local dir="$1" head="$2" review_commit="$3"
|
|
mkdir -p "$dir"
|
|
cat > "$dir/last.pr.json" <<EOF
|
|
{"headRefOid":"$head","reviews":[{"submittedAt":"2026-09-25T12:00:00Z","state":"COMMENTED","author":{"login":"copilot"},"commit":{"oid":"$review_commit"},"body":"Test review body"}]}
|
|
EOF
|
|
echo '[]' > "$dir/last.inline.json"
|
|
echo '[]' > "$dir/last.issue.json"
|
|
}
|
|
|
|
# The state a fresh push is in: the PR exists, no check has registered yet.
|
|
# A watcher keyed on "nothing pending" calls this finished and green.
|
|
mk "$work/none" last.json <<'EOF'
|
|
{"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[]}
|
|
EOF
|
|
run_case "no checks registered yet is not success" 3 "$work/none"
|
|
|
|
# conclusion is a STRING, empty while running -- never null. A null test
|
|
# matches nothing, so every check looks finished.
|
|
mk "$work/running" last.json <<'EOF'
|
|
{"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"UNSTABLE","statusCheckRollup":[
|
|
{"name":"build","status":"IN_PROGRESS","conclusion":""},
|
|
{"name":"test","status":"COMPLETED","conclusion":"SUCCESS"}]}
|
|
EOF
|
|
run_case "a running check with an empty conclusion is not finished" 3 "$work/running"
|
|
|
|
mk "$work/green" last.json <<'EOF'
|
|
{"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[
|
|
{"name":"build","status":"COMPLETED","conclusion":"SUCCESS"},
|
|
{"name":"skipped","status":"COMPLETED","conclusion":"SKIPPED"}]}
|
|
EOF
|
|
mk_reviews_empty "$work/green" "aaaaaaaaaaaa"
|
|
run_case "all completed and green" 0 "$work/green"
|
|
|
|
# ── Reviews integration ────────────────────────────────────────────────────
|
|
# pr-watch now calls pr-reviews after checks are green. Test that it works.
|
|
mk "$work/green-reviews" last.json <<'EOF'
|
|
{"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[
|
|
{"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]}
|
|
EOF
|
|
mk_reviews_with_review "$work/green-reviews" "aaaaaaaaaaaa" "aaaaaaaaaaaa"
|
|
run_case_capture "green with reviews shows review output" 0 "$work/green-reviews" "$work/out1.txt"
|
|
if grep -q "pr-reviews:" "$work/out1.txt"; then
|
|
ok "green with reviews prints pr-reviews output"
|
|
else
|
|
bad "green with reviews prints pr-reviews output" "no pr-reviews output" "pr-reviews: in output"
|
|
fi
|
|
|
|
# --no-reviews skips the review printout
|
|
run_case_capture "--no-reviews skips review output" 0 "$work/green-reviews" "$work/out2.txt" --no-reviews
|
|
if grep -q "pr-reviews:" "$work/out2.txt"; then
|
|
bad "--no-reviews skips review output" "pr-reviews in output" "no pr-reviews"
|
|
else
|
|
ok "--no-reviews actually skips review output"
|
|
fi
|
|
|
|
# pr-reviews failure: if pr-reviews fails, pr-watch exits 6 (not 0)
|
|
# Create a green fixture with MALFORMED review fixtures so pr-reviews fails
|
|
mk "$work/green-bad-reviews" last.json <<'EOF'
|
|
{"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[
|
|
{"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]}
|
|
EOF
|
|
# Create malformed JSON that will cause jq to fail in pr-reviews
|
|
echo 'NOT VALID JSON' > "$work/green-bad-reviews/last.pr.json"
|
|
echo '[]' > "$work/green-bad-reviews/last.inline.json"
|
|
echo '[]' > "$work/green-bad-reviews/last.issue.json"
|
|
run_case "pr-reviews failure exits 6" 6 "$work/green-bad-reviews"
|
|
|
|
mk "$work/red" last.json <<'EOF'
|
|
{"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"UNSTABLE","statusCheckRollup":[
|
|
{"name":"build","status":"COMPLETED","conclusion":"FAILURE"}]}
|
|
EOF
|
|
run_case "a failed check fails" 1 "$work/red"
|
|
|
|
mk "$work/conflict" last.json <<'EOF'
|
|
{"headRefOid":"aaaaaaaaaaaa","mergeable":"CONFLICTING","mergeStateStatus":"DIRTY","statusCheckRollup":[
|
|
{"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]}
|
|
EOF
|
|
run_case "green checks on a conflicting PR still fail" 1 "$work/conflict"
|
|
|
|
# The one that cost a real fix: a verdict about a tree that no longer exists
|
|
# reads as a pass. PR #67 merged at a head that predated its correction.
|
|
mkdir -p "$work/moved"
|
|
cat > "$work/moved/1.json" <<'EOF'
|
|
{"headRefOid":"aaaaaaaaaaaa","mergeable":"MERGEABLE","mergeStateStatus":"UNSTABLE","statusCheckRollup":[
|
|
{"name":"build","status":"IN_PROGRESS","conclusion":""}]}
|
|
EOF
|
|
cat > "$work/moved/last.json" <<'EOF'
|
|
{"headRefOid":"bbbbbbbbbbbb","mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","statusCheckRollup":[
|
|
{"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]}
|
|
EOF
|
|
run_case "a head that moves mid-wait is not a pass" 2 "$work/moved" --timeout 60
|
|
|
|
run_case "--expect refuses a different head" 2 "$work/green" --expect bbbbbbbb
|
|
run_case "--expect accepts a prefix of the head" 0 "$work/green" --expect aaaa
|
|
|
|
# improvise-edpl. GitHub computes mergeability lazily: for a while after the
|
|
# base moves it reports UNKNOWN, and PRs #172 and #179 both exited 0 there.
|
|
# This is the shape both had: every check green, mergeability unresolved.
|
|
green_rollup='[{"name":"build","status":"COMPLETED","conclusion":"SUCCESS"}]'
|
|
red_rollup='[{"name":"build","status":"COMPLETED","conclusion":"FAILURE"}]'
|
|
# $1 dir, $2 file, $3 mergeable, $4 mergeStateStatus, $5 rollup (JSON values)
|
|
state() {
|
|
mkdir -p "$1"
|
|
printf '{"headRefOid":"aaaaaaaaaaaa","mergeable":%s,"mergeStateStatus":%s,"statusCheckRollup":%s}\n' \
|
|
"$3" "$4" "$5" > "$1/$2"
|
|
}
|
|
|
|
state "$work/unk-conflict" 1.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup"
|
|
state "$work/unk-conflict" last.json '"CONFLICTING"' '"DIRTY"' "$green_rollup"
|
|
run_case "UNKNOWN that resolves to CONFLICTING fails" 1 "$work/unk-conflict" --timeout 60
|
|
|
|
state "$work/unk-clean" 1.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup"
|
|
state "$work/unk-clean" 2.json '"MERGEABLE"' '"UNKNOWN"' "$green_rollup"
|
|
state "$work/unk-clean" last.json '"MERGEABLE"' '"CLEAN"' "$green_rollup"
|
|
run_case "UNKNOWN that resolves to MERGEABLE/CLEAN passes" 0 "$work/unk-clean" --timeout 60
|
|
|
|
state "$work/unk-forever" last.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup"
|
|
run_case "UNKNOWN until the timeout is a timeout, not a pass" 3 "$work/unk-forever"
|
|
|
|
# A failed check fails whatever mergeability says; waiting on it would turn a
|
|
# known red into a timeout.
|
|
state "$work/unk-red" last.json '"UNKNOWN"' '"UNKNOWN"' "$red_rollup"
|
|
run_case "a failed check fails without waiting on mergeability" 1 "$work/unk-red" --timeout 0
|
|
|
|
# The whole table, enumerated rather than hand-picked, every check green. The
|
|
# rule it states is in context/code/tooling.md, "pr-watch and mergeability".
|
|
for m in MERGEABLE CONFLICTING UNKNOWN; do
|
|
for s in BEHIND BLOCKED CLEAN DIRTY DRAFT HAS_HOOKS UNKNOWN UNSTABLE SOMETHING_NEW null; do
|
|
if [ "$m" = CONFLICTING ]; then want=1
|
|
elif [ "$m" = UNKNOWN ] || [ "$s" = UNKNOWN ]; then want=3
|
|
else case "$s" in CLEAN|UNSTABLE|HAS_HOOKS) want=0 ;; BLOCKED) want=3 ;; *) want=1 ;; esac
|
|
fi
|
|
sv="\"$s\""; [ "$s" = null ] && sv=null
|
|
state "$work/table-$m-$s" last.json "\"$m\"" "$sv" "$green_rollup"
|
|
# Add review fixtures for cases that exit 0 (pr-watch calls pr-reviews)
|
|
if [ "$want" = 0 ]; then
|
|
mk_reviews_empty "$work/table-$m-$s" "aaaaaaaaaaaa"
|
|
fi
|
|
run_case "table: $m/$s" "$want" "$work/table-$m-$s" --timeout 0
|
|
done
|
|
done
|
|
|
|
# improvise-zlj2. The required "Browser harness" check comes from a job that
|
|
# `needs: build`, so its check run does not exist until build finishes. In that
|
|
# gap every registered check is green and GitHub says MERGEABLE/BLOCKED. The
|
|
# rollup below is PR #189's, verbatim; the gap states drop or rewind entries.
|
|
cat > "$work/rollup.jsonl" <<'EOF'
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:41:53Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406461","name":"Build all bundles","startedAt":"2026-09-23T04:40:15Z","status":"COMPLETED","workflowName":"CI"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047406233","name":"plan","startedAt":"2026-09-23T04:40:14Z","status":"COMPLETED","workflowName":"Release"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047482378","name":"build-local-artifacts (${{ join(matrix.targets, ', ') }})","startedAt":"2026-09-23T04:40:32Z","status":"COMPLETED","workflowName":"Release"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:42:22Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406506","name":"Test and lint (host)","startedAt":"2026-09-23T04:40:16Z","status":"COMPLETED","workflowName":"CI"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047482733","name":"build-global-artifacts","startedAt":"2026-09-23T04:40:32Z","status":"COMPLETED","workflowName":"Release"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:40:40Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406409","name":"wasm32 check (browser bundle crates)","startedAt":"2026-09-23T04:40:15Z","status":"COMPLETED","workflowName":"CI"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047482801","name":"host","startedAt":"2026-09-23T04:40:33Z","status":"COMPLETED","workflowName":"Release"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:40:25Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406415","name":"Browser JS unit tests (node --test)","startedAt":"2026-09-23T04:40:14Z","status":"COMPLETED","workflowName":"CI"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:40:32Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261564/job/107047483313","name":"announce","startedAt":"2026-09-23T04:40:33Z","status":"COMPLETED","workflowName":"Release"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:41:17Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406264","name":"Knowledge base consistency","startedAt":"2026-09-23T04:40:14Z","status":"COMPLETED","workflowName":"CI"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:40:39Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047406373","name":"Lint the harness crate","startedAt":"2026-09-23T04:40:14Z","status":"COMPLETED","workflowName":"CI"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:44:17Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107047753280","name":"Browser harness (OPFS contract + shell)","startedAt":"2026-09-23T04:41:56Z","status":"COMPLETED","workflowName":"CI"}
|
|
{"__typename":"CheckRun","completedAt":"2026-09-23T04:44:18Z","conclusion":"SKIPPED","detailsUrl":"https://github.com/fiddlerwoaroof/improvise/actions/runs/35819261571/job/107048259684","name":"Deploy to GitHub Pages","startedAt":"2026-09-23T04:44:18Z","status":"COMPLETED","workflowName":"CI"}
|
|
EOF
|
|
harness='Browser harness (OPFS contract + shell)'
|
|
deploy='Deploy to GitHub Pages'
|
|
# $1 dir, $2 file, $3 mergeable, $4 state, $5 jq filter over the rollup array
|
|
gap_state() {
|
|
state "$1" "$2" "\"$3\"" "\"$4\"" "$(jq -cs "$5" "$work/rollup.jsonl")"
|
|
}
|
|
build_running='map(if .name == "Build all bundles" then .status = "IN_PROGRESS" | .conclusion = "" else . end)'
|
|
not_yet="map(select(.name != \"$harness\" and .name != \"$deploy\"))"
|
|
harness_running="map(if .name == \"$harness\" then .status = \"IN_PROGRESS\" | .conclusion = \"\" else . end) | map(select(.name != \"$deploy\"))"
|
|
gap_state "$work/gap" 1.json MERGEABLE BLOCKED "$not_yet | $build_running"
|
|
gap_state "$work/gap" 2.json MERGEABLE BLOCKED "$not_yet"
|
|
gap_state "$work/gap" 3.json MERGEABLE BLOCKED "$harness_running"
|
|
gap_state "$work/gap" last.json MERGEABLE CLEAN '.'
|
|
mk_reviews_empty "$work/gap" "aaaaaaaaaaaa"
|
|
run_case "the needs-gap before the required check registers is not a failure" 0 "$work/gap" --timeout 60
|
|
|
|
gap_state "$work/gap-never" last.json MERGEABLE BLOCKED "$not_yet"
|
|
run_case "a required check that never registers is a timeout" 3 "$work/gap-never"
|
|
|
|
# improvise-9pfs. Here a failed REQUIRED check reports MERGEABLE/BLOCKED, not
|
|
# UNSTABLE. The table above holds only green rollups, so this red was unpinned.
|
|
gap_state "$work/blocked-red" last.json MERGEABLE BLOCKED \
|
|
'map(if .name == "Test and lint (host)" then .conclusion = "FAILURE" else . end)'
|
|
run_case "a failed required check under MERGEABLE/BLOCKED fails without waiting" 1 \
|
|
"$work/blocked-red" --timeout 0
|
|
|
|
# improvise-1qaa/29w2. Each ended fixture is a shape GitHub returned live (#189,
|
|
# #74, #46); the last is the one a waiting loop would sit on.
|
|
# $1 dir, $2 file, $3 head, $4 state, $5 mergeable, $6 mergeStateStatus
|
|
ended() {
|
|
mkdir -p "$1"
|
|
printf '{"headRefOid":"%s","state":"%s","mergeable":"%s","mergeStateStatus":"%s","statusCheckRollup":%s}\n' \
|
|
"$3" "$4" "$5" "$6" "$green_rollup" > "$1/$2"
|
|
}
|
|
for shape in "MERGED UNKNOWN UNKNOWN" "CLOSED CONFLICTING DIRTY" "CLOSED MERGEABLE BLOCKED"; do
|
|
set -- $shape
|
|
d="$work/ended-$1-$3"
|
|
ended "$d" last.json aaaaaaaaaaaa "$1" "$2" "$3"
|
|
run_case "a $1 PR at $2/$3 ends the watch as no longer open" 5 "$d" --timeout 5
|
|
polls="$(cat "$d/.poll" 2>/dev/null || echo 0)"
|
|
[ "$polls" = 1 ] && ok "a $1 PR at $2/$3 is read once, not waited on" \
|
|
|| bad "a $1 PR at $2/$3 is read once, not waited on (polls)" "$polls" 1
|
|
done
|
|
|
|
# improvise-24lz. A merge at a head the watcher never saw is the PR #67 hazard,
|
|
# so a moved head outranks a closed PR, with or without --expect.
|
|
state "$work/moved-merged" 1.json '"MERGEABLE"' '"UNSTABLE"' \
|
|
'[{"name":"build","status":"IN_PROGRESS","conclusion":""}]'
|
|
ended "$work/moved-merged" last.json bbbbbbbbbbbb MERGED UNKNOWN UNKNOWN
|
|
run_case "a head that moves and then merges is not reported as merged" 2 \
|
|
"$work/moved-merged" --timeout 60
|
|
ended "$work/merged-elsewhere" last.json bbbbbbbbbbbb MERGED UNKNOWN UNKNOWN
|
|
run_case "a PR merged at a head other than --expect is a head mismatch" 2 \
|
|
"$work/merged-elsewhere" --expect aaaa --timeout 5
|
|
|
|
# improvise-h3ni. The head must stay pinned across the UNKNOWN wait, or a
|
|
# verdict about b is reported as a verdict about a.
|
|
state "$work/unk-moved" 1.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup"
|
|
state "$work/unk-moved" 2.json '"UNKNOWN"' '"UNKNOWN"' "$green_rollup"
|
|
state "$work/unk-moved" last.json '"MERGEABLE"' '"CLEAN"' "$green_rollup"
|
|
for f in 2.json last.json; do
|
|
sed 's/aaaaaaaaaaaa/bbbbbbbbbbbb/' "$work/unk-moved/$f" > "$work/unk-moved/$f.new"
|
|
mv -f "$work/unk-moved/$f.new" "$work/unk-moved/$f"
|
|
done
|
|
run_case "a head that moves during the UNKNOWN wait is not a pass" 2 "$work/unk-moved" --timeout 60
|
|
|
|
printf '\npr-watch-test: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|