Edward Langley
168a28d2c5
improvise-52eq: the AGENTS.md recipe searched `bd show` output for "not found" and printed "all resolve" when there was none, so a bd that was missing, failing or never run passed the audit. scripts/bd-audit-ids keeps the file scan and exclusions, then `comm -23`s the cited ids against `bd list --all --limit 0 --json`. A dead id is exit 1 and is named. A bd that does not answer is exit 3, never a pass. scripts/bd-audit-ids-test.sh has eight cases on a throwaway repo with a fake bd. The old recipe as a rival fails 7 of the 8. Disabling the comparison fails the dead-id case, and an empty exclusion list fails the pass case. CI runs it with the other named suites. The "One environment note" now reports which forms the guard refused and allowed, as observed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
88 lines
3.5 KiB
Bash
Executable File
88 lines
3.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check_the_checker for scripts/bd-audit-ids. A throwaway git repo and a fake
|
|
# bd; no network, no real tracker. See context/test-evidence/tooling.md.
|
|
set -uo pipefail
|
|
|
|
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
AUDIT="$here/bd-audit-ids"
|
|
work="$(mktemp -d)"
|
|
trap 'rm -rf "$work"' EXIT
|
|
|
|
pass=0; fail=0
|
|
ok() { printf ' ok %s\n' "$1"; pass=$((pass+1)); }
|
|
bad() { printf ' FAIL %s (%s)\n' "$1" "$2"; fail=$((fail+1)); }
|
|
|
|
# The fake bd answers `list` from $FAKE_BD_OUT and exits $FAKE_BD_RC; any
|
|
# other subcommand is an error, so the audit cannot pass by calling `show`.
|
|
mkdir -p "$work/bin"
|
|
cat > "$work/bin/bd" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
[ "${1:-}" = list ] || { echo "fake bd: unexpected '$*'" >&2; exit 2; }
|
|
[ -f "${FAKE_BD_OUT:-}" ] && cat "$FAKE_BD_OUT"
|
|
exit "${FAKE_BD_RC:-0}"
|
|
EOF
|
|
chmod +x "$work/bin/bd"
|
|
|
|
repo="$work/repo"
|
|
mkdir -p "$repo/context" "$repo/harness/serve"
|
|
git -C "$repo" init -q
|
|
printf 'serve/\n' > "$repo/harness/.gitignore"
|
|
# A live id, and a crate name the exclusion list must drop: without the
|
|
# exclusion, improvise-core would be reported dead and the first case fails.
|
|
printf 'See improvise-ab12 in improvise-core.\n' > "$repo/context/notes.md"
|
|
# Ignored build output with a dead id. The audit reads git's file list, so
|
|
# this must never be reported (improvise-8haj).
|
|
printf 'improvise-gone9\n' > "$repo/harness/serve/copy.md"
|
|
|
|
printf '[{"id":"improvise-ab12"},{"id":"improvise-cd34"}]\n' > "$work/known.json"
|
|
printf '' > "$work/empty"
|
|
printf '[]\n' > "$work/none.json"
|
|
|
|
# $1 name, $2 expected exit, $3 grep -E pattern the output must contain;
|
|
# the caller sets FAKE_BD_* and BD_AUDIT_BD in the environment.
|
|
run_case() {
|
|
local name="$1" want="$2" pat="$3"
|
|
(cd "$repo" && "$AUDIT") > "$work/out" 2>&1
|
|
local got=$?
|
|
if [ "$got" != "$want" ]; then bad "$name" "exit $got, wanted $want: $(head -c 200 "$work/out")"
|
|
elif ! grep -Eq "$pat" "$work/out"; then bad "$name" "no /$pat/ in: $(head -c 200 "$work/out")"
|
|
else ok "$name"; fi
|
|
}
|
|
|
|
export BD_AUDIT_BD="$work/bin/bd"
|
|
|
|
FAKE_BD_OUT="$work/known.json" FAKE_BD_RC=0 \
|
|
run_case "every cited id known is a pass" 0 'all 1 cited ids resolve'
|
|
|
|
printf 'Also improvise-zzzq.\n' > "$repo/context/dead.md"
|
|
FAKE_BD_OUT="$work/known.json" FAKE_BD_RC=0 \
|
|
run_case "a dead id is exit 1, and named" 1 '^ improvise-zzzq$'
|
|
rm -f "$repo/context/dead.md"
|
|
|
|
# The three shapes of "bd never answered". The recipe this replaced reported
|
|
# "all resolve" for every one of them (improvise-52eq).
|
|
FAKE_BD_OUT="$work/empty" FAKE_BD_RC=0 \
|
|
run_case "a bd that prints nothing is not a pass" 3 'did not print a JSON list|returned no issues'
|
|
FAKE_BD_OUT="$work/none.json" FAKE_BD_RC=0 \
|
|
run_case "a bd that lists no issues is not a pass" 3 'returned no issues'
|
|
FAKE_BD_OUT="$work/known.json" FAKE_BD_RC=1 \
|
|
run_case "a bd that fails is not a pass" 3 'list. failed'
|
|
BD_AUDIT_BD="$work/bin/no-such-bd" \
|
|
run_case "a missing bd is not a pass" 3 'is not on PATH'
|
|
|
|
# The live repo, with the fake bd listing nothing: if this passed, a real
|
|
# audit whose bd fell silent would pass too.
|
|
(cd "$here/.." && FAKE_BD_OUT="$work/none.json" "$AUDIT") > "$work/out" 2>&1
|
|
got=$?
|
|
[ "$got" = 3 ] && ok "the real tree does not pass against a silent bd" \
|
|
|| bad "the real tree does not pass against a silent bd" "exit $got"
|
|
|
|
"$AUDIT" --help > "$work/out" 2>&1
|
|
got=$?
|
|
[ "$got" = 4 ] && grep -q 'POSITIVE' "$work/out" \
|
|
&& ok "--help is exit 4 and states the positive comparison" \
|
|
|| bad "--help is exit 4 and states the positive comparison" "exit $got"
|
|
|
|
printf '\nbd-audit-ids-test: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|