Merge gh/main (b4779ed) into tooling/dak5-ci-suites
Resolved context/test-evidence/tooling.md: gh/main's version, with #226's bd-audit-ids entry, plus this branch's dak5 and maax entries. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
This commit is contained in:
Edward Langley
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# The dead-ID audit: every improvise-<id> the tree cites must be a ticket bd
|
||||
# knows. See AGENTS.md, "Cite a ticket ID you have read back" (improvise-52eq).
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage: bd-audit-ids
|
||||
|
||||
Run from anywhere inside the repository. Takes no arguments.
|
||||
|
||||
exit 0 every cited id is a ticket bd lists
|
||||
exit 1 at least one cited id is not; each is printed
|
||||
exit 3 bd could not answer: missing, failing, or listing nothing
|
||||
exit 4 usage error, or the file scan itself failed
|
||||
|
||||
The comparison is POSITIVE: a cited id passes only by appearing in what
|
||||
`bd list --all` returns. The recipe this replaced looked for bd's "not found"
|
||||
lines and reported "all resolve" when there were none, so a bd that never ran
|
||||
read as a clean audit.
|
||||
|
||||
THIS SCRIPT'S `$?` MUST BE THE ONE YOU READ. Nothing may follow it in the same
|
||||
pipeline or `;` list.
|
||||
EOF
|
||||
exit 4
|
||||
}
|
||||
[ $# -eq 0 ] || usage
|
||||
|
||||
BD="${BD_AUDIT_BD:-bd}"
|
||||
ROOTS=(context README.md AGENTS.md CLAUDE.md crates src tests examples web
|
||||
harness docs .agents .claude)
|
||||
# Crate, package, artifact and file names that are shaped like ids. Anchored,
|
||||
# because an unanchored `gp` would silently exclude any real ticket id starting with it.
|
||||
EXCLUDE='^improvise-(core|formula|io|protocol|command|gp|ws|worker|wasm|harness|registry|report|side|web)$'
|
||||
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
git ls-files -z --cached --others --exclude-standard -- "${ROOTS[@]}" > "$work/files"
|
||||
|
||||
# xargs exits 123 when one grep batch matched nothing, which is not an error.
|
||||
set +e
|
||||
xargs -0 grep -d skip -ohE 'improvise-[a-z0-9]{3,8}\b' < "$work/files" > "$work/raw"
|
||||
rc=$?
|
||||
set -e
|
||||
[ "$rc" -eq 0 ] || [ "$rc" -eq 123 ] || { echo "bd-audit-ids: grep failed ($rc)" >&2; exit 4; }
|
||||
sort -u "$work/raw" | { grep -vE "$EXCLUDE" || true; } > "$work/cited"
|
||||
|
||||
if ! command -v "$BD" >/dev/null 2>&1; then
|
||||
echo "bd-audit-ids: '$BD' is not on PATH, so nothing was checked." >&2
|
||||
exit 3
|
||||
fi
|
||||
if ! "$BD" list --all --limit 0 --json > "$work/known.json" 2> "$work/bd.err"; then
|
||||
echo "bd-audit-ids: '$BD list' failed, so nothing was checked:" >&2
|
||||
cat "$work/bd.err" >&2
|
||||
exit 3
|
||||
fi
|
||||
if ! jq -r '.[].id' "$work/known.json" 2>/dev/null | sort -u > "$work/known"; then
|
||||
echo "bd-audit-ids: '$BD list' did not print a JSON list of issues." >&2
|
||||
exit 3
|
||||
fi
|
||||
if [ ! -s "$work/known" ]; then
|
||||
echo "bd-audit-ids: '$BD list' returned no issues; refusing to call that a pass." >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
comm -23 "$work/cited" "$work/known" > "$work/dead"
|
||||
if [ -s "$work/dead" ]; then
|
||||
echo "bd-audit-ids: cited but not a ticket bd knows:"
|
||||
sed 's/^/ /' "$work/dead"
|
||||
exit 1
|
||||
fi
|
||||
printf 'bd-audit-ids: all %d cited ids resolve (bd knows %d)\n' \
|
||||
"$(wc -l < "$work/cited" | tr -d ' ')" "$(wc -l < "$work/known" | tr -d ' ')"
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# check_the_checker for scripts/bd-audit-ids. A throwaway git repo and a fake
|
||||
# bd; no network, no real tracker. See context/test-evidence/tooling.md.
|
||||
set -uo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
AUDIT="$here/bd-audit-ids"
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
|
||||
pass=0; fail=0
|
||||
ok() { printf ' ok %s\n' "$1"; pass=$((pass+1)); }
|
||||
bad() { printf ' FAIL %s (%s)\n' "$1" "$2"; fail=$((fail+1)); }
|
||||
|
||||
# The fake bd answers `list` from $FAKE_BD_OUT and exits $FAKE_BD_RC; any
|
||||
# other subcommand is an error, so the audit cannot pass by calling `show`.
|
||||
mkdir -p "$work/bin"
|
||||
cat > "$work/bin/bd" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
[ "${1:-}" = list ] || { echo "fake bd: unexpected '$*'" >&2; exit 2; }
|
||||
[ -f "${FAKE_BD_OUT:-}" ] && cat "$FAKE_BD_OUT"
|
||||
exit "${FAKE_BD_RC:-0}"
|
||||
EOF
|
||||
chmod +x "$work/bin/bd"
|
||||
|
||||
repo="$work/repo"
|
||||
mkdir -p "$repo/context" "$repo/harness/serve"
|
||||
git -C "$repo" init -q
|
||||
printf 'serve/\n' > "$repo/harness/.gitignore"
|
||||
# A live id, and a crate name the exclusion list must drop: without the
|
||||
# exclusion, improvise-core would be reported dead and the first case fails.
|
||||
printf 'See improvise-ab12 in improvise-core.\n' > "$repo/context/notes.md"
|
||||
# Ignored build output with a dead id. The audit reads git's file list, so
|
||||
# this must never be reported (improvise-8haj).
|
||||
printf 'improvise-gone9\n' > "$repo/harness/serve/copy.md"
|
||||
|
||||
printf '[{"id":"improvise-ab12"},{"id":"improvise-cd34"}]\n' > "$work/known.json"
|
||||
printf '' > "$work/empty"
|
||||
printf '[]\n' > "$work/none.json"
|
||||
|
||||
# $1 name, $2 expected exit, $3 grep -E pattern the output must contain;
|
||||
# the caller sets FAKE_BD_* and BD_AUDIT_BD in the environment.
|
||||
run_case() {
|
||||
local name="$1" want="$2" pat="$3"
|
||||
(cd "$repo" && "$AUDIT") > "$work/out" 2>&1
|
||||
local got=$?
|
||||
if [ "$got" != "$want" ]; then bad "$name" "exit $got, wanted $want: $(head -c 200 "$work/out")"
|
||||
elif ! grep -Eq "$pat" "$work/out"; then bad "$name" "no /$pat/ in: $(head -c 200 "$work/out")"
|
||||
else ok "$name"; fi
|
||||
}
|
||||
|
||||
export BD_AUDIT_BD="$work/bin/bd"
|
||||
|
||||
FAKE_BD_OUT="$work/known.json" FAKE_BD_RC=0 \
|
||||
run_case "every cited id known is a pass" 0 'all 1 cited ids resolve'
|
||||
|
||||
printf 'Also improvise-zzzq.\n' > "$repo/context/dead.md"
|
||||
FAKE_BD_OUT="$work/known.json" FAKE_BD_RC=0 \
|
||||
run_case "a dead id is exit 1, and named" 1 '^ improvise-zzzq$'
|
||||
rm -f "$repo/context/dead.md"
|
||||
|
||||
# The three shapes of "bd never answered". The recipe this replaced reported
|
||||
# "all resolve" for every one of them (improvise-52eq).
|
||||
FAKE_BD_OUT="$work/empty" FAKE_BD_RC=0 \
|
||||
run_case "a bd that prints nothing is not a pass" 3 'did not print a JSON list|returned no issues'
|
||||
FAKE_BD_OUT="$work/none.json" FAKE_BD_RC=0 \
|
||||
run_case "a bd that lists no issues is not a pass" 3 'returned no issues'
|
||||
FAKE_BD_OUT="$work/known.json" FAKE_BD_RC=1 \
|
||||
run_case "a bd that fails is not a pass" 3 'list. failed'
|
||||
BD_AUDIT_BD="$work/bin/no-such-bd" \
|
||||
run_case "a missing bd is not a pass" 3 'is not on PATH'
|
||||
|
||||
# The live repo, with the fake bd listing nothing: if this passed, a real
|
||||
# audit whose bd fell silent would pass too.
|
||||
(cd "$here/.." && FAKE_BD_OUT="$work/none.json" "$AUDIT") > "$work/out" 2>&1
|
||||
got=$?
|
||||
[ "$got" = 3 ] && ok "the real tree does not pass against a silent bd" \
|
||||
|| bad "the real tree does not pass against a silent bd" "exit $got"
|
||||
|
||||
"$AUDIT" --help > "$work/out" 2>&1
|
||||
got=$?
|
||||
[ "$got" = 4 ] && grep -q 'POSITIVE' "$work/out" \
|
||||
&& ok "--help is exit 4 and states the positive comparison" \
|
||||
|| bad "--help is exit 4 and states the positive comparison" "exit $got"
|
||||
|
||||
printf '\nbd-audit-ids-test: %d passed, %d failed\n' "$pass" "$fail"
|
||||
[ "$fail" -eq 0 ]
|
||||
Reference in New Issue
Block a user