Edward Langley
3405fc4def
loadProgram REPLACES a whole predicate, so adding one clause meant re-sending every existing clause of it through an agent's context window. Measured on the wave-3 batch: ~113KB read and ~113KB written to deliver ~25KB of new text. That sets the behaviour -- writing becomes a thing you batch and do rarely -- and the batch is where a short payload silently drops the rest of a predicate, which has cost this repo 22 decisions and 19 doctrines once already. scripts/kb reads clauses as DATA and never consults them, so the cost of adding a clause is the clause. Canonical form is read_term in, portray_clause out -- measured, not designed: all 495 existing facts round-trip byte-identically and the form is a fixed point in one pass. Rules move to one line, which is why the reformat lands here rather than earlier: the MCP would re-expand them on its next save and the two would thrash. trealla moves into the devShell. Reaching it through `nix shell` per invocation took the tool's first real use past a two-minute timeout for a write that takes 0.42s with tpl on PATH. Four guards, each added because injecting its fault showed the tool accepting what it should refuse. None was found by reading: - an unparseable delta dropped trealla into its REPL, which HANGS with a terminal and EXITS 0 without one -- a refusal that reads as acceptance - `-g main,halt` exits 0 on a throw AND on a failure, because a goal that fails never reaches the halt beside it; halt(2) alone exits 2, so the default is the hazard. The goal now guarantees a halt on every path (0/2/1) - `42.` parses, is not callable, and landed as a fact whose head is a number; kb_consistent has no opinion about stray terms - fmt discarded the merge's exit code and would have committed a failed canonicalization Plus the two that were designed in: the result is read back and required to be the clauses intended (variant equality -- == compares variable identity and reports every rule as different), and kb_consistent runs on the CANDIDATE, so a bad delta leaves the knowledge base untouched rather than needing a revert. scripts/kb-test.sh proves all nine by injection. context/code/kb-tool.md carries the argument, including what it does not reach. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FC2jVn8Au7E9JPkw6bwXXH
143 lines
4.6 KiB
Bash
Executable File
143 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check_the_checker for scripts/kb: every refusal is proved by injecting the
|
|
# fault it names, and every refusal must leave the knowledge base UNCHANGED.
|
|
set -uo pipefail
|
|
|
|
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
repo="$(cd "$here/.." && pwd)"
|
|
KB="$repo/prolog-sessions/kb.pl"
|
|
work="$(mktemp -d)"
|
|
trap 'rm -rf "$work"' EXIT
|
|
|
|
pass=0
|
|
fail=0
|
|
ok() { printf ' ok %s\n' "$1"; pass=$((pass+1)); }
|
|
bad() { printf ' FAIL %s\n' "$1"; fail=$((fail+1)); }
|
|
|
|
# Each case runs against its own copy, so a bug cannot reach the real file.
|
|
fresh() { cp "$KB" "$work/kb.pl"; }
|
|
kb() { KB_FILE="$work/kb.pl" "$here/kb" "$@"; }
|
|
unchanged() { diff -q "$KB" "$work/kb.pl" >/dev/null; }
|
|
|
|
echo "kb-test: the refusals"
|
|
|
|
# 1. A duplicate key is refused, and nothing is written.
|
|
fresh
|
|
cat > "$work/dup.pl" <<'EOF'
|
|
doctrine(bug_first,'a second, contradicting text for a name that already exists').
|
|
EOF
|
|
if kb add "$work/dup.pl" >/dev/null 2>&1; then
|
|
bad "add accepted a duplicate key"
|
|
elif unchanged; then
|
|
ok "add refuses a duplicate key and leaves the kb unchanged"
|
|
else
|
|
bad "add refused a duplicate key but MODIFIED the kb"
|
|
fi
|
|
|
|
# 2. replace is the door that duplicate goes through, and it replaces rather
|
|
# than appending -- a count that grew would be two clauses for one name.
|
|
fresh
|
|
before="$(kb count | grep '^doctrine/2=')"
|
|
if kb replace "$work/dup.pl" >/dev/null 2>&1; then
|
|
after="$(kb count | grep '^doctrine/2=')"
|
|
if [ "$before" = "$after" ]; then
|
|
ok "replace overwrites in place ($after)"
|
|
else
|
|
bad "replace changed the clause count: $before -> $after"
|
|
fi
|
|
else
|
|
bad "replace refused a duplicate key"
|
|
fi
|
|
|
|
# 3. A delta that does not parse is refused.
|
|
fresh
|
|
printf 'doctrine(broken, unterminated\n' > "$work/bad.pl"
|
|
if kb add "$work/bad.pl" >/dev/null 2>&1; then
|
|
bad "add accepted an unparseable delta"
|
|
elif unchanged; then
|
|
ok "add refuses an unparseable delta and leaves the kb unchanged"
|
|
else
|
|
bad "add refused an unparseable delta but MODIFIED the kb"
|
|
fi
|
|
|
|
# 4. A delta that parses but breaks kb_consistent is refused. This is the one
|
|
# that matters: the candidate is checked BEFORE it becomes the kb.
|
|
fresh
|
|
cat > "$work/illegal.pl" <<'EOF'
|
|
crate_depends_on(improvise_core,improvise_command).
|
|
EOF
|
|
if kb add "$work/illegal.pl" >/dev/null 2>&1; then
|
|
bad "add accepted a delta that breaks kb_consistent"
|
|
elif unchanged; then
|
|
ok "add refuses a kb_consistent violation and leaves the kb unchanged"
|
|
else
|
|
bad "add refused a violation but MODIFIED the kb"
|
|
fi
|
|
|
|
# 4b. A delta trealla cannot even take apart must refuse, not crash into a
|
|
# success. `-g main,halt` exits 0 on a throw, so this is the guard for it.
|
|
fresh
|
|
printf '42.\n' > "$work/notaclause.pl"
|
|
if kb add "$work/notaclause.pl" >/dev/null 2>&1; then
|
|
bad "add accepted a term that is not a clause"
|
|
elif unchanged; then
|
|
ok "add refuses a non-clause term and leaves the kb unchanged"
|
|
else
|
|
bad "add refused a non-clause term but MODIFIED the kb"
|
|
fi
|
|
|
|
echo "kb-test: the happy paths"
|
|
|
|
# 5. A genuinely new clause lands, and the count moves by exactly one.
|
|
fresh
|
|
cat > "$work/new.pl" <<'EOF'
|
|
doctrine(a_name_that_is_certainly_not_taken,'text for a brand new doctrine').
|
|
EOF
|
|
before="$(kb count | grep '^doctrine/2=' | cut -d= -f2)"
|
|
if kb add "$work/new.pl" >/dev/null 2>&1; then
|
|
after="$(kb count | grep '^doctrine/2=' | cut -d= -f2)"
|
|
if [ "$((before + 1))" = "$after" ]; then
|
|
ok "add lands one new clause ($before -> $after)"
|
|
else
|
|
bad "add moved the count wrongly: $before -> $after"
|
|
fi
|
|
else
|
|
bad "add refused a valid new clause"
|
|
fi
|
|
|
|
# 6. The new clause is readable back as itself -- a count can move without the
|
|
# content being what was asked for.
|
|
if KB_FILE="$work/kb.pl" "$here/kb" query \
|
|
"doctrine(a_name_that_is_certainly_not_taken,T), T == 'text for a brand new doctrine'" \
|
|
>/dev/null 2>&1; then
|
|
ok "the added clause reads back with the text it was given"
|
|
else
|
|
bad "the added clause is not readable back as itself"
|
|
fi
|
|
|
|
# 7. Canonical form is a fixed point: a second fmt changes nothing.
|
|
fresh
|
|
kb fmt >/dev/null 2>&1
|
|
cp "$work/kb.pl" "$work/once.pl"
|
|
kb fmt >/dev/null 2>&1
|
|
if diff -q "$work/once.pl" "$work/kb.pl" >/dev/null; then
|
|
ok "fmt is idempotent"
|
|
else
|
|
bad "fmt is not idempotent -- every add would churn the file"
|
|
fi
|
|
|
|
# 8. Canonicalizing loses no clause. A serializer that drops a predicate is the
|
|
# wipe-the-kb failure, so the count is checked across the reformat.
|
|
fresh
|
|
b="$(kb count)"
|
|
kb fmt >/dev/null 2>&1
|
|
a="$(kb count)"
|
|
if [ "$b" = "$a" ]; then
|
|
ok "fmt preserves every clause count"
|
|
else
|
|
bad "fmt changed the counts"
|
|
fi
|
|
|
|
printf '\nkb-test: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|