fix: scan harness/, and make the orphan check real rather than claimed

Both findings from the review on #33, and the second is the sharper one.

The checker skipped harness/opfs-contract/src/lib.rs — a shipped Rust source
tree with 14 runs over the limit, including a 22-line module doc — so it could
have reported success while the convention was violated. harness/ is now in the
default roots.

And design-principles claimed a renamed test was made detectable because 'the
checker looks for both halves'. It did not: it counted comment lines and listed
doctest fences, and nothing read context/test-evidence/ at all. So the cost the
convention says is controlled was uncontrolled, in the same commit that
introduced the convention — a reviewed paragraph is not evidence, and this one
was mine.

--check-evidence now requires every '### name' heading to name a test that
exists. One direction only: a test may have no evidence (most do not), but
evidence naming a test nobody can find is a dangling explanation.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Edward Langley
2026-09-20 09:16:41 -07:00
parent 9494c14044
commit f4b89fa5e3
2 changed files with 56 additions and 4 deletions
+52 -2
View File
@@ -51,14 +51,64 @@ def rust_files(roots):
yield os.path.join(dirpath, f)
EVIDENCE_DIR = os.path.join("context", "test-evidence")
def check_evidence(roots):
"""Every `### name` in context/test-evidence/ must name a test that exists.
One direction only, deliberately: a test may have no evidence (most do
not), but evidence naming a test nobody can find is an orphan -- which is
the cost design-principles names for splitting the narrative from the test.
"""
if not os.path.isdir(EVIDENCE_DIR):
print("no context/test-evidence/ yet: nothing to check")
return 0
claimed = {}
for dirpath, _, filenames in os.walk(EVIDENCE_DIR):
for f in sorted(filenames):
if not f.endswith(".md"):
continue
path = os.path.join(dirpath, f)
for n, raw in enumerate(open(path, encoding="utf8", errors="replace"), 1):
s = raw.strip()
if s.startswith("### "):
claimed.setdefault(s[4:].strip().strip("`"), []).append(f"{path}:{n}")
defined = set()
for path in rust_files(roots):
for raw in open(path, encoding="utf8", errors="replace"):
s = raw.strip()
if s.startswith("fn ") or s.startswith("pub fn ") or s.startswith("async fn "):
name = s.split("fn ", 1)[1].split("(")[0].split("<")[0].strip()
defined.add(name)
orphans = {k: v for k, v in claimed.items() if k not in defined}
if not orphans:
print(f"test-evidence: {len(claimed)} entries, all name a test that exists")
return 0
print(f"{len(orphans)} test-evidence entries name no test:\n", file=sys.stderr)
for name, where in sorted(orphans.items()):
print(f" {name} ({', '.join(where)})", file=sys.stderr)
return 1
def main():
ap = argparse.ArgumentParser()
ap.add_argument("roots", nargs="*", default=["src", "crates", "tests", "examples"])
ap.add_argument("roots", nargs="*",
default=["src", "crates", "tests", "examples", "harness"])
ap.add_argument("--limit", type=int, default=LIMIT)
ap.add_argument("--list-doctests", action="store_true",
help="list doc comments containing a ``` fence and exit")
ap.add_argument("--check-evidence", action="store_true",
help="every context/test-evidence heading must name a test that exists")
args = ap.parse_args()
roots = [r for r in (args.roots or ["src", "crates", "tests", "examples"]) if os.path.isdir(r)]
default_roots = ["src", "crates", "tests", "examples", "harness"]
roots = [r for r in (args.roots or default_roots) if os.path.isdir(r)]
if args.check_evidence:
return check_evidence(roots)
if args.list_doctests:
for path in sorted(rust_files(roots)):