fix: scan harness/, and make the orphan check real rather than claimed
Both findings from the review on #33, and the second is the sharper one. The checker skipped harness/opfs-contract/src/lib.rs — a shipped Rust source tree with 14 runs over the limit, including a 22-line module doc — so it could have reported success while the convention was violated. harness/ is now in the default roots. And design-principles claimed a renamed test was made detectable because 'the checker looks for both halves'. It did not: it counted comment lines and listed doctest fences, and nothing read context/test-evidence/ at all. So the cost the convention says is controlled was uncontrolled, in the same commit that introduced the convention — a reviewed paragraph is not evidence, and this one was mine. --check-evidence now requires every '### name' heading to name a test that exists. One direction only: a test may have no evidence (most do not), but evidence naming a test nobody can find is a dangling explanation. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -51,14 +51,64 @@ def rust_files(roots):
|
||||
yield os.path.join(dirpath, f)
|
||||
|
||||
|
||||
EVIDENCE_DIR = os.path.join("context", "test-evidence")
|
||||
|
||||
|
||||
def check_evidence(roots):
|
||||
"""Every `### name` in context/test-evidence/ must name a test that exists.
|
||||
|
||||
One direction only, deliberately: a test may have no evidence (most do
|
||||
not), but evidence naming a test nobody can find is an orphan -- which is
|
||||
the cost design-principles names for splitting the narrative from the test.
|
||||
"""
|
||||
if not os.path.isdir(EVIDENCE_DIR):
|
||||
print("no context/test-evidence/ yet: nothing to check")
|
||||
return 0
|
||||
|
||||
claimed = {}
|
||||
for dirpath, _, filenames in os.walk(EVIDENCE_DIR):
|
||||
for f in sorted(filenames):
|
||||
if not f.endswith(".md"):
|
||||
continue
|
||||
path = os.path.join(dirpath, f)
|
||||
for n, raw in enumerate(open(path, encoding="utf8", errors="replace"), 1):
|
||||
s = raw.strip()
|
||||
if s.startswith("### "):
|
||||
claimed.setdefault(s[4:].strip().strip("`"), []).append(f"{path}:{n}")
|
||||
|
||||
defined = set()
|
||||
for path in rust_files(roots):
|
||||
for raw in open(path, encoding="utf8", errors="replace"):
|
||||
s = raw.strip()
|
||||
if s.startswith("fn ") or s.startswith("pub fn ") or s.startswith("async fn "):
|
||||
name = s.split("fn ", 1)[1].split("(")[0].split("<")[0].strip()
|
||||
defined.add(name)
|
||||
|
||||
orphans = {k: v for k, v in claimed.items() if k not in defined}
|
||||
if not orphans:
|
||||
print(f"test-evidence: {len(claimed)} entries, all name a test that exists")
|
||||
return 0
|
||||
print(f"{len(orphans)} test-evidence entries name no test:\n", file=sys.stderr)
|
||||
for name, where in sorted(orphans.items()):
|
||||
print(f" {name} ({', '.join(where)})", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("roots", nargs="*", default=["src", "crates", "tests", "examples"])
|
||||
ap.add_argument("roots", nargs="*",
|
||||
default=["src", "crates", "tests", "examples", "harness"])
|
||||
ap.add_argument("--limit", type=int, default=LIMIT)
|
||||
ap.add_argument("--list-doctests", action="store_true",
|
||||
help="list doc comments containing a ``` fence and exit")
|
||||
ap.add_argument("--check-evidence", action="store_true",
|
||||
help="every context/test-evidence heading must name a test that exists")
|
||||
args = ap.parse_args()
|
||||
roots = [r for r in (args.roots or ["src", "crates", "tests", "examples"]) if os.path.isdir(r)]
|
||||
default_roots = ["src", "crates", "tests", "examples", "harness"]
|
||||
roots = [r for r in (args.roots or default_roots) if os.path.isdir(r)]
|
||||
|
||||
if args.check_evidence:
|
||||
return check_evidence(roots)
|
||||
|
||||
if args.list_doctests:
|
||||
for path in sorted(rust_files(roots)):
|
||||
|
||||
Reference in New Issue
Block a user