Make --check-evidence additive, so no clean exit skips the comment cap

scripts/check-comment-runs.py early-returned on args.check_evidence, so the
flagged run answered the evidence question and said nothing at all about the
comment-run cap. The flagged form is the expensive one -- it needs a compiled
workspace -- which makes it the one a human runs alone; three agents hit this
independently in one campaign, and in one case a local gate went green while
CI went red on a three-line doc comment (improvise-ijql).

The cap moves into check_cap(), and --check-evidence now runs it first and
returns the worse of the two verdicts. The ticket's cheaper alternative -- keep
the early return and print a line saying the cap did not run -- is declined and
the reason is recorded: a provenance line qualifies an answer the run gave,
where a disclaimer qualifies one it did not, while still exiting 0. Exit 0 is
what CI branches on and what a human reads as clean, so the rule is stated over
the exit code: a tree with a long comment run must not pass any invocation that
can exit 0.

The guard is _dispatch_self_test, which runs the script as a subprocess against
a temporary tree -- the only level at which this is visible, since every unit in
the file was correct and the defect was in main()'s dispatch. Both arms were
demonstrated by injection: restoring the early return fails
"DISPATCH: --check-evidence: exited 0 over a four-line comment run", and forcing
the cap to always report a violation fails the clean arm for both invocations,
which is what stops an always-fail from satisfying the first.

Measured: the cap adds 0.29s to a 6.5s --check-evidence run.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FC2jVn8Au7E9JPkw6bwXXH
This commit is contained in:
Edward Langley
2026-09-20 22:14:39 -07:00
parent bc35ac51c1
commit da564dc4c1
3 changed files with 155 additions and 18 deletions
+76 -18
View File
@@ -24,6 +24,7 @@ import os
import re
import subprocess
import sys
import tempfile
LIMIT = 2
@@ -521,6 +522,29 @@ def check_evidence(roots, rust_listing=None, rust_ignored=None):
return 1
def check_cap(roots, limit):
"""The comment-run cap. Every invocation that can exit 0 runs this.
Why no flag selects it away: context/code/tooling.md §2, "A flag adds".
"""
violations = []
for path in sorted(rust_files(roots)):
lines = open(path, encoding="utf8", errors="replace").read().split("\n")
for start, length in comment_runs(lines):
if length > limit:
violations.append((path, start, length))
if not violations:
print(f"comment runs: none longer than {limit} lines")
return 0
worst = sorted(violations, key=lambda v: -v[2])
print(f"{len(violations)} comment runs longer than {limit} lines:\n", file=sys.stderr)
for path, start, length in worst:
print(f" {path}:{start} {length} lines", file=sys.stderr)
return 1
SELF_TEST_CASES = [
("a plain run of three", 3, """
// one
@@ -941,6 +965,50 @@ def _evidence_self_test():
return failures
SELF_TEST_DISPATCH = [
("bare", []),
("--check-evidence", ["--check-evidence",
"--rust-tests", "listing.txt",
"--rust-tests-ignored", "ignored.txt"]),
]
_DISPATCH_LISTING = "\nrunning 1 test\na::b::c: test\n\n1 test, 0 benchmarks\n"
_DISPATCH_IGNORED = "\nrunning 0 tests\n\n0 tests, 0 benchmarks\n"
_DISPATCH_CAPPED = "// one\n// two\n// three\n// four\nfn x() {}\n"
_DISPATCH_CLEAN = "// one\n// two\nfn x() {}\n"
def _dispatch_case(tmp, body, extra):
os.makedirs(os.path.join(tmp, "src"), exist_ok=True)
open(os.path.join(tmp, "src", "f.rs"), "w", encoding="utf8").write(body)
open(os.path.join(tmp, "listing.txt"), "w", encoding="utf8").write(_DISPATCH_LISTING)
open(os.path.join(tmp, "ignored.txt"), "w", encoding="utf8").write(_DISPATCH_IGNORED)
cmd = [sys.executable, os.path.abspath(__file__), "src"] + extra
return subprocess.run(cmd, cwd=tmp, capture_output=True, text=True)
def _dispatch_self_test():
"""Every invocation that can exit 0 answers the cap (improvise-ijql).
The fault is a four-line run; the clean arm stops an always-fail passing.
See context/code/tooling.md §2, "A flag adds a question".
"""
failures = []
for name, extra in SELF_TEST_DISPATCH:
with tempfile.TemporaryDirectory() as tmp:
run = _dispatch_case(tmp, _DISPATCH_CAPPED, extra)
if run.returncode == 0:
failures.append(f" DISPATCH: {name}: exited 0 over a four-line comment run")
elif "f.rs" not in run.stderr:
failures.append(f" DISPATCH: {name}: failed without naming the file")
with tempfile.TemporaryDirectory() as tmp:
run = _dispatch_case(tmp, _DISPATCH_CLEAN, extra)
if run.returncode != 0:
failures.append(f" DISPATCH: {name}: refused a clean tree "
f"({run.returncode}): {run.stderr.strip()[:200]}")
return failures
def self_test():
"""Check the checker against every bypass it has already been fixed for.
@@ -958,6 +1026,7 @@ def self_test():
if runs:
failures.append(f" FALSE POSITIVE: {name}: flagged {runs}")
failures += _evidence_self_test()
failures += _dispatch_self_test()
if failures:
print("checker self-test FAILED:", file=sys.stderr)
print("\n".join(failures), file=sys.stderr)
@@ -966,7 +1035,8 @@ def self_test():
f"{len(SELF_TEST_CLEAN)} legitimate shapes allowed, "
f"{len(SELF_TEST_EVIDENCE)} evidence verdicts, "
f"{len(SELF_TEST_LISTING) + len(SELF_TEST_HEADINGS)} listing readings, "
f"{len(SELF_TEST_AMBIGUITY)} name resolutions")
f"{len(SELF_TEST_AMBIGUITY)} name resolutions, "
f"{len(SELF_TEST_DISPATCH)} invocations answer the cap")
return 0
@@ -1012,7 +1082,10 @@ def main():
print("error: --rust-tests and --rust-tests-ignored come together; one alone "
"silently drops the ignored-test rule", file=sys.stderr)
return 2
return check_evidence(roots, args.rust_tests, args.rust_tests_ignored)
# Additive, not selective: this flag used to early-return, so the
# expensive form a human runs alone exited 0 over a capped file.
capped = check_cap(roots, args.limit)
return max(capped, check_evidence(roots, args.rust_tests, args.rust_tests_ignored))
if args.list_doctests:
for path in sorted(rust_files(roots)):
@@ -1022,22 +1095,7 @@ def main():
print(f"{path}:{n}")
return 0
violations = []
for path in sorted(rust_files(roots)):
lines = open(path, encoding="utf8", errors="replace").read().split("\n")
for start, length in comment_runs(lines):
if length > args.limit:
violations.append((path, start, length))
if not violations:
print(f"comment runs: none longer than {args.limit} lines")
return 0
worst = sorted(violations, key=lambda v: -v[2])
print(f"{len(violations)} comment runs longer than {args.limit} lines:\n", file=sys.stderr)
for path, start, length in worst:
print(f" {path}:{start} {length} lines", file=sys.stderr)
return 1
return check_cap(roots, args.limit)
if __name__ == "__main__":