Make --check-evidence additive, so no clean exit skips the comment cap
scripts/check-comment-runs.py early-returned on args.check_evidence, so the flagged run answered the evidence question and said nothing at all about the comment-run cap. The flagged form is the expensive one -- it needs a compiled workspace -- which makes it the one a human runs alone; three agents hit this independently in one campaign, and in one case a local gate went green while CI went red on a three-line doc comment (improvise-ijql). The cap moves into check_cap(), and --check-evidence now runs it first and returns the worse of the two verdicts. The ticket's cheaper alternative -- keep the early return and print a line saying the cap did not run -- is declined and the reason is recorded: a provenance line qualifies an answer the run gave, where a disclaimer qualifies one it did not, while still exiting 0. Exit 0 is what CI branches on and what a human reads as clean, so the rule is stated over the exit code: a tree with a long comment run must not pass any invocation that can exit 0. The guard is _dispatch_self_test, which runs the script as a subprocess against a temporary tree -- the only level at which this is visible, since every unit in the file was correct and the defect was in main()'s dispatch. Both arms were demonstrated by injection: restoring the early return fails "DISPATCH: --check-evidence: exited 0 over a four-line comment run", and forcing the cap to always report a violation fails the clean arm for both invocations, which is what stops an always-fail from satisfying the first. Measured: the cap adds 0.29s to a 6.5s --check-evidence run. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FC2jVn8Au7E9JPkw6bwXXH
This commit is contained in:
@@ -24,6 +24,7 @@ import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
LIMIT = 2
|
||||
|
||||
@@ -521,6 +522,29 @@ def check_evidence(roots, rust_listing=None, rust_ignored=None):
|
||||
return 1
|
||||
|
||||
|
||||
def check_cap(roots, limit):
|
||||
"""The comment-run cap. Every invocation that can exit 0 runs this.
|
||||
|
||||
Why no flag selects it away: context/code/tooling.md §2, "A flag adds".
|
||||
"""
|
||||
violations = []
|
||||
for path in sorted(rust_files(roots)):
|
||||
lines = open(path, encoding="utf8", errors="replace").read().split("\n")
|
||||
for start, length in comment_runs(lines):
|
||||
if length > limit:
|
||||
violations.append((path, start, length))
|
||||
|
||||
if not violations:
|
||||
print(f"comment runs: none longer than {limit} lines")
|
||||
return 0
|
||||
|
||||
worst = sorted(violations, key=lambda v: -v[2])
|
||||
print(f"{len(violations)} comment runs longer than {limit} lines:\n", file=sys.stderr)
|
||||
for path, start, length in worst:
|
||||
print(f" {path}:{start} {length} lines", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
SELF_TEST_CASES = [
|
||||
("a plain run of three", 3, """
|
||||
// one
|
||||
@@ -941,6 +965,50 @@ def _evidence_self_test():
|
||||
return failures
|
||||
|
||||
|
||||
SELF_TEST_DISPATCH = [
|
||||
("bare", []),
|
||||
("--check-evidence", ["--check-evidence",
|
||||
"--rust-tests", "listing.txt",
|
||||
"--rust-tests-ignored", "ignored.txt"]),
|
||||
]
|
||||
|
||||
_DISPATCH_LISTING = "\nrunning 1 test\na::b::c: test\n\n1 test, 0 benchmarks\n"
|
||||
_DISPATCH_IGNORED = "\nrunning 0 tests\n\n0 tests, 0 benchmarks\n"
|
||||
_DISPATCH_CAPPED = "// one\n// two\n// three\n// four\nfn x() {}\n"
|
||||
_DISPATCH_CLEAN = "// one\n// two\nfn x() {}\n"
|
||||
|
||||
|
||||
def _dispatch_case(tmp, body, extra):
|
||||
os.makedirs(os.path.join(tmp, "src"), exist_ok=True)
|
||||
open(os.path.join(tmp, "src", "f.rs"), "w", encoding="utf8").write(body)
|
||||
open(os.path.join(tmp, "listing.txt"), "w", encoding="utf8").write(_DISPATCH_LISTING)
|
||||
open(os.path.join(tmp, "ignored.txt"), "w", encoding="utf8").write(_DISPATCH_IGNORED)
|
||||
cmd = [sys.executable, os.path.abspath(__file__), "src"] + extra
|
||||
return subprocess.run(cmd, cwd=tmp, capture_output=True, text=True)
|
||||
|
||||
|
||||
def _dispatch_self_test():
|
||||
"""Every invocation that can exit 0 answers the cap (improvise-ijql).
|
||||
|
||||
The fault is a four-line run; the clean arm stops an always-fail passing.
|
||||
See context/code/tooling.md §2, "A flag adds a question".
|
||||
"""
|
||||
failures = []
|
||||
for name, extra in SELF_TEST_DISPATCH:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run = _dispatch_case(tmp, _DISPATCH_CAPPED, extra)
|
||||
if run.returncode == 0:
|
||||
failures.append(f" DISPATCH: {name}: exited 0 over a four-line comment run")
|
||||
elif "f.rs" not in run.stderr:
|
||||
failures.append(f" DISPATCH: {name}: failed without naming the file")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
run = _dispatch_case(tmp, _DISPATCH_CLEAN, extra)
|
||||
if run.returncode != 0:
|
||||
failures.append(f" DISPATCH: {name}: refused a clean tree "
|
||||
f"({run.returncode}): {run.stderr.strip()[:200]}")
|
||||
return failures
|
||||
|
||||
|
||||
def self_test():
|
||||
"""Check the checker against every bypass it has already been fixed for.
|
||||
|
||||
@@ -958,6 +1026,7 @@ def self_test():
|
||||
if runs:
|
||||
failures.append(f" FALSE POSITIVE: {name}: flagged {runs}")
|
||||
failures += _evidence_self_test()
|
||||
failures += _dispatch_self_test()
|
||||
if failures:
|
||||
print("checker self-test FAILED:", file=sys.stderr)
|
||||
print("\n".join(failures), file=sys.stderr)
|
||||
@@ -966,7 +1035,8 @@ def self_test():
|
||||
f"{len(SELF_TEST_CLEAN)} legitimate shapes allowed, "
|
||||
f"{len(SELF_TEST_EVIDENCE)} evidence verdicts, "
|
||||
f"{len(SELF_TEST_LISTING) + len(SELF_TEST_HEADINGS)} listing readings, "
|
||||
f"{len(SELF_TEST_AMBIGUITY)} name resolutions")
|
||||
f"{len(SELF_TEST_AMBIGUITY)} name resolutions, "
|
||||
f"{len(SELF_TEST_DISPATCH)} invocations answer the cap")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -1012,7 +1082,10 @@ def main():
|
||||
print("error: --rust-tests and --rust-tests-ignored come together; one alone "
|
||||
"silently drops the ignored-test rule", file=sys.stderr)
|
||||
return 2
|
||||
return check_evidence(roots, args.rust_tests, args.rust_tests_ignored)
|
||||
# Additive, not selective: this flag used to early-return, so the
|
||||
# expensive form a human runs alone exited 0 over a capped file.
|
||||
capped = check_cap(roots, args.limit)
|
||||
return max(capped, check_evidence(roots, args.rust_tests, args.rust_tests_ignored))
|
||||
|
||||
if args.list_doctests:
|
||||
for path in sorted(rust_files(roots)):
|
||||
@@ -1022,22 +1095,7 @@ def main():
|
||||
print(f"{path}:{n}")
|
||||
return 0
|
||||
|
||||
violations = []
|
||||
for path in sorted(rust_files(roots)):
|
||||
lines = open(path, encoding="utf8", errors="replace").read().split("\n")
|
||||
for start, length in comment_runs(lines):
|
||||
if length > args.limit:
|
||||
violations.append((path, start, length))
|
||||
|
||||
if not violations:
|
||||
print(f"comment runs: none longer than {args.limit} lines")
|
||||
return 0
|
||||
|
||||
worst = sorted(violations, key=lambda v: -v[2])
|
||||
print(f"{len(violations)} comment runs longer than {args.limit} lines:\n", file=sys.stderr)
|
||||
for path, start, length in worst:
|
||||
print(f" {path}:{start} {length} lines", file=sys.stderr)
|
||||
return 1
|
||||
return check_cap(roots, args.limit)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user