Edward Langley
338d0287d2
Seventh silent success, found by CC-command on PR #41, and the THIRD instance in this campaign of one class: matching a substring where an exact token was meant. 'test' in s also matched '#[cfg(test)]', so every cfg-gated fixture was accepted as a test by --check-evidence — a gate written to reject exactly those. Two plain fixtures cleared it: Host::model and Sessions::apply_caught_up. The first instance was the bd dead-ID audit's unanchored exclusion list, where a real ticket improvise-gp12 would have been silently excluded because it starts with a crate name. The second was --check-evidence accepting any fn. This is the third. Now matched by exact attribute path (test, tokio::test, async_std::test, wasm_bindgen_test). Fault-injected: a #[cfg(test)] fn under a ### heading is reported as an orphan, a #[test] fn is not. Re-ran over the five merged units: 215 entries still pass, so none of them relied on the hole. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
239 lines
10 KiB
Python
Executable File
239 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Fail if any .rs file carries more than two consecutive comment lines.
|
|
|
|
The rule (user, 2026-09-20): reasoning lives in context/ prose, not in code.
|
|
A comment run is at most two lines, optionally naming a context/ file for the
|
|
rest. See context/code/ for where the prose went, and improvise-9c8c for why.
|
|
|
|
Counts `///`, `//!` and `//` alike. Attributes are not comments and are
|
|
ignored.
|
|
|
|
A ``` fence SUSPENDS the count rather than ending the run: the fenced body is
|
|
exempt (a doctest is a running test, not prose) but the prose on either side
|
|
belongs to ONE run and is summed. Two lines above a fence and two below is a
|
|
run of four, which is over the limit -- and that is deliberate, because a
|
|
reader meets it as one uninterrupted comment block. See --list-doctests.
|
|
|
|
Note rustdoc executes a bare ``` or ```rust fence and does NOT execute
|
|
```text, ```ignore or ```no_run. Only a runnable fence earns the exemption;
|
|
the rest is prose wearing a fence and the cap applies to it.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
|
|
LIMIT = 2
|
|
|
|
|
|
def comment_runs(lines):
|
|
"""Yield (start_line_number, length) for each run, skipping runnable doctests.
|
|
|
|
Fence state and exemption are TWO facts and must be tracked separately.
|
|
Deciding both from one flag meant a non-runnable fence's bare closing ```
|
|
looked runnable and opened an exemption, so prose after a ```text block
|
|
evaded the cap entirely (found by review on PR #42 -- the sixth silent
|
|
success in this file, and the one introduced by fixing the fourth).
|
|
"""
|
|
run_start = None
|
|
run_len = 0
|
|
in_fence = False # are we between an opening and a closing fence?
|
|
fence_exempt = False # is THIS fence's body a doctest rustdoc will run?
|
|
for n, raw in enumerate(lines, 1):
|
|
s = raw.strip()
|
|
is_comment = s.startswith("//")
|
|
if is_comment and s.lstrip("/!").strip().startswith("```"):
|
|
if in_fence:
|
|
# The DELIMITERS belong to the block they close, so a runnable
|
|
# fence's closing ``` is exempt too -- otherwise it lands in the
|
|
# prose run and a two-line doc reads as three.
|
|
exempt_here = fence_exempt
|
|
in_fence = False
|
|
fence_exempt = False
|
|
else:
|
|
info = s.lstrip("/!").strip()[3:].strip().lower()
|
|
# rustdoc runs a bare or `rust` fence and does not run `text`,
|
|
# `ignore` or `no_run`. Only a runnable body earns the exemption.
|
|
in_fence = True
|
|
fence_exempt = info in ("", "rust") or info.startswith("rust,")
|
|
exempt_here = fence_exempt
|
|
if exempt_here:
|
|
continue
|
|
# A non-runnable fence line is prose and counts like any other.
|
|
if run_start is None:
|
|
run_start, run_len = n, 0
|
|
run_len += 1
|
|
continue
|
|
if is_comment and in_fence and fence_exempt:
|
|
continue
|
|
if is_comment:
|
|
if run_start is None:
|
|
run_start, run_len = n, 0
|
|
run_len += 1
|
|
else:
|
|
if run_start is not None and run_len > 0:
|
|
yield run_start, run_len
|
|
run_start, run_len = None, 0
|
|
if run_start is not None and run_len > 0:
|
|
yield run_start, run_len
|
|
|
|
|
|
def rust_files(roots):
|
|
for root in roots:
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
dirnames[:] = [d for d in dirnames if d != "target"]
|
|
for f in filenames:
|
|
if f.endswith(".rs"):
|
|
yield os.path.join(dirpath, f)
|
|
|
|
|
|
_TEST_ATTRS = ("test", "tokio::test", "async_std::test", "wasm_bindgen_test")
|
|
|
|
|
|
def _is_test_attribute(line):
|
|
"""True only for an attribute that actually marks a test.
|
|
|
|
Matched by exact path, never by substring: `#[cfg(test)]` contains "test"
|
|
and marks a compilation condition, not a test.
|
|
"""
|
|
body = line.strip()
|
|
if not body.startswith("#["):
|
|
return False
|
|
body = body[2:].rstrip()
|
|
body = body[:-1] if body.endswith("]") else body
|
|
return body.strip().strip(":") in _TEST_ATTRS
|
|
|
|
|
|
EVIDENCE_DIR = os.path.join("context", "test-evidence")
|
|
|
|
|
|
def check_evidence(roots):
|
|
"""Every `### name` in context/test-evidence/ must name a test that exists.
|
|
|
|
One direction only, deliberately: a test may have no evidence (most do
|
|
not), but evidence naming a test nobody can find is an orphan -- which is
|
|
the cost design-principles names for splitting the narrative from the test.
|
|
"""
|
|
if not os.path.isdir(EVIDENCE_DIR):
|
|
print("no context/test-evidence/ yet: nothing to check")
|
|
return 0
|
|
|
|
claimed = {}
|
|
for dirpath, _, filenames in os.walk(EVIDENCE_DIR):
|
|
for f in sorted(filenames):
|
|
if not f.endswith(".md"):
|
|
continue
|
|
path = os.path.join(dirpath, f)
|
|
for n, raw in enumerate(open(path, encoding="utf8", errors="replace"), 1):
|
|
s = raw.strip()
|
|
if s.startswith("### "):
|
|
claimed.setdefault(s[4:].strip().strip("`"), []).append(f"{path}:{n}")
|
|
|
|
# A heading must name a TEST, not merely a function. Accepting any `fn`
|
|
# let a `###` naming a fixture or a helper pass, so an evidence file could
|
|
# report itself satisfied while naming nothing that runs (found on PR #39).
|
|
defined = set()
|
|
for path in rust_files(roots):
|
|
lines = open(path, encoding="utf8", errors="replace").read().split("\n")
|
|
annotated = False
|
|
# A `proptest!` block accepts its bodies as tests -- but only INSIDE the
|
|
# block. A bare flag was never cleared, so the first proptest! in a file
|
|
# made every later fn, helpers included, count as a test: a silent
|
|
# success in the gate itself (found by review on PR #40). Track the
|
|
# block's braces and leave when they balance.
|
|
proptest_depth = None
|
|
for raw in lines:
|
|
s = raw.strip()
|
|
if proptest_depth is not None:
|
|
proptest_depth += raw.count("{") - raw.count("}")
|
|
if proptest_depth <= 0:
|
|
proptest_depth = None
|
|
elif s.startswith("proptest!"):
|
|
proptest_depth = raw.count("{") - raw.count("}")
|
|
if proptest_depth <= 0:
|
|
proptest_depth = None if "{" in raw else 0
|
|
in_proptest = proptest_depth is not None
|
|
# ANCHOR the attribute. `"test" in s` also matched `#[cfg(test)]`,
|
|
# so every cfg-gated fixture cleared a gate written to reject
|
|
# exactly those -- the third instance of the unanchored-substring
|
|
# class in this campaign, after the bd audit's exclusion list.
|
|
if s.startswith("#[") and _is_test_attribute(s):
|
|
annotated = True
|
|
continue
|
|
if s.startswith("fn ") or s.startswith("pub fn ") or s.startswith("async fn "):
|
|
if annotated or in_proptest:
|
|
name = s.split("fn ", 1)[1].split("(")[0].split("<")[0].strip()
|
|
defined.add(name)
|
|
annotated = False
|
|
elif s and not s.startswith("#[") and not s.startswith("//"):
|
|
annotated = False
|
|
|
|
orphans = {k: v for k, v in claimed.items() if k not in defined}
|
|
if not orphans:
|
|
print(f"test-evidence: {len(claimed)} entries, all name a test that exists")
|
|
return 0
|
|
print(f"{len(orphans)} test-evidence entries name no test:\n", file=sys.stderr)
|
|
for name, where in sorted(orphans.items()):
|
|
print(f" {name} ({', '.join(where)})", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("roots", nargs="*",
|
|
default=["src", "crates", "tests", "examples", "harness"])
|
|
ap.add_argument("--limit", type=int, default=LIMIT)
|
|
ap.add_argument("--list-doctests", action="store_true",
|
|
help="list doc comments containing a ``` fence and exit")
|
|
ap.add_argument("--check-evidence", action="store_true",
|
|
help="every context/test-evidence heading must name a test that exists")
|
|
args = ap.parse_args()
|
|
default_roots = ["src", "crates", "tests", "examples", "harness"]
|
|
requested = args.roots or default_roots
|
|
# A path that is not a directory must FAIL. Filtering it away made
|
|
# `check-comment-runs.py some/file.rs` walk nothing and print success --
|
|
# a filter that over-matches fails in the direction that looks like
|
|
# passing, which is the one direction a checker must not fail in.
|
|
roots = [r for r in requested if os.path.isdir(r)]
|
|
missing = [r for r in requested if not os.path.isdir(r)]
|
|
if missing and args.roots:
|
|
for r in missing:
|
|
kind = "not a directory" if os.path.exists(r) else "does not exist"
|
|
print(f"error: {r}: {kind} (pass directories, not files)", file=sys.stderr)
|
|
return 2
|
|
if not roots:
|
|
print("error: nothing to scan", file=sys.stderr)
|
|
return 2
|
|
|
|
if args.check_evidence:
|
|
return check_evidence(roots)
|
|
|
|
if args.list_doctests:
|
|
for path in sorted(rust_files(roots)):
|
|
for n, raw in enumerate(open(path, encoding="utf8", errors="replace"), 1):
|
|
s = raw.strip()
|
|
if s.startswith(("///", "//!")) and s.lstrip("/!").strip().startswith("```"):
|
|
print(f"{path}:{n}")
|
|
return 0
|
|
|
|
violations = []
|
|
for path in sorted(rust_files(roots)):
|
|
lines = open(path, encoding="utf8", errors="replace").read().split("\n")
|
|
for start, length in comment_runs(lines):
|
|
if length > args.limit:
|
|
violations.append((path, start, length))
|
|
|
|
if not violations:
|
|
print(f"comment runs: none longer than {args.limit} lines")
|
|
return 0
|
|
|
|
worst = sorted(violations, key=lambda v: -v[2])
|
|
print(f"{len(violations)} comment runs longer than {args.limit} lines:\n", file=sys.stderr)
|
|
for path, start, length in worst:
|
|
print(f" {path}:{start} {length} lines", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|