#!/usr/bin/env python3 """Fail if any .rs file carries more than two consecutive comment lines. The rule (user, 2026-09-20): reasoning lives in context/ prose, not in code. A comment run is at most two lines, optionally naming a context/ file for the rest. See context/code/ for where the prose went, and improvise-9c8c for why. Counts `///`, `//!` and `//` alike. Attributes are not comments and are ignored. A ``` fence SUSPENDS the count rather than ending the run: the fenced body is exempt (a doctest is a running test, not prose) but the prose on either side belongs to ONE run and is summed. Two lines above a fence and two below is a run of four, which is over the limit -- and that is deliberate, because a reader meets it as one uninterrupted comment block. See --list-doctests. Note rustdoc executes a bare ``` or ```rust fence and does NOT execute ```text, ```ignore or ```no_run. Only a runnable fence earns the exemption; the rest is prose wearing a fence and the cap applies to it. """ import argparse import os import re import subprocess import sys import tempfile LIMIT = 2 _RAW_STR = re.compile(r'(?:b|c)?r(#*)"') _CHAR_LIT = re.compile(r"'(?:\\.|[^'\\])'") _DOC_ATTR = re.compile(r"#!?\[\s*doc\s*=") def scan(lines): """Classify each line, respecting Rust strings and NESTED block comments. Returns per line: has_code, has_comment, the `//` body, the CODE-ONLY text (no string or comment content), whether comment content leads the line, and whether the line began inside a block comment. See context/code/tooling.md. """ out = [] in_block = 0 in_raw = None in_str = False for raw in lines: i, n = 0, len(raw) started_in_block = in_block > 0 has_code = False has_comment = started_in_block body = None code = [] leads = None while i < n: if in_block: has_comment = True leads = True if leads is None else leads if raw.startswith("/*", i): in_block += 1 i += 2 elif raw.startswith("*/", i): in_block -= 1 i += 2 else: i += 1 continue if in_raw is not None: closer = '"' + "#" * in_raw if raw.startswith(closer, i): in_raw = None i += len(closer) else: i += 1 has_code = True leads = False if leads is None else leads continue if in_str: has_code = True leads = False if leads is None else leads if raw[i] == "\\": i += 2 continue if raw[i] == '"': in_str = False i += 1 continue if raw[i].isspace(): i += 1 continue if raw.startswith("//", i): has_comment = True leads = True if leads is None else leads if not has_code: body = raw[i:] break if raw.startswith("/*", i): has_comment = True leads = True if leads is None else leads in_block = 1 i += 2 continue m = _RAW_STR.match(raw, i) if m: in_raw = len(m.group(1)) has_code = True leads = False if leads is None else leads code.append('"') i = m.end() continue m = _CHAR_LIT.match(raw, i) if m: has_code = True leads = False if leads is None else leads i = m.end() continue if raw[i] == '"': in_str = True has_code = True leads = False if leads is None else leads i += 1 continue has_code = True leads = False if leads is None else leads code.append(raw[i]) i += 1 out.append((has_code, has_comment, body, "".join(code), bool(leads), started_in_block)) return out def _kind(body): if body is None: return "//" if body.startswith("//!"): return "//!" return "///" if body.startswith("///") else "//" def comment_runs(lines): """Yield (start_line_number, length) for each run, skipping runnable doctests. Every question here is asked of the SCANNER, never of the raw line: an attribute's bracket depth is counted over code only, because `#[cfg(feature = "[")]` otherwise exempted the whole file. See context/code/tooling.md. """ scanned = scan(lines) run_start, run_len, run_kind = None, 0, None in_fence = fence_exempt = False attr_depth = 0 attr_inner = False inner_pending = False def add(n, kind): nonlocal run_start, run_len, run_kind if run_start is None: run_start, run_len, run_kind = n, 0, kind run_len += 1 for n, (has_code, has_comment, body, code, leads, in_block_at_start) in enumerate(scanned, 1): # A line that CLOSES a block comment carries comment text even when code # follows it; a line that opens one after code does not. is_comment = has_comment and (not has_code or (in_block_at_start and leads)) code_s = code.strip() if attr_depth > 0: attr_depth += code.count("[") - code.count("]") if is_comment: add(n, _kind(body)) if attr_depth <= 0: attr_depth = 0 inner_pending = inner_pending or attr_inner continue if not is_comment and (code_s.startswith("#[") or code_s.startswith("#![")): # `#[doc = "..."]` and `#![doc = "..."]` ARE comments; `#[doc(hidden)]` # is not, which an unanchored `#[doc` prefix could not tell apart. if _DOC_ATTR.match(code_s): add(n, "//!" if code_s.startswith("#![") else "///") continue inner = code_s.startswith("#![") depth = code.count("[") - code.count("]") if depth > 0: attr_depth, attr_inner = depth, inner continue inner_pending = inner_pending or inner continue if is_comment: kind = _kind(body) # An inner attribute ends a run only where the DOC KIND changes: a # crate doc and an item doc are two blocks, `//!` either side is one. if inner_pending: inner_pending = False if run_kind is not None and kind != run_kind and run_start is not None and run_len > 0: yield run_start, run_len run_start, run_len, run_kind = None, 0, None doc = kind in ("///", "//!") text = body.lstrip("/!").strip() if body is not None else "" if doc and text.startswith("```"): if in_fence: exempt_here = fence_exempt in_fence = fence_exempt = False else: info = text[3:].strip().lower() in_fence = True fence_exempt = info in ("", "rust") or info.startswith("rust,") exempt_here = fence_exempt if exempt_here: continue add(n, kind) continue if doc and in_fence and fence_exempt: continue add(n, kind) else: in_fence = fence_exempt = False inner_pending = False if run_start is not None and run_len > 0: yield run_start, run_len run_start, run_len, run_kind = None, 0, None if run_start is not None and run_len > 0: yield run_start, run_len def rust_files(roots): for root in roots: for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [d for d in dirnames if d != "target"] for f in filenames: if f.endswith(".rs"): yield os.path.join(dirpath, f) _TEST_ATTRS = ("test", "tokio::test", "async_std::test", "wasm_bindgen_test") def _is_test_attribute(line): """True only for an attribute that actually marks a test. Matched by exact path, never by substring: `#[cfg(test)]` contains "test" and marks a compilation condition, not a test. """ body = line.strip() if not body.startswith("#["): return False body = body[2:].rstrip() body = body[:-1] if body.endswith("]") else body return body.strip().strip(":") in _TEST_ATTRS EVIDENCE_DIR = os.path.join("context", "test-evidence") EVIDENCE_CITE = "test-evidence" JS_ROOTS = [os.path.join("harness", "tests"), os.path.join("web", "tests")] _RUST_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)*$") _QUOTED_NAME = re.compile(r'^"(.+)"$') # Only a RUNNABLE LEAF test. `.skip`, `.todo`, `.describe` and `.only` are all # excluded: the first three name no executed assertion, and `.only` disables # every OTHER test in its file, so a heading satisfied by one is a heading # satisfied by a file that stopped running (review, PR #69). _JS_TEST = re.compile( r"""(?:^|[^\w.])(?:test|it)\s*\(\s*(['"`])(.+?)\1""" ) def evidence_claims(text): """Yield ("rust"|"js"|"prose", name, lineno) for every heading in a file. `### ` and `#### ` alike, because the browser entries have to use `####` and were invisible to a check that collected only `### ` (improvise-5ray). A heading is a CLAIM when it is shaped like a test name -- a bare identifier (Rust) or a quoted string (JS). Anything else is prose about the section: `Helper - x`, `Note - y`, a file's own scope. That shape rule is the whole discrimination, and rewriting a claim as prose retires it with nothing checking the prose is honest (improvise-9za9). The mechanism is the diff a reviewer reads, not the printed prose count, which nothing stores. See context/code/tooling.md. """ for n, raw in enumerate(text.split("\n"), 1): s = raw.strip() if not (s.startswith("### ") or s.startswith("#### ")): continue body = s.lstrip("#").strip().strip("`").strip() m = _QUOTED_NAME.match(body) if m: yield "js", m.group(1), n elif _RUST_NAME.match(body): yield "rust", body, n else: yield "prose", body, n _CARGO_LISTED = re.compile(r"^(\S+): test$") def cargo_test_names(listing, ignored_listing=""): """The FULL PATHS cargo says it will run, from `cargo test -- --list`. Paths and not leaves: six leaf names collide in this workspace, so a leaf is not an identity. See context/code/tooling.md and improvise-rm4s. """ def paths(text): return { m.group(1) for m in (_CARGO_LISTED.match(line.strip()) for line in text.split("\n")) if m } return paths(listing) - paths(ignored_listing) def _leaf(path): return path.split("::")[-1] def _cargo_listing(extra): cmd = ["cargo", "test", "--workspace", "--", "--list", *extra] try: run = subprocess.run(cmd, capture_output=True, text=True) except OSError as e: raise SystemExit(f"error: cannot run cargo ({e}).\n" "--check-evidence needs a toolchain, or --rust-tests FILE " "holding a saved `cargo test --workspace -- --list`.") if run.returncode != 0: raise SystemExit(f"error: {' '.join(cmd)} failed:\n{run.stderr}") return run.stdout def rust_citations(text): """The tests here whose doc comment cites the evidence dir. A citation obliges only a TEST. Accepting any `fn` let a `###` naming a fixture or a helper pass, so an evidence file could report itself satisfied while naming nothing that runs (found on PR #39). """ citing = set() annotated = False doc = [] # A `proptest!` block accepts its bodies as tests -- but only INSIDE the # block. A bare flag was never cleared, so the first proptest! in a file # made every later fn, helpers included, count as a test: a silent # success in the gate itself (found by review on PR #40). Track the # block's braces and leave when they balance. proptest_depth = None for raw in text.split("\n"): s = raw.strip() if proptest_depth is not None: proptest_depth += raw.count("{") - raw.count("}") if proptest_depth <= 0: proptest_depth = None elif s.startswith("proptest!"): proptest_depth = raw.count("{") - raw.count("}") if proptest_depth <= 0: proptest_depth = None if "{" in raw else 0 in_proptest = proptest_depth is not None if s.startswith("//"): doc.append(s) continue # ANCHOR the attribute. `"test" in s` also matched `#[cfg(test)]`, # so every cfg-gated fixture cleared a gate written to reject # exactly those -- the third instance of the unanchored-substring # class in this campaign, after the bd audit's exclusion list. if s.startswith("#[") and _is_test_attribute(s): annotated = True continue if s.startswith("fn ") or s.startswith("pub fn ") or s.startswith("async fn "): if annotated or in_proptest: name = s.split("fn ", 1)[1].split("(")[0].split("<")[0].strip() if any(EVIDENCE_CITE in d for d in doc): citing.add(name) annotated = False doc = [] elif s and not s.startswith("#["): annotated = False doc = [] return citing def js_tests(text): """(tests defined here, tests whose preceding comment cites the evidence dir). The JS half exists because `defined` held only Rust `fn` names, so a `####` naming a Playwright or node test could never have resolved even if the heading had been collected -- the convention that kept the checker quiet was the same one that made it blind (improvise-5ray). """ defined, citing = set(), set() doc = [] for raw in text.split("\n"): s = raw.strip() m = _JS_TEST.search(raw) if m: defined.add(m.group(2)) if any(EVIDENCE_CITE in d for d in doc): citing.add(m.group(2)) doc = [] continue if s.startswith("//") or s.startswith("*") or s.startswith("/*"): doc.append(s) elif s: doc = [] return defined, citing def js_files(roots): for root in roots: for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [d for d in dirnames if d != "node_modules"] for f in filenames: if f.endswith(".spec.js") or f.endswith(".test.js"): yield os.path.join(dirpath, f) def _duplicate_claims(claims, by_leaf, defined): """Two headings naming ONE test, keyed by the test they resolve to. Resolved and not spelled, so `### f` beside `### m::tests::f` is caught too: the heading is the key a reader looks a test up by, and two entries under one key means the first one found is taken as current. See context/code/tooling.md. """ def canonical(kind, name): if kind == "js" or name in defined["rust"]: return name matches = by_leaf.get(name, set()) return next(iter(matches)) if len(matches) == 1 else name grouped = {} for (kind, name), where in claims.items(): grouped.setdefault((kind, canonical(kind, name)), []).extend(where) def in_file_order(loc): path, _, n = loc.rpartition(":") return (path, int(n) if n.isdigit() else 0) return {key: sorted(where, key=in_file_order) for key, where in grouped.items() if len(where) > 1} def evidence_verdict(claims, defined, citing): """(orphans, unstated, ambiguous, duplicates), keyed by ("rust"|"js", name). `orphans` is evidence naming a test nobody can find. `unstated` is the reverse obligation (improvise-lyyc): a test that POINTS AT the evidence file must appear in it, or a reader following the pointer arrives nowhere. `ambiguous` is a bare leaf that two tests answer to -- a refusal and not a warning, because a heading satisfied by a test in another module answering another question is a pass while the gate does nothing. `duplicates` is the mirror image: two headings answered by one test, where every copy passes and the reader takes whichever comes first as current (improvise-hsoe). Also a refusal, on the improvise-rm4s precedent. Only that narrower reverse obligation, because most tests have no evidence and should not be forced to. What none of the four can see: an entry can exist and still describe its subject backwards. """ by_leaf = {} for path in defined["rust"]: by_leaf.setdefault(_leaf(path), set()).add(path) orphans, ambiguous = {}, {} for key, where in claims.items(): kind, name = key if kind == "js": if name not in defined["js"]: orphans[key] = where continue if name in defined["rust"]: continue matches = by_leaf.get(name, set()) if len(matches) == 1: continue (ambiguous if matches else orphans)[key] = where # A citation is a bare `fn` name, so it is satisfied by a QUALIFIED heading # naming that leaf -- qualifying a heading must not orphan its own test. claimed = {(kind, _leaf(name)) for kind, name in claims} unstated = {} for kind, names in citing.items(): for name in names: if (kind, name) not in claimed: unstated.setdefault((kind, name), []).extend(sorted(names[name])) return orphans, unstated, ambiguous, _duplicate_claims(claims, by_leaf, defined) def check_evidence(roots, rust_listing=None, rust_ignored=None): """Both directions over context/test-evidence/. See evidence_verdict. A MISSING evidence directory is not an early return: it used to be, and it exited 0 while 170 citations pointed at nothing (review of PR #83). """ claims, prose = {}, 0 for dirpath, _, filenames in os.walk(EVIDENCE_DIR): for f in sorted(filenames): if not f.endswith(".md"): continue path = os.path.join(dirpath, f) text = open(path, encoding="utf8", errors="replace").read() for kind, name, n in evidence_claims(text): if kind == "prose": prose += 1 else: claims.setdefault((kind, name), []).append(f"{path}:{n}") if rust_listing is None: source = "cargo" rust_names = cargo_test_names(_cargo_listing([]), _cargo_listing(["--ignored"])) else: source = f"{rust_listing} + {rust_ignored}" rust_names = cargo_test_names(open(rust_listing, encoding="utf8").read(), open(rust_ignored, encoding="utf8").read()) defined = {"rust": rust_names, "js": set()} citing = {"rust": {}, "js": {}} for path in rust_files(roots): for name in rust_citations(open(path, encoding="utf8", errors="replace").read()): citing["rust"].setdefault(name, []).append(path) for path in js_files(JS_ROOTS): d, c = js_tests(open(path, encoding="utf8", errors="replace").read()) defined["js"] |= d for name in c: citing["js"].setdefault(name, []).append(path) orphans, unstated, ambiguous, duplicates = evidence_verdict(claims, defined, citing) if not orphans and not unstated and not ambiguous and not duplicates: print(f"test-evidence: {len(claims)} entries, all name a test that runs " f"and no test is named twice (names from {source}); " f"{sum(len(v) for v in citing.values())} tests cite the file and all have one; " f"{prose} prose headings claim nothing") return 0 if orphans: print(f"{len(orphans)} test-evidence entries name no test:\n", file=sys.stderr) for (kind, name), where in sorted(orphans.items()): print(f" [{kind}] {name} ({', '.join(where)})", file=sys.stderr) if ambiguous: by_leaf = {} for path in defined["rust"]: by_leaf.setdefault(_leaf(path), set()).add(path) print(f"\n{len(ambiguous)} test-evidence entries name a leaf that is not unique; " "write the module path:\n", file=sys.stderr) for (kind, name), where in sorted(ambiguous.items()): print(f" [{kind}] {name} ({', '.join(where)})", file=sys.stderr) for path in sorted(by_leaf[name]): print(f" -> {path}", file=sys.stderr) if duplicates: print(f"\n{len(duplicates)} tests are named by more than one test-evidence " "heading; merge them or give each its own test:\n", file=sys.stderr) for (kind, name), where in sorted(duplicates.items()): print(f" [{kind}] {name} ({', '.join(where)})", file=sys.stderr) if unstated: print(f"\n{len(unstated)} tests cite {EVIDENCE_DIR} and have no entry there:\n", file=sys.stderr) for (kind, name), where in sorted(unstated.items()): print(f" [{kind}] {name} ({', '.join(where)})", file=sys.stderr) return 1 def check_cap(roots, limit): """The comment-run cap. Every invocation that can exit 0 runs this. Why no flag selects it away: context/code/tooling.md §2, "A flag adds". """ violations = [] for path in sorted(rust_files(roots)): lines = open(path, encoding="utf8", errors="replace").read().split("\n") for start, length in comment_runs(lines): if length > limit: violations.append((path, start, length)) if not violations: print(f"comment runs: none longer than {limit} lines") return 0 worst = sorted(violations, key=lambda v: -v[2]) print(f"{len(violations)} comment runs longer than {limit} lines:\n", file=sys.stderr) for path, start, length in worst: print(f" {path}:{start} {length} lines", file=sys.stderr) return 1 SELF_TEST_CASES = [ ("a plain run of three", 3, """ // one // two // three fn f() {} """), ("a block comment of six", 6, """ /* one two three four five */ fn f() {} """), ("a rustdoc block comment of six", 6, """ /** one * two * three * four * five */ fn f() {} """), ("an unclosed fence must not exempt the rest of the file", 6, """ /// opens a fence and never closes it /// ``` /// let x = 1; fn a() {} /// one /// two /// three /// four /// five /// six fn b() {} """), ("an attribute must not split a run", 4, """ /// one /// two #[inline] /// three /// four fn c() {} """), ("a non-runnable fence is prose and counts", 4, """ /// one /// ```text /// two /// ``` fn d() {} """), ("a fence in a PLAIN // comment earns no exemption", 7, """ // ``` // one // two // three // four // five // ``` fn f() {} """), ("a block comment opened after code on the same line", 5, """ fn f() {} /* one two three four five */ """), ("nested block comments do not close early", 7, """ /* /* */ one two three four five */ fn f() {} """), ("a multi-line attribute does not split a run", 4, """ /// one /// two #[cfg(all( unix, target_pointer_width = "64" ))] /// three /// four fn f() {} """), ("an attribute inside a fence does not reopen it", 5, """ /// one /// ``` /// let x = 1; #[inline] /// ``` /// hidden A /// hidden B /// hidden C /// hidden D fn g() {} """), ("#[doc = ...] is the one attribute that is a comment", 3, """ #[doc = "one"] #[doc = "two"] #[doc = "three"] fn f() {} """), ("#![doc = ...] is the inner spelling of the same thing", 5, """ #![doc = "one"] #![doc = "two"] #![doc = "three"] #![doc = "four"] #![doc = "five"] fn f() {} """), ("an attribute holding a bracket in a string exempts nothing", 6, """ #[cfg(feature = "[")] fn g() {} // a // b // c // d // e // f fn f() {} """), ("an inner attribute between two //! blocks does not split them", 4, """ //! one //! two #![allow(clippy::needless_return)] //! three //! four fn f() {} """), ("a block comment closing on a line with code still counts", 3, """ /* one two three */ fn f() {} """), ("a char literal holding a quote does not open a string", 3, """ pub const C: char = '"'; // one // two // three fn f() {} """), ] SELF_TEST_CLEAN = [ ("two lines is the limit", """ // one // two fn f() {} """), ("a runnable doctest is exempt", """ /// one /// ``` /// let x = 1; /// assert_eq!(x, 1); /// ``` /// two fn f() {} """), ("a closed fence does not leak into a later comment", """ /// one /// ``` /// let x = 1; /// ``` fn a() {} // two // three fn b() {} """), ("a block comment inside a raw string is not a comment", """ pub const S: &str = r#"x"y /* not a comment one two three four "#; fn a() {} """), ("// inside a raw string is not a comment", """ pub const S: &str = r#"x"y // not a comment // nor this // nor this // nor this "#; fn a() {} """), ("a crate doc and an item doc are separated by the inner attribute", """ //! crate one //! crate two #![allow(clippy::needless_return)] /// item one /// item two pub fn go() {} """), ("// inside a multi-line plain string is not a comment", """ pub const S: &str = "open \\ // one // two // three close"; fn a() {} """), ("#[doc(hidden)] is an attribute, not a comment", """ /// one /// two #[doc(hidden)] pub fn f() {} """), ("an attribute's brackets close, so later code still ends a run", """ /// one /// two #[cfg(all( unix, ))] fn f() {} /// three /// four fn g() {} """), ] # (name, listing, ignored listing, the paths that RUN). # # Three source shapes defeated the old scrape -- a test in a string literal, one # behind a cfg feature, one behind cfg(target_arch = "wasm32") -- and an ignored # test defeats a listing read without its `--ignored` twin (improvise-rm4s). SELF_TEST_LISTING = [ ("the module path is kept, because a leaf is not an identity", "help::tests::a_real_test: test\n", "", ["help::tests::a_real_test"]), ("an ignored test is one that does not run", "a::runs: test\na::skipped: test\n", "a::skipped: test\n", ["a::runs"]), ("cargo's prose is not a test name", " Running unittests src/lib.rs (target/debug/deps/x-1)\n" "2 tests, 0 benchmarks\n", "", []), ] # (name, evidence markdown, rust source, js source, cargo listing, orphans, unstated). # # The evidence checker had no self-test at all, which is how both halves of # improvise-5ray and improvise-lyyc survived: the shape that silenced the # checker was the shape it could not see. Each case below is an injection. SELF_TEST_EVIDENCE = [ ("a #### heading naming no test is an orphan", '#### "a test that does not exist anywhere"\n', "", "", "", ["a test that does not exist anywhere"], []), ("a #### heading naming a JS test that exists is satisfied", '#### "a peer that cannot be reached is named"\n', "", 'test("a peer that cannot be reached is named", async () => {});\n', "", [], []), ("a quoted heading can never be satisfied by a Rust fn of that name", '#### "a_rust_name"\n', "#[test]\nfn a_rust_name() {}\n", "", "m::a_rust_name: test\n", ["a_rust_name"], []), ("a prose #### heading claims nothing", "#### Helper — `addressed`\n#### Note — the basis_check module\n" "#### harness/tests/shell.spec.js — the file's own scope\n", "", "", "", [], []), ("a ### naming a helper rather than a test is still an orphan", "### a_helper\n", "fn a_helper() {}\n", "", "", ["a_helper"], []), ("a test that cites the evidence file and has no entry is caught", "", "/// See `context/test-evidence/clients.md`.\n#[test]\nfn a_cited_test() {}\n", "", "m::a_cited_test: test\n", [], ["a_cited_test"]), ("the same test with an entry is satisfied", "### a_cited_test\n", "/// See `context/test-evidence/clients.md`.\n#[test]\nfn a_cited_test() {}\n", "", "m::a_cited_test: test\n", [], []), ("a NON-test fn citing the file is under no obligation", "", "/// See `context/test-evidence/clients.md`.\nfn a_helper() {}\n", "", "", [], []), ("a citation belonging to the fn ABOVE does not attach to the next one", "### cited\n", "/// See `context/test-evidence/clients.md`.\n#[test]\nfn cited() {}\n" "\n#[test]\nfn uncited() {}\n", "", "m::cited: test\nm::uncited: test\n", [], []), ("a heading cannot be satisfied by a skipped or todo test", '#### "a skipped one"\n#### "a todo one"\n', "", 'test.skip("a skipped one", () => {});\ntest.todo("a todo one");\n', "", ["a skipped one", "a todo one"], []), ("a heading cannot be satisfied by a test.only, which disables its file", '#### "the only one left running"\n', "", 'test.only("the only one left running", () => {});\n', "", ["the only one left running"], []), ("a heading cannot be satisfied by an empty suite's name", '#### "a suite with nothing in it"\n', "", 'test.describe("a suite with nothing in it", () => {});\n', "", ["a suite with nothing in it"], []), ("a test nested in a suite still satisfies its heading", '#### "a nested test"\n', "", 'test.describe("a group", () => {\n test("a nested test", () => {});\n});\n', "", [], []), ("a JS test that cites the evidence file and has no entry is caught", "", "", '// See context/test-evidence/clients.md.\ntest("a cited spec", async () => {});\n', "", [], ["a cited spec"]), ] # (name, evidence markdown, the kind of each heading in order). # # Prose is a third KIND rather than a heading the collector drops, so retiring a # claim by rewriting it as prose moves the reported count (improvise-9za9). SELF_TEST_HEADINGS = [ ("a heading is classified three ways, prose included", '#### Helper — `addressed`\n### a_rust_name\n#### "a js name"\n', ["prose", "rust", "js"]), ("a module-qualified heading is a Rust claim, not prose", "### ui::project::tests::a_load_that_fails_is_not_a_rebase\n", ["rust"]), ] # (name, evidence markdown, cargo listing, orphans, ambiguous). # # Six leaf names collide in this workspace and two were cited, so a heading was # satisfiable by a test in another module answering another question -- renaming # the one an entry describes left the checker at exit 0 (review of PR #83). SELF_TEST_AMBIGUITY = [ ("a leaf two tests answer to is refused, not silently resolved", "### a_load_that_fails_is_not_a_rebase\n", "ui::host::tests::a_load_that_fails_is_not_a_rebase: test\n" "ui::project::tests::a_load_that_fails_is_not_a_rebase: test\n", [], ["a_load_that_fails_is_not_a_rebase"]), ("qualifying it resolves it, and to the one it names", "### ui::project::tests::a_load_that_fails_is_not_a_rebase\n", "ui::host::tests::a_load_that_fails_is_not_a_rebase: test\n" "ui::project::tests::a_load_that_fails_is_not_a_rebase: test\n", [], []), ("a qualified heading naming a module that does not have it is an orphan", "### ui::nowhere::tests::a_load_that_fails_is_not_a_rebase\n", "ui::host::tests::a_load_that_fails_is_not_a_rebase: test\n", ["ui::nowhere::tests::a_load_that_fails_is_not_a_rebase"], []), ("a unique leaf still resolves unqualified", "### a_lonely_test\n", "m::tests::a_lonely_test: test\n", [], []), ] # (name, evidence markdown, cargo listing, js source, the tests named twice). # # A heading that names a test that RUNS passes -- and a second heading naming # the same one passes just as well, so an entry superseded by its successor # reads as current to whoever meets it first (improvise-hsoe, found on PR #92). # Every case here is an injection; the clean arms stop an always-fail passing. SELF_TEST_DUPLICATES = [ ("one heading twice in one file is refused", "### a_real_test\n### a_real_test\n", "m::tests::a_real_test: test\n", "", ["m::tests::a_real_test"]), ("the qualified and bare spellings of one test are the same heading", "### a_real_test\n### m::tests::a_real_test\n", "m::tests::a_real_test: test\n", "", ["m::tests::a_real_test"]), # A BARE heading, because two qualified ones resolve by the exact-match arm # and never reach the ambiguity guard this case is named for (review, PR #96). ("a bare heading over an ambiguous leaf is not a duplicate of a qualified one", "### shared\n### m::tests::shared\n### n::tests::shared\n", "m::tests::shared: test\nn::tests::shared: test\n", "", []), ("a JS heading twice is refused too", '#### "a spec"\n#### "a spec"\n', "", 'test("a spec", () => {});\n', ["a spec"]), ("two headings naming two tests are not duplicates", "### one\n### two\n", "m::one: test\nm::two: test\n", "", []), ("prose headings repeat freely -- they claim no test", "### Helper - x\n### Helper - x\n", "", "", []), ] def _duplicate_self_test(): failures = [] for name, md, listing, js, want in SELF_TEST_DUPLICATES: claims = {} for kind, cname, n in evidence_claims(md): if kind != "prose": claims.setdefault((kind, cname), []).append(f"fixture:{n}") jd, _ = js_tests(js) defined = {"rust": cargo_test_names(listing), "js": jd} *_, duplicates = evidence_verdict(claims, defined, {"rust": {}, "js": {}}) got = sorted(k[1] for k in duplicates) if got != sorted(want): failures.append(f" DUPLICATES: {name}: expected {sorted(want)}, saw {got}") return failures def _ambiguity_self_test(): failures = _duplicate_self_test() for name, md, listing, want_o, want_a in SELF_TEST_AMBIGUITY: claims = {} for kind, cname, n in evidence_claims(md): if kind != "prose": claims.setdefault((kind, cname), []).append(f"fixture:{n}") defined = {"rust": cargo_test_names(listing), "js": set()} orphans, _, ambiguous, _dupes = evidence_verdict( claims, defined, {"rust": {}, "js": {}}) got_o = sorted(k[1] for k in orphans) got_a = sorted(k[1] for k in ambiguous) if got_o != sorted(want_o): failures.append(f" ORPHANS: {name}: expected {sorted(want_o)}, saw {got_o}") if got_a != sorted(want_a): failures.append(f" AMBIGUOUS: {name}: expected {sorted(want_a)}, saw {got_a}") return failures def _listing_self_test(): failures = [] for name, listing, ignored, want in SELF_TEST_LISTING: got = sorted(cargo_test_names(listing, ignored)) if got != sorted(want): failures.append(f" LISTING: {name}: expected {sorted(want)}, saw {got}") for name, md, want in SELF_TEST_HEADINGS: got = [kind for kind, _, _ in evidence_claims(md)] if got != want: failures.append(f" HEADINGS: {name}: expected {want}, saw {got}") return failures def _evidence_self_test(): failures = _listing_self_test() + _ambiguity_self_test() for name, md, rs, js, listing, want_orphans, want_unstated in SELF_TEST_EVIDENCE: claims = {} for kind, cname, n in evidence_claims(md): if kind != "prose": claims.setdefault((kind, cname), []).append(f"fixture:{n}") rc = rust_citations(rs) jd, jc = js_tests(js) defined = {"rust": cargo_test_names(listing), "js": jd} citing = {"rust": {k: ["fixture.rs"] for k in rc}, "js": {k: ["fixture.js"] for k in jc}} orphans, unstated, ambiguous, duplicates = evidence_verdict(claims, defined, citing) if duplicates: failures.append(f" DUPLICATES: {name}: unexpected {sorted(k[1] for k in duplicates)}") got_o = sorted(k[1] for k in orphans) got_u = sorted(k[1] for k in unstated) if got_o != sorted(want_orphans): failures.append(f" ORPHANS: {name}: expected {sorted(want_orphans)}, saw {got_o}") if got_u != sorted(want_unstated): failures.append(f" UNSTATED: {name}: expected {sorted(want_unstated)}, saw {got_u}") if ambiguous: failures.append(f" AMBIGUOUS: {name}: unexpected {sorted(k[1] for k in ambiguous)}") return failures SELF_TEST_DISPATCH = [ ("bare", []), ("--check-evidence", ["--check-evidence", "--rust-tests", "listing.txt", "--rust-tests-ignored", "ignored.txt"]), ] _DISPATCH_LISTING = "\nrunning 1 test\na::b::c: test\n\n1 test, 0 benchmarks\n" _DISPATCH_IGNORED = "\nrunning 0 tests\n\n0 tests, 0 benchmarks\n" _DISPATCH_CAPPED = "// one\n// two\n// three\n// four\nfn x() {}\n" _DISPATCH_CLEAN = "// one\n// two\nfn x() {}\n" def _dispatch_case(tmp, body, extra): os.makedirs(os.path.join(tmp, "src"), exist_ok=True) open(os.path.join(tmp, "src", "f.rs"), "w", encoding="utf8").write(body) open(os.path.join(tmp, "listing.txt"), "w", encoding="utf8").write(_DISPATCH_LISTING) open(os.path.join(tmp, "ignored.txt"), "w", encoding="utf8").write(_DISPATCH_IGNORED) cmd = [sys.executable, os.path.abspath(__file__), "src"] + extra return subprocess.run(cmd, cwd=tmp, capture_output=True, text=True) def _dispatch_self_test(): """Every invocation that can exit 0 answers the cap (improvise-ijql). The fault is a four-line run; the clean arm stops an always-fail passing. See context/code/tooling.md §2, "A flag adds a question". """ failures = [] for name, extra in SELF_TEST_DISPATCH: with tempfile.TemporaryDirectory() as tmp: run = _dispatch_case(tmp, _DISPATCH_CAPPED, extra) if run.returncode == 0: failures.append(f" DISPATCH: {name}: exited 0 over a four-line comment run") elif not ("comment runs longer than" in run.stderr and "f.rs" in run.stderr): failures.append(f" DISPATCH: {name}: failed, but not about the cap: " f"{run.stderr.strip()[:200]}") with tempfile.TemporaryDirectory() as tmp: run = _dispatch_case(tmp, _DISPATCH_CLEAN, extra) if run.returncode != 0: failures.append(f" DISPATCH: {name}: refused a clean tree " f"({run.returncode}): {run.stderr.strip()[:200]}") return failures def self_test(): """Check the checker against every bypass it has already been fixed for. Ten silent successes have been found in this file, every one failing in the direction that looks like passing. See context/code/tooling.md. """ failures = [] for name, expected, src in SELF_TEST_CASES: runs = [n for n in comment_runs(src.split("\n")) if n[1] > LIMIT] longest = max((n[1] for n in runs), default=0) if longest != expected: failures.append(f" MISSED: {name}: expected a run of {expected}, saw {longest}") for name, src in SELF_TEST_CLEAN: runs = [n for n in comment_runs(src.split("\n")) if n[1] > LIMIT] if runs: failures.append(f" FALSE POSITIVE: {name}: flagged {runs}") failures += _evidence_self_test() failures += _dispatch_self_test() if failures: print("checker self-test FAILED:", file=sys.stderr) print("\n".join(failures), file=sys.stderr) return 1 print(f"checker self-test: {len(SELF_TEST_CASES)} violations caught, " f"{len(SELF_TEST_CLEAN)} legitimate shapes allowed, " f"{len(SELF_TEST_EVIDENCE)} evidence verdicts, " f"{len(SELF_TEST_LISTING) + len(SELF_TEST_HEADINGS)} listing readings, " f"{len(SELF_TEST_AMBIGUITY)} name resolutions, " f"{len(SELF_TEST_DUPLICATES)} duplicate-heading verdicts, " f"{len(SELF_TEST_DISPATCH)} invocations answer the cap") return 0 def main(): ap = argparse.ArgumentParser() ap.add_argument("roots", nargs="*", default=["src", "crates", "tests", "examples", "harness"]) ap.add_argument("--limit", type=int, default=LIMIT) ap.add_argument("--list-doctests", action="store_true", help="list doc comments containing a ``` fence and exit") ap.add_argument("--check-evidence", action="store_true", help="every context/test-evidence heading must name a test that exists") ap.add_argument("--rust-tests", metavar="FILE", help="a saved `cargo test --workspace -- --list`; the default runs it. " "Requires --rust-tests-ignored: one file alone would skip the " "ignored-test rule entirely") ap.add_argument("--rust-tests-ignored", metavar="FILE", help="the matching `cargo test --workspace -- --list --ignored`") ap.add_argument("--self-test", action="store_true", help="check the checker against the bypasses it has been fixed for") args = ap.parse_args() if args.self_test: return self_test() default_roots = ["src", "crates", "tests", "examples", "harness"] requested = args.roots or default_roots # A path that is not a directory must FAIL. Filtering it away made # `check-comment-runs.py some/file.rs` walk nothing and print success -- # a filter that over-matches fails in the direction that looks like # passing, which is the one direction a checker must not fail in. roots = [r for r in requested if os.path.isdir(r)] missing = [r for r in requested if not os.path.isdir(r)] if missing and args.roots: for r in missing: kind = "not a directory" if os.path.exists(r) else "does not exist" print(f"error: {r}: {kind} (pass directories, not files)", file=sys.stderr) return 2 if not roots: print("error: nothing to scan", file=sys.stderr) return 2 if args.check_evidence: if bool(args.rust_tests) != bool(args.rust_tests_ignored): print("error: --rust-tests and --rust-tests-ignored come together; one alone " "silently drops the ignored-test rule", file=sys.stderr) return 2 # Additive, not selective: this flag used to early-return, so the # expensive form a human runs alone exited 0 over a capped file. capped = check_cap(roots, args.limit) return max(capped, check_evidence(roots, args.rust_tests, args.rust_tests_ignored)) if args.list_doctests: for path in sorted(rust_files(roots)): for n, raw in enumerate(open(path, encoding="utf8", errors="replace"), 1): s = raw.strip() if s.startswith(("///", "//!")) and s.lstrip("/!").strip().startswith("```"): print(f"{path}:{n}") return 0 return check_cap(roots, args.limit) if __name__ == "__main__": sys.exit(main())