improvise/scripts/check-comment-runs.py

#!/usr/bin/env python3
"""Fail if any .rs file carries more than two consecutive comment lines.

The rule (user, 2026-09-20): reasoning lives in context/ prose, not in code.
A comment run is at most two lines, optionally naming a context/ file for the
rest. See context/code/ for where the prose went, and improvise-9c8c for why.

Counts `///`, `//!` and `//` alike. Attributes are not comments and are
ignored.

A ``` fence SUSPENDS the count rather than ending the run: the fenced body is
exempt (a doctest is a running test, not prose) but the prose on either side
belongs to ONE run and is summed. Two lines above a fence and two below is a
run of four, which is over the limit -- and that is deliberate, because a
reader meets it as one uninterrupted comment block. See --list-doctests.

Note rustdoc executes a bare ``` or ```rust fence and does NOT execute
```text, ```ignore or ```no_run. Only a runnable fence earns the exemption;
the rest is prose wearing a fence and the cap applies to it.
"""

import argparse
import os
import re
import subprocess
import sys
import tempfile

LIMIT = 2


_RAW_STR = re.compile(r'(?:b|c)?r(#*)"')
_CHAR_LIT = re.compile(r"'(?:\\.|[^'\\])'")
_DOC_ATTR = re.compile(r"#!?\[\s*doc\s*=")


def scan(lines):
    """Classify each line, respecting Rust strings and NESTED block comments.

    Returns per line: has_code, has_comment, the `//` body, the CODE-ONLY text
    (no string or comment content), whether comment content leads the line, and
    whether the line began inside a block comment. See context/code/tooling.md.
    """
    out = []
    in_block = 0
    in_raw = None
    in_str = False
    for raw in lines:
        i, n = 0, len(raw)
        started_in_block = in_block > 0
        has_code = False
        has_comment = started_in_block
        body = None
        code = []
        leads = None
        while i < n:
            if in_block:
                has_comment = True
                leads = True if leads is None else leads
                if raw.startswith("/*", i):
                    in_block += 1
                    i += 2
                elif raw.startswith("*/", i):
                    in_block -= 1
                    i += 2
                else:
                    i += 1
                continue
            if in_raw is not None:
                closer = '"' + "#" * in_raw
                if raw.startswith(closer, i):
                    in_raw = None
                    i += len(closer)
                else:
                    i += 1
                has_code = True
                leads = False if leads is None else leads
                continue
            if in_str:
                has_code = True
                leads = False if leads is None else leads
                if raw[i] == "\\":
                    i += 2
                    continue
                if raw[i] == '"':
                    in_str = False
                i += 1
                continue
            if raw[i].isspace():
                i += 1
                continue
            if raw.startswith("//", i):
                has_comment = True
                leads = True if leads is None else leads
                if not has_code:
                    body = raw[i:]
                break
            if raw.startswith("/*", i):
                has_comment = True
                leads = True if leads is None else leads
                in_block = 1
                i += 2
                continue
            m = _RAW_STR.match(raw, i)
            if m:
                in_raw = len(m.group(1))
                has_code = True
                leads = False if leads is None else leads
                code.append('"')
                i = m.end()
                continue
            m = _CHAR_LIT.match(raw, i)
            if m:
                has_code = True
                leads = False if leads is None else leads
                i = m.end()
                continue
            if raw[i] == '"':
                in_str = True
                has_code = True
                leads = False if leads is None else leads
                i += 1
                continue
            has_code = True
            leads = False if leads is None else leads
            code.append(raw[i])
            i += 1
        out.append((has_code, has_comment, body, "".join(code), bool(leads), started_in_block))
    return out


def _kind(body):
    if body is None:
        return "//"
    if body.startswith("//!"):
        return "//!"
    return "///" if body.startswith("///") else "//"


def comment_runs(lines):
    """Yield (start_line_number, length) for each run, skipping runnable doctests.

    Every question here is asked of the SCANNER, never of the raw line: an
    attribute's bracket depth is counted over code only, because `#[cfg(feature
    = "[")]` otherwise exempted the whole file. See context/code/tooling.md.
    """
    scanned = scan(lines)

    run_start, run_len, run_kind = None, 0, None
    in_fence = fence_exempt = False
    attr_depth = 0
    attr_inner = False
    inner_pending = False

    def add(n, kind):
        nonlocal run_start, run_len, run_kind
        if run_start is None:
            run_start, run_len, run_kind = n, 0, kind
        run_len += 1

    for n, (has_code, has_comment, body, code, leads, in_block_at_start) in enumerate(scanned, 1):
        # A line that CLOSES a block comment carries comment text even when code
        # follows it; a line that opens one after code does not.
        is_comment = has_comment and (not has_code or (in_block_at_start and leads))
        code_s = code.strip()

        if attr_depth > 0:
            attr_depth += code.count("[") - code.count("]")
            if is_comment:
                add(n, _kind(body))
            if attr_depth <= 0:
                attr_depth = 0
                inner_pending = inner_pending or attr_inner
            continue

        if not is_comment and (code_s.startswith("#[") or code_s.startswith("#![")):
            # `#[doc = "..."]` and `#![doc = "..."]` ARE comments; `#[doc(hidden)]`
            # is not, which an unanchored `#[doc` prefix could not tell apart.
            if _DOC_ATTR.match(code_s):
                add(n, "//!" if code_s.startswith("#![") else "///")
                continue
            inner = code_s.startswith("#![")
            depth = code.count("[") - code.count("]")
            if depth > 0:
                attr_depth, attr_inner = depth, inner
                continue
            inner_pending = inner_pending or inner
            continue

        if is_comment:
            kind = _kind(body)
            # An inner attribute ends a run only where the DOC KIND changes: a
            # crate doc and an item doc are two blocks, `//!` either side is one.
            if inner_pending:
                inner_pending = False
                if run_kind is not None and kind != run_kind and run_start is not None and run_len > 0:
                    yield run_start, run_len
                    run_start, run_len, run_kind = None, 0, None
            doc = kind in ("///", "//!")
            text = body.lstrip("/!").strip() if body is not None else ""
            if doc and text.startswith("```"):
                if in_fence:
                    exempt_here = fence_exempt
                    in_fence = fence_exempt = False
                else:
                    info = text[3:].strip().lower()
                    in_fence = True
                    fence_exempt = info in ("", "rust") or info.startswith("rust,")
                    exempt_here = fence_exempt
                if exempt_here:
                    continue
                add(n, kind)
                continue
            if doc and in_fence and fence_exempt:
                continue
            add(n, kind)
        else:
            in_fence = fence_exempt = False
            inner_pending = False
            if run_start is not None and run_len > 0:
                yield run_start, run_len
            run_start, run_len, run_kind = None, 0, None
    if run_start is not None and run_len > 0:
        yield run_start, run_len


def rust_files(roots):
    for root in roots:
        for dirpath, dirnames, filenames in os.walk(root):
            dirnames[:] = [d for d in dirnames if d != "target"]
            for f in filenames:
                if f.endswith(".rs"):
                    yield os.path.join(dirpath, f)


_TEST_ATTRS = ("test", "tokio::test", "async_std::test", "wasm_bindgen_test")


def _is_test_attribute(line):
    """True only for an attribute that actually marks a test.

    Matched by exact path, never by substring: `#[cfg(test)]` contains "test"
    and marks a compilation condition, not a test.
    """
    body = line.strip()
    if not body.startswith("#["):
        return False
    body = body[2:].rstrip()
    body = body[:-1] if body.endswith("]") else body
    return body.strip().strip(":") in _TEST_ATTRS


EVIDENCE_DIR = os.path.join("context", "test-evidence")
EVIDENCE_CITE = "test-evidence"
JS_ROOTS = [os.path.join("harness", "tests"), os.path.join("web", "tests")]

_RUST_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)*$")
_QUOTED_NAME = re.compile(r'^"(.+)"$')
# Only a RUNNABLE LEAF test. `.skip`, `.todo`, `.describe` and `.only` are all
# excluded: the first three name no executed assertion, and `.only` disables
# every OTHER test in its file, so a heading satisfied by one is a heading
# satisfied by a file that stopped running (review, PR #69).
_JS_TEST = re.compile(
    r"""(?:^|[^\w.])(?:test|it)\s*\(\s*(['"`])(.+?)\1"""
)


def evidence_claims(text):
    """Yield ("rust"|"js"|"prose", name, lineno) for every heading in a file.

    `### ` and `#### ` alike, because the browser entries have to use `####`
    and were invisible to a check that collected only `### ` (improvise-5ray).
    A heading is a CLAIM when it is shaped like a test name -- a bare
    identifier (Rust) or a quoted string (JS). Anything else is prose about
    the section: `Helper - x`, `Note - y`, a file's own scope. That shape rule
    is the whole discrimination, and rewriting a claim as prose retires it with
    nothing checking the prose is honest (improvise-9za9). The mechanism is the
    diff a reviewer reads, not the printed prose count, which nothing stores.
    See context/code/tooling.md.
    """
    for n, raw in enumerate(text.split("\n"), 1):
        s = raw.strip()
        if not (s.startswith("### ") or s.startswith("#### ")):
            continue
        body = s.lstrip("#").strip().strip("`").strip()
        m = _QUOTED_NAME.match(body)
        if m:
            yield "js", m.group(1), n
        elif _RUST_NAME.match(body):
            yield "rust", body, n
        else:
            yield "prose", body, n


_CARGO_LISTED = re.compile(r"^(\S+): test$")


def cargo_test_names(listing, ignored_listing=""):
    """The FULL PATHS cargo says it will run, from `cargo test -- --list`.

    Paths and not leaves: six leaf names collide in this workspace, so a leaf
    is not an identity. See context/code/tooling.md and improvise-rm4s.
    """
    def paths(text):
        return {
            m.group(1)
            for m in (_CARGO_LISTED.match(line.strip()) for line in text.split("\n"))
            if m
        }

    return paths(listing) - paths(ignored_listing)


def _leaf(path):
    return path.split("::")[-1]


def _cargo_listing(extra):
    cmd = ["cargo", "test", "--workspace", "--", "--list", *extra]
    try:
        run = subprocess.run(cmd, capture_output=True, text=True)
    except OSError as e:
        raise SystemExit(f"error: cannot run cargo ({e}).\n"
                         "--check-evidence needs a toolchain, or --rust-tests FILE "
                         "holding a saved `cargo test --workspace -- --list`.")
    if run.returncode != 0:
        raise SystemExit(f"error: {' '.join(cmd)} failed:\n{run.stderr}")
    return run.stdout


def rust_citations(text):
    """The tests here whose doc comment cites the evidence dir.

    A citation obliges only a TEST. Accepting any `fn` let a `###` naming a
    fixture or a helper pass, so an evidence file could report itself satisfied
    while naming nothing that runs (found on PR #39).
    """
    citing = set()
    annotated = False
    doc = []
    # A `proptest!` block accepts its bodies as tests -- but only INSIDE the
    # block. A bare flag was never cleared, so the first proptest! in a file
    # made every later fn, helpers included, count as a test: a silent
    # success in the gate itself (found by review on PR #40). Track the
    # block's braces and leave when they balance.
    proptest_depth = None
    for raw in text.split("\n"):
        s = raw.strip()
        if proptest_depth is not None:
            proptest_depth += raw.count("{") - raw.count("}")
            if proptest_depth <= 0:
                proptest_depth = None
        elif s.startswith("proptest!"):
            proptest_depth = raw.count("{") - raw.count("}")
            if proptest_depth <= 0:
                proptest_depth = None if "{" in raw else 0
        in_proptest = proptest_depth is not None
        if s.startswith("//"):
            doc.append(s)
            continue
        # ANCHOR the attribute. `"test" in s` also matched `#[cfg(test)]`,
        # so every cfg-gated fixture cleared a gate written to reject
        # exactly those -- the third instance of the unanchored-substring
        # class in this campaign, after the bd audit's exclusion list.
        if s.startswith("#[") and _is_test_attribute(s):
            annotated = True
            continue
        if s.startswith("fn ") or s.startswith("pub fn ") or s.startswith("async fn "):
            if annotated or in_proptest:
                name = s.split("fn ", 1)[1].split("(")[0].split("<")[0].strip()
                if any(EVIDENCE_CITE in d for d in doc):
                    citing.add(name)
            annotated = False
            doc = []
        elif s and not s.startswith("#["):
            annotated = False
            doc = []
    return citing


def js_tests(text):
    """(tests defined here, tests whose preceding comment cites the evidence dir).

    The JS half exists because `defined` held only Rust `fn` names, so a
    `####` naming a Playwright or node test could never have resolved even if
    the heading had been collected -- the convention that kept the checker
    quiet was the same one that made it blind (improvise-5ray).
    """
    defined, citing = set(), set()
    doc = []
    for raw in text.split("\n"):
        s = raw.strip()
        m = _JS_TEST.search(raw)
        if m:
            defined.add(m.group(2))
            if any(EVIDENCE_CITE in d for d in doc):
                citing.add(m.group(2))
            doc = []
            continue
        if s.startswith("//") or s.startswith("*") or s.startswith("/*"):
            doc.append(s)
        elif s:
            doc = []
    return defined, citing


def js_files(roots):
    for root in roots:
        for dirpath, dirnames, filenames in os.walk(root):
            dirnames[:] = [d for d in dirnames if d != "node_modules"]
            for f in filenames:
                if f.endswith(".spec.js") or f.endswith(".test.js"):
                    yield os.path.join(dirpath, f)


def _duplicate_claims(claims, by_leaf, defined):
    """Two headings naming ONE test, keyed by the test they resolve to.

    Resolved and not spelled, so `### f` beside `### m::tests::f` is caught
    too: the heading is the key a reader looks a test up by, and two entries
    under one key means the first one found is taken as current. See
    context/code/tooling.md.
    """
    def canonical(kind, name):
        if kind == "js" or name in defined["rust"]:
            return name
        matches = by_leaf.get(name, set())
        return next(iter(matches)) if len(matches) == 1 else name

    grouped = {}
    for (kind, name), where in claims.items():
        grouped.setdefault((kind, canonical(kind, name)), []).extend(where)
    def in_file_order(loc):
        path, _, n = loc.rpartition(":")
        return (path, int(n) if n.isdigit() else 0)

    return {key: sorted(where, key=in_file_order)
            for key, where in grouped.items() if len(where) > 1}


def evidence_verdict(claims, defined, citing):
    """(orphans, unstated, ambiguous, duplicates), keyed by ("rust"|"js", name).

    `orphans` is evidence naming a test nobody can find. `unstated` is the
    reverse obligation (improvise-lyyc): a test that POINTS AT the evidence
    file must appear in it, or a reader following the pointer arrives nowhere.
    `ambiguous` is a bare leaf that two tests answer to -- a refusal and not a
    warning, because a heading satisfied by a test in another module answering
    another question is a pass while the gate does nothing.
    `duplicates` is the mirror image: two headings answered by one test, where
    every copy passes and the reader takes whichever comes first as current
    (improvise-hsoe). Also a refusal, on the improvise-rm4s precedent.
    Only that narrower reverse obligation, because most tests have no evidence
    and should not be forced to. What none of the four can see: an entry can
    exist and still describe its subject backwards.
    """
    by_leaf = {}
    for path in defined["rust"]:
        by_leaf.setdefault(_leaf(path), set()).add(path)

    orphans, ambiguous = {}, {}
    for key, where in claims.items():
        kind, name = key
        if kind == "js":
            if name not in defined["js"]:
                orphans[key] = where
            continue
        if name in defined["rust"]:
            continue
        matches = by_leaf.get(name, set())
        if len(matches) == 1:
            continue
        (ambiguous if matches else orphans)[key] = where

    # A citation is a bare `fn` name, so it is satisfied by a QUALIFIED heading
    # naming that leaf -- qualifying a heading must not orphan its own test.
    claimed = {(kind, _leaf(name)) for kind, name in claims}
    unstated = {}
    for kind, names in citing.items():
        for name in names:
            if (kind, name) not in claimed:
                unstated.setdefault((kind, name), []).extend(sorted(names[name]))
    return orphans, unstated, ambiguous, _duplicate_claims(claims, by_leaf, defined)


def check_evidence(roots, rust_listing=None, rust_ignored=None):
    """Both directions over context/test-evidence/. See evidence_verdict.

    A MISSING evidence directory is not an early return: it used to be, and it
    exited 0 while 170 citations pointed at nothing (review of PR #83).
    """
    claims, prose = {}, 0
    for dirpath, _, filenames in os.walk(EVIDENCE_DIR):
        for f in sorted(filenames):
            if not f.endswith(".md"):
                continue
            path = os.path.join(dirpath, f)
            text = open(path, encoding="utf8", errors="replace").read()
            for kind, name, n in evidence_claims(text):
                if kind == "prose":
                    prose += 1
                else:
                    claims.setdefault((kind, name), []).append(f"{path}:{n}")

    if rust_listing is None:
        source = "cargo"
        rust_names = cargo_test_names(_cargo_listing([]), _cargo_listing(["--ignored"]))
    else:
        source = f"{rust_listing} + {rust_ignored}"
        rust_names = cargo_test_names(open(rust_listing, encoding="utf8").read(),
                                      open(rust_ignored, encoding="utf8").read())

    defined = {"rust": rust_names, "js": set()}
    citing = {"rust": {}, "js": {}}
    for path in rust_files(roots):
        for name in rust_citations(open(path, encoding="utf8", errors="replace").read()):
            citing["rust"].setdefault(name, []).append(path)
    for path in js_files(JS_ROOTS):
        d, c = js_tests(open(path, encoding="utf8", errors="replace").read())
        defined["js"] |= d
        for name in c:
            citing["js"].setdefault(name, []).append(path)

    orphans, unstated, ambiguous, duplicates = evidence_verdict(claims, defined, citing)
    if not orphans and not unstated and not ambiguous and not duplicates:
        print(f"test-evidence: {len(claims)} entries, all name a test that runs "
              f"and no test is named twice (names from {source}); "
              f"{sum(len(v) for v in citing.values())} tests cite the file and all have one; "
              f"{prose} prose headings claim nothing")
        return 0
    if orphans:
        print(f"{len(orphans)} test-evidence entries name no test:\n", file=sys.stderr)
        for (kind, name), where in sorted(orphans.items()):
            print(f"  [{kind}] {name}  ({', '.join(where)})", file=sys.stderr)
    if ambiguous:
        by_leaf = {}
        for path in defined["rust"]:
            by_leaf.setdefault(_leaf(path), set()).add(path)
        print(f"\n{len(ambiguous)} test-evidence entries name a leaf that is not unique; "
              "write the module path:\n", file=sys.stderr)
        for (kind, name), where in sorted(ambiguous.items()):
            print(f"  [{kind}] {name}  ({', '.join(where)})", file=sys.stderr)
            for path in sorted(by_leaf[name]):
                print(f"         -> {path}", file=sys.stderr)
    if duplicates:
        print(f"\n{len(duplicates)} tests are named by more than one test-evidence "
              "heading; merge them or give each its own test:\n", file=sys.stderr)
        for (kind, name), where in sorted(duplicates.items()):
            print(f"  [{kind}] {name}  ({', '.join(where)})", file=sys.stderr)
    if unstated:
        print(f"\n{len(unstated)} tests cite {EVIDENCE_DIR} and have no entry there:\n",
              file=sys.stderr)
        for (kind, name), where in sorted(unstated.items()):
            print(f"  [{kind}] {name}  ({', '.join(where)})", file=sys.stderr)
    return 1


def check_cap(roots, limit):
    """The comment-run cap. Every invocation that can exit 0 runs this.

    Why no flag selects it away: context/code/tooling.md §2, "A flag adds".
    """
    violations = []
    for path in sorted(rust_files(roots)):
        lines = open(path, encoding="utf8", errors="replace").read().split("\n")
        for start, length in comment_runs(lines):
            if length > limit:
                violations.append((path, start, length))

    if not violations:
        print(f"comment runs: none longer than {limit} lines")
        return 0

    worst = sorted(violations, key=lambda v: -v[2])
    print(f"{len(violations)} comment runs longer than {limit} lines:\n", file=sys.stderr)
    for path, start, length in worst:
        print(f"  {path}:{start}  {length} lines", file=sys.stderr)
    return 1


SELF_TEST_CASES = [
    ("a plain run of three", 3, """
// one
// two
// three
fn f() {}
"""),
    ("a block comment of six", 6, """
/* one
   two
   three
   four
   five
*/
fn f() {}
"""),
    ("a rustdoc block comment of six", 6, """
/** one
 * two
 * three
 * four
 * five
 */
fn f() {}
"""),
    ("an unclosed fence must not exempt the rest of the file", 6, """
/// opens a fence and never closes it
/// ```
/// let x = 1;
fn a() {}

/// one
/// two
/// three
/// four
/// five
/// six
fn b() {}
"""),
    ("an attribute must not split a run", 4, """
/// one
/// two
#[inline]
/// three
/// four
fn c() {}
"""),
    ("a non-runnable fence is prose and counts", 4, """
/// one
/// ```text
/// two
/// ```
fn d() {}
"""),
    ("a fence in a PLAIN // comment earns no exemption", 7, """
// ```
// one
// two
// three
// four
// five
// ```
fn f() {}
"""),
    ("a block comment opened after code on the same line", 5, """
fn f() {} /* one
two
three
four
five
*/
"""),
    ("nested block comments do not close early", 7, """
/* /* */
one
two
three
four
five
*/
fn f() {}
"""),
    ("a multi-line attribute does not split a run", 4, """
/// one
/// two
#[cfg(all(
    unix,
    target_pointer_width = "64"
))]
/// three
/// four
fn f() {}
"""),
    ("an attribute inside a fence does not reopen it", 5, """
/// one
/// ```
/// let x = 1;
#[inline]
/// ```
/// hidden A
/// hidden B
/// hidden C
/// hidden D
fn g() {}
"""),
    ("#[doc = ...] is the one attribute that is a comment", 3, """
#[doc = "one"]
#[doc = "two"]
#[doc = "three"]
fn f() {}
"""),
    ("#![doc = ...] is the inner spelling of the same thing", 5, """
#![doc = "one"]
#![doc = "two"]
#![doc = "three"]
#![doc = "four"]
#![doc = "five"]
fn f() {}
"""),
    ("an attribute holding a bracket in a string exempts nothing", 6, """
#[cfg(feature = "[")]
fn g() {}
// a
// b
// c
// d
// e
// f
fn f() {}
"""),
    ("an inner attribute between two //! blocks does not split them", 4, """
//! one
//! two
#![allow(clippy::needless_return)]
//! three
//! four
fn f() {}
"""),
    ("a block comment closing on a line with code still counts", 3, """
/* one
two
three */ fn f() {}
"""),
    ("a char literal holding a quote does not open a string", 3, """
pub const C: char = '"';
// one
// two
// three
fn f() {}
"""),
]

SELF_TEST_CLEAN = [
    ("two lines is the limit", """
// one
// two
fn f() {}
"""),
    ("a runnable doctest is exempt", """
/// one
/// ```
/// let x = 1;
/// assert_eq!(x, 1);
/// ```
/// two
fn f() {}
"""),
    ("a closed fence does not leak into a later comment", """
/// one
/// ```
/// let x = 1;
/// ```
fn a() {}

// two
// three
fn b() {}
"""),
    ("a block comment inside a raw string is not a comment", """
pub const S: &str = r#"x"y /* not a comment
one
two
three
four
"#;
fn a() {}
"""),
    ("// inside a raw string is not a comment", """
pub const S: &str = r#"x"y // not a comment
// nor this
// nor this
// nor this
"#;
fn a() {}
"""),
    ("a crate doc and an item doc are separated by the inner attribute", """
//! crate one
//! crate two
#![allow(clippy::needless_return)]
/// item one
/// item two
pub fn go() {}
"""),
    ("// inside a multi-line plain string is not a comment", """
pub const S: &str = "open \\
// one
// two
// three
close";
fn a() {}
"""),
    ("#[doc(hidden)] is an attribute, not a comment", """
/// one
/// two
#[doc(hidden)]
pub fn f() {}
"""),
    ("an attribute's brackets close, so later code still ends a run", """
/// one
/// two
#[cfg(all(
    unix,
))]
fn f() {}
/// three
/// four
fn g() {}
"""),
]


# (name, listing, ignored listing, the paths that RUN).
#
# Three source shapes defeated the old scrape -- a test in a string literal, one
# behind a cfg feature, one behind cfg(target_arch = "wasm32") -- and an ignored
# test defeats a listing read without its `--ignored` twin (improvise-rm4s).
SELF_TEST_LISTING = [
    ("the module path is kept, because a leaf is not an identity",
     "help::tests::a_real_test: test\n", "", ["help::tests::a_real_test"]),

    ("an ignored test is one that does not run",
     "a::runs: test\na::skipped: test\n", "a::skipped: test\n", ["a::runs"]),

    ("cargo's prose is not a test name",
     "     Running unittests src/lib.rs (target/debug/deps/x-1)\n"
     "2 tests, 0 benchmarks\n", "", []),
]


# (name, evidence markdown, rust source, js source, cargo listing, orphans, unstated).
#
# The evidence checker had no self-test at all, which is how both halves of
# improvise-5ray and improvise-lyyc survived: the shape that silenced the
# checker was the shape it could not see. Each case below is an injection.
SELF_TEST_EVIDENCE = [
    ("a #### heading naming no test is an orphan",
     '#### "a test that does not exist anywhere"\n',
     "", "", "", ["a test that does not exist anywhere"], []),

    ("a #### heading naming a JS test that exists is satisfied",
     '#### "a peer that cannot be reached is named"\n',
     "", 'test("a peer that cannot be reached is named", async () => {});\n',
     "", [], []),

    ("a quoted heading can never be satisfied by a Rust fn of that name",
     '#### "a_rust_name"\n',
     "#[test]\nfn a_rust_name() {}\n", "", "m::a_rust_name: test\n",
     ["a_rust_name"], []),

    ("a prose #### heading claims nothing",
     "#### Helper — `addressed`\n#### Note — the basis_check module\n"
     "#### harness/tests/shell.spec.js — the file's own scope\n",
     "", "", "", [], []),

    ("a ### naming a helper rather than a test is still an orphan",
     "### a_helper\n", "fn a_helper() {}\n", "", "", ["a_helper"], []),

    ("a test that cites the evidence file and has no entry is caught",
     "", "/// See `context/test-evidence/clients.md`.\n#[test]\nfn a_cited_test() {}\n",
     "", "m::a_cited_test: test\n", [], ["a_cited_test"]),

    ("the same test with an entry is satisfied",
     "### a_cited_test\n",
     "/// See `context/test-evidence/clients.md`.\n#[test]\nfn a_cited_test() {}\n",
     "", "m::a_cited_test: test\n", [], []),

    ("a NON-test fn citing the file is under no obligation",
     "", "/// See `context/test-evidence/clients.md`.\nfn a_helper() {}\n", "",
     "", [], []),

    ("a citation belonging to the fn ABOVE does not attach to the next one",
     "### cited\n",
     "/// See `context/test-evidence/clients.md`.\n#[test]\nfn cited() {}\n"
     "\n#[test]\nfn uncited() {}\n",
     "", "m::cited: test\nm::uncited: test\n", [], []),

    ("a heading cannot be satisfied by a skipped or todo test",
     '#### "a skipped one"\n#### "a todo one"\n', "",
     'test.skip("a skipped one", () => {});\ntest.todo("a todo one");\n',
     "", ["a skipped one", "a todo one"], []),

    ("a heading cannot be satisfied by a test.only, which disables its file",
     '#### "the only one left running"\n', "",
     'test.only("the only one left running", () => {});\n',
     "", ["the only one left running"], []),

    ("a heading cannot be satisfied by an empty suite's name",
     '#### "a suite with nothing in it"\n', "",
     'test.describe("a suite with nothing in it", () => {});\n',
     "", ["a suite with nothing in it"], []),

    ("a test nested in a suite still satisfies its heading",
     '#### "a nested test"\n', "",
     'test.describe("a group", () => {\n  test("a nested test", () => {});\n});\n',
     "", [], []),

    ("a JS test that cites the evidence file and has no entry is caught",
     "", "",
     '// See context/test-evidence/clients.md.\ntest("a cited spec", async () => {});\n',
     "", [], ["a cited spec"]),
]


# (name, evidence markdown, the kind of each heading in order).
#
# Prose is a third KIND rather than a heading the collector drops, so retiring a
# claim by rewriting it as prose moves the reported count (improvise-9za9).
SELF_TEST_HEADINGS = [
    ("a heading is classified three ways, prose included",
     '#### Helper — `addressed`\n### a_rust_name\n#### "a js name"\n',
     ["prose", "rust", "js"]),

    ("a module-qualified heading is a Rust claim, not prose",
     "### ui::project::tests::a_load_that_fails_is_not_a_rebase\n", ["rust"]),
]


# (name, evidence markdown, cargo listing, orphans, ambiguous).
#
# Six leaf names collide in this workspace and two were cited, so a heading was
# satisfiable by a test in another module answering another question -- renaming
# the one an entry describes left the checker at exit 0 (review of PR #83).
SELF_TEST_AMBIGUITY = [
    ("a leaf two tests answer to is refused, not silently resolved",
     "### a_load_that_fails_is_not_a_rebase\n",
     "ui::host::tests::a_load_that_fails_is_not_a_rebase: test\n"
     "ui::project::tests::a_load_that_fails_is_not_a_rebase: test\n",
     [], ["a_load_that_fails_is_not_a_rebase"]),

    ("qualifying it resolves it, and to the one it names",
     "### ui::project::tests::a_load_that_fails_is_not_a_rebase\n",
     "ui::host::tests::a_load_that_fails_is_not_a_rebase: test\n"
     "ui::project::tests::a_load_that_fails_is_not_a_rebase: test\n",
     [], []),

    ("a qualified heading naming a module that does not have it is an orphan",
     "### ui::nowhere::tests::a_load_that_fails_is_not_a_rebase\n",
     "ui::host::tests::a_load_that_fails_is_not_a_rebase: test\n",
     ["ui::nowhere::tests::a_load_that_fails_is_not_a_rebase"], []),

    ("a unique leaf still resolves unqualified",
     "### a_lonely_test\n", "m::tests::a_lonely_test: test\n", [], []),
]


# (name, evidence markdown, cargo listing, js source, the tests named twice).
#
# A heading that names a test that RUNS passes -- and a second heading naming
# the same one passes just as well, so an entry superseded by its successor
# reads as current to whoever meets it first (improvise-hsoe, found on PR #92).
# Every case here is an injection; the clean arms stop an always-fail passing.
SELF_TEST_DUPLICATES = [
    ("one heading twice in one file is refused",
     "### a_real_test\n### a_real_test\n", "m::tests::a_real_test: test\n", "",
     ["m::tests::a_real_test"]),

    ("the qualified and bare spellings of one test are the same heading",
     "### a_real_test\n### m::tests::a_real_test\n", "m::tests::a_real_test: test\n", "",
     ["m::tests::a_real_test"]),

    # A BARE heading, because two qualified ones resolve by the exact-match arm
    # and never reach the ambiguity guard this case is named for (review, PR #96).
    ("a bare heading over an ambiguous leaf is not a duplicate of a qualified one",
     "### shared\n### m::tests::shared\n### n::tests::shared\n",
     "m::tests::shared: test\nn::tests::shared: test\n", "", []),

    ("a JS heading twice is refused too",
     '#### "a spec"\n#### "a spec"\n', "", 'test("a spec", () => {});\n', ["a spec"]),

    ("two headings naming two tests are not duplicates",
     "### one\n### two\n", "m::one: test\nm::two: test\n", "", []),

    ("prose headings repeat freely -- they claim no test",
     "### Helper - x\n### Helper - x\n", "", "", []),
]


def _duplicate_self_test():
    failures = []
    for name, md, listing, js, want in SELF_TEST_DUPLICATES:
        claims = {}
        for kind, cname, n in evidence_claims(md):
            if kind != "prose":
                claims.setdefault((kind, cname), []).append(f"fixture:{n}")
        jd, _ = js_tests(js)
        defined = {"rust": cargo_test_names(listing), "js": jd}
        *_, duplicates = evidence_verdict(claims, defined, {"rust": {}, "js": {}})
        got = sorted(k[1] for k in duplicates)
        if got != sorted(want):
            failures.append(f"  DUPLICATES: {name}: expected {sorted(want)}, saw {got}")
    return failures


def _ambiguity_self_test():
    failures = _duplicate_self_test()
    for name, md, listing, want_o, want_a in SELF_TEST_AMBIGUITY:
        claims = {}
        for kind, cname, n in evidence_claims(md):
            if kind != "prose":
                claims.setdefault((kind, cname), []).append(f"fixture:{n}")
        defined = {"rust": cargo_test_names(listing), "js": set()}
        orphans, _, ambiguous, _dupes = evidence_verdict(
            claims, defined, {"rust": {}, "js": {}})
        got_o = sorted(k[1] for k in orphans)
        got_a = sorted(k[1] for k in ambiguous)
        if got_o != sorted(want_o):
            failures.append(f"  ORPHANS: {name}: expected {sorted(want_o)}, saw {got_o}")
        if got_a != sorted(want_a):
            failures.append(f"  AMBIGUOUS: {name}: expected {sorted(want_a)}, saw {got_a}")
    return failures


def _listing_self_test():
    failures = []
    for name, listing, ignored, want in SELF_TEST_LISTING:
        got = sorted(cargo_test_names(listing, ignored))
        if got != sorted(want):
            failures.append(f"  LISTING: {name}: expected {sorted(want)}, saw {got}")
    for name, md, want in SELF_TEST_HEADINGS:
        got = [kind for kind, _, _ in evidence_claims(md)]
        if got != want:
            failures.append(f"  HEADINGS: {name}: expected {want}, saw {got}")
    return failures


def _evidence_self_test():
    failures = _listing_self_test() + _ambiguity_self_test()
    for name, md, rs, js, listing, want_orphans, want_unstated in SELF_TEST_EVIDENCE:
        claims = {}
        for kind, cname, n in evidence_claims(md):
            if kind != "prose":
                claims.setdefault((kind, cname), []).append(f"fixture:{n}")
        rc = rust_citations(rs)
        jd, jc = js_tests(js)
        defined = {"rust": cargo_test_names(listing), "js": jd}
        citing = {"rust": {k: ["fixture.rs"] for k in rc},
                  "js": {k: ["fixture.js"] for k in jc}}
        orphans, unstated, ambiguous, duplicates = evidence_verdict(claims, defined, citing)
        if duplicates:
            failures.append(f"  DUPLICATES: {name}: unexpected {sorted(k[1] for k in duplicates)}")
        got_o = sorted(k[1] for k in orphans)
        got_u = sorted(k[1] for k in unstated)
        if got_o != sorted(want_orphans):
            failures.append(f"  ORPHANS: {name}: expected {sorted(want_orphans)}, saw {got_o}")
        if got_u != sorted(want_unstated):
            failures.append(f"  UNSTATED: {name}: expected {sorted(want_unstated)}, saw {got_u}")
        if ambiguous:
            failures.append(f"  AMBIGUOUS: {name}: unexpected {sorted(k[1] for k in ambiguous)}")
    return failures


SELF_TEST_DISPATCH = [
    ("bare", []),
    ("--check-evidence", ["--check-evidence",
                          "--rust-tests", "listing.txt",
                          "--rust-tests-ignored", "ignored.txt"]),
]

_DISPATCH_LISTING = "\nrunning 1 test\na::b::c: test\n\n1 test, 0 benchmarks\n"
_DISPATCH_IGNORED = "\nrunning 0 tests\n\n0 tests, 0 benchmarks\n"
_DISPATCH_CAPPED = "// one\n// two\n// three\n// four\nfn x() {}\n"
_DISPATCH_CLEAN = "// one\n// two\nfn x() {}\n"


def _dispatch_case(tmp, body, extra):
    os.makedirs(os.path.join(tmp, "src"), exist_ok=True)
    open(os.path.join(tmp, "src", "f.rs"), "w", encoding="utf8").write(body)
    open(os.path.join(tmp, "listing.txt"), "w", encoding="utf8").write(_DISPATCH_LISTING)
    open(os.path.join(tmp, "ignored.txt"), "w", encoding="utf8").write(_DISPATCH_IGNORED)
    cmd = [sys.executable, os.path.abspath(__file__), "src"] + extra
    return subprocess.run(cmd, cwd=tmp, capture_output=True, text=True)


def _dispatch_self_test():
    """Every invocation that can exit 0 answers the cap (improvise-ijql).

    The fault is a four-line run; the clean arm stops an always-fail passing.
    See context/code/tooling.md §2, "A flag adds a question".
    """
    failures = []
    for name, extra in SELF_TEST_DISPATCH:
        with tempfile.TemporaryDirectory() as tmp:
            run = _dispatch_case(tmp, _DISPATCH_CAPPED, extra)
            if run.returncode == 0:
                failures.append(f"  DISPATCH: {name}: exited 0 over a four-line comment run")
            elif not ("comment runs longer than" in run.stderr and "f.rs" in run.stderr):
                failures.append(f"  DISPATCH: {name}: failed, but not about the cap: "
                                f"{run.stderr.strip()[:200]}")
        with tempfile.TemporaryDirectory() as tmp:
            run = _dispatch_case(tmp, _DISPATCH_CLEAN, extra)
            if run.returncode != 0:
                failures.append(f"  DISPATCH: {name}: refused a clean tree "
                                f"({run.returncode}): {run.stderr.strip()[:200]}")
    return failures


def self_test():
    """Check the checker against every bypass it has already been fixed for.

    Ten silent successes have been found in this file, every one failing in the
    direction that looks like passing. See context/code/tooling.md.
    """
    failures = []
    for name, expected, src in SELF_TEST_CASES:
        runs = [n for n in comment_runs(src.split("\n")) if n[1] > LIMIT]
        longest = max((n[1] for n in runs), default=0)
        if longest != expected:
            failures.append(f"  MISSED: {name}: expected a run of {expected}, saw {longest}")
    for name, src in SELF_TEST_CLEAN:
        runs = [n for n in comment_runs(src.split("\n")) if n[1] > LIMIT]
        if runs:
            failures.append(f"  FALSE POSITIVE: {name}: flagged {runs}")
    failures += _evidence_self_test()
    failures += _dispatch_self_test()
    if failures:
        print("checker self-test FAILED:", file=sys.stderr)
        print("\n".join(failures), file=sys.stderr)
        return 1
    print(f"checker self-test: {len(SELF_TEST_CASES)} violations caught, "
          f"{len(SELF_TEST_CLEAN)} legitimate shapes allowed, "
          f"{len(SELF_TEST_EVIDENCE)} evidence verdicts, "
          f"{len(SELF_TEST_LISTING) + len(SELF_TEST_HEADINGS)} listing readings, "
          f"{len(SELF_TEST_AMBIGUITY)} name resolutions, "
          f"{len(SELF_TEST_DUPLICATES)} duplicate-heading verdicts, "
          f"{len(SELF_TEST_DISPATCH)} invocations answer the cap")
    return 0


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("roots", nargs="*",
                    default=["src", "crates", "tests", "examples", "harness"])
    ap.add_argument("--limit", type=int, default=LIMIT)
    ap.add_argument("--list-doctests", action="store_true",
                    help="list doc comments containing a ``` fence and exit")
    ap.add_argument("--check-evidence", action="store_true",
                    help="every context/test-evidence heading must name a test that exists")
    ap.add_argument("--rust-tests", metavar="FILE",
                    help="a saved `cargo test --workspace -- --list`; the default runs it. "
                         "Requires --rust-tests-ignored: one file alone would skip the "
                         "ignored-test rule entirely")
    ap.add_argument("--rust-tests-ignored", metavar="FILE",
                    help="the matching `cargo test --workspace -- --list --ignored`")
    ap.add_argument("--self-test", action="store_true",
                    help="check the checker against the bypasses it has been fixed for")
    args = ap.parse_args()
    if args.self_test:
        return self_test()
    default_roots = ["src", "crates", "tests", "examples", "harness"]
    requested = args.roots or default_roots
    # A path that is not a directory must FAIL. Filtering it away made
    # `check-comment-runs.py some/file.rs` walk nothing and print success --
    # a filter that over-matches fails in the direction that looks like
    # passing, which is the one direction a checker must not fail in.
    roots = [r for r in requested if os.path.isdir(r)]
    missing = [r for r in requested if not os.path.isdir(r)]
    if missing and args.roots:
        for r in missing:
            kind = "not a directory" if os.path.exists(r) else "does not exist"
            print(f"error: {r}: {kind} (pass directories, not files)", file=sys.stderr)
        return 2
    if not roots:
        print("error: nothing to scan", file=sys.stderr)
        return 2

    if args.check_evidence:
        if bool(args.rust_tests) != bool(args.rust_tests_ignored):
            print("error: --rust-tests and --rust-tests-ignored come together; one alone "
                  "silently drops the ignored-test rule", file=sys.stderr)
            return 2
        # Additive, not selective: this flag used to early-return, so the
        # expensive form a human runs alone exited 0 over a capped file.
        capped = check_cap(roots, args.limit)
        return max(capped, check_evidence(roots, args.rust_tests, args.rust_tests_ignored))

    if args.list_doctests:
        for path in sorted(rust_files(roots)):
            for n, raw in enumerate(open(path, encoding="utf8", errors="replace"), 1):
                s = raw.strip()
                if s.startswith(("///", "//!")) and s.lstrip("/!").strip().startswith("```"):
                    print(f"{path}:{n}")
        return 0

    return check_cap(roots, args.limit)


if __name__ == "__main__":
    sys.exit(main())