Edward Langley
901f7fadc5
Main's PR #287 added a workflow-level defaults shell, and the refusal made every suite read as unrun. The two cases that pinned it are deleted; one shows a step under workflow and job defaults is counted. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
767 lines
34 KiB
Python
Executable File
767 lines
34 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""pages-deploy must `needs` every other job in the workflow.
|
|
|
|
The rule that does not go stale is "every check job"; a subset is what let
|
|
README name three of seven while the rest read as ungated (improvise-x7cq).
|
|
Stating it was not enough -- a new job was added during review of PR #83 and
|
|
nothing noticed, so the rule is a gate. See context/code/tooling.md.
|
|
|
|
Second rule: every suite `scripts/checkers.sh --list` knows must run in some
|
|
CI step. ci.yml names its suites by hand, so a new `*-test.sh` ran locally and
|
|
never in CI until someone remembered to add it (improvise-dak5). Each job runs
|
|
`checkers.sh --ci <job>`, whose table lives in checkers.sh; no shell is parsed
|
|
here, and what a row runs is asked of `--would-run` (improvise-maax, st8q).
|
|
|
|
`--self-test` checks the checker: each case is a graph this must refuse.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
WORKFLOW = ".github/workflows/ci.yml"
|
|
DEPLOY = "pages-deploy"
|
|
|
|
_JOB = re.compile(r"^ ([A-Za-z0-9_-]+):$")
|
|
_JOB_OPEN = re.compile(r"^ ([A-Za-z0-9_-]+):(\s|$)")
|
|
_NEEDS = re.compile(r"^ needs:\s*\[(.*)\]\s*$")
|
|
|
|
|
|
def read_graph(text):
|
|
"""(all job names in order, the names `pages-deploy` needs).
|
|
|
|
Both come off indentation, which is what a workflow's own shape is. A job
|
|
key is two spaces; `needs:` is four INSIDE the job it belongs to, so the
|
|
current job is tracked rather than the first `needs:` taken. A job opens on
|
|
`_JOB_OPEN`, wider than `_JOB`: this reader refuses nothing, so it reads wide
|
|
(improvise-j49r).
|
|
"""
|
|
jobs, needs, current = [], None, None
|
|
in_jobs = False
|
|
for raw in text.split("\n"):
|
|
if raw.startswith("jobs:"):
|
|
in_jobs = True
|
|
continue
|
|
if not in_jobs:
|
|
continue
|
|
m = _JOB_OPEN.match(raw)
|
|
if m:
|
|
current = m.group(1)
|
|
jobs.append(current)
|
|
continue
|
|
m = _NEEDS.match(raw)
|
|
if m and current == DEPLOY:
|
|
needs = [n.strip() for n in m.group(1).split(",") if n.strip()]
|
|
return jobs, needs
|
|
|
|
|
|
def verdict(jobs, needs):
|
|
"""The complaints, as strings. Empty means the graph is sound."""
|
|
if DEPLOY not in jobs:
|
|
return [f"{DEPLOY} is not a job in this workflow"]
|
|
if needs is None:
|
|
return [f"{DEPLOY} has no `needs:` list, so nothing gates the deploy"]
|
|
expected = {j for j in jobs if j != DEPLOY}
|
|
missing = sorted(expected - set(needs))
|
|
unknown = sorted(set(needs) - expected)
|
|
out = [f"{DEPLOY} does not need `{j}`, so a failing {j} cannot stop a deploy"
|
|
for j in missing]
|
|
out += [f"{DEPLOY} needs `{j}`, which is not a job in this workflow" for j in unknown]
|
|
return out
|
|
|
|
|
|
_JOB_KEY = re.compile(r"^ ([A-Za-z_][\w-]*):(.*)$")
|
|
_STEP_START = re.compile(r"^ - ")
|
|
_STEP_KEY = re.compile(r"^ (?:- | )([A-Za-z_][\w-]*):(.*)$")
|
|
_ANY_KEY = re.compile(r"^(\s*)(?:- )?([A-Za-z_][\w-]*):(.*)$")
|
|
_CI_CALL = re.compile(r"^(?:nix shell \S+ --command )?\./scripts/checkers\.sh --ci ([\w-]+)$")
|
|
_DISQUALIFY = ("if", "continue-on-error", "shell")
|
|
|
|
|
|
_NAME = r"[A-Za-z_][\w-]*"
|
|
_TOP_KEY = re.compile(rf"^{_NAME}:(\s|$)")
|
|
_NEEDS_VALUE = re.compile(rf"^(|{_NAME}|\[\s*{_NAME}(\s*,\s*{_NAME})*\s*\])$")
|
|
_NEEDS_ITEM = re.compile(rf"^ - {_NAME}$")
|
|
_COMMENT = re.compile(r"\s#")
|
|
_UNSAFE_CHAR = re.compile("[\x00-\x08\x0b-\x1f\x7f-\x9f\u2028\u2029]")
|
|
|
|
|
|
def _readable(line, indent, in_steps, in_needs, job_keyed):
|
|
"""Whether a structural line inside `jobs:` is one this gate can read. Anything
|
|
else is refused, not parsed: context/code/tooling.md (improvise-w0j4, rjaa)."""
|
|
if indent == 2:
|
|
return bool(_JOB.match(line))
|
|
if indent == 4:
|
|
m = _JOB_KEY.match(line)
|
|
return bool(m) and (m.group(1) != "needs"
|
|
or bool(_NEEDS_VALUE.match(_inline_value(m.group(2)))))
|
|
if in_needs:
|
|
return bool(_NEEDS_ITEM.match(line))
|
|
if in_steps:
|
|
return indent in (6, 8) and bool(_STEP_KEY.match(line))
|
|
return indent > 4 and job_keyed
|
|
|
|
|
|
def _inline_value(rest):
|
|
"""A key's inline value. No comment is stripped: a line carrying one is
|
|
refused, so every matcher reads the same string (improvise-j49r)."""
|
|
return rest.strip()
|
|
|
|
|
|
_FLOW_LINE = re.compile(r"""^(\[[^\[\]{}"'#]*\]|\{[^\[\]{}"'#]*\})$""")
|
|
_BLOCK_HEADER = re.compile(r"^[|>][-+0-9]*$")
|
|
|
|
|
|
def _plain_value(value):
|
|
"""Whether a value is one of the few shapes the gate reads, all ending on their own
|
|
line; anything else is refused: context/code/tooling.md (improvise-0wn3)."""
|
|
if value == "" or _BLOCK_HEADER.match(value) or _FLOW_LINE.match(value):
|
|
return True
|
|
if value[0] in "\"'":
|
|
return not _opens_quote(value)
|
|
return value[0] not in "[]{}&*!|>%@`?,"
|
|
|
|
|
|
def _line_value(raw):
|
|
"""The value a line carries: a key's, a list item's, or the whole line's."""
|
|
k = _ANY_KEY.match(raw)
|
|
if k:
|
|
return _inline_value(k.group(3))
|
|
body = raw.strip()
|
|
return body[2:].strip() if body.startswith("- ") else body
|
|
|
|
|
|
def _opens_quote(value):
|
|
"""True when a quoted inline value is not closed on its own line."""
|
|
if value[:1] not in ("'", '"') or len(value) < 2:
|
|
return value[:1] in ("'", '"')
|
|
q, body = value[0], value[1:]
|
|
if q == "'":
|
|
return not body.replace("''", "").endswith("'")
|
|
return not re.sub(r'\\.', "", body).endswith('"')
|
|
|
|
|
|
def ci_steps(text):
|
|
"""(job, value, disqualifiers) for every step of every job, and every line
|
|
mentioning scripts/checkers.sh that is not a step's whole `run:` value.
|
|
|
|
Nothing is read as shell. A key with an inline value owns every following
|
|
line indented deeper, which is how a block scalar, a folded value, a plain
|
|
continuation or a heredoc inside one is skipped whole (improvise-st8q)."""
|
|
steps, stray, unread = [], [], []
|
|
job, in_steps, job_flags, step = None, False, set(), None
|
|
in_jobs, job_keyed = False, False
|
|
owner, owner_step = None, None
|
|
needs, if_jobs, in_needs = {}, set(), False
|
|
|
|
def close_step():
|
|
if step is not None:
|
|
steps.append(step)
|
|
|
|
for n, raw in enumerate(text.split("\n"), 1):
|
|
if _UNSAFE_CHAR.search(raw):
|
|
unread.append((n, "char"))
|
|
continue
|
|
indent = len(raw) - len(raw.lstrip())
|
|
if owner is not None and (not raw.strip() or indent > owner):
|
|
if "scripts/checkers.sh" in raw:
|
|
stray.append(n)
|
|
if owner_step is not None and raw.strip():
|
|
owner_step["multiline"] = True
|
|
continue
|
|
owner, owner_step = None, None
|
|
if not raw.strip() or raw.lstrip().startswith("#"):
|
|
continue
|
|
if indent == 0:
|
|
in_jobs = raw.startswith("jobs:")
|
|
readable = bool(_TOP_KEY.match(raw))
|
|
else:
|
|
readable = not in_jobs or indent > 8 or _readable(raw, indent, in_steps,
|
|
in_needs, job_keyed)
|
|
k = _ANY_KEY.match(raw)
|
|
shaped = _plain_value(_line_value(raw))
|
|
if _COMMENT.search(raw) or not shaped or not readable:
|
|
unread.append((n, "comment" if _COMMENT.search(raw) else
|
|
"plain" if not readable else "value"))
|
|
m = _JOB.match(raw)
|
|
if m:
|
|
close_step()
|
|
step = None
|
|
job, in_steps, job_flags, in_needs = m.group(1), False, set(), False
|
|
job_keyed = False
|
|
continue
|
|
if in_needs and re.match(r"^ - ", raw):
|
|
needs[job].add(_inline_value(raw.strip()[2:]))
|
|
continue
|
|
m = _JOB_KEY.match(raw)
|
|
if m and job is not None:
|
|
job_keyed = True
|
|
close_step()
|
|
step = None
|
|
key, value = m.group(1), _inline_value(m.group(2))
|
|
in_steps = key == "steps" and value == ""
|
|
in_needs = key == "needs" and value == ""
|
|
if key == "needs":
|
|
needs[job] = {x.strip() for x in value.strip("[]").split(",") if x.strip()}
|
|
if key == "if":
|
|
if_jobs.add(job)
|
|
if key in _DISQUALIFY:
|
|
job_flags.add(key)
|
|
for s in steps:
|
|
if s["job"] == job:
|
|
s["flags"].add("job " + key)
|
|
if in_steps and _STEP_START.match(raw):
|
|
close_step()
|
|
step = {"job": job, "run": None, "flags": {"job " + f for f in job_flags},
|
|
"line": n}
|
|
m = _STEP_KEY.match(raw) if in_steps and step is not None else None
|
|
if m:
|
|
key, value = m.group(1), _inline_value(m.group(2))
|
|
if key == "run":
|
|
step["run"] = value
|
|
if "scripts/checkers.sh" in raw and not _CI_CALL.match(value):
|
|
stray.append(n)
|
|
elif key in _DISQUALIFY:
|
|
step["flags"].add(key)
|
|
elif "scripts/checkers.sh" in raw:
|
|
stray.append(n)
|
|
elif "scripts/checkers.sh" in raw:
|
|
stray.append(n)
|
|
if k and _inline_value(k.group(3)):
|
|
owner = len(k.group(1)) + (2 if raw.lstrip().startswith("- ") else 0)
|
|
owner_step = step if m and m.group(1) == "run" else None
|
|
close_step()
|
|
for s in steps:
|
|
s["flags"] |= {f"needs {j}, which has if" for j in _needed(s["job"], needs) & if_jobs}
|
|
return steps, stray, unread
|
|
|
|
|
|
def _needed(job, needs):
|
|
"""Every job `job` needs, transitively: a skipped one skips it too (improvise-bbwa)."""
|
|
seen, todo = set(), list(needs.get(job, ()))
|
|
while todo:
|
|
j = todo.pop()
|
|
if j not in seen:
|
|
seen.add(j)
|
|
todo += needs.get(j, ())
|
|
return seen
|
|
|
|
|
|
def would_run(args):
|
|
"""What checkers.sh itself says these arguments run. Its argument rules and
|
|
its CI table are not restated here (improvise-maax)."""
|
|
out = subprocess.run(["scripts/checkers.sh", "--would-run", *args],
|
|
capture_output=True, text=True).stdout
|
|
ran, missing = set(), set()
|
|
for line in out.split("\n"):
|
|
if line.startswith("would-run: "):
|
|
ran.add(line[len("would-run: "):])
|
|
elif line.startswith("missing: "):
|
|
missing.add(line[len("missing: "):])
|
|
return ran, missing
|
|
|
|
|
|
def _unrun(s):
|
|
return (f"`{s}` is a checkers.sh suite that no CI step runs; add it to a row of "
|
|
"ci_suites in scripts/checkers.sh")
|
|
|
|
|
|
def _unread(n, kind="plain"):
|
|
if kind == "char":
|
|
return (f"ci.yml line {n} holds a control character or a line break YAML sees and "
|
|
"this gate does not (U+0085, U+2028, U+2029); remove it")
|
|
if kind == "value":
|
|
return (f"ci.yml line {n} has a value that is not one plain line (a multi-line "
|
|
"quote or flow, a nested or quoted flow, an anchor, alias or tag); spell "
|
|
"it plainly")
|
|
if kind == "comment":
|
|
return (f"ci.yml line {n} has a ` #`, which YAML may read as a comment; this gate "
|
|
"reads no comments, so put it on a line of its own")
|
|
return (f"ci.yml line {n} is a key or `needs:` value this gate cannot read as a "
|
|
"plain name, so it cannot trust the graph; spell it plainly")
|
|
|
|
|
|
def suite_verdict(suites, text):
|
|
"""The complaints: suites no counted step runs, and every step or line that
|
|
does not count, each named so the fix is obvious rather than silent."""
|
|
steps, stray, unread = ci_steps(text)
|
|
out, ran, missing = [_unread(n, c) for n, c in unread], set(), set()
|
|
for s in steps:
|
|
m = _CI_CALL.match(s["run"] or "")
|
|
if not m:
|
|
continue
|
|
if m.group(1) != s["job"]:
|
|
out.append(f"job `{s['job']}` runs `checkers.sh --ci {m.group(1)}`; "
|
|
"a job runs only its own row of the CI table")
|
|
elif s.get("multiline"):
|
|
out.append(f"job `{s['job']}`'s `checkers.sh --ci` step's run: continues "
|
|
"onto another line, so it does not count")
|
|
elif s["flags"]:
|
|
out.append(f"job `{s['job']}`'s `checkers.sh --ci` step is conditional "
|
|
f"({', '.join(sorted(s['flags']))}), so it does not count")
|
|
else:
|
|
r, mi = would_run(["--ci", s["job"]])
|
|
ran |= r
|
|
missing |= mi
|
|
out += [f"ci.yml line {n} mentions scripts/checkers.sh outside a step whose "
|
|
"whole `run:` is `./scripts/checkers.sh --ci <job>`" for n in stray]
|
|
out += [_unrun(s) for s in sorted(set(suites) - ran)]
|
|
out += [f"checkers.sh's CI table names `{s}`, which is not a suite"
|
|
for s in sorted(missing)]
|
|
return out
|
|
|
|
|
|
def read_workflow(path):
|
|
"""The workflow's text. Universal newlines fold CR and CRLF into the one break the
|
|
gate splits on; every other break YAML knows is refused (improvise-xpzh)."""
|
|
return open(path, encoding="utf8").read()
|
|
|
|
|
|
def list_suites():
|
|
"""The suites, from checkers.sh itself, so its naming rule has one home."""
|
|
out = subprocess.run(["scripts/checkers.sh", "--list"], capture_output=True,
|
|
text=True, check=True).stdout
|
|
return [s for s in out.split("\n") if s.strip()]
|
|
|
|
|
|
def _wf(*jobs):
|
|
return "jobs:\n" + "".join(jobs)
|
|
|
|
|
|
def _job(name, *steps, head=""):
|
|
return f" {name}:\n runs-on: x\n{head} steps:\n" + "".join(steps)
|
|
|
|
|
|
_BUILD = _job("build", " - run: ./scripts/checkers.sh --ci build\n")
|
|
_REQ = ["assemble-stamp", "pr-watch"]
|
|
_NO_PW = [_unrun("pr-watch")]
|
|
|
|
|
|
def _stray(n):
|
|
return (f"ci.yml line {n} mentions scripts/checkers.sh outside a step whose "
|
|
"whole `run:` is `./scripts/checkers.sh --ci <job>`")
|
|
|
|
|
|
_GATE = " gate:\n if: false\n steps:\n - run: echo\n"
|
|
_KB_RUN = " - run: ./scripts/checkers.sh --ci kb\n"
|
|
_YAML_ONLY_BREAKS = ("\x85", "\u2028", "\u2029")
|
|
_INNER_RUN = " run: ./scripts/checkers.sh --ci kb\n"
|
|
_NEEDS_GATE = ["job `kb`'s `checkers.sh --ci` step is conditional (needs gate, which has if), "
|
|
"so it does not count"] + _NO_PW
|
|
|
|
|
|
def _unreadable(name, text, *markers, rest=(), kind="plain"):
|
|
"""A case whose workflow spells a key or a `needs:` value the gate must refuse."""
|
|
lines = text.split("\n")
|
|
at = [next(i for i, l in enumerate(lines, 1) if m in l) for m in markers]
|
|
return (name, _REQ, text, [_unread(n, kind) for n in at] + list(rest))
|
|
|
|
|
|
# (name, suites to require, workflow text, the complaints it must draw). Every
|
|
# case asks the REAL checkers.sh table, where `kb`'s row holds pr-watch.
|
|
SUITE_SELF_TEST = [
|
|
("two jobs each running their own row are accepted", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: nix shell nixpkgs#trealla --command "
|
|
"./scripts/checkers.sh --ci kb\n")), []),
|
|
|
|
_unreadable("a trailing comment on the counted run: line is refused (j49r)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN.rstrip("\n") + " # the kb row\n")),
|
|
"# the kb row", rest=[_stray(9)] + _NO_PW, kind="comment"),
|
|
|
|
("a job that never calls its row leaves its suites unrun", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: echo nothing\n")), _NO_PW),
|
|
|
|
("a call inside a heredoc in a block run: does not count (st8q)", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: |\n cat >> x <<'EOF'\n"
|
|
" ./scripts/checkers.sh --ci kb\n EOF\n")),
|
|
[_stray(11)] + _NO_PW),
|
|
|
|
("a folded run: does not count (st8q)", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: >\n ./scripts/checkers.sh --ci kb\n")),
|
|
[_stray(10)] + _NO_PW),
|
|
|
|
("a plain continuation line does not count", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: echo\n ./scripts/checkers.sh --ci kb\n")),
|
|
[_stray(10)] + _NO_PW),
|
|
|
|
_unreadable("a run: line inside a multi-line quoted value: the quote is refused (6zf1)",
|
|
_wf(_BUILD, _job("kb", " - name: \"a\n"
|
|
" run: ./scripts/checkers.sh --ci kb\n b\"\n")),
|
|
'- name: "a', kind="value", rest=[_unread(11)]),
|
|
|
|
("a whole job inside a top-level block scalar does not count", _REQ,
|
|
_wf(_BUILD) + "notes: |\n kb:\n steps:\n"
|
|
" - run: ./scripts/checkers.sh --ci kb\n",
|
|
[_stray(9)] + _NO_PW),
|
|
|
|
("a step-shaped line inside a job-level block scalar does not count", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: echo\n",
|
|
head=" notes: |\n - run: ./scripts/checkers.sh --ci kb\n")),
|
|
[_stray(9)] + _NO_PW),
|
|
|
|
("a step-shaped item under a job key other than steps: does not count", _REQ,
|
|
_wf(_BUILD, " kb:\n services:\n - run: ./scripts/checkers.sh --ci kb\n"
|
|
" steps:\n - run: echo\n"),
|
|
[_stray(8)] + _NO_PW),
|
|
|
|
("the call with a continuation line does not count (1be2)", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: ./scripts/checkers.sh --ci kb\n"
|
|
" || true\n")),
|
|
["job `kb`'s `checkers.sh --ci` step's run: continues onto another line, "
|
|
"so it does not count"] + _NO_PW),
|
|
|
|
("a job that needs a job under if: does not count (bbwa)", _REQ,
|
|
_wf(_BUILD, " gate:\n if: false\n steps:\n - run: echo\n",
|
|
_job("kb", " - run: ./scripts/checkers.sh --ci kb\n",
|
|
head=" needs: [build, gate]\n")),
|
|
["job `kb`'s `checkers.sh --ci` step is conditional (needs gate, which has if), "
|
|
"so it does not count"] + _NO_PW),
|
|
|
|
("the same, with needs: written as a list", _REQ,
|
|
_wf(_BUILD, " gate:\n if: false\n steps:\n - run: echo\n",
|
|
_job("kb", " - run: ./scripts/checkers.sh --ci kb\n",
|
|
head=" needs:\n - gate\n")),
|
|
["job `kb`'s `checkers.sh --ci` step is conditional (needs gate, which has if), "
|
|
"so it does not count"] + _NO_PW),
|
|
|
|
("the 1be2 continuation exactly one column past the key does not count (w0j4)", _REQ,
|
|
_wf(_BUILD, _job("kb", _KB_RUN + " || true\n")),
|
|
["job `kb`'s `checkers.sh --ci` step's run: continues onto another line, "
|
|
"so it does not count"] + _NO_PW),
|
|
|
|
("control: a padded flow list is still read (w0j4)", _REQ,
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=" needs: [ gate ]\n")),
|
|
_NEEDS_GATE),
|
|
|
|
("control: a plain scalar needs: is still read (w0j4)", _REQ,
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=" needs: gate\n")),
|
|
_NEEDS_GATE),
|
|
|
|
_unreadable("a quoted name in a flow needs: is refused (w0j4)",
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=' needs: ["gate"]\n')),
|
|
'needs: ["gate"]'),
|
|
|
|
_unreadable("a quoted scalar needs: is refused (w0j4)",
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=" needs: 'gate'\n")),
|
|
"needs: 'gate'"),
|
|
|
|
_unreadable("a quoted item in a block needs: is refused (w0j4)",
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=' needs:\n - "gate"\n')),
|
|
'- "gate"'),
|
|
|
|
_unreadable("an anchored flow needs: is refused (w0j4)",
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=" needs: &d [gate]\n")),
|
|
"needs: &d"),
|
|
|
|
_unreadable("an anchored block needs: is refused (w0j4)",
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=" needs: &d\n - gate\n")),
|
|
"needs: &d"),
|
|
|
|
_unreadable("a flow needs: over two lines is refused (w0j4)",
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=" needs: [build,\n gate]\n")),
|
|
"needs: [build,"),
|
|
|
|
_unreadable("a quoted job key is refused (w0j4)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN, head=' "if": false\n')),
|
|
'"if"'),
|
|
|
|
_unreadable("a quoted step key is refused (w0j4)",
|
|
_wf(_BUILD, _job("kb", " - \"if\": false\n"
|
|
" run: ./scripts/checkers.sh --ci kb\n")),
|
|
'"if"'),
|
|
|
|
_unreadable("a single-quoted continue-on-error is refused (w0j4)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN, head=" 'continue-on-error': true\n")),
|
|
"'continue-on-error'"),
|
|
|
|
_unreadable("a key with a space before its colon is refused (w0j4)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN, head=" if : false\n")),
|
|
"if : false"),
|
|
|
|
_unreadable("a quoted job name is refused (w0j4)",
|
|
_wf(_BUILD, ' "gate":\n if: false\n steps:\n - run: echo\n',
|
|
_job("kb", _KB_RUN)),
|
|
'"gate"',
|
|
rest=["job `build`'s `checkers.sh --ci` step is conditional (job if), "
|
|
"so it does not count", _unrun("assemble-stamp")]),
|
|
|
|
_unreadable("a quoted top-level key is refused (w0j4)",
|
|
'"defaults":\n run:\n shell: bash\n' + _wf(_BUILD, _job("kb", _KB_RUN)),
|
|
'"defaults"'),
|
|
|
|
("control: a job key's block content at indent 6 is content (rjaa)", _REQ,
|
|
_wf(_BUILD, _job("kb", _KB_RUN, head=" env:\n FOO: bar\n")), []),
|
|
|
|
_unreadable("a needed job whose keys sit at indent 6 is refused (rjaa)",
|
|
_wf(_BUILD, " gate:\n if: false\n steps:\n - run: echo\n",
|
|
_job("kb", _KB_RUN, head=" needs: gate\n")),
|
|
" if: false", " steps:", " - run: echo"),
|
|
|
|
_unreadable("a needed job whose keys sit at indent 5 is refused (rjaa)",
|
|
_wf(_BUILD, " gate:\n if: false\n steps:\n - run: echo\n",
|
|
_job("kb", _KB_RUN, head=" needs: [gate]\n")),
|
|
" if: false", " steps:", " - run: echo"),
|
|
|
|
_unreadable("a job between, with its needs: at indent 6, is refused (rjaa)",
|
|
_wf(_BUILD, _GATE,
|
|
" mid:\n needs: gate\n steps:\n - run: echo\n",
|
|
_job("kb", _KB_RUN, head=" needs: mid\n")),
|
|
" needs: gate", " steps:", " - run: echo"),
|
|
|
|
_unreadable("an explicit ? key is refused (w0j4)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN, head=" ? if\n : false\n")),
|
|
"? if", ": false"),
|
|
|
|
("the same, two needs: away", _REQ,
|
|
_wf(_BUILD, " gate:\n if: false\n steps:\n - run: echo\n",
|
|
" mid:\n needs: gate\n steps:\n - run: echo\n",
|
|
_job("kb", " - run: ./scripts/checkers.sh --ci kb\n",
|
|
head=" needs: [mid]\n")),
|
|
["job `kb`'s `checkers.sh --ci` step is conditional (needs gate, which has if), "
|
|
"so it does not count"] + _NO_PW),
|
|
|
|
("a step under if: does not count (tvte)", _REQ,
|
|
_wf(_BUILD, _job("kb", " - if: false\n run: ./scripts/checkers.sh --ci kb\n")),
|
|
["job `kb`'s `checkers.sh --ci` step is conditional (if), so it does not count"]
|
|
+ _NO_PW),
|
|
|
|
("a job under if: does not count (tvte)", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: ./scripts/checkers.sh --ci kb\n",
|
|
head=" if: false\n")),
|
|
["job `kb`'s `checkers.sh --ci` step is conditional (job if), so it does not count"]
|
|
+ _NO_PW),
|
|
|
|
("a job-level if: after its steps still disqualifies them", _REQ,
|
|
_wf(_BUILD, " kb:\n steps:\n - run: ./scripts/checkers.sh --ci kb\n"
|
|
" if: false\n"),
|
|
["job `kb`'s `checkers.sh --ci` step is conditional (job if), so it does not count"]
|
|
+ _NO_PW),
|
|
|
|
("a step under a defaults: run: shell: is counted (the user's ruling)", _REQ,
|
|
"defaults:\n run:\n shell: nix develop .#ci --command bash -e {0}\n"
|
|
+ _wf(_BUILD, _job("kb", _KB_RUN, head=" defaults:\n run:\n shell: bash\n")),
|
|
[]),
|
|
|
|
("continue-on-error does not count", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: ./scripts/checkers.sh --ci kb\n"
|
|
" continue-on-error: true\n")),
|
|
["job `kb`'s `checkers.sh --ci` step is conditional (continue-on-error), "
|
|
"so it does not count"] + _NO_PW),
|
|
|
|
("shell control flow around the call does not count", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: true || ./scripts/checkers.sh --ci kb\n")),
|
|
[_stray(9)] + _NO_PW),
|
|
|
|
("a job running another job's row is refused", _REQ,
|
|
_wf(_BUILD, _job("kb", " - run: ./scripts/checkers.sh --ci build\n")),
|
|
["job `kb` runs `checkers.sh --ci build`; a job runs only its own row of the CI table"]
|
|
+ _NO_PW),
|
|
|
|
("a bare mention in a step name is refused", _REQ,
|
|
_wf(_BUILD, _job("kb", " - name: runs scripts/checkers.sh\n"
|
|
" run: ./scripts/checkers.sh --ci kb\n")),
|
|
[_stray(9)]),
|
|
|
|
_unreadable("a job header with a trailing comment is refused (j49r)",
|
|
_wf(_BUILD, " test:\n runs-on: x\n",
|
|
" gate: # skipped on purpose\n if: false\n steps:\n - run: echo\n",
|
|
_job("kb", _KB_RUN, head=" needs: gate\n")),
|
|
"gate: # skipped", kind="comment"),
|
|
|
|
_unreadable("a closed quote followed by a comment is refused (j49r)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN, head=" env:\n A: \"x\" # note\n")
|
|
+ " if: false\n", " after:\n steps:\n - run: echo \"z\"\n"),
|
|
'"x" # note', kind="comment",
|
|
rest=["job `kb`'s `checkers.sh --ci` step is conditional (job if), "
|
|
"so it does not count"] + _NO_PW),
|
|
|
|
_unreadable("a # inside a quoted value, below indent 8, is refused (j49r)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN + " - uses: x\n with:\n"
|
|
" foo: \"x # y\"\n")
|
|
+ " if: false\n", " after:\n steps:\n - run: echo \"z\"\n"),
|
|
'"x # y"', kind="comment",
|
|
rest=["job `kb`'s `checkers.sh --ci` step is conditional (job if), "
|
|
"so it does not count"] + _NO_PW),
|
|
|
|
_unreadable("a trailing comment on a step is refused (j49r)",
|
|
_wf(_BUILD, _job("kb", " - uses: actions/checkout@v7 # pin\n" + _KB_RUN)),
|
|
"# pin", kind="comment"),
|
|
|
|
_unreadable("a one-line run: with a ` #` inside quotes is refused (j49r)",
|
|
_wf(_BUILD, _job("kb", " - run: echo \"a # b\"\n" + _KB_RUN)),
|
|
'"a # b"', kind="comment"),
|
|
|
|
_unreadable("a multi-line double-quoted value is refused, escaped quote and all (6zf1)",
|
|
_wf(_BUILD, " gate:\n runs-on: x\n name: \"a\n it\\\" is\n b: \"\n"
|
|
" if: false\n steps:\n - run: echo \"z\"\n",
|
|
_job("kb", _KB_RUN, head=" needs: gate\n")),
|
|
'name: "a', kind="value", rest=_NEEDS_GATE),
|
|
|
|
_unreadable("a multi-line single-quoted value is refused, doubled quote and all (6zf1)",
|
|
_wf(_BUILD, " gate:\n runs-on: x\n name: 'a\n it''s\n b: '\n"
|
|
" if: false\n steps:\n - run: echo 'z'\n",
|
|
_job("kb", _KB_RUN, head=" needs: gate\n")),
|
|
"name: 'a", kind="value", rest=_NEEDS_GATE),
|
|
|
|
_unreadable("an opening line ending in an escaped \\\" still opens a quote (6zf1)",
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=" needs: gate\n name: \"a\\\"\n"
|
|
" needs: ok\n x: y\"\n")),
|
|
'name: "a', kind="value"),
|
|
|
|
_unreadable("an opening line ending in a doubled '' still opens a quote (6zf1)",
|
|
_wf(_BUILD, _GATE, _job("kb", _KB_RUN, head=" needs: gate\n name: 'it''\n"
|
|
" needs: ok\n x: y'\n")),
|
|
"name: 'it", kind="value"),
|
|
|
|
_unreadable("a flow list whose quote runs onto the next lines is refused (0wn3)",
|
|
_wf(_BUILD, _job("kb", " - name: [a, \"b\n" + _INNER_RUN + " c: x\"]\n")),
|
|
'name: [a, "b', kind="value"),
|
|
|
|
_unreadable("the same with a single-quoted string (0wn3)",
|
|
_wf(_BUILD, _job("kb", " - name: [a, 'b\n" + _INNER_RUN + " c: x']\n")),
|
|
"name: [a, 'b", kind="value"),
|
|
|
|
_unreadable("the same as a flow map (0wn3)",
|
|
_wf(_BUILD, _job("kb", " - name: {a: \"b\n" + _INNER_RUN + " c: x\"}\n")),
|
|
'name: {a: "b', kind="value"),
|
|
|
|
_unreadable("a flow list that ends in its closer with a quote still open is refused (0wn3)",
|
|
_wf(_BUILD, _job("kb", " - name: [a, \"b]\n" + _INNER_RUN + " c: x\"]\n")),
|
|
'name: [a, "b]', kind="value"),
|
|
|
|
_unreadable("a flow value opened on a line of its own is refused too (0wn3)",
|
|
_wf(_BUILD, _job("kb", " - name:\n [a, \"b\n" + _INNER_RUN
|
|
+ " c: x\"]\n")),
|
|
'[a, "b', kind="value"),
|
|
|
|
_unreadable("an anchored multi-line flow is refused (0wn3)",
|
|
_wf(_BUILD, _job("kb", " - name: &a [x, \"b\n" + _INNER_RUN + " c: x\"]\n")),
|
|
'name: &a', kind="value"),
|
|
|
|
_unreadable("a tagged multi-line flow is refused (0wn3)",
|
|
_wf(_BUILD, _job("kb", " - name: !!seq [x, \"b\n" + _INNER_RUN
|
|
+ " c: x\"]\n")),
|
|
'name: !!seq', kind="value"),
|
|
|
|
*[_unreadable(f"a U+{ord(sep):04X} inside a line is refused (xpzh)",
|
|
_wf(_BUILD, " kb:\n runs-on: x" + sep + " if: false\n steps:\n" + _KB_RUN),
|
|
sep, kind="char")
|
|
for sep in _YAML_ONLY_BREAKS],
|
|
|
|
_unreadable("a line break inside a block scalar's body is refused too (xpzh)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN + " - run: |\n echo a\u2028 if: false\n")),
|
|
"echo a", kind="char"),
|
|
|
|
_unreadable("a line break inside a full-line comment is refused too (xpzh)",
|
|
_wf(_BUILD, " kb:\n runs-on: x\n # note\u2028 if: false\n steps:\n"
|
|
+ _KB_RUN), "# note", kind="char"),
|
|
|
|
("a YAML comment mentioning checkers.sh is fine", _REQ,
|
|
_wf(_BUILD, _job("kb", " # every suite goes through scripts/checkers.sh\n"
|
|
" - run: ./scripts/checkers.sh --ci kb\n")), []),
|
|
]
|
|
|
|
|
|
# (name, file bytes as text, the complaints it must draw), read back through
|
|
# read_workflow, because what a CR means depends on that read (improvise-xpzh).
|
|
FILE_SELF_TEST = [
|
|
("control: CRLF line ends are folded by the read and accepted (xpzh)",
|
|
_wf(_BUILD, _job("kb", _KB_RUN)).replace("\n", "\r\n"), []),
|
|
|
|
("control: a lone CR is a line break, as YAML reads it (xpzh)",
|
|
_wf(_BUILD, " kb:\n runs-on: x\r if: false\n steps:\n" + _KB_RUN),
|
|
["job `kb`'s `checkers.sh --ci` step is conditional (job if), so it does not count"]
|
|
+ _NO_PW),
|
|
]
|
|
|
|
|
|
# (name, workflow text, the complaints it must draw).
|
|
SELF_TEST = [
|
|
("a job nobody gates is named",
|
|
"jobs:\n test:\n x: 1\n zz-new-check:\n x: 1\n"
|
|
" pages-deploy:\n needs: [test]\n",
|
|
[f"{DEPLOY} does not need `zz-new-check`, so a failing zz-new-check cannot stop a deploy"]),
|
|
|
|
("the complete graph is accepted",
|
|
"jobs:\n test:\n x: 1\n wasm:\n x: 1\n"
|
|
" pages-deploy:\n needs: [test, wasm]\n", []),
|
|
|
|
("a needs naming a job that does not exist is caught",
|
|
"jobs:\n test:\n x: 1\n pages-deploy:\n needs: [test, ghost]\n",
|
|
[f"{DEPLOY} needs `ghost`, which is not a job in this workflow"]),
|
|
|
|
("a deploy with no needs at all is caught",
|
|
"jobs:\n test:\n x: 1\n pages-deploy:\n x: 1\n",
|
|
[f"{DEPLOY} has no `needs:` list, so nothing gates the deploy"]),
|
|
|
|
("a job header with a trailing comment is still a job (j49r)",
|
|
"jobs:\n test:\n x: 1\n gate: # skipped on purpose\n x: 1\n"
|
|
" pages-deploy:\n needs: [test]\n",
|
|
[f"{DEPLOY} does not need `gate`, so a failing gate cannot stop a deploy"]),
|
|
|
|
# `needs:` belongs to the job it is indented under. Taking the first one
|
|
# seen would read ANOTHER job's list as the deploy's and pass.
|
|
("another job's needs is not mistaken for the deploy's",
|
|
"jobs:\n test:\n x: 1\n browser:\n needs: [test]\n"
|
|
" pages-deploy:\n x: 1\n",
|
|
[f"{DEPLOY} has no `needs:` list, so nothing gates the deploy"]),
|
|
]
|
|
|
|
|
|
def self_test():
|
|
failures = []
|
|
for name, text, want in SELF_TEST:
|
|
jobs, needs = read_graph(text)
|
|
got = verdict(jobs, needs)
|
|
if got != want:
|
|
failures.append(f" {name}: expected {want}, saw {got}")
|
|
for name, suites, text, want in SUITE_SELF_TEST:
|
|
got = suite_verdict(suites, text)
|
|
if got != want:
|
|
failures.append(f" {name}: expected {want}, saw {got}")
|
|
for name, content, want in FILE_SELF_TEST:
|
|
with tempfile.TemporaryDirectory() as d:
|
|
path = os.path.join(d, "ci.yml")
|
|
with open(path, "w", encoding="utf8", newline="") as f:
|
|
f.write(content)
|
|
got = suite_verdict(_REQ, read_workflow(path))
|
|
if got != want:
|
|
failures.append(f" {name}: expected {want}, saw {got}")
|
|
if failures:
|
|
print("ci-graph self-test FAILED:", file=sys.stderr)
|
|
print("\n".join(failures), file=sys.stderr)
|
|
return 1
|
|
print(f"ci-graph self-test: {len(SELF_TEST)} graphs, "
|
|
f"{len(SUITE_SELF_TEST)} suite lists and {len(FILE_SELF_TEST)} files judged")
|
|
return 0
|
|
|
|
|
|
def main():
|
|
if "--self-test" in sys.argv[1:]:
|
|
return self_test()
|
|
text = read_workflow(WORKFLOW)
|
|
jobs, needs = read_graph(text)
|
|
if len(jobs) < 2:
|
|
print(f"error: {WORKFLOW}: found {len(jobs)} jobs; the parse is wrong",
|
|
file=sys.stderr)
|
|
return 2
|
|
suites = list_suites()
|
|
if not suites:
|
|
print("error: checkers.sh --list named no suites; the listing is wrong",
|
|
file=sys.stderr)
|
|
return 2
|
|
complaints = verdict(jobs, needs) + suite_verdict(suites, text)
|
|
if complaints:
|
|
for c in complaints:
|
|
print(f"ci-graph: {c}", file=sys.stderr)
|
|
return 1
|
|
print(f"ci-graph: {DEPLOY} needs all {len(jobs) - 1} other jobs; "
|
|
f"all {len(suites)} checkers.sh suites run in CI")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|