#!/usr/bin/env node
// Generate the documentation site from context/ into OUT_DIR (default site/docs).
// The argument and the rationale live in context/code/doc-site.md.
//
// NOT standalone: the link check resolves ../index.html and ../app/ on disk, so
// OUT_DIR's parent must already be an assembled site. Run web/assemble.sh, which
// invokes this last for that reason. Bare `node scripts/build-doc-site.mjs out`
// exits 1 on 48 outward links, and that strictness is the point — the hole it
// replaced was counting those links as checked while skipping them.
//
// The reorder also moved WHERE a failure shows: a dead link now leaves a site/
// that looks complete with a partial docs/, where before it left one with no
// app/. Contained only because assemble.sh is `set -eu` and CI gates on it.
import { readFileSync, writeFileSync, mkdirSync, rmSync, readdirSync, existsSync, statSync } from "node:fs";
import { dirname, join, relative } from "node:path";
import { fileURLToPath } from "node:url";
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
const out = process.argv[2] || join(root, "site", "docs");
const token = process.argv[3] || "";
const ref = /^[0-9a-f]{40}$/.test(token) ? token : "main";
const REPO = "https://github.com/fiddlerwoaroof/improvise";
// ---------------------------------------------------------------------------
// The manifest. A doc is published only by being named here — see
// context/code/doc-site.md for what was left out and why.
const SECTIONS = [
{
id: "start",
title: "Start here",
blurb:
"What the project is for, and the principles the code is held to.",
docs: [
["docs/design-notes.md", "design-notes", "Design notes"],
["context/design-principles.md", "design-principles", "Design principles"],
["context/repo-map.md", "repo-map", "Repository map"],
],
},
{
id: "adr",
title: "Decision records",
blurb: "Decisions with consequences, including the options that were rejected.",
docs: [
["context/adr/001-concurrent-edits.md", "adr/001-concurrent-edits", "ADR 001 — Concurrent edits"],
["context/adr/002-naming-a-cell.md", "adr/002-naming-a-cell", "ADR 002 — Naming a cell"],
["context/adr/003-live-backed-models.md", "adr/003-live-backed-models", "ADR 003 — Live-backed models"],
["context/adr/004-faces-and-themes.md", "adr/004-faces-and-themes", "ADR 004 — Faces"],
["context/adr/005-exact-decimal-arithmetic.md", "adr/005-exact-decimal-arithmetic", "ADR 005 — Exact decimal arithmetic (proposed)"],
["context/adr/007-stable-identities.md", "adr/007-stable-identities", "ADR 007 — Stable identities"],
],
},
{
id: "code",
title: "The arguments, by area",
blurb:
"The reasoning that used to live in code comments. Written for a reader with the source open.",
docs: [
["context/code/core.md", "code/core", "improvise-core"],
["context/code/formula-and-io.md", "code/formula-and-io", "improvise-formula and improvise-io"],
["context/code/protocol.md", "code/protocol", "improvise-protocol"],
["context/code/command.md", "code/command", "improvise-command"],
["context/code/clients.md", "code/clients", "The clients"],
["context/code/tui.md", "code/tui", "The TUI"],
["context/code/cli.md", "code/cli", "The headless CLI"],
["context/code/gp.md", "code/gp", "gp-tree and improvise-gp"],
["context/code/numtower.md", "code/numtower", "numtower, the numeric tower"],
["context/code/kb-tool.md", "code/kb-tool", "The kb tool"],
["context/code/tooling.md", "code/tooling", "The repository's own checkers"],
],
},
{
id: "evidence",
title: "Test evidence",
blurb:
"For each test whose narrative is longer than its code: what broke, what fault was injected, and why the test can fail. Contributor reference, not a tour.",
docs: [
["context/test-evidence/core.md", "test-evidence/core", "improvise-core"],
["context/test-evidence/formula-and-io.md", "test-evidence/formula-and-io", "improvise-formula and improvise-io"],
["context/test-evidence/protocol.md", "test-evidence/protocol", "improvise-protocol"],
["context/test-evidence/command.md", "test-evidence/command", "improvise-command"],
["context/test-evidence/clients.md", "test-evidence/clients", "The clients"],
["context/test-evidence/tui.md", "test-evidence/tui", "The TUI"],
["context/test-evidence/cli.md", "test-evidence/cli", "The headless CLI"],
["context/test-evidence/gp.md", "test-evidence/gp", "gp-tree and improvise-gp"],
["context/test-evidence/numtower.md", "test-evidence/numtower", "numtower"],
["context/test-evidence/tooling.md", "test-evidence/tooling", "The repository's own gates"],
["context/test-evidence/layout-cost.md", "test-evidence/layout-cost", "Layout and keypress cost"],
["context/test-evidence/scale-bisect.md", "test-evidence/scale-bisect", "Scale bisect"],
],
},
{
id: "meta",
title: "About this site",
blurb: "What is published here, what is not, and why.",
docs: [["context/code/doc-site.md", "doc-site", "About this site"]],
},
];
const PAGE_BY_SOURCE = new Map();
for (const s of SECTIONS) for (const [src, slug] of s.docs) PAGE_BY_SOURCE.set(src, `${slug}.html`);
// An omission is a DECISION, so it is restated here beside the manifest rather
// than only in doc-site.md section 2. An allowlist and not a count: a new
// omission has to be argued in code, and a vanished one has to be removed --
// a stale entry asserting a reason for a file nobody can find is the failure a
// dead ticket id is (improvise-res9).
const OMITTED = new Map([
["context/plan.md", "a prompt, not a plan; cites a context/SPEC.md that no longer exists"],
["context/cloud-agents.md", "operational notes for agents, not about the program"],
["context/code/pr-review.md", "how an agent reads a PR's reviews; about this repo's process, not about the program"],
]);
// Derived, because the manifest only ever grows and a doc added without a line
// here publishes nothing and says nothing: build green, link check green, doc
// simply absent. This runs at module scope, before any page is generated, so
// it answers the manifest question before the outward-link check answers a
// different one -- a missing doc is actionable wherever you ran the generator.
{
const onDisk = [];
(function walk(d) {
for (const e of readdirSync(join(root, d), { withFileTypes: true })) {
const rel = `${d}/${e.name}`;
if (e.isDirectory()) walk(rel);
else if (e.name.endsWith(".md")) onDisk.push(rel);
}
})("context");
onDisk.push("docs/design-notes.md");
const unaccounted = onDisk.filter((f) => !PAGE_BY_SOURCE.has(f) && !OMITTED.has(f));
const vanished = [...OMITTED.keys()].filter((f) => !existsSync(join(root, f)));
if (unaccounted.length || vanished.length) {
for (const f of unaccounted) console.error(`doc-site: ${f} is neither published nor a named omission`);
for (const f of vanished) console.error(`doc-site: ${f} is listed as omitted but does not exist`);
process.exit(1);
}
}
// The generator's link check resolves ../index.html and ../app/ against the
// filesystem, so it must run once the rest of the tree is there. PR #63's
// appended loop happened to land above it; one landing below would make the
// outward-link check pass against a half-built tree, silently.
{
const cmds = readFileSync(join(root, "web", "assemble.sh"), "utf8")
.split("\n")
.map((l) => l.trim())
.filter((l) => l && !l.startsWith("#"));
// An INVOCATION, not a mention: `echo "see scripts/build-doc-site.mjs"` after
// a real command satisfied `.includes` while running nothing (review, PR #83).
if (!/^(node|sh)\s.*build-doc-site\.mjs/.test(cmds[cmds.length - 1])) {
console.error("doc-site: web/assemble.sh must invoke this generator LAST;");
console.error(`doc-site: its last command is instead: ${cmds[cmds.length - 1]}`);
process.exit(1);
}
}
// ---------------------------------------------------------------------------
// The ticket index. `.beads/issues.jsonl` is a passive export and lags the Dolt
// database, so an id it does not carry is rendered as plain text rather than a
// link that 404s.
function readTickets() {
const path = join(root, ".beads", "issues.jsonl");
const map = new Map();
if (!existsSync(path)) return map;
for (const line of readFileSync(path, "utf8").split("\n")) {
if (!line.trim()) continue;
let o;
try {
o = JSON.parse(line);
} catch {
continue;
}
if (o && o._type === "issue" && typeof o.id === "string" && typeof o.title === "string") {
map.set(o.id, { title: o.title, status: o.status || "unknown", kind: o.issue_type || o.type || "" });
}
}
return map;
}
const TICKETS = readTickets();
// Tokens that look like an id but name a crate or a package. Anchored, because
// an unanchored alternative is a prefix match that silently excludes a real id.
const NON_TICKET = new Set(
"core formula io protocol command gp ws worker wasm harness registry side".split(" "),
);
// ---------------------------------------------------------------------------
// Markdown. A deliberately small subset, covering exactly what the corpus uses;
// anything unrecognised becomes paragraph text, so no input line is ever lost.
const esc = (s) =>
s
.replace(/&/g, "&")
.replace(//g, ">")
.replace(/"/g, """)
.replace(/'/g, "'");
// A TOC entry must not nest an anchor inside the one wrapping it: a heading
// whose text is a path or a ticket id already carries an , and the parser
// closes the outer one early, leaving an empty unclickable stub.
const stripTags = (html) => html.replace(/<[^>]+>/g, "");
function slugify(s) {
return (
s
.toLowerCase()
.replace(/<[^>]+>/g, "")
.replace(/&[a-z]+;/g, "")
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "") || "section"
);
}
// A code span whose whole content is a repo path becomes a link: to the
// generated page when the path is a published doc, to GitHub otherwise. The
// path must exist on disk, so a stale citation stays plain text.
function pathHref(p, depth) {
const clean = p.replace(/^\.\//, "");
if (!/^[A-Za-z0-9_.\/-]+$/.test(clean) || !/[\/.]/.test(clean)) return null;
// A span of punctuation alone (`.`, `-`, `/`) is prose, not a path.
if (!/[A-Za-z0-9]/.test(clean)) return null;
if (clean.includes("..") || clean.startsWith("/")) return null;
if (PAGE_BY_SOURCE.has(clean)) return "../".repeat(depth) + PAGE_BY_SOURCE.get(clean);
const abs = join(root, clean);
if (!existsSync(abs)) return null;
// GitHub serves a directory under /tree/, not /blob/; relying on its
// redirect is a 404 waiting on a change of behaviour we do not control.
const kind = statSync(abs).isDirectory() ? "tree" : "blob";
return `${REPO}/${kind}/${ref}/${clean.replace(/\/$/, "")}`;
}
function inline(text, ctx) {
// Code spans first: a backtick run opens a span only if a run of the same
// length closes it, so an unmatched ``` in prose stays literal.
const parts = [];
let i = 0;
while (i < text.length) {
if (text[i] === "`") {
let n = 0;
while (text[i + n] === "`") n++;
const fence = "`".repeat(n);
const end = text.indexOf(fence, i + n);
const after = end >= 0 ? end + n : -1;
if (end >= 0 && (after >= text.length || text[after] !== "`")) {
parts.push({ code: text.slice(i + n, end) });
i = after;
continue;
}
}
const next = text.indexOf("`", i + 1);
parts.push({ text: text.slice(i, next < 0 ? text.length : next) });
i = next < 0 ? text.length : next;
}
// Code spans stand aside as placeholders while emphasis is matched, because
// the corpus routinely opens `**` on one side of a code span and closes it on
// the other; matching per segment silently drops that bold.
const held = [];
let s = parts
.map((p) => {
if (p.code === undefined) return esc(p.text);
const body = `${esc(p.code)}`;
const href = pathHref(p.code.trim(), ctx.depth);
held.push(href ? `${body}` : body);
return `\u0000${held.length - 1}\u0000`;
})
.join("");
s = s.replace(/\[([^\]]+)\]\(([^)\s]+)\)/g, (_m, label, href) => `${label}`);
// Strong first, and its body may contain a single `*` pair: `[^*]+` cannot
// match `**a *b* c**`, and the leftover `*` then pairs across the bold.
s = s.replace(/\*\*((?:[^*]|\*(?!\*))+)\*\*/g, "$1");
s = s.replace(/\*([^*]+)\*/g, "$1");
s = s.replace(/\bimprovise-([a-z0-9]{3,8})\b/g, (m, tail) => {
if (NON_TICKET.has(tail)) return m;
ctx.cited.add(m);
if (!TICKETS.has(m)) {
ctx.unresolved.add(m);
return `${m}`;
}
const t = TICKETS.get(m);
const href = `${"../".repeat(ctx.depth)}tickets.html#${m}`;
return `${m}`;
});
return s.replace(/\u0000(\d+)\u0000/g, (_m, n) => held[Number(n)]);
}
// Block parse. Returns { html, toc }.
function renderMarkdown(src, ctx) {
const lines = src.split("\n");
const toc = [];
const seen = new Map();
const html = blocks(lines);
return { html: html.join("\n"), toc };
function heading(level, text) {
const rendered = inline(text, ctx);
let id = slugify(text);
const n = (seen.get(id) || 0) + 1;
seen.set(id, n);
if (n > 1) id = `${id}-${n}`;
ctx.anchors.add(id);
if (level <= 3) toc.push({ level, id, text: stripTags(rendered) });
const mark = level > 1 ? `#` : "";
return `${mark}${rendered}`;
}
function blocks(ls) {
const outHtml = [];
let i = 0;
let para = [];
const flush = () => {
if (para.length) outHtml.push(`
${inline(para.join(" "), ctx)}
`);
para = [];
};
while (i < ls.length) {
const line = ls[i];
const fence = line.match(/^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_-]*)\s*$/);
if (fence) {
flush();
const close = new RegExp(`^\\s*${fence[2][0] === "~" ? "~" : "\`"}{3,}\\s*$`);
const body = [];
i++;
while (i < ls.length && !close.test(ls[i])) {
body.push(ls[i]);
i++;
}
i++;
outHtml.push(`
${esc(body.join("\n"))}
`);
continue;
}
const h = line.match(/^(#{1,6})\s+(.*)$/);
if (h) {
flush();
outHtml.push(heading(h[1].length, h[2].trim()));
i++;
continue;
}
if (/^\s*(-{3,}|\*{3,}|_{3,})\s*$/.test(line) && !para.length) {
flush();
outHtml.push("");
i++;
continue;
}
if (/^\s*\|/.test(line) && i + 1 < ls.length && /^\s*\|[\s:|-]+\|?\s*$/.test(ls[i + 1])) {
flush();
const rows = [];
while (i < ls.length && /^\s*\|/.test(ls[i])) {
rows.push(ls[i]);
i++;
}
outHtml.push(table(rows));
continue;
}
if (/^\s*>/.test(line)) {
flush();
const body = [];
while (i < ls.length && /^\s*>/.test(ls[i])) {
body.push(ls[i].replace(/^\s*>\s?/, ""));
i++;
}
outHtml.push(`
${blocks(body).join("\n")}
`);
continue;
}
const li = line.match(/^(\s*)([-*+]|\d{1,3}\.)\s+(.*)$/);
if (li) {
flush();
const [listHtml, next] = list(ls, i, li[1].length);
outHtml.push(listHtml);
i = next;
continue;
}
if (!line.trim()) {
flush();
i++;
continue;
}
para.push(line.trim());
i++;
}
flush();
return outHtml;
}
// An item's continuation lines are dedented by that item's own marker width,
// so a nested list keeps its relative indentation and reparses as a list.
function list(ls, start, indent) {
const ordered = /^\s*\d{1,3}\./.test(ls[start]);
const items = [];
let i = start;
let first = null;
while (i < ls.length) {
const m = ls[i].match(/^(\s*)([-*+]|(\d{1,3})\.)\s+(.*)$/);
if (m && m[1].length === indent) {
if (ordered && first === null) first = Number(m[3]);
items.push({ num: m[3] ? Number(m[3]) : null, cw: m[0].length - m[4].length, body: [m[4]] });
i++;
continue;
}
if (!items.length) break;
const cur = items[items.length - 1];
if (!ls[i].trim()) {
let j = i;
while (j < ls.length && !ls[j].trim()) j++;
if (j >= ls.length) break;
// A blank line ends the list only if what follows is neither the next
// item nor a continuation of the open one.
const isSibling = /^(\s*)([-*+]|\d{1,3}\.)\s+/.test(ls[j]) && ls[j].search(/\S/) === indent;
if (isSibling) {
i = j;
continue;
}
if (ls[j].search(/\S/) > indent) {
cur.body.push("");
i = j;
continue;
}
break;
}
const col = ls[i].search(/\S/);
if (col <= indent) break;
cur.body.push(ls[i].slice(Math.min(col, cur.cw)));
i++;
}
const tag = ordered ? "ol" : "ul";
const attr = ordered && first !== null && first !== 1 ? ` start="${first}"` : "";
const rendered = items
.map((it) => {
const inner = blocks(it.body);
const body = inner.length === 1 && inner[0].startsWith("
")
? inner[0].slice(3, -4)
: inner.join("\n");
const id = ctx.listItemId && it.num !== null ? ctx.listItemId(it.num) : null;
if (id) ctx.anchors.add(id);
return `${body}`;
})
.join("\n");
return [`<${tag}${attr}>\n${rendered}\n`, i];
}
// A cell boundary is an UNESCAPED pipe outside a code span. Splitting the raw
// row on every `|` shreds `\|` and this repo's own `|Time Period|` syntax into
// extra columns, which no column count can detect because the row still parses.
function splitRow(r) {
const out = [];
let cur = "";
let tick = 0;
for (let i = 0; i < r.length; i++) {
const c = r[i];
if (c === "\\" && r[i + 1] === "|" && !tick) {
cur += "|";
i++;
continue;
}
if (c === "`") {
let n = 0;
while (r[i + n] === "`") n++;
if (!tick) tick = n;
else if (tick === n) tick = 0;
cur += "`".repeat(n);
i += n - 1;
continue;
}
if (c === "|" && !tick) {
out.push(cur);
cur = "";
continue;
}
cur += c;
}
out.push(cur);
if (out.length && !out[0].trim()) out.shift();
if (out.length && !out[out.length - 1].trim()) out.pop();
return out;
}
function table(rows) {
const cells = (r) => splitRow(r.trim()).map((c) => inline(c.trim(), ctx));
const head = cells(rows[0]);
const body = rows.slice(2).map(cells);
return (
`
`;
// Do NOT "simplify" this to : the sidebar must force it OPEN, which
// `display` cannot do, and the desktop nav collapses. doc-site.md §7.
const parts = [
`");
return parts.join("");
}
function page({ slug, title, body, depth }) {
const up = "../".repeat(depth);
return `
${esc(title)} — improvise docs
${nav(slug, depth)}
${body}
`;
}
function write(rel, content) {
const p = join(out, rel);
mkdirSync(dirname(p), { recursive: true });
writeFileSync(p, content);
return p;
}
// ---------------------------------------------------------------------------
rmSync(out, { recursive: true, force: true });
mkdirSync(out, { recursive: true });
write("style.css", CSS);
const cited = new Set();
const unresolved = new Set();
const written = [];
for (const section of SECTIONS) {
for (const [src, slug, title] of section.docs) {
const depth = slug.split("/").length - 1;
const ctx = { depth, cited, unresolved, anchors: new Set() };
if (slug === "repo-map") ctx.listItemId = (n) => (n >= 1 && n <= 200 ? `gotcha-${n}` : null);
const md = readFileSync(join(root, src), "utf8");
const { html, toc } = renderMarkdown(md, ctx);
const tocHtml = toc.length
? `On this page
`
: "";
// Derived from the anchors actually emitted. A hardcoded count fails closed
// on a deleted gotcha (the link check catches it) and SILENTLY on an added
// one, which is the direction the list moves in.
const gotchas = [...ctx.anchors]
.map((a) => /^gotcha-(\d+)$/.exec(a))
.filter(Boolean)
.map((m) => Number(m[1]))
.sort((a, b) => a - b);
const jump = gotchas.length
? `
Cited in the prose; the committed tracker export does not carry it.
`;
return `
${id}
${esc(t.status)}
${esc(t.title)}
`;
})
.join("");
const body = `
Ticket index
The prose on this site cites issue ids like improvise-9c8c. They are this project's own tracker ids, kept in-repo with beads; there is no public issue URL to link to, so this page is what they resolve to. Titles come from .beads/issues.jsonl, a passive export that lags the database — an id it does not carry is listed here without a title and is not linked from the prose.
${ids.length} ids are cited, of which ${ids.length - unresolved.size} resolve.
Id
Status
Title
${rows}
`;
written.push({ rel: "tickets.html", html: page({ slug: "tickets", title: "Ticket index", depth: 0, body }), anchors: ctx.anchors });
}
// Landing page for the docs section.
{
// The two conventions go through `inline` rather than being hand-written, so
// the sentence describing the linking demonstrates it instead of asserting it.
const conventions = inline(
"Ids like `improvise-9c8c` are tracker issues and link to the ticket index below. " +
"Paths like `crates/improvise-core/src/model/types.rs` link to the source on GitHub at `" +
ref +
"`.",
{ depth: 0, cited, unresolved, anchors: new Set() },
);
const cards = SECTIONS.map(
(s) =>
`
These are the working documents of the improvise repository, published as written. They are addressed to someone reading or changing the code: the arguments behind the architecture, the decisions that were made and the ones that were rejected, and the record of what each load-bearing test is guarding against.
Nothing here is a tutorial. For what the program does, see the project home or run the browser demo.