feat(tooling): repo-stats computes its churn tables in improvise (improvise-1xfb.5.1)

code-maat's CSVs and a per-file churn CSV go into one directory, one
sparse model is imported from it, and a checked-in improvise script
builds and exports the views the report prints.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze

scripts/repo-stats

@@ -1,14 +1,14 @@
 #!/usr/bin/env python3
 """Churn, coupling and doc-growth report over a window of history.
 
-code-maat does coupling, age, abs-churn and soc; this adds the by-area split,
+code-maat's analyses and this script's per-file churn go into one directory of
-the prose-to-code and deletion ratios, doc sizes and coupling flags. Report
+CSVs; improvise imports them into one model and computes the report's tables.
-only: it gates nothing. See context/code/tooling.md section 7.
+See context/code/tooling.md section 7.
 """
 
 import argparse
 import csv
-import io
+import datetime
 import json
 import os
 import re
@@ -32,12 +32,24 @@ AREAS = [
     ("rust src", re.compile(r"\.(rs|pest)$")),
 ]
 OTHER = "other"
-AREA_NAMES = [a for a, _ in AREAS] + [OTHER]
 PROSE = {"context/code", "context/test-evidence", "other context", "repo-map",
          "agent docs and skills"}
 CODE = {"rust src", "rust tests", "scripts/CI", "web and harness"}
 AGENT_RULES = "agent docs and skills"
 
+# One CSV per code-maat analysis, plus churn.csv. Each stem becomes an item of
+# the model's File category, which the .improvise script pages on.
+ANALYSES = ["coupling", "abs-churn", "age", "soc"]
+CATEGORIES = ["commit", "date", "week", "entity", "area", "kind", "coupled",
+              "crosses-crate", "agent-rules"]
+MEASURES = ["added", "deleted", "commits", "degree", "average-revs", "age-months", "soc"]
+TABLES = ["by-area", "by-kind", "area-by-week", "by-day"]
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+
+class ImproviseFailed(Exception):
+    """improvise refused the pipeline's input: a product regression, not a statistic."""
+
 
 def area_of(path):
     for name, rx in AREAS:
@@ -46,6 +58,10 @@ def area_of(path):
     return OTHER
 
 
+def kind_of(area):
+    return "prose" if area in PROSE else "code" if area in CODE else "other"
+
+
 def crate_of(path):
     """The crate a path belongs to, or None for a path in no crate."""
     m = re.match(r"^crates/([^/]+)/", path)
@@ -79,36 +95,127 @@ def git2_log(repo, sha, since):
                "--date=short", "--pretty=format:--%h--%ad--%aN", "--no-renames")
 
 
-def parse_log(text):
+def is_header(line):
-    """(commits, [(path, added, deleted)]). A binary file's '-' counts as 0."""
+    return line.startswith("--") and line.count("--") >= 3
-    commits, rows = 0, []
+
-    for line in text.split("\n"):
+
-        if line.startswith("--") and line.count("--") >= 3:
+def churn_rows(log):
-            commits += 1
+    """One row per (commit, file). A commit is written `g<sha>`, because improvise
+    reads an all-digit or exponent-shaped sha as a number (tooling.md section 7)."""
+    rows, commit, date = [], None, None
+    for line in log.split("\n"):
+        if is_header(line):
+            _, commit, date, _ = line.split("--", 3)
         elif "\t" in line:
             a, d, path = line.split("\t", 2)
-            rows.append((path, int(a) if a != "-" else 0, int(d) if d != "-" else 0))
+            year, week, _ = datetime.date.fromisoformat(date).isocalendar()
-    return commits, rows
+            area = area_of(path)
+            rows.append({"commit": "g" + commit, "date": date,
+                         "week": f"{year}-W{week:02d}", "entity": path, "area": area,
+                         "kind": kind_of(area), "added": 0 if a == "-" else int(a),
+                         "deleted": 0 if d == "-" else int(d)})
+    return rows
 
 
-def ratio(num, den):
+def write_csv(path, header, rows):
-    return round(num / den, 3) if den else None
+    with open(path, "w", newline="") as f:
+        w = csv.DictWriter(f, fieldnames=header)
+        w.writeheader()
+        w.writerows(rows)
 
 
-def churn_by_area(rows):
+def read_csv(path):
-    areas = {n: {"added": 0, "deleted": 0} for n in AREA_NAMES}
+    with open(path, newline="") as f:
-    for path, a, d in rows:
+        return list(csv.DictReader(f))
-        areas[area_of(path)]["added"] += a
-        areas[area_of(path)]["deleted"] += d
-    for v in areas.values():
-        v["deletion_ratio"] = ratio(v["deleted"], v["added"])
-    return areas
 
 
-def prose_to_code(areas):
+def code_maat(logfile, analysis, out, *extra):
-    prose = sum(areas[a]["added"] for a in PROSE)
+    subprocess.run(["code-maat", "-l", logfile, "-c", "git2", "-a", analysis,
-    code = sum(areas[a]["added"] for a in CODE)
+                    "-o", out, *extra], capture_output=True, text=True, check=True)
-    return {"prose_added": prose, "code_added": code, "ratio": ratio(prose, code)}
+
+
+def flag_coupling(path):
+    """code-maat's coupling.csv, plus the two flags code-maat has no notion of."""
+    rows = read_csv(path)
+    for r in rows:
+        a, b = crate_of(r["entity"]), crate_of(r["coupled"])
+        r["crosses-crate"] = "yes" if a and b and a != b else "no"
+        both = area_of(r["entity"]) == area_of(r["coupled"]) == AGENT_RULES
+        r["agent-rules"] = "yes" if both else "no"
+    write_csv(path, ["entity", "coupled", "degree", "average-revs", "crosses-crate",
+                     "agent-rules"], rows)
+
+
+def build_dir(out, log, min_revs, today):
+    """The directory improvise imports: churn.csv and one CSV per analysis."""
+    rows = churn_rows(log)
+    write_csv(os.path.join(out, "churn.csv"), list(rows[0]), rows)
+    with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f:
+        f.write(log + "\n")
+    try:
+        revs = ["--min-revs", str(min_revs), "--min-shared-revs", str(min_revs)]
+        extra = {"coupling": revs, "age": ["--age-time-now", today]}
+        for a in ANALYSES:
+            code_maat(f.name, a, os.path.join(out, f"{a}.csv"), *extra.get(a, []))
+    finally:
+        os.unlink(f.name)
+    flag_coupling(os.path.join(out, "coupling.csv"))
+    for name in ["churn"] + ANALYSES:
+        pad_categories(os.path.join(out, f"{name}.csv"))
+
+
+def pad_categories(path):
+    """Every category column, `-` where this CSV has none: improvise skips a record
+    that lacks any category's value, so a sparse merge imports nothing (tooling.md)."""
+    rows = read_csv(path)
+    with open(path, newline="") as f:
+        header = next(csv.reader(f))
+    for r in rows:
+        for c in CATEGORIES:
+            r.setdefault(c, "-")
+    write_csv(path, header + [c for c in CATEGORIES if c not in header], rows)
+
+
+def improvise(out, *args):
+    exe = os.environ.get("IMPROVISE", "improvise")
+    p = subprocess.run([exe, *args], cwd=out, capture_output=True, text=True)
+    if p.returncode != 0 or "warning:" in p.stderr:
+        raise ImproviseFailed(f"improvise {' '.join(args)} exited {p.returncode}:\n"
+                              f"{p.stderr}")
+
+
+def run_improvise(out):
+    """Import the directory into one model, then build its views and export them.
+    Naming the files is interim until `improvise import <dir>` (improvise-lpzu)."""
+    files = ["churn.csv"] + [f"{a}.csv" for a in ANALYSES]
+    flags = [f"--category={c}" for c in CATEGORIES] + [f"--measure={m}" for m in MEASURES]
+    improvise(out, "import", *files, "--no-wizard", *flags, "--name=Repo stats",
+              "--formula=DeletionRatio = deleted / added FORMAT .3",
+              "-o", "repo-stats.improv")
+    os.makedirs(os.path.join(out, "report"), exist_ok=True)
+    with open(os.path.join(out, "per-file-views.improvise"), "w") as f:
+        f.write(per_file_views(["churn"] + ANALYSES))
+    improvise(out, "script", "per-file-views.improvise", "-f", "repo-stats.improv")
+    improvise(out, "script", os.path.join(HERE, "repo-stats.improvise"),
+              "-f", "repo-stats.improv")
+    return {t: read_rows(os.path.join(out, "report", f"{t}.csv")) for t in TABLES}
+
+
+def per_file_views(names):
+    """A records view per source CSV, paged to it. Interim: improvise-lpzu's folder
+    import makes these itself, showing only each file's own columns."""
+    lines = []
+    for n in names:
+        lines += [f'create-view "{n}"', f'switch-view "{n}"', "set-axis _Index row",
+                  "set-axis _Dim column", "set-axis File page", f"set-page File {n}",
+                  "set-axis _Measure none"]
+        lines += [f"set-axis {c} none" for c in CATEGORIES]
+    return "\n".join(lines) + "\n"
+
+
+def read_rows(path):
+    with open(path, newline="") as f:
+        return list(csv.reader(f))
 
 
 def gotchas(text):
@@ -149,64 +256,44 @@ def doc_sizes(repo, sha):
     }
 
 
-def code_maat(logfile, analysis, *extra):
+def report(repo, ref, since, min_revs, out):
-    out = subprocess.run(["code-maat", "-l", logfile, "-c", "git2", "-a", analysis,
-                          *extra], capture_output=True, text=True, check=True).stdout
-    return list(csv.DictReader(io.StringIO(out)))
-
-
-def flag_coupling(rows):
-    for r in rows:
-        a, b = crate_of(r["entity"]), crate_of(r["coupled"])
-        r["crosses_crate"] = bool(a and b and a != b)
-        r["agent_rules"] = area_of(r["entity"]) == area_of(r["coupled"]) == AGENT_RULES
-    return rows
-
-
-def maat(log, min_revs, today):
-    if not log.strip():
-        return {"coupling": [], "abs_churn": [], "age": [], "soc": []}
-    with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f:
-        f.write(log + "\n")
-    try:
-        revs = ["--min-revs", str(min_revs), "--min-shared-revs", str(min_revs)]
-        return {
-            "coupling": flag_coupling(code_maat(f.name, "coupling", *revs)),
-            "abs_churn": code_maat(f.name, "abs-churn"),
-            "age": code_maat(f.name, "age", "--age-time-now", today),
-            "soc": code_maat(f.name, "soc"),
-        }
-    finally:
-        os.unlink(f.name)
-
-
-def report(repo, ref, since, min_revs):
     name, sha = resolve_ref(repo, ref)
     log = git2_log(repo, sha, since)
-    commits, rows = parse_log(log)
+    commits = sum(1 for line in log.split("\n") if is_header(line))
-    areas = churn_by_area(rows)
     today = git(repo, "show", "-s", "--date=short", "--format=%cd", sha).strip()
-    return {
+    r = {"ref": name, "sha": sha, "since": since, "commits": commits, "out": out,
-        "ref": name, "sha": sha, "since": since, "commits": commits,
+         "docs": doc_sizes(repo, sha), "tables": {}, "coupling": [], "soc": [], "age": []}
-        "areas": areas, "prose_to_code": prose_to_code(areas),
+    if commits:
-        "docs": doc_sizes(repo, sha), "code_maat": maat(log, min_revs, today),
+        build_dir(out, log, min_revs, today)
-    }
+        r["tables"] = run_improvise(out)
+        for a in ("coupling", "soc", "age"):
+            r[a] = read_csv(os.path.join(out, f"{a}.csv"))
+    return r
 
 
-def fmt_ratio(r):
+def table(rows):
-    return "-" if r is None else f"{r:.3f}"
+    """A CSV improvise exported, as aligned text; the first column left-aligned."""
+    n = max(len(r) for r in rows)
+    widths = [max(len(r[i]) for r in rows if i < len(r)) for i in range(n)]
+    return [("  " + "  ".join((c.ljust if i == 0 else c.rjust)(widths[i])
+                              for i, c in enumerate(r))).rstrip() for r in rows]
+
+
+TITLES = {
+    "by-area": "Churn by area (improvise)",
+    "by-kind": "Prose, code and other; ProseToCode = prose / code (improvise)",
+    "area-by-week": "Lines added by area and ISO week (improvise)",
+    "by-day": "Lines added per day, churn.csv beside code-maat's abs-churn (improvise)",
+}
 
 
 def text(r, top):
     out = [f"repo-stats: {r['ref']} ({r['sha'][:10]}), since {r['since']}, "
-           f"{r['commits']} non-merge commits", "", "Churn by area"]
+           f"{r['commits']} non-merge commits"]
-    out.append(f"  {'area':<24}{'added':>9}{'deleted':>9}  del/add")
+    for t in TABLES:
-    for n, v in r["areas"].items():
+        if r["tables"].get(t):
-        out.append(f"  {n:<24}{v['added']:>9}{v['deleted']:>9}  "
+            out += ["", TITLES[t]] + table(r["tables"][t])
-                   f"{fmt_ratio(v['deletion_ratio'])}")
+    out += ["", "Doc sizes (lines)"]
-    p = r["prose_to_code"]
-    out += ["", f"Prose to code (added lines): {p['prose_added']} / {p['code_added']}"
-                f" = {fmt_ratio(p['ratio'])}", "", "Doc sizes (lines)"]
     out += [f"  {n:>6}  {path}" for path, n in r["docs"]["context_lines"].items()]
     rm, te = r["docs"]["repo_map"], r["docs"]["test_evidence"]
     lg = rm["longest"]
@@ -214,20 +301,16 @@ def text(r, top):
         f"; longest is {lg['number']} ({lg['lines']} lines, {lg['chars']} chars)"
         if lg else ""))
     out.append(f"  test-evidence entries: {te['entries']}")
-    m = r["code_maat"]
     out += ["", f"Change coupling (code-maat; top {top})"]
-    for c in m["coupling"][:top]:
+    for c in r["coupling"][:top]:
-        flags = [f for f, on in (("CROSSES-CRATE", c["crosses_crate"]),
+        flags = [f for f, k in (("CROSSES-CRATE", "crosses-crate"),
-                                 ("AGENT-RULES", c["agent_rules"])) if on]
+                                ("AGENT-RULES", "agent-rules")) if c[k] == "yes"]
         out.append(f"  {c['degree']:>4}% {c['average-revs']:>4} revs  {c['entity']}"
                    f" <-> {c['coupled']}" + (f"  [{', '.join(flags)}]" if flags else ""))
     out += ["", f"Sum of coupling (code-maat; top {top})"]
-    out += [f"  {s['soc']:>6}  {s['entity']}" for s in m["soc"][:top]]
+    out += [f"  {s['soc']:>6}  {s['entity']}" for s in r["soc"][:top]]
-    out += ["", "Absolute churn per day (code-maat)"]
+    out += ["", f"Code age in months (code-maat; first {top})"]
-    out += [f"  {c['date']}  +{c['added']} -{c['deleted']}  {c['commits']} commits"
+    out += [f"  {a['age-months']:>4}  {a['entity']}" for a in r["age"][:top]]
-            for c in m["abs_churn"]]
-    out += ["", f"Code age in months (code-maat; oldest {top})"]
-    out += [f"  {a['age-months']:>4}  {a['entity']}" for a in m["age"][:top]]
     return "\n".join(out)
 
 
@@ -238,12 +321,20 @@ def main():
     ap.add_argument("--min-revs", type=int, default=5,
                     help="code-maat --min-revs and --min-shared-revs (default 5)")
     ap.add_argument("--top", type=int, default=15, help="rows per ranked table")
+    ap.add_argument("--out", default="repo-stats",
+                    help="the directory for the CSVs, the model and its exports")
     ap.add_argument("--json", action="store_true", help="emit JSON instead of text")
     ap.add_argument("-C", dest="repo", default=".", help="the repository (default .)")
     a = ap.parse_args()
-    r = report(a.repo, a.ref, a.since, a.min_revs)
+    os.makedirs(a.out, exist_ok=True)
+    try:
+        r = report(a.repo, a.ref, a.since, a.min_revs, os.path.abspath(a.out))
+    except ImproviseFailed as e:
+        print(f"repo-stats: {e}", file=sys.stderr)
+        return 3
     print(json.dumps(r, indent=2) if a.json else text(r, a.top))
+    return 0
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

scripts/repo-stats.improvise

@@ -0,0 +1,87 @@
+# The views scripts/repo-stats builds over the model it imported, run by
+# `improvise script` with the output directory as the working directory.
+# Every `export` here is a table the text report prints. See
+# context/code/tooling.md section 7.
+
+# The prose-to-code ratio is a formula over the kind category's own items.
+add-formula kind "ProseToCode = prose / code FORMAT .3"
+
+# Churn by area, with the deletion ratio (a _Measure formula from the import).
+create-view "By area"
+switch-view "By area"
+set-axis File page
+set-page File churn
+set-axis area row
+set-axis _Measure column
+set-axis kind none
+set-axis week none
+set-axis date none
+set-axis commit none
+set-axis entity none
+set-axis coupled none
+set-axis crosses-crate none
+set-axis agent-rules none
+toggle-prune-empty
+set-format .0
+export report/by-area.csv
+
+# Prose, code and other, and the ratio of the first two.
+create-view "By kind"
+switch-view "By kind"
+set-axis File page
+set-page File churn
+set-axis kind row
+set-axis _Measure column
+set-axis area none
+set-axis week none
+set-axis date none
+set-axis commit none
+set-axis entity none
+set-axis coupled none
+set-axis crosses-crate none
+set-axis agent-rules none
+toggle-prune-empty
+set-format .0
+export report/by-kind.csv
+
+# Lines added per area per ISO week.
+create-view "Area by week"
+switch-view "Area by week"
+set-axis File page
+set-page File churn
+set-axis area row
+set-axis week column
+set-axis _Measure page
+set-page _Measure added
+set-axis kind none
+set-axis date none
+set-axis commit none
+set-axis entity none
+set-axis coupled none
+set-axis crosses-crate none
+set-axis agent-rules none
+toggle-prune-empty
+set-format .0
+export report/area-by-week.csv
+
+# Lines added per day, from both sources side by side: this script's churn
+# rows and code-maat's abs-churn. The two columns must agree.
+create-view "By day"
+switch-view "By day"
+set-axis date row
+set-axis File column
+set-axis _Measure page
+set-page _Measure added
+set-axis area none
+set-axis kind none
+set-axis week none
+set-axis commit none
+set-axis entity none
+set-axis coupled none
+set-axis crosses-crate none
+set-axis agent-rules none
+toggle-prune-empty
+set-format .0
+export report/by-day.csv
+
+switch-view "By area"