fiddlerwoaroof/improvise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tooling): repo-stats computes its churn tables in improvise (improvise-1xfb.5.1)

Browse Source 
code-maat's CSVs and a per-file churn CSV go into one directory, one
sparse model is imported from it, and a checked-in improvise script
builds and exports the views the report prints.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
This commit is contained in:
Claude (agent-stats)
2026-09-24 20:43:53 -07:00
committed by Edward Langley
parent 73a8556937
commit 2cd246780b
2 changed files with 266 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/repo-stats
+179 -88
View File
@@ -1,14 +1,14 @@
#!/usr/bin/env python3
"""Churn, coupling and doc-growth report over a window of history.
code-maat does coupling, age, abs-churn and soc; this adds the by-area split,
the prose-to-code and deletion ratios, doc sizes and coupling flags. Report
only: it gates nothing. See context/code/tooling.md section 7.
code-maat's analyses and this script's per-file churn go into one directory of
CSVs; improvise imports them into one model and computes the report's tables.
See context/code/tooling.md section 7.
"""
import argparse
import csv
import io
import datetime
import json
import os
import re
@@ -32,12 +32,24 @@ AREAS = [
("rust src", re.compile(r"\.(rs|pest)$")),
]
OTHER = "other"
AREA_NAMES = [a for a, _ in AREAS] + [OTHER]
PROSE = {"context/code", "context/test-evidence", "other context", "repo-map",
"agent docs and skills"}
CODE = {"rust src", "rust tests", "scripts/CI", "web and harness"}
AGENT_RULES = "agent docs and skills"
# One CSV per code-maat analysis, plus churn.csv. Each stem becomes an item of
# the model's File category, which the .improvise script pages on.
ANALYSES = ["coupling", "abs-churn", "age", "soc"]
CATEGORIES = ["commit", "date", "week", "entity", "area", "kind", "coupled",
"crosses-crate", "agent-rules"]
MEASURES = ["added", "deleted", "commits", "degree", "average-revs", "age-months", "soc"]
TABLES = ["by-area", "by-kind", "area-by-week", "by-day"]
HERE = os.path.dirname(os.path.abspath(__file__))
class ImproviseFailed(Exception):
"""improvise refused the pipeline's input: a product regression, not a statistic."""
def area_of(path):
for name, rx in AREAS:
@@ -46,6 +58,10 @@ def area_of(path):
return OTHER
def kind_of(area):
return "prose" if area in PROSE else "code" if area in CODE else "other"
def crate_of(path):
"""The crate a path belongs to, or None for a path in no crate."""
m = re.match(r"^crates/([^/]+)/", path)
@@ -79,36 +95,127 @@ def git2_log(repo, sha, since):
"--date=short", "--pretty=format:--%h--%ad--%aN", "--no-renames")
def parse_log(text):
"""(commits, [(path, added, deleted)]). A binary file's '-' counts as 0."""
commits, rows = 0, []
for line in text.split("\n"):
if line.startswith("--") and line.count("--") >= 3:
commits += 1
def is_header(line):
return line.startswith("--") and line.count("--") >= 3
def churn_rows(log):
"""One row per (commit, file). A commit is written `g<sha>`, because improvise
reads an all-digit or exponent-shaped sha as a number (tooling.md section 7)."""
rows, commit, date = [], None, None
for line in log.split("\n"):
if is_header(line):
_, commit, date, _ = line.split("--", 3)
elif "\t" in line:
a, d, path = line.split("\t", 2)
rows.append((path, int(a) if a != "-" else 0, int(d) if d != "-" else 0))
return commits, rows
year, week, _ = datetime.date.fromisoformat(date).isocalendar()
area = area_of(path)
rows.append({"commit": "g" + commit, "date": date,
"week": f"{year}-W{week:02d}", "entity": path, "area": area,
"kind": kind_of(area), "added": 0 if a == "-" else int(a),
"deleted": 0 if d == "-" else int(d)})
return rows
def ratio(num, den):
return round(num / den, 3) if den else None
def write_csv(path, header, rows):
with open(path, "w", newline="") as f:
w = csv.DictWriter(f, fieldnames=header)
w.writeheader()
w.writerows(rows)
def churn_by_area(rows):
areas = {n: {"added": 0, "deleted": 0} for n in AREA_NAMES}
for path, a, d in rows:
areas[area_of(path)]["added"] += a
areas[area_of(path)]["deleted"] += d
for v in areas.values():
v["deletion_ratio"] = ratio(v["deleted"], v["added"])
return areas
def read_csv(path):
with open(path, newline="") as f:
return list(csv.DictReader(f))
def prose_to_code(areas):
prose = sum(areas[a]["added"] for a in PROSE)
code = sum(areas[a]["added"] for a in CODE)
return {"prose_added": prose, "code_added": code, "ratio": ratio(prose, code)}
def code_maat(logfile, analysis, out, *extra):
subprocess.run(["code-maat", "-l", logfile, "-c", "git2", "-a", analysis,
"-o", out, *extra], capture_output=True, text=True, check=True)
def flag_coupling(path):
"""code-maat's coupling.csv, plus the two flags code-maat has no notion of."""
rows = read_csv(path)
for r in rows:
a, b = crate_of(r["entity"]), crate_of(r["coupled"])
r["crosses-crate"] = "yes" if a and b and a != b else "no"
both = area_of(r["entity"]) == area_of(r["coupled"]) == AGENT_RULES
r["agent-rules"] = "yes" if both else "no"
write_csv(path, ["entity", "coupled", "degree", "average-revs", "crosses-crate",
"agent-rules"], rows)
def build_dir(out, log, min_revs, today):
"""The directory improvise imports: churn.csv and one CSV per analysis."""
rows = churn_rows(log)
write_csv(os.path.join(out, "churn.csv"), list(rows[0]), rows)
with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f:
f.write(log + "\n")
try:
revs = ["--min-revs", str(min_revs), "--min-shared-revs", str(min_revs)]
extra = {"coupling": revs, "age": ["--age-time-now", today]}
for a in ANALYSES:
code_maat(f.name, a, os.path.join(out, f"{a}.csv"), *extra.get(a, []))
finally:
os.unlink(f.name)
flag_coupling(os.path.join(out, "coupling.csv"))
for name in ["churn"] + ANALYSES:
pad_categories(os.path.join(out, f"{name}.csv"))
def pad_categories(path):
"""Every category column, `-` where this CSV has none: improvise skips a record
that lacks any category's value, so a sparse merge imports nothing (tooling.md)."""
rows = read_csv(path)
with open(path, newline="") as f:
header = next(csv.reader(f))
for r in rows:
for c in CATEGORIES:
r.setdefault(c, "-")
write_csv(path, header + [c for c in CATEGORIES if c not in header], rows)
def improvise(out, *args):
exe = os.environ.get("IMPROVISE", "improvise")
p = subprocess.run([exe, *args], cwd=out, capture_output=True, text=True)
if p.returncode != 0 or "warning:" in p.stderr:
raise ImproviseFailed(f"improvise {' '.join(args)} exited {p.returncode}:\n"
f"{p.stderr}")
def run_improvise(out):
"""Import the directory into one model, then build its views and export them.
Naming the files is interim until `improvise import <dir>` (improvise-lpzu)."""
files = ["churn.csv"] + [f"{a}.csv" for a in ANALYSES]
flags = [f"--category={c}" for c in CATEGORIES] + [f"--measure={m}" for m in MEASURES]
improvise(out, "import", *files, "--no-wizard", *flags, "--name=Repo stats",
"--formula=DeletionRatio = deleted / added FORMAT .3",
"-o", "repo-stats.improv")
os.makedirs(os.path.join(out, "report"), exist_ok=True)
with open(os.path.join(out, "per-file-views.improvise"), "w") as f:
f.write(per_file_views(["churn"] + ANALYSES))
improvise(out, "script", "per-file-views.improvise", "-f", "repo-stats.improv")
improvise(out, "script", os.path.join(HERE, "repo-stats.improvise"),
"-f", "repo-stats.improv")
return {t: read_rows(os.path.join(out, "report", f"{t}.csv")) for t in TABLES}
def per_file_views(names):
"""A records view per source CSV, paged to it. Interim: improvise-lpzu's folder
import makes these itself, showing only each file's own columns."""
lines = []
for n in names:
lines += [f'create-view "{n}"', f'switch-view "{n}"', "set-axis _Index row",
"set-axis _Dim column", "set-axis File page", f"set-page File {n}",
"set-axis _Measure none"]
lines += [f"set-axis {c} none" for c in CATEGORIES]
return "\n".join(lines) + "\n"
def read_rows(path):
with open(path, newline="") as f:
return list(csv.reader(f))
def gotchas(text):
@@ -149,64 +256,44 @@ def doc_sizes(repo, sha):
}
def code_maat(logfile, analysis, *extra):
out = subprocess.run(["code-maat", "-l", logfile, "-c", "git2", "-a", analysis,
*extra], capture_output=True, text=True, check=True).stdout
return list(csv.DictReader(io.StringIO(out)))
def flag_coupling(rows):
for r in rows:
a, b = crate_of(r["entity"]), crate_of(r["coupled"])
r["crosses_crate"] = bool(a and b and a != b)
r["agent_rules"] = area_of(r["entity"]) == area_of(r["coupled"]) == AGENT_RULES
return rows
def maat(log, min_revs, today):
if not log.strip():
return {"coupling": [], "abs_churn": [], "age": [], "soc": []}
with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f:
f.write(log + "\n")
try:
revs = ["--min-revs", str(min_revs), "--min-shared-revs", str(min_revs)]
return {
"coupling": flag_coupling(code_maat(f.name, "coupling", *revs)),
"abs_churn": code_maat(f.name, "abs-churn"),
"age": code_maat(f.name, "age", "--age-time-now", today),
"soc": code_maat(f.name, "soc"),
}
finally:
os.unlink(f.name)
def report(repo, ref, since, min_revs):
def report(repo, ref, since, min_revs, out):
name, sha = resolve_ref(repo, ref)
log = git2_log(repo, sha, since)
commits, rows = parse_log(log)
areas = churn_by_area(rows)
commits = sum(1 for line in log.split("\n") if is_header(line))
today = git(repo, "show", "-s", "--date=short", "--format=%cd", sha).strip()
return {
"ref": name, "sha": sha, "since": since, "commits": commits,
"areas": areas, "prose_to_code": prose_to_code(areas),
"docs": doc_sizes(repo, sha), "code_maat": maat(log, min_revs, today),
}
r = {"ref": name, "sha": sha, "since": since, "commits": commits, "out": out,
"docs": doc_sizes(repo, sha), "tables": {}, "coupling": [], "soc": [], "age": []}
if commits:
build_dir(out, log, min_revs, today)
r["tables"] = run_improvise(out)
for a in ("coupling", "soc", "age"):
r[a] = read_csv(os.path.join(out, f"{a}.csv"))
return r
def fmt_ratio(r):
return "-" if r is None else f"{r:.3f}"
def table(rows):
"""A CSV improvise exported, as aligned text; the first column left-aligned."""
n = max(len(r) for r in rows)
widths = [max(len(r[i]) for r in rows if i < len(r)) for i in range(n)]
return [(" " + " ".join((c.ljust if i == 0 else c.rjust)(widths[i])
for i, c in enumerate(r))).rstrip() for r in rows]
TITLES = {
"by-area": "Churn by area (improvise)",
"by-kind": "Prose, code and other; ProseToCode = prose / code (improvise)",
"area-by-week": "Lines added by area and ISO week (improvise)",
"by-day": "Lines added per day, churn.csv beside code-maat's abs-churn (improvise)",
}
def text(r, top):
out = [f"repo-stats: {r['ref']} ({r['sha'][:10]}), since {r['since']}, "
f"{r['commits']} non-merge commits", "", "Churn by area"]
out.append(f" {'area':<24}{'added':>9}{'deleted':>9} del/add")
for n, v in r["areas"].items():
out.append(f" {n:<24}{v['added']:>9}{v['deleted']:>9} "
f"{fmt_ratio(v['deletion_ratio'])}")
p = r["prose_to_code"]
out += ["", f"Prose to code (added lines): {p['prose_added']} / {p['code_added']}"
f" = {fmt_ratio(p['ratio'])}", "", "Doc sizes (lines)"]
f"{r['commits']} non-merge commits"]
for t in TABLES:
if r["tables"].get(t):
out += ["", TITLES[t]] + table(r["tables"][t])
out += ["", "Doc sizes (lines)"]
out += [f" {n:>6} {path}" for path, n in r["docs"]["context_lines"].items()]
rm, te = r["docs"]["repo_map"], r["docs"]["test_evidence"]
lg = rm["longest"]
@@ -214,20 +301,16 @@ def text(r, top):
f"; longest is {lg['number']} ({lg['lines']} lines, {lg['chars']} chars)"
if lg else ""))
out.append(f" test-evidence entries: {te['entries']}")
m = r["code_maat"]
out += ["", f"Change coupling (code-maat; top {top})"]
for c in m["coupling"][:top]:
flags = [f for f, on in (("CROSSES-CRATE", c["crosses_crate"]),
("AGENT-RULES", c["agent_rules"])) if on]
for c in r["coupling"][:top]:
flags = [f for f, k in (("CROSSES-CRATE", "crosses-crate"),
("AGENT-RULES", "agent-rules")) if c[k] == "yes"]
out.append(f" {c['degree']:>4}% {c['average-revs']:>4} revs {c['entity']}"
f" <-> {c['coupled']}" + (f" [{', '.join(flags)}]" if flags else ""))
out += ["", f"Sum of coupling (code-maat; top {top})"]
out += [f" {s['soc']:>6} {s['entity']}" for s in m["soc"][:top]]
out += ["", "Absolute churn per day (code-maat)"]
out += [f" {c['date']} +{c['added']} -{c['deleted']} {c['commits']} commits"
for c in m["abs_churn"]]
out += ["", f"Code age in months (code-maat; oldest {top})"]
out += [f" {a['age-months']:>4} {a['entity']}" for a in m["age"][:top]]
out += [f" {s['soc']:>6} {s['entity']}" for s in r["soc"][:top]]
out += ["", f"Code age in months (code-maat; first {top})"]
out += [f" {a['age-months']:>4} {a['entity']}" for a in r["age"][:top]]
return "\n".join(out)
@@ -238,12 +321,20 @@ def main():
ap.add_argument("--min-revs", type=int, default=5,
help="code-maat --min-revs and --min-shared-revs (default 5)")
ap.add_argument("--top", type=int, default=15, help="rows per ranked table")
ap.add_argument("--out", default="repo-stats",
help="the directory for the CSVs, the model and its exports")
ap.add_argument("--json", action="store_true", help="emit JSON instead of text")
ap.add_argument("-C", dest="repo", default=".", help="the repository (default .)")
a = ap.parse_args()
r = report(a.repo, a.ref, a.since, a.min_revs)
os.makedirs(a.out, exist_ok=True)
try:
r = report(a.repo, a.ref, a.since, a.min_revs, os.path.abspath(a.out))
except ImproviseFailed as e:
print(f"repo-stats: {e}", file=sys.stderr)
return 3
print(json.dumps(r, indent=2) if a.json else text(r, a.top))
return 0
if __name__ == "__main__":
main()
sys.exit(main())
scripts/repo-stats.improvise
+87
View File
@@ -0,0 +1,87 @@
# The views scripts/repo-stats builds over the model it imported, run by
# `improvise script` with the output directory as the working directory.
# Every `export` here is a table the text report prints. See
# context/code/tooling.md section 7.
# The prose-to-code ratio is a formula over the kind category's own items.
add-formula kind "ProseToCode = prose / code FORMAT .3"
# Churn by area, with the deletion ratio (a _Measure formula from the import).
create-view "By area"
switch-view "By area"
set-axis File page
set-page File churn
set-axis area row
set-axis _Measure column
set-axis kind none
set-axis week none
set-axis date none
set-axis commit none
set-axis entity none
set-axis coupled none
set-axis crosses-crate none
set-axis agent-rules none
toggle-prune-empty
set-format .0
export report/by-area.csv
# Prose, code and other, and the ratio of the first two.
create-view "By kind"
switch-view "By kind"
set-axis File page
set-page File churn
set-axis kind row
set-axis _Measure column
set-axis area none
set-axis week none
set-axis date none
set-axis commit none
set-axis entity none
set-axis coupled none
set-axis crosses-crate none
set-axis agent-rules none
toggle-prune-empty
set-format .0
export report/by-kind.csv
# Lines added per area per ISO week.
create-view "Area by week"
switch-view "Area by week"
set-axis File page
set-page File churn
set-axis area row
set-axis week column
set-axis _Measure page
set-page _Measure added
set-axis kind none
set-axis date none
set-axis commit none
set-axis entity none
set-axis coupled none
set-axis crosses-crate none
set-axis agent-rules none
toggle-prune-empty
set-format .0
export report/area-by-week.csv
# Lines added per day, from both sources side by side: this script's churn
# rows and code-maat's abs-churn. The two columns must agree.
create-view "By day"
switch-view "By day"
set-axis date row
set-axis File column
set-axis _Measure page
set-page _Measure added
set-axis area none
set-axis kind none
set-axis week none
set-axis commit none
set-axis entity none
set-axis coupled none
set-axis crosses-crate none
set-axis agent-rules none
toggle-prune-empty
set-format .0
export report/by-day.csv
switch-view "By area"