diff --git a/scripts/repo-stats b/scripts/repo-stats index 7a8cecb5..35a45a0e 100755 --- a/scripts/repo-stats +++ b/scripts/repo-stats @@ -1,14 +1,14 @@ #!/usr/bin/env python3 """Churn, coupling and doc-growth report over a window of history. -code-maat does coupling, age, abs-churn and soc; this adds the by-area split, -the prose-to-code and deletion ratios, doc sizes and coupling flags. Report -only: it gates nothing. See context/code/tooling.md section 7. +code-maat's analyses and this script's per-file churn go into one directory of +CSVs; improvise imports them into one model and computes the report's tables. +See context/code/tooling.md section 7. """ import argparse import csv -import io +import datetime import json import os import re @@ -32,12 +32,24 @@ AREAS = [ ("rust src", re.compile(r"\.(rs|pest)$")), ] OTHER = "other" -AREA_NAMES = [a for a, _ in AREAS] + [OTHER] PROSE = {"context/code", "context/test-evidence", "other context", "repo-map", "agent docs and skills"} CODE = {"rust src", "rust tests", "scripts/CI", "web and harness"} AGENT_RULES = "agent docs and skills" +# One CSV per code-maat analysis, plus churn.csv. Each stem becomes an item of +# the model's File category, which the .improvise script pages on. +ANALYSES = ["coupling", "abs-churn", "age", "soc"] +CATEGORIES = ["commit", "date", "week", "entity", "area", "kind", "coupled", + "crosses-crate", "agent-rules"] +MEASURES = ["added", "deleted", "commits", "degree", "average-revs", "age-months", "soc"] +TABLES = ["by-area", "by-kind", "area-by-week", "by-day"] +HERE = os.path.dirname(os.path.abspath(__file__)) + + +class ImproviseFailed(Exception): + """improvise refused the pipeline's input: a product regression, not a statistic.""" + def area_of(path): for name, rx in AREAS: @@ -46,6 +58,10 @@ def area_of(path): return OTHER +def kind_of(area): + return "prose" if area in PROSE else "code" if area in CODE else "other" + + def crate_of(path): """The crate a path belongs to, or None for a path in no crate.""" m = re.match(r"^crates/([^/]+)/", path) @@ -79,36 +95,127 @@ def git2_log(repo, sha, since): "--date=short", "--pretty=format:--%h--%ad--%aN", "--no-renames") -def parse_log(text): - """(commits, [(path, added, deleted)]). A binary file's '-' counts as 0.""" - commits, rows = 0, [] - for line in text.split("\n"): - if line.startswith("--") and line.count("--") >= 3: - commits += 1 +def is_header(line): + return line.startswith("--") and line.count("--") >= 3 + + +def churn_rows(log): + """One row per (commit, file). A commit is written `g`, because improvise + reads an all-digit or exponent-shaped sha as a number (tooling.md section 7).""" + rows, commit, date = [], None, None + for line in log.split("\n"): + if is_header(line): + _, commit, date, _ = line.split("--", 3) elif "\t" in line: a, d, path = line.split("\t", 2) - rows.append((path, int(a) if a != "-" else 0, int(d) if d != "-" else 0)) - return commits, rows + year, week, _ = datetime.date.fromisoformat(date).isocalendar() + area = area_of(path) + rows.append({"commit": "g" + commit, "date": date, + "week": f"{year}-W{week:02d}", "entity": path, "area": area, + "kind": kind_of(area), "added": 0 if a == "-" else int(a), + "deleted": 0 if d == "-" else int(d)}) + return rows -def ratio(num, den): - return round(num / den, 3) if den else None +def write_csv(path, header, rows): + with open(path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=header) + w.writeheader() + w.writerows(rows) -def churn_by_area(rows): - areas = {n: {"added": 0, "deleted": 0} for n in AREA_NAMES} - for path, a, d in rows: - areas[area_of(path)]["added"] += a - areas[area_of(path)]["deleted"] += d - for v in areas.values(): - v["deletion_ratio"] = ratio(v["deleted"], v["added"]) - return areas +def read_csv(path): + with open(path, newline="") as f: + return list(csv.DictReader(f)) -def prose_to_code(areas): - prose = sum(areas[a]["added"] for a in PROSE) - code = sum(areas[a]["added"] for a in CODE) - return {"prose_added": prose, "code_added": code, "ratio": ratio(prose, code)} +def code_maat(logfile, analysis, out, *extra): + subprocess.run(["code-maat", "-l", logfile, "-c", "git2", "-a", analysis, + "-o", out, *extra], capture_output=True, text=True, check=True) + + +def flag_coupling(path): + """code-maat's coupling.csv, plus the two flags code-maat has no notion of.""" + rows = read_csv(path) + for r in rows: + a, b = crate_of(r["entity"]), crate_of(r["coupled"]) + r["crosses-crate"] = "yes" if a and b and a != b else "no" + both = area_of(r["entity"]) == area_of(r["coupled"]) == AGENT_RULES + r["agent-rules"] = "yes" if both else "no" + write_csv(path, ["entity", "coupled", "degree", "average-revs", "crosses-crate", + "agent-rules"], rows) + + +def build_dir(out, log, min_revs, today): + """The directory improvise imports: churn.csv and one CSV per analysis.""" + rows = churn_rows(log) + write_csv(os.path.join(out, "churn.csv"), list(rows[0]), rows) + with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f: + f.write(log + "\n") + try: + revs = ["--min-revs", str(min_revs), "--min-shared-revs", str(min_revs)] + extra = {"coupling": revs, "age": ["--age-time-now", today]} + for a in ANALYSES: + code_maat(f.name, a, os.path.join(out, f"{a}.csv"), *extra.get(a, [])) + finally: + os.unlink(f.name) + flag_coupling(os.path.join(out, "coupling.csv")) + for name in ["churn"] + ANALYSES: + pad_categories(os.path.join(out, f"{name}.csv")) + + +def pad_categories(path): + """Every category column, `-` where this CSV has none: improvise skips a record + that lacks any category's value, so a sparse merge imports nothing (tooling.md).""" + rows = read_csv(path) + with open(path, newline="") as f: + header = next(csv.reader(f)) + for r in rows: + for c in CATEGORIES: + r.setdefault(c, "-") + write_csv(path, header + [c for c in CATEGORIES if c not in header], rows) + + +def improvise(out, *args): + exe = os.environ.get("IMPROVISE", "improvise") + p = subprocess.run([exe, *args], cwd=out, capture_output=True, text=True) + if p.returncode != 0 or "warning:" in p.stderr: + raise ImproviseFailed(f"improvise {' '.join(args)} exited {p.returncode}:\n" + f"{p.stderr}") + + +def run_improvise(out): + """Import the directory into one model, then build its views and export them. + Naming the files is interim until `improvise import ` (improvise-lpzu).""" + files = ["churn.csv"] + [f"{a}.csv" for a in ANALYSES] + flags = [f"--category={c}" for c in CATEGORIES] + [f"--measure={m}" for m in MEASURES] + improvise(out, "import", *files, "--no-wizard", *flags, "--name=Repo stats", + "--formula=DeletionRatio = deleted / added FORMAT .3", + "-o", "repo-stats.improv") + os.makedirs(os.path.join(out, "report"), exist_ok=True) + with open(os.path.join(out, "per-file-views.improvise"), "w") as f: + f.write(per_file_views(["churn"] + ANALYSES)) + improvise(out, "script", "per-file-views.improvise", "-f", "repo-stats.improv") + improvise(out, "script", os.path.join(HERE, "repo-stats.improvise"), + "-f", "repo-stats.improv") + return {t: read_rows(os.path.join(out, "report", f"{t}.csv")) for t in TABLES} + + +def per_file_views(names): + """A records view per source CSV, paged to it. Interim: improvise-lpzu's folder + import makes these itself, showing only each file's own columns.""" + lines = [] + for n in names: + lines += [f'create-view "{n}"', f'switch-view "{n}"', "set-axis _Index row", + "set-axis _Dim column", "set-axis File page", f"set-page File {n}", + "set-axis _Measure none"] + lines += [f"set-axis {c} none" for c in CATEGORIES] + return "\n".join(lines) + "\n" + + +def read_rows(path): + with open(path, newline="") as f: + return list(csv.reader(f)) def gotchas(text): @@ -149,64 +256,44 @@ def doc_sizes(repo, sha): } -def code_maat(logfile, analysis, *extra): - out = subprocess.run(["code-maat", "-l", logfile, "-c", "git2", "-a", analysis, - *extra], capture_output=True, text=True, check=True).stdout - return list(csv.DictReader(io.StringIO(out))) - - -def flag_coupling(rows): - for r in rows: - a, b = crate_of(r["entity"]), crate_of(r["coupled"]) - r["crosses_crate"] = bool(a and b and a != b) - r["agent_rules"] = area_of(r["entity"]) == area_of(r["coupled"]) == AGENT_RULES - return rows - - -def maat(log, min_revs, today): - if not log.strip(): - return {"coupling": [], "abs_churn": [], "age": [], "soc": []} - with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as f: - f.write(log + "\n") - try: - revs = ["--min-revs", str(min_revs), "--min-shared-revs", str(min_revs)] - return { - "coupling": flag_coupling(code_maat(f.name, "coupling", *revs)), - "abs_churn": code_maat(f.name, "abs-churn"), - "age": code_maat(f.name, "age", "--age-time-now", today), - "soc": code_maat(f.name, "soc"), - } - finally: - os.unlink(f.name) - - -def report(repo, ref, since, min_revs): +def report(repo, ref, since, min_revs, out): name, sha = resolve_ref(repo, ref) log = git2_log(repo, sha, since) - commits, rows = parse_log(log) - areas = churn_by_area(rows) + commits = sum(1 for line in log.split("\n") if is_header(line)) today = git(repo, "show", "-s", "--date=short", "--format=%cd", sha).strip() - return { - "ref": name, "sha": sha, "since": since, "commits": commits, - "areas": areas, "prose_to_code": prose_to_code(areas), - "docs": doc_sizes(repo, sha), "code_maat": maat(log, min_revs, today), - } + r = {"ref": name, "sha": sha, "since": since, "commits": commits, "out": out, + "docs": doc_sizes(repo, sha), "tables": {}, "coupling": [], "soc": [], "age": []} + if commits: + build_dir(out, log, min_revs, today) + r["tables"] = run_improvise(out) + for a in ("coupling", "soc", "age"): + r[a] = read_csv(os.path.join(out, f"{a}.csv")) + return r -def fmt_ratio(r): - return "-" if r is None else f"{r:.3f}" +def table(rows): + """A CSV improvise exported, as aligned text; the first column left-aligned.""" + n = max(len(r) for r in rows) + widths = [max(len(r[i]) for r in rows if i < len(r)) for i in range(n)] + return [(" " + " ".join((c.ljust if i == 0 else c.rjust)(widths[i]) + for i, c in enumerate(r))).rstrip() for r in rows] + + +TITLES = { + "by-area": "Churn by area (improvise)", + "by-kind": "Prose, code and other; ProseToCode = prose / code (improvise)", + "area-by-week": "Lines added by area and ISO week (improvise)", + "by-day": "Lines added per day, churn.csv beside code-maat's abs-churn (improvise)", +} def text(r, top): out = [f"repo-stats: {r['ref']} ({r['sha'][:10]}), since {r['since']}, " - f"{r['commits']} non-merge commits", "", "Churn by area"] - out.append(f" {'area':<24}{'added':>9}{'deleted':>9} del/add") - for n, v in r["areas"].items(): - out.append(f" {n:<24}{v['added']:>9}{v['deleted']:>9} " - f"{fmt_ratio(v['deletion_ratio'])}") - p = r["prose_to_code"] - out += ["", f"Prose to code (added lines): {p['prose_added']} / {p['code_added']}" - f" = {fmt_ratio(p['ratio'])}", "", "Doc sizes (lines)"] + f"{r['commits']} non-merge commits"] + for t in TABLES: + if r["tables"].get(t): + out += ["", TITLES[t]] + table(r["tables"][t]) + out += ["", "Doc sizes (lines)"] out += [f" {n:>6} {path}" for path, n in r["docs"]["context_lines"].items()] rm, te = r["docs"]["repo_map"], r["docs"]["test_evidence"] lg = rm["longest"] @@ -214,20 +301,16 @@ def text(r, top): f"; longest is {lg['number']} ({lg['lines']} lines, {lg['chars']} chars)" if lg else "")) out.append(f" test-evidence entries: {te['entries']}") - m = r["code_maat"] out += ["", f"Change coupling (code-maat; top {top})"] - for c in m["coupling"][:top]: - flags = [f for f, on in (("CROSSES-CRATE", c["crosses_crate"]), - ("AGENT-RULES", c["agent_rules"])) if on] + for c in r["coupling"][:top]: + flags = [f for f, k in (("CROSSES-CRATE", "crosses-crate"), + ("AGENT-RULES", "agent-rules")) if c[k] == "yes"] out.append(f" {c['degree']:>4}% {c['average-revs']:>4} revs {c['entity']}" f" <-> {c['coupled']}" + (f" [{', '.join(flags)}]" if flags else "")) out += ["", f"Sum of coupling (code-maat; top {top})"] - out += [f" {s['soc']:>6} {s['entity']}" for s in m["soc"][:top]] - out += ["", "Absolute churn per day (code-maat)"] - out += [f" {c['date']} +{c['added']} -{c['deleted']} {c['commits']} commits" - for c in m["abs_churn"]] - out += ["", f"Code age in months (code-maat; oldest {top})"] - out += [f" {a['age-months']:>4} {a['entity']}" for a in m["age"][:top]] + out += [f" {s['soc']:>6} {s['entity']}" for s in r["soc"][:top]] + out += ["", f"Code age in months (code-maat; first {top})"] + out += [f" {a['age-months']:>4} {a['entity']}" for a in r["age"][:top]] return "\n".join(out) @@ -238,12 +321,20 @@ def main(): ap.add_argument("--min-revs", type=int, default=5, help="code-maat --min-revs and --min-shared-revs (default 5)") ap.add_argument("--top", type=int, default=15, help="rows per ranked table") + ap.add_argument("--out", default="repo-stats", + help="the directory for the CSVs, the model and its exports") ap.add_argument("--json", action="store_true", help="emit JSON instead of text") ap.add_argument("-C", dest="repo", default=".", help="the repository (default .)") a = ap.parse_args() - r = report(a.repo, a.ref, a.since, a.min_revs) + os.makedirs(a.out, exist_ok=True) + try: + r = report(a.repo, a.ref, a.since, a.min_revs, os.path.abspath(a.out)) + except ImproviseFailed as e: + print(f"repo-stats: {e}", file=sys.stderr) + return 3 print(json.dumps(r, indent=2) if a.json else text(r, a.top)) + return 0 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/repo-stats.improvise b/scripts/repo-stats.improvise new file mode 100644 index 00000000..ce08f088 --- /dev/null +++ b/scripts/repo-stats.improvise @@ -0,0 +1,87 @@ +# The views scripts/repo-stats builds over the model it imported, run by +# `improvise script` with the output directory as the working directory. +# Every `export` here is a table the text report prints. See +# context/code/tooling.md section 7. + +# The prose-to-code ratio is a formula over the kind category's own items. +add-formula kind "ProseToCode = prose / code FORMAT .3" + +# Churn by area, with the deletion ratio (a _Measure formula from the import). +create-view "By area" +switch-view "By area" +set-axis File page +set-page File churn +set-axis area row +set-axis _Measure column +set-axis kind none +set-axis week none +set-axis date none +set-axis commit none +set-axis entity none +set-axis coupled none +set-axis crosses-crate none +set-axis agent-rules none +toggle-prune-empty +set-format .0 +export report/by-area.csv + +# Prose, code and other, and the ratio of the first two. +create-view "By kind" +switch-view "By kind" +set-axis File page +set-page File churn +set-axis kind row +set-axis _Measure column +set-axis area none +set-axis week none +set-axis date none +set-axis commit none +set-axis entity none +set-axis coupled none +set-axis crosses-crate none +set-axis agent-rules none +toggle-prune-empty +set-format .0 +export report/by-kind.csv + +# Lines added per area per ISO week. +create-view "Area by week" +switch-view "Area by week" +set-axis File page +set-page File churn +set-axis area row +set-axis week column +set-axis _Measure page +set-page _Measure added +set-axis kind none +set-axis date none +set-axis commit none +set-axis entity none +set-axis coupled none +set-axis crosses-crate none +set-axis agent-rules none +toggle-prune-empty +set-format .0 +export report/area-by-week.csv + +# Lines added per day, from both sources side by side: this script's churn +# rows and code-maat's abs-churn. The two columns must agree. +create-view "By day" +switch-view "By day" +set-axis date row +set-axis File column +set-axis _Measure page +set-page _Measure added +set-axis area none +set-axis kind none +set-axis week none +set-axis commit none +set-axis entity none +set-axis coupled none +set-axis crosses-crate none +set-axis agent-rules none +toggle-prune-empty +set-format .0 +export report/by-day.csv + +switch-view "By area"