fiddlerwoaroof/improvise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(tooling): the repo-stats suite runs the whole pipeline over its fixture (improvise-1xfb.5.1)

Browse Source 
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
This commit is contained in:
Claude (agent-stats)
2026-09-24 20:46:21 -07:00
committed by Edward Langley
parent 2cd246780b
commit 20335ee616
1 changed files with 72 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/repo-stats-test.sh
+72 -38
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
# check_the_checker for scripts/repo-stats (improvise-1xfb.5.1), against a fixture
# repo whose every number is known by construction. See context/test-evidence/tooling.md.
# check_the_checker for scripts/repo-stats (improvise-1xfb.5.1): the whole pipeline,
# code-maat and improvise included, over a fixture repo whose every number is known.
# See context/test-evidence/tooling.md.
set -uo pipefail
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -12,6 +13,15 @@ pass=0; fail=0
ok() { printf ' ok %s\n' "$1"; pass=$((pass+1)); }
bad() { printf ' FAIL %s (%s)\n' "$1" "$2"; fail=$((fail+1)); }
# The improvise under test: on PATH in CI (`nix shell .#improvise`), else this tree's.
if command -v improvise >/dev/null; then
IMPROVISE="$(command -v improvise)"
else
cargo build --quiet --manifest-path "$here/../Cargo.toml" --bin improvise || exit 1
IMPROVISE="$here/../target/debug/improvise"
fi
export IMPROVISE
# $1 name, $2 jq expression over $work/out.json, $3 the value it must print
is() {
local got
@@ -19,6 +29,11 @@ is() {
[ "$got" = "$3" ] && ok "$1" || bad "$1" "got $got, wanted $3"
}
# $1 name, $2 table, $3 row label, $4 the row's cells after the label, as JSON
row() {
is "$1" ".tables[\"$2\"][]|select(.[0]==\"$3\")|.[1:]" "$4"
}
# $1 name, $2 fixed-string the text report must contain
says() {
grep -Fq -- "$2" "$work/out.txt" && ok "$1" || bad "$1" "no '$2' in the text report"
@@ -52,7 +67,7 @@ put crates/a/tests/t.rs 4 x2
put AGENTS.md 3 x2; put .agents/skills/wave/agent-rules.md 3 x2
commit 2026-06-03
# One file in each remaining area.
# One file in each remaining area. The repo map is 13 lines.
put context/code/x.md 8 x3
mkdir -p "$repo/context/test-evidence"
printf '### t_one\ntext\n#### t_two\n## not an entry\n### t_three\n' \
@@ -70,8 +85,6 @@ cat > "$repo/context/repo-map.md" <<'EOF'
---
## File Inventory
4. not a gotcha, because the section has ended
EOF
put context/plan.md 2 x3
@@ -81,7 +94,7 @@ put web/app.js 9 x3; put harness/x.js 2 x3
put Cargo.lock 11 x3
put crates/a/src/grammar.pest 3 x3
commit 2026-06-04
g tag before-shrink 2>/dev/null || true
g tag before-shrink
# A side branch merged with a merge commit: the side commit counts, the merge does not.
g checkout -q -b side
@@ -90,61 +103,82 @@ g checkout -q main
GIT_AUTHOR_DATE=2026-06-06T12:00:00Z GIT_COMMITTER_DATE=2026-06-06T12:00:00Z \
g merge -q --no-ff -m merge side
# A shrink, so one area deletes more than it adds in a commit.
put web/app.js 3 x4; commit 2026-06-07
# A shrink in the next ISO week, so one area deletes and one row spans two weeks.
put web/app.js 3 x4; commit 2026-06-09
# $1 output dir, $2 "text" to also write the text report, rest: extra args.
run() {
"$STATS" -C "$repo" --since 2026-06-01 --min-revs 2 "$@" --json > "$work/out.json" 2> "$work/err" \
local out="$1" text="$2"; shift 2
"$STATS" -C "$repo" --since 2026-06-01 --min-revs 2 --out "$out" "$@" --json \
> "$work/out.json" 2> "$work/err" \
|| { bad "repo-stats $* ran" "$(cat "$work/err")"; return 1; }
"$STATS" -C "$repo" --since 2026-06-01 --min-revs 2 "$@" > "$work/out.txt" 2>&1 \
[ "$text" = text ] || return 0
"$STATS" -C "$repo" --since 2026-06-01 --min-revs 2 --out "$out" "$@" \
> "$work/out.txt" 2>&1 \
|| { bad "repo-stats $* ran (text)" "$(cat "$work/out.txt")"; return 1; }
}
if run; then
if run "$work/stats" text; then
is "no gh/main in the fixture, so the ref is HEAD" '.ref' '"HEAD"'
is "the merge commit is not counted, the side commit is" '.commits' '5'
is "rust src is .rs and .pest outside tests" '.areas["rust src"]|[.added,.deleted]' '[35,20]'
is "rust tests are the tests/ files" '.areas["rust tests"]|[.added,.deleted]' '[8,4]'
is "agent docs and skills" '.areas["agent docs and skills"]|[.added,.deleted]' '[12,6]'
is "context/code" '.areas["context/code"].added' '8'
is "context/test-evidence" '.areas["context/test-evidence"].added' '5'
is "repo-map is its own area" '.areas["repo-map"].added' '15'
is "other context" '.areas["other context"].added' '2'
is "kb.pl" '.areas["kb.pl"].added' '5'
is "scripts/CI takes .github" '.areas["scripts/CI"].added' '8'
is "web and harness" '.areas["web and harness"]|[.added,.deleted]' '[14,9]'
is "a lock file is generated" '.areas.generated.added' '11'
is "the pre-window README counts nowhere" '.areas.other|[.added,.deleted]' '[4,0]'
is "deletion ratio is deleted over added" '.areas["rust src"].deletion_ratio' '0.571'
is "an area that deletes nothing has a deletion ratio of 0" \
'.areas["kb.pl"].deletion_ratio' '0.0'
is "prose to code is prose added over code added" \
'.prose_to_code|[.prose_added,.code_added,.ratio]' '[42,65,0.646]'
row "rust src is .rs and .pest outside tests" by-area "rust src" '["35","20","0.571"]'
row "rust tests are the tests/ files" by-area "rust tests" '["8","4","0.500"]'
row "agent docs and skills" by-area "agent docs and skills" '["12","6","0.500"]'
row "context/code" by-area "context/code" '["8","0","0.000"]'
row "context/test-evidence" by-area "context/test-evidence" '["5","0","0.000"]'
row "repo-map is its own area" by-area "repo-map" '["13","0","0.000"]'
row "other context" by-area "other context" '["2","0","0.000"]'
row "kb.pl" by-area "kb.pl" '["5","0","0.000"]'
row "scripts/CI takes .github" by-area "scripts/CI" '["8","0","0.000"]'
row "web and harness, and its deletion ratio" by-area "web and harness" '["14","9","0.643"]'
row "a lock file is generated" by-area "generated" '["11","0","0.000"]'
row "the pre-window README counts nowhere" by-area "other" '["4","0","0.000"]'
row "prose is the context areas and agent docs" by-kind prose '["40","6","0.150"]'
row "code is rust, scripts/CI and web" by-kind code '["65","33","0.508"]'
row "prose to code is prose over code" by-kind ProseToCode '["0.615","0.182",""]'
is "weeks are columns, first-seen first" '.tables["area-by-week"][0][1:]' '["2026-W24","2026-W23"]'
row "an area is split by ISO week" area-by-week "web and harness" '["3","11"]'
is "every day's churn.csv total equals code-maat's abs-churn" \
'[.tables["by-day"][1:][]|.[1]==.[2]]|all' 'true'
row "a day's total is every file changed that day" by-day 2026-06-04 '["66","66"]'
is "gotchas end with their section" '.docs.repo_map.gotchas' '3'
is "the longest gotcha is measured over its lines" \
'.docs.repo_map.longest|[.number,.lines]' '[2,2]'
is "test-evidence entries are ### and #### headings" '.docs.test_evidence.entries' '3'
is "context doc lines" '.docs.context_lines["context/repo-map.md"]' '15'
is "coupling finds the ten co-changed pairs" '.code_maat.coupling|length' '10'
is "coupling finds the ten co-changed pairs" '.coupling|length' '10'
is "two crates' files cross a crate boundary" \
'[.code_maat.coupling[]|select(.crosses_crate)|[.entity,.coupled]|sort]' \
'[.coupling[]|select(.["crosses-crate"]=="yes")|[.entity,.coupled]|sort]' \
'[["crates/a/src/lib.rs","crates/b/src/lib.rs"],["crates/a/tests/t.rs","crates/b/src/lib.rs"]]'
is "two agent-rule docs are flagged" \
'[.code_maat.coupling[]|select(.agent_rules)|[.entity,.coupled]|sort]' \
'[.coupling[]|select(.["agent-rules"]=="yes")|[.entity,.coupled]|sort]' \
'[[".agents/skills/wave/agent-rules.md","AGENTS.md"]]'
is "abs-churn has one row per day" '.code_maat.abs_churn|length' '5'
is "soc sums a file's co-changes over its commits" \
'.code_maat.soc[]|select(.entity=="crates/a/src/lib.rs")|.soc' '"8"'
says "the text report flags a crate crossing" "[CROSSES-CRATE]"
says "the text report flags agent-rule docs" "[AGENT-RULES]"
says "the text report prints the ratio" "Prose to code (added lines): 42 / 65 = 0.646"
says "the text report prints improvise's ratio row" "ProseToCode"
views="$(grep -c '^## View: ' "$work/stats/repo-stats.improv")"
[ "$views" = 10 ] && ok "the model keeps a view per file and the report's four" \
|| bad "the model keeps a view per file and the report's four" "$views views"
fi
# The window is scoped to the ref: at the tag, the shrink has not happened.
if run --ref before-shrink; then
is "--ref scopes the log" '.areas["web and harness"]|[.added,.deleted]' '[11,0]'
if run "$work/at-tag" json --ref before-shrink; then
row "--ref scopes the log" by-area "web and harness" '["11","0","0.000"]'
is "--ref names the ref it read" '.ref' '"before-shrink"'
fi
# improvise refusing the pipeline is exit 3, whatever the numbers were.
mkdir -p "$work/bin"
printf '#!/usr/bin/env bash\necho "Parse error: no" >&2\nexit 1\n' > "$work/bin/refuses"
printf '#!/usr/bin/env bash\necho "warning: Skipped 3 of 3 records" >&2\nexit 0\n' \
> "$work/bin/warns"
chmod +x "$work/bin/refuses" "$work/bin/warns"
for stub in refuses warns; do
IMPROVISE="$work/bin/$stub" "$STATS" -C "$repo" --since 2026-06-01 --out "$work/$stub" \
> /dev/null 2>&1
got=$?
[ "$got" = 3 ] && ok "an improvise that $stub fails the pipeline with exit 3" \
|| bad "an improvise that $stub fails the pipeline with exit 3" "exit $got"
done
printf '\nrepo-stats: %d passed, %d failed\n' "$pass" "$fail"
[ "$fail" = 0 ]