From 20335ee6163da4fffd8b78ef473efd9883b8b0d9 Mon Sep 17 00:00:00 2001 From: "Claude (agent-stats)" Date: Thu, 24 Sep 2026 20:46:21 -0700 Subject: [PATCH] test(tooling): the repo-stats suite runs the whole pipeline over its fixture (improvise-1xfb.5.1) Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze --- scripts/repo-stats-test.sh | 110 ++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 38 deletions(-) diff --git a/scripts/repo-stats-test.sh b/scripts/repo-stats-test.sh index 4b82901b..ee69e055 100755 --- a/scripts/repo-stats-test.sh +++ b/scripts/repo-stats-test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# check_the_checker for scripts/repo-stats (improvise-1xfb.5.1), against a fixture -# repo whose every number is known by construction. See context/test-evidence/tooling.md. +# check_the_checker for scripts/repo-stats (improvise-1xfb.5.1): the whole pipeline, +# code-maat and improvise included, over a fixture repo whose every number is known. +# See context/test-evidence/tooling.md. set -uo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -12,6 +13,15 @@ pass=0; fail=0 ok() { printf ' ok %s\n' "$1"; pass=$((pass+1)); } bad() { printf ' FAIL %s (%s)\n' "$1" "$2"; fail=$((fail+1)); } +# The improvise under test: on PATH in CI (`nix shell .#improvise`), else this tree's. +if command -v improvise >/dev/null; then + IMPROVISE="$(command -v improvise)" +else + cargo build --quiet --manifest-path "$here/../Cargo.toml" --bin improvise || exit 1 + IMPROVISE="$here/../target/debug/improvise" +fi +export IMPROVISE + # $1 name, $2 jq expression over $work/out.json, $3 the value it must print is() { local got @@ -19,6 +29,11 @@ is() { [ "$got" = "$3" ] && ok "$1" || bad "$1" "got $got, wanted $3" } +# $1 name, $2 table, $3 row label, $4 the row's cells after the label, as JSON +row() { + is "$1" ".tables[\"$2\"][]|select(.[0]==\"$3\")|.[1:]" "$4" +} + # $1 name, $2 fixed-string the text report must contain says() { grep -Fq -- "$2" "$work/out.txt" && ok "$1" || bad "$1" "no '$2' in the text report" @@ -52,7 +67,7 @@ put crates/a/tests/t.rs 4 x2 put AGENTS.md 3 x2; put .agents/skills/wave/agent-rules.md 3 x2 commit 2026-06-03 -# One file in each remaining area. +# One file in each remaining area. The repo map is 13 lines. put context/code/x.md 8 x3 mkdir -p "$repo/context/test-evidence" printf '### t_one\ntext\n#### t_two\n## not an entry\n### t_three\n' \ @@ -70,8 +85,6 @@ cat > "$repo/context/repo-map.md" <<'EOF' --- -## File Inventory - 4. not a gotcha, because the section has ended EOF put context/plan.md 2 x3 @@ -81,7 +94,7 @@ put web/app.js 9 x3; put harness/x.js 2 x3 put Cargo.lock 11 x3 put crates/a/src/grammar.pest 3 x3 commit 2026-06-04 -g tag before-shrink 2>/dev/null || true +g tag before-shrink # A side branch merged with a merge commit: the side commit counts, the merge does not. g checkout -q -b side @@ -90,61 +103,82 @@ g checkout -q main GIT_AUTHOR_DATE=2026-06-06T12:00:00Z GIT_COMMITTER_DATE=2026-06-06T12:00:00Z \ g merge -q --no-ff -m merge side -# A shrink, so one area deletes more than it adds in a commit. -put web/app.js 3 x4; commit 2026-06-07 +# A shrink in the next ISO week, so one area deletes and one row spans two weeks. +put web/app.js 3 x4; commit 2026-06-09 +# $1 output dir, $2 "text" to also write the text report, rest: extra args. run() { - "$STATS" -C "$repo" --since 2026-06-01 --min-revs 2 "$@" --json > "$work/out.json" 2> "$work/err" \ + local out="$1" text="$2"; shift 2 + "$STATS" -C "$repo" --since 2026-06-01 --min-revs 2 --out "$out" "$@" --json \ + > "$work/out.json" 2> "$work/err" \ || { bad "repo-stats $* ran" "$(cat "$work/err")"; return 1; } - "$STATS" -C "$repo" --since 2026-06-01 --min-revs 2 "$@" > "$work/out.txt" 2>&1 \ + [ "$text" = text ] || return 0 + "$STATS" -C "$repo" --since 2026-06-01 --min-revs 2 --out "$out" "$@" \ + > "$work/out.txt" 2>&1 \ || { bad "repo-stats $* ran (text)" "$(cat "$work/out.txt")"; return 1; } } -if run; then +if run "$work/stats" text; then is "no gh/main in the fixture, so the ref is HEAD" '.ref' '"HEAD"' is "the merge commit is not counted, the side commit is" '.commits' '5' - is "rust src is .rs and .pest outside tests" '.areas["rust src"]|[.added,.deleted]' '[35,20]' - is "rust tests are the tests/ files" '.areas["rust tests"]|[.added,.deleted]' '[8,4]' - is "agent docs and skills" '.areas["agent docs and skills"]|[.added,.deleted]' '[12,6]' - is "context/code" '.areas["context/code"].added' '8' - is "context/test-evidence" '.areas["context/test-evidence"].added' '5' - is "repo-map is its own area" '.areas["repo-map"].added' '15' - is "other context" '.areas["other context"].added' '2' - is "kb.pl" '.areas["kb.pl"].added' '5' - is "scripts/CI takes .github" '.areas["scripts/CI"].added' '8' - is "web and harness" '.areas["web and harness"]|[.added,.deleted]' '[14,9]' - is "a lock file is generated" '.areas.generated.added' '11' - is "the pre-window README counts nowhere" '.areas.other|[.added,.deleted]' '[4,0]' - is "deletion ratio is deleted over added" '.areas["rust src"].deletion_ratio' '0.571' - is "an area that deletes nothing has a deletion ratio of 0" \ - '.areas["kb.pl"].deletion_ratio' '0.0' - is "prose to code is prose added over code added" \ - '.prose_to_code|[.prose_added,.code_added,.ratio]' '[42,65,0.646]' + row "rust src is .rs and .pest outside tests" by-area "rust src" '["35","20","0.571"]' + row "rust tests are the tests/ files" by-area "rust tests" '["8","4","0.500"]' + row "agent docs and skills" by-area "agent docs and skills" '["12","6","0.500"]' + row "context/code" by-area "context/code" '["8","0","0.000"]' + row "context/test-evidence" by-area "context/test-evidence" '["5","0","0.000"]' + row "repo-map is its own area" by-area "repo-map" '["13","0","0.000"]' + row "other context" by-area "other context" '["2","0","0.000"]' + row "kb.pl" by-area "kb.pl" '["5","0","0.000"]' + row "scripts/CI takes .github" by-area "scripts/CI" '["8","0","0.000"]' + row "web and harness, and its deletion ratio" by-area "web and harness" '["14","9","0.643"]' + row "a lock file is generated" by-area "generated" '["11","0","0.000"]' + row "the pre-window README counts nowhere" by-area "other" '["4","0","0.000"]' + row "prose is the context areas and agent docs" by-kind prose '["40","6","0.150"]' + row "code is rust, scripts/CI and web" by-kind code '["65","33","0.508"]' + row "prose to code is prose over code" by-kind ProseToCode '["0.615","0.182",""]' + is "weeks are columns, first-seen first" '.tables["area-by-week"][0][1:]' '["2026-W24","2026-W23"]' + row "an area is split by ISO week" area-by-week "web and harness" '["3","11"]' + is "every day's churn.csv total equals code-maat's abs-churn" \ + '[.tables["by-day"][1:][]|.[1]==.[2]]|all' 'true' + row "a day's total is every file changed that day" by-day 2026-06-04 '["66","66"]' is "gotchas end with their section" '.docs.repo_map.gotchas' '3' is "the longest gotcha is measured over its lines" \ '.docs.repo_map.longest|[.number,.lines]' '[2,2]' is "test-evidence entries are ### and #### headings" '.docs.test_evidence.entries' '3' - is "context doc lines" '.docs.context_lines["context/repo-map.md"]' '15' - is "coupling finds the ten co-changed pairs" '.code_maat.coupling|length' '10' + is "coupling finds the ten co-changed pairs" '.coupling|length' '10' is "two crates' files cross a crate boundary" \ - '[.code_maat.coupling[]|select(.crosses_crate)|[.entity,.coupled]|sort]' \ + '[.coupling[]|select(.["crosses-crate"]=="yes")|[.entity,.coupled]|sort]' \ '[["crates/a/src/lib.rs","crates/b/src/lib.rs"],["crates/a/tests/t.rs","crates/b/src/lib.rs"]]' is "two agent-rule docs are flagged" \ - '[.code_maat.coupling[]|select(.agent_rules)|[.entity,.coupled]|sort]' \ + '[.coupling[]|select(.["agent-rules"]=="yes")|[.entity,.coupled]|sort]' \ '[[".agents/skills/wave/agent-rules.md","AGENTS.md"]]' - is "abs-churn has one row per day" '.code_maat.abs_churn|length' '5' - is "soc sums a file's co-changes over its commits" \ - '.code_maat.soc[]|select(.entity=="crates/a/src/lib.rs")|.soc' '"8"' says "the text report flags a crate crossing" "[CROSSES-CRATE]" says "the text report flags agent-rule docs" "[AGENT-RULES]" - says "the text report prints the ratio" "Prose to code (added lines): 42 / 65 = 0.646" + says "the text report prints improvise's ratio row" "ProseToCode" + views="$(grep -c '^## View: ' "$work/stats/repo-stats.improv")" + [ "$views" = 10 ] && ok "the model keeps a view per file and the report's four" \ + || bad "the model keeps a view per file and the report's four" "$views views" fi # The window is scoped to the ref: at the tag, the shrink has not happened. -if run --ref before-shrink; then - is "--ref scopes the log" '.areas["web and harness"]|[.added,.deleted]' '[11,0]' +if run "$work/at-tag" json --ref before-shrink; then + row "--ref scopes the log" by-area "web and harness" '["11","0","0.000"]' is "--ref names the ref it read" '.ref' '"before-shrink"' fi +# improvise refusing the pipeline is exit 3, whatever the numbers were. +mkdir -p "$work/bin" +printf '#!/usr/bin/env bash\necho "Parse error: no" >&2\nexit 1\n' > "$work/bin/refuses" +printf '#!/usr/bin/env bash\necho "warning: Skipped 3 of 3 records" >&2\nexit 0\n' \ + > "$work/bin/warns" +chmod +x "$work/bin/refuses" "$work/bin/warns" +for stub in refuses warns; do + IMPROVISE="$work/bin/$stub" "$STATS" -C "$repo" --since 2026-06-01 --out "$work/$stub" \ + > /dev/null 2>&1 + got=$? + [ "$got" = 3 ] && ok "an improvise that $stub fails the pipeline with exit 3" \ + || bad "an improvise that $stub fails the pipeline with exit 3" "exit $got" +done + printf '\nrepo-stats: %d passed, %d failed\n' "$pass" "$fail" [ "$fail" = 0 ]