Ed Langley
932ff408e6
tooling: pr-reviews timeout case asserts WAIT+1 polls, and a hang fails (7fl8)
200 lines
7.2 KiB
Bash
Executable File
200 lines
7.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check_the_checker for scripts/checkers.sh. Every case injects a fault that
|
|
# a bare `./scripts/X-test.sh` cannot see, and asserts the runner does.
|
|
# See context/test-evidence/tooling.md.
|
|
set -uo pipefail
|
|
|
|
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
RUNNER="$here/checkers.sh"
|
|
work="$(mktemp -d)"
|
|
trap 'rm -rf "$work"' EXIT
|
|
|
|
pass=0; fail=0
|
|
ok() { printf ' ok %s\n' "$1"; pass=$((pass+1)); }
|
|
bad() { printf ' FAIL %s (%s)\n' "$1" "$2"; fail=$((fail+1)); }
|
|
|
|
# A sandbox is a copy of the runner beside fixture suites, so `$HERE` inside it
|
|
# is the sandbox: the shebang scan and the suite list are both the fixtures'.
|
|
sandbox() {
|
|
local d="$work/$1"
|
|
mkdir -p "$d"
|
|
cp "$RUNNER" "$d/checkers.sh"
|
|
chmod +x "$d/checkers.sh"
|
|
printf '%s' "$d"
|
|
}
|
|
|
|
# A suite is only discovered when the thing it checks is present, so every
|
|
# fixture gets a stub subject beside it.
|
|
suite() {
|
|
local d="$1" name="$2"
|
|
cat > "$d/$name-test.sh"
|
|
chmod +x "$d/$name-test.sh"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$d/$name.sh"
|
|
chmod +x "$d/$name.sh"
|
|
}
|
|
|
|
# $1 name, $2 expected exit, rest: the command. Nothing follows the runner in
|
|
# its own pipeline, which is the rule its own header states.
|
|
expect() {
|
|
local what="$1" want="$2"; shift 2
|
|
"$@" > "$work/out" 2>&1
|
|
local got=$?
|
|
[ "$got" = "$want" ] && ok "$what" || bad "$what" "exit $got, wanted $want"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. The fault this runner exists for. `nix develop` exports `name`; a suite
|
|
# that reads an unset variable under `set -u` is therefore SILENT in the dev
|
|
# shell and fatal in CI. That is improvise-f9qf, reproduced exactly.
|
|
d="$(sandbox setu)"
|
|
suite "$d" masked <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
echo "read: $name"
|
|
EOF
|
|
export name=nix-shell-env
|
|
expect "a suite reading an inherited dev-shell variable fails under the runner" \
|
|
1 "$d/checkers.sh" masked
|
|
|
|
# The same suite run the way the old gate ran it PASSES, which is what makes
|
|
# the case above discriminating rather than a test of `set -u`.
|
|
"$d/masked-test.sh" > "$work/out" 2>&1
|
|
direct=$?
|
|
[ "$direct" = 0 ] \
|
|
&& ok "and passes when run directly with that variable exported" \
|
|
|| bad "and passes when run directly with that variable exported" \
|
|
"exit $direct, wanted 0"
|
|
unset name
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. The bash-5 `local` semantics. A multi-assignment `local` does not read
|
|
# left to right on bash 5, so this suite passes on macOS bash 3.2 and dies
|
|
# under `set -u` on bash 5 -- PR #133's 19/19 local against 2/19 in CI.
|
|
d="$(sandbox bash5)"
|
|
suite "$d" vers <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
f() { local a="$1" b="$a/x"; echo "$b"; }
|
|
f one
|
|
EOF
|
|
expect "a bash-3.2-only construct fails under the runner's pinned bash" \
|
|
1 "$d/checkers.sh" vers
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. The shebang rule. `#!/bin/sh` is dash in CI and bash-in-sh-mode on macOS.
|
|
d="$(sandbox shebang)"
|
|
suite "$d" fine <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
printf '#!/bin/sh\nexit 0\n' > "$d/legacy.sh"
|
|
chmod +x "$d/legacy.sh"
|
|
expect "a #!/bin/sh script in the scripts directory is refused" 1 \
|
|
"$d/checkers.sh" fine
|
|
grep -q 'wanted #!/usr/bin/env bash' "$work/out" \
|
|
&& ok "and the refusal names the shebang it wanted" \
|
|
|| bad "and the refusal names the shebang it wanted" "no such message"
|
|
|
|
# Without the injected file the same sandbox passes, so the case above is not
|
|
# passing on some other complaint.
|
|
rm -f "$d/legacy.sh"
|
|
expect "and the same sandbox passes once it is removed" 0 "$d/checkers.sh" fine
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. A failing suite is reported, and a passing one is not.
|
|
d="$(sandbox rc)"
|
|
suite "$d" red <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 1
|
|
EOF
|
|
expect "a failing suite fails the runner" 1 "$d/checkers.sh" red
|
|
|
|
d="$(sandbox green)"
|
|
suite "$d" blue <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
expect "a passing suite passes the runner" 0 "$d/checkers.sh" blue
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. No bash >= 5 and no nix is exit 3, not a silent run under whatever is
|
|
# there. This is the arm that would have refused the macOS system bash.
|
|
d="$(sandbox oldbash)"
|
|
suite "$d" any <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
mkdir -p "$work/fakebin"
|
|
cat > "$work/fakebin/bash" <<'EOF'
|
|
#!/bin/sh
|
|
case "$*" in *BASH_VERSINFO*) echo 3; exit 0 ;; esac
|
|
exit 0
|
|
EOF
|
|
chmod +x "$work/fakebin/bash"
|
|
# The runner is invoked through a REAL bash, so the fake one below is only
|
|
# what its own `command -v bash` probe finds -- not its interpreter.
|
|
# utilbin carries only the utilities the runner itself needs, and no nix, so
|
|
# the fake bash is the only candidate and the nix fallback is absent. Linked
|
|
# one by one rather than taking /usr/bin:/bin, which on NixOS hold only `env`
|
|
# and `sh` -- the runner died on `dirname` there, exit 1 instead of 3.
|
|
mkdir -p "$work/utilbin"
|
|
for tool in dirname; do
|
|
ln -sf "$(command -v "$tool")" "$work/utilbin/$tool"
|
|
done
|
|
env PATH="$work/fakebin:$work/utilbin" HOME="$HOME" \
|
|
"${BASH:-bash}" "$d/checkers.sh" any > "$work/out" 2>&1
|
|
got=$?
|
|
[ "$got" = 3 ] && ok "an old bash with no nix is exit 3" \
|
|
|| bad "an old bash with no nix is exit 3" "exit $got, wanted 3"
|
|
|
|
# 5b. The floor is 5.1, not 5: `wait -n -p` is 5.1 (improvise-fd0j). The 5.1
|
|
# control proves the 5.0 refusal is the version and not the fake itself.
|
|
for v in "5 0:3:a bash 5.0 with no nix is exit 3, not 5.x" \
|
|
"5 1:0:a bash 5.1 is accepted"; do
|
|
printf '#!/bin/sh\ncase "$*" in *BASH_VERSINFO*) echo %s; exit 0 ;; esac\nexit 0\n' \
|
|
"${v%%:*}" > "$work/fakebin/bash"
|
|
want="${v#*:}"; name="${want#*:}"; want="${want%%:*}"
|
|
env PATH="$work/fakebin:/usr/bin:/bin" HOME="$HOME" \
|
|
"${BASH:-bash}" "$d/checkers.sh" any > "$work/out" 2>&1
|
|
got=$?
|
|
[ "$got" = "$want" ] && ok "$name" || bad "$name" "exit $got, wanted $want"
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 6. A named suite that does not exist is a usage error, not a silent no-op.
|
|
d="$(sandbox missing)"
|
|
suite "$d" present <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
expect "an unknown suite name is exit 4" 4 "$d/checkers.sh" absent
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 7. --list names every suite in the directory and nothing else.
|
|
d="$(sandbox listing)"
|
|
suite "$d" alpha <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
suite "$d" beta <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
"$d/checkers.sh" --list > "$work/out" 2>&1
|
|
listed="$(tr '\n' ' ' < "$work/out")"
|
|
[ "$listed" = "alpha beta " ] && ok "--list names exactly the suites present" \
|
|
|| bad "--list names exactly the suites present" "got [$listed]"
|
|
|
|
# A test RUNNER also ends in -test.sh. It is not a suite, and running it as
|
|
# one is how `scripts/node-test.sh` printed its usage inside a green gate.
|
|
printf '#!/usr/bin/env bash\nexit 4\n' > "$d/gamma-test.sh"
|
|
chmod +x "$d/gamma-test.sh"
|
|
"$d/checkers.sh" --list > "$work/out" 2>&1
|
|
listed="$(tr '\n' ' ' < "$work/out")"
|
|
[ "$listed" = "alpha beta " ] \
|
|
&& ok "an X-test.sh with no X beside it is not a suite" \
|
|
|| bad "an X-test.sh with no X beside it is not a suite" "got [$listed]"
|
|
|
|
printf '\ncheckers-test: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|