improvise/scripts/checkers.sh

#!/usr/bin/env bash
# The one entry point for every check_the_checker in scripts/. CI calls THIS,
# and so does the local gate, so a suite cannot be green under one shell and
# red under the other. Reasoning: context/code/tooling.md section 3.
#
# It does two things a bare `./scripts/X-test.sh` does not. It runs every
# suite under the bash running it (the dev shell's), and it:
#
#   1. scrubs the environment down to an allowlist, because `nix develop`
#      exports `name`, `out`, `system`, `shell`, `builder` and two dozen more
#      generically-named variables that CI does not have -- a `set -u` fault
#      reading one of them is SILENT inside the dev shell and fatal in CI;
#   2. refuses a shell script in scripts/ whose shebang is not
#      `#!/usr/bin/env bash`, so the sh/bash split cannot come back.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

usage() {
  cat >&2 <<'EOF'
usage: checkers.sh [--list] [--self-test] [--ci <job>] [name ...]

  name        a suite to run, spelled without the -test.sh suffix
              (e.g. `pr-reviews` for scripts/pr-reviews-test.sh).
              With no name, every suite in scripts/ runs.
  --list      print the suite names and exit
  --self-test run scripts/checkers-test.sh, which injects the faults this
              runner exists to catch
  --ci <job>  run the suites the CI table gives that job of
              .github/workflows/ci.yml, and nothing else. Each CI job runs
              exactly this, so the table is the one list of what CI runs
  --would-run as the FIRST argument only: run nothing, and print
              `would-run: <suite>` (or `missing: <name>`) for each suite
              the rest of the arguments would run. check-ci-graph.py
              asks this, so the argument rules have one home

exit 0  every suite passed
exit 1  a suite failed, or a shebang is not `#!/usr/bin/env bash`
exit 4  usage error

THIS SCRIPT'S `$?` MUST BE THE ONE YOU READ. Nothing may follow it in the same
pipeline or `;` list -- a trailing `echo "exit=$?"` takes the status.
EOF
}

# The allowlist. A variable not named here does not reach a suite, which is the
# whole point: the dev shell's `name` masked improvise-f9qf's fault for a day.
# CHECKERS_KEEP adds names, space-separated, for a caller that needs one.
ALLOW="HOME TMPDIR TMP TEMP TERM LANG LC_ALL USER LOGNAME
       CI GITHUB_ACTIONS GITHUB_TOKEN GH_TOKEN
       NIX_PATH NIX_REMOTE NIX_SSL_CERT_FILE SSL_CERT_FILE
       XDG_CACHE_HOME XDG_CONFIG_HOME XDG_DATA_HOME"

# Every shell script in scripts/ says `#!/usr/bin/env bash` and nothing else.
# `#!/bin/sh` is dash in CI and bash 3.2 in sh-mode on macOS: a second platform
# split, undetected for the same reason the first one was.
check_shebangs() {
  local bad=0 f first
  for f in "$HERE"/*; do
    [ -f "$f" ] || continue
    first="$(head -1 "$f")"
    case "$first" in
      '#!'*sh|'#!'*sh\ *)
        case "$first" in
          '#!/usr/bin/env bash') ;;
          *) printf 'checkers: %s has shebang %s, wanted #!/usr/bin/env bash\n' \
               "${f##*/}" "$first" >&2; bad=1 ;;
        esac ;;
    esac
  done
  return $bad
}

# A check_the_checker is named after the thing it checks, so `X-test.sh` is a
# suite only when `X` or `X.sh` is here too. That excludes `node-test.sh`,
# which is a test RUNNER whose own checker is `node-test-test.sh`.
suites() {
  local f n
  for f in "$HERE"/*-test.sh; do
    [ -f "$f" ] || continue
    n="${f##*/}"; n="${n%-test.sh}"
    if [ -f "$HERE/$n" ] || [ -f "$HERE/$n.sh" ]; then printf '%s\n' "$n"; fi
  done
}

# $1 the suite file, run by the bash running this script, with PATH unchanged.
run_suite() {
  local file="$1"
  local -a env_args=(-i)
  local v
  for v in $ALLOW ${CHECKERS_KEEP:-}; do
    if [ -n "${!v+set}" ]; then env_args+=("$v=${!v}"); fi
  done
  env_args+=("PATH=$PATH")
  env "${env_args[@]}" "$BASH" "$file"
}

# The CI table: the suites each job of .github/workflows/ci.yml runs, by that
# job's `checkers.sh --ci <job>` step. check-ci-graph.py holds it to --list.
ci_suites() {
  case "$1" in
    build) echo assemble-stamp ;;
    js) echo node-test ;;
    # Needs the improvise binary, which only this job builds (tooling.md section 7).
    nix-build) echo repo-stats ;;
    # check-harness-lock must never read "could not resolve" as "stale" (improvise-zs2r).
    kb) echo kb checkers pr-watch release-preflight pr-reviews fmt-workspaces bd-audit-ids check-harness-lock ;;
    *) return 1 ;;
  esac
}

main() {
  local self_test=0 would_run=0 ci_job=""
  local -a names=()
  # Only as the FIRST argument, so a CI step spelling it anywhere is an error.
  if [ "${1:-}" = --would-run ]; then would_run=1; shift; fi
  while [ $# -gt 0 ]; do
    case "$1" in
      --list) suites; return 0 ;;
      --self-test) self_test=1 ;;
      --ci) ci_job="${2:-}"; [ -n "$ci_job" ] || { usage; return 4; }; shift ;;
      -h|--help) usage; return 4 ;;
      -*) printf 'checkers: unknown option %s\n' "$1" >&2; usage; return 4 ;;
      *) names+=("$1") ;;
    esac
    shift
  done

  if [ -n "$ci_job" ]; then
    if [ "$self_test" = 1 ] || [ ${#names[@]} -gt 0 ]; then
      echo 'checkers: --ci takes a job and nothing else' >&2; return 4
    fi
    local table
    table="$(ci_suites "$ci_job")" || {
      printf 'checkers: no CI job %s in the table\n' "$ci_job" >&2; return 4; }
    read -r -a names <<< "$table"
  elif [ "$self_test" = 1 ]; then
    names=(checkers)
  elif [ ${#names[@]} -eq 0 ]; then
    while IFS= read -r line; do names+=("$line"); done < <(suites)
  fi

  local n file
  if [ "$would_run" = 1 ]; then
    for n in ${names[@]+"${names[@]}"}; do
      if [ -f "$HERE/$n-test.sh" ]; then printf 'would-run: %s\n' "$n"
      else printf 'missing: %s\n' "$n"; fi
    done
    return 0
  fi

  printf 'checkers: bash %s at %s\n' "$BASH_VERSION" "$BASH"
  local rc=0
  check_shebangs || rc=1

  for n in ${names[@]+"${names[@]}"}; do
    file="$HERE/$n-test.sh"
    if [ ! -f "$file" ]; then
      printf 'checkers: no such suite: %s\n' "$n" >&2
      rc=4; continue
    fi
    printf '\n=== %s ===\n' "$n"
    run_suite "$file" || rc=1
  done
  return $rc
}

main "$@"