test(11.1-06): plant one fixture per docs checker rule and run them in the gate

- testdata/clean passes every check; 63 violation cases each overlay one
  fault and a want.txt naming the single problem it must produce
- TestPlantedViolations also plants the forbidden name (split literals)
  and symlink escapes at run time; TestBuildOutputGuard covers --out
- check-phase11.1.sh --self-test runs the corpus through a go test -json
  detector that refuses FAIL, SKIP, zero tests and no tests to run
- the self-test scratch copy now carries examples/ and README.md, so its
  baseline sees the example command literals the real tree sees
This commit is contained in:
Jakub Zych
2026-09-30 23:52:19 +02:00
parent 4c68d31862
commit 1d38e73f03
154 changed files with 1998 additions and 1 deletions

View File

@@ -152,6 +152,71 @@ run_go() {
echo "phase11.1 go passed"
}
# detect_json reads a go test -json log. It refuses a build failure, any
# failed test or package, any skipped test, "no tests to run", a run with
# zero passing tests, and any required "<package> <Test>" pair (from
# PHASE11_1_REQUIRE, newline separated) that did not PASS.
detect_json() {
python3 - "$1" <<'PY'
import json, os, sys
path = sys.argv[1]
require = [r.strip() for r in os.environ.get("PHASE11_1_REQUIRE", "").splitlines() if r.strip()]
passed = set()
with open(path, encoding="utf-8", errors="replace") as fh:
for raw in fh:
line = raw.strip()
if not line.startswith("{"):
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
print("refuse: non-json test output", file=sys.stderr)
sys.exit(4)
action, test, pkg = ev.get("Action"), ev.get("Test") or "", ev.get("Package") or ""
if action == "build-fail" or (action == "fail" and ev.get("FailedBuild")):
print(f"refuse: build failed in {pkg}", file=sys.stderr)
sys.exit(1)
if action == "output" and "no tests to run" in (ev.get("Output") or ""):
print(f"refuse: no tests to run in {pkg}", file=sys.stderr)
sys.exit(3)
if action == "skip" and test:
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
sys.exit(2)
if action == "fail":
print(f"refuse: failed {pkg} {test}".rstrip(), file=sys.stderr)
sys.exit(1)
if action == "pass" and test:
passed.add(f"{pkg} {test}")
if not passed:
print("refuse: zero tests", file=sys.stderr)
sys.exit(3)
missing = [r for r in require if r not in passed]
if missing:
print("refuse: named tests did not pass (missing, renamed or filtered out): " + ", ".join(missing), file=sys.stderr)
sys.exit(5)
PY
}
# go_json_named runs `go test -json -count=1 -run '^(names)$'` for one
# package and requires every named test to PASS.
go_json_named() {
local pkg="$1" log import names name require="" code=0
shift
import="$(cd "$ROOT" && go list -f '{{.ImportPath}}' "$pkg")" || refuse "go list $pkg failed" || return 1
names="$(IFS='|'; echo "$*")"
for name in "$@"; do
require+="$import $name"$'\n'
done
log="$(mktemp)"
(cd "$ROOT" && go test -json -count=1 "$pkg" -run "^($names)\$" >"$log" 2>&1) || code=$?
if ! PHASE11_1_REQUIRE="$require" detect_json "$log"; then
rm -f "$log"
return 1
fi
rm -f "$log"
[ "$code" -eq 0 ] || refuse "go test $pkg exited $code"
}
# expect_refusal runs docs:build --check on the scratch root and requires a
# non-zero exit whose output names the rule.
expect_refusal() {
@@ -181,7 +246,10 @@ run_self_test() {
summer="$tmp/summer"
build_summer "$summer"
mkdir -p "$scratch" "$pristine"
cp -R "$ROOT/docs" "$ROOT/modules" "$ROOT/go.mod" "$ROOT/go.sum" "$scratch/"
cp -R "$ROOT/docs" "$ROOT/modules" "$ROOT/go.mod" "$ROOT/go.sum" "$ROOT/README.md" "$scratch/"
# The command checker reads bonfire.Command literals under examples/
# (the tracked files only: examples/hello/bin holds built binaries).
git -C "$ROOT" ls-files -z examples | tar -C "$ROOT" --null -T - -cf - | tar -C "$scratch" -xf -
cp -R "$ROOT/docs" "$pristine/"
(cd "$scratch" && GOWORK=off "$summer" docs:build --check --root "$scratch" >/dev/null) ||
@@ -225,6 +293,12 @@ run_self_test() {
expect_refusal 'unknown callout' 'callout: unknown type DANGER (use NOTE, TIP or WARNING)' "$summer" "$scratch"
restore "$pristine" "$scratch" docs/index.md
# The planted-violation corpus: one fixture per checker rule, each of
# which must fail for its own rule, and the clean fixture. Skips, zero
# matches and "no tests to run" refuse.
go_json_named ./internal/docsite TestPlantedViolations TestCleanFixture ||
refuse "self-test: the planted-violation corpus did not pass"
echo "phase11.1 self-test passed"
}