Files
summercms/scripts/check-phase10.sh
Jakub Zych 7ac6c05ab3 chore(12-04): expect only the updated broadcast golden to skip
The album store handler asserts the created golden now; updated stays pending
until the update handler lands.
2026-10-02 14:01:32 +02:00

467 lines
19 KiB
Bash
Executable File

#!/usr/bin/env bash
# Phase 10 fail-closed gate (Admin Vue SPA). Every stage exits non-zero on a
# failing command, a go test run that fails, skips or matches zero tests, a
# named test that did not run, OpenAPI or dist drift, a hygiene violation or
# an evidence gap. --self-test proves each detector fails closed.
#
# Allow-list: none. The fonoteka.go parity failures TestMigrateSeedsCanonicalGenres
# and TestSchemaMatchesPHPSnapshot, accepted until then, pass since fonoteka.go
# 21c0f12 (fix(09): update parity expectations for the backend admin schema),
# and the detector refuses an allow-listed failure that passes. KNOWN_APP_FAILURES
# stays as the one place to name a future known failure, with a reason.
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
APP="$(cd "$ROOT/../fonoteka.go" && pwd)"
PHASE_DIR="$ROOT/.planning/phases/10-admin-vue-spa"
REVIEW="$PHASE_DIR/10-SECURITY-REVIEW.md"
VALIDATION="$PHASE_DIR/10-VALIDATION.md"
APP_PLUGINS=(./plugins/golem15/fonoteka/... ./plugins/golem15/user/...)
KNOWN_APP_FAILURES=""
# Phase 11 (11-06) records the created/updated broadcast goldens from PHP and
# reports them as skipped until Phase 12 asserts them; check-phase11.sh
# requires exactly these skips with this text. Phase 12 (12-04) asserts the
# created golden through the album store handler; updated stays pending
# until the update handler lands.
APP_PENDING_SKIPS="TestBroadcastGoldens/updated"
APP_PENDING_TEXT="pending: Phase 12"
usage() {
cat >&2 <<'EOF'
usage:
check-phase10.sh --self-test
check-phase10.sh --go
check-phase10.sh --security
check-phase10.sh --postgres
check-phase10.sh --spa
check-phase10.sh --openapi
check-phase10.sh --dist
check-phase10.sh --hygiene
check-phase10.sh --evidence
check-phase10.sh --all
EOF
exit 2
}
# phase10_detect reads go test -json. Exit 1 fail, 2 skip, 3 zero tests,
# 4 non-JSON, 5 a required test did not pass, 6 an allow-listed failure now
# passes. PHASE10_REQUIRE lists test names that must pass; PHASE10_ALLOW lists
# "package:Test" failures that are accepted (and must still fail).
phase10_detect() {
python3 - "$1" <<'PY'
import json, os, sys
path = sys.argv[1]
allow = set(os.environ.get("PHASE10_ALLOW", "").split())
require = set(os.environ.get("PHASE10_REQUIRE", "").split())
expect_skip = set(os.environ.get("PHASE10_EXPECT_SKIP", "").split())
skip_text = os.environ.get("PHASE10_SKIP_TEXT", "")
skip_output = {}
skipped = set()
passed = set()
failed_tests = {}
failed_pkgs = []
build_failed = False
with open(path, encoding="utf-8", errors="replace") as fh:
for raw in fh:
line = raw.strip()
if not line.startswith("{"):
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
print("refuse: non-json test output", file=sys.stderr)
sys.exit(4)
action = ev.get("Action")
test = ev.get("Test") or ""
pkg = ev.get("Package") or ""
if action == "build-fail":
build_failed = True
if action == "output" and test in expect_skip:
skip_output.setdefault(test, []).append(ev.get("Output") or "")
if action == "skip" and test:
# A later phase's documented pending test (for example the Phase
# 12 broadcast goldens) may skip, but only with its pending text.
if test in expect_skip and skip_text and any(skip_text in o for o in skip_output.get(test, [])):
skipped.add(test)
continue
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
sys.exit(2)
if action == "fail":
if ev.get("FailedBuild"):
build_failed = True
if test:
failed_tests.setdefault(pkg, []).append(test)
else:
failed_pkgs.append(pkg)
if action == "pass" and test:
passed.add(test)
top = test.split("/", 1)[0]
if f"{pkg}:{top}" in allow and "/" not in test:
print(f"refuse: allow-listed failure {pkg} {test} now passes; remove it from the gate", file=sys.stderr)
sys.exit(6)
if build_failed:
print("refuse: build failed", file=sys.stderr)
sys.exit(1)
accepted = []
for pkg, tests in failed_tests.items():
for test in tests:
top = test.split("/", 1)[0]
if f"{pkg}:{top}" in allow:
accepted.append(f"{pkg} {test}")
continue
print(f"refuse: failed {pkg} {test}", file=sys.stderr)
sys.exit(1)
for pkg in failed_pkgs:
if not failed_tests.get(pkg):
print(f"refuse: failed {pkg or 'unknown package'}", file=sys.stderr)
sys.exit(1)
for item in sorted(set(accepted)):
print(f"known pre-existing failure (deferred-items.md): {item}", file=sys.stderr)
missing = sorted(name for name in require if name not in passed)
if missing:
print("refuse: required tests did not pass: " + ", ".join(missing), file=sys.stderr)
sys.exit(5)
unexpected_passes = sorted(expect_skip & passed)
missing_skips = sorted(expect_skip - skipped)
if unexpected_passes:
print("refuse: expected pending tests passed: " + ", ".join(unexpected_passes), file=sys.stderr)
sys.exit(7)
if missing_skips:
print("refuse: expected pending skips did not occur: " + ", ".join(missing_skips), file=sys.stderr)
sys.exit(7)
if not passed:
print("refuse: zero tests", file=sys.stderr)
sys.exit(3)
PY
}
# phase10_go DIR PKGS... [-run REGEX] runs go test -json through the detector.
# The go test exit status is trusted only when no failure was allow-listed.
phase10_go() {
local dir="$1"
shift
local log err
log="$(mktemp)"
err="$(mktemp)"
set +e
(cd "$dir" && go test -json -count=1 "$@") >"$log" 2>"$err"
local rc=$?
set -e
local dc=0
phase10_detect "$log" || dc=$?
if [[ "$dc" -ne 0 || ("$rc" -ne 0 && -z "${PHASE10_ALLOW:-}") ]]; then
cat "$err" >&2 || true
tail -n 40 "$log" >&2 || true
rm -f "$log" "$err"
echo "refuse: go test $* in $dir (test=$rc detect=$dc)" >&2
exit 1
fi
rm -f "$log" "$err"
}
# phase10_tests DIR PKG TEST... requires every named test to run and pass.
phase10_tests() {
local dir="$1" pkg="$2"
shift 2
local names="$*"
local regex="^($(tr ' ' '|' <<<"$names"))\$"
PHASE10_REQUIRE="$names" phase10_go "$dir" "$pkg" -run "$regex"
}
expect_detect() {
local name="$1" want="$2" payload="$3"
local log dc=0
log="$(mktemp)"
printf '%s\n' "$payload" >"$log"
phase10_detect "$log" 2>/dev/null || dc=$?
rm -f "$log"
if [[ "$dc" -ne "$want" ]]; then
echo "refuse: self-test $name: detector exit $dc, want $want" >&2
exit 1
fi
}
# The directories the hygiene stage scans, copied for the self-test.
HYGIENE_DIRS=(admin/src admin/tests admin/openapi admin/package.json modules/boardwalk modules/cabana modules/phrasebook)
run_self_test() {
bash -n "${BASH_SOURCE[0]}"
expect_detect pass 0 '{"Action":"pass","Package":"p","Test":"TestPhase10Coverage"}'
expect_detect fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
{"Action":"fail","Package":"p","Test":"TestPhase10CSRF"}'
expect_detect skip 2 '{"Action":"skip","Package":"p","Test":"TestPhase10AssembledAcceptance"}'
PHASE10_EXPECT_SKIP="TestG/created" PHASE10_SKIP_TEXT="pending: Phase 12" expect_detect pending-skip 0 \
'{"Action":"output","Package":"p","Test":"TestG/created","Output":"pending: Phase 12 asserts it\n"}
{"Action":"skip","Package":"p","Test":"TestG/created"}
{"Action":"pass","Package":"p","Test":"TestG"}'
PHASE10_EXPECT_SKIP="TestG/created" PHASE10_SKIP_TEXT="pending: Phase 12" expect_detect pending-skip-without-text 2 \
'{"Action":"skip","Package":"p","Test":"TestG/created"}
{"Action":"pass","Package":"p","Test":"TestG"}'
PHASE10_EXPECT_SKIP="TestG/created" PHASE10_SKIP_TEXT="pending: Phase 12" expect_detect pending-skip-missing 7 \
'{"Action":"pass","Package":"p","Test":"TestOther"}'
PHASE10_EXPECT_SKIP="TestG/created" PHASE10_SKIP_TEXT="pending: Phase 12" expect_detect pending-skip-passes 7 \
'{"Action":"pass","Package":"p","Test":"TestG/created"}'
expect_detect zero 3 '{"Action":"pass","Package":"git.golem15.com/golem15/summercms/modules/cabana"}'
expect_detect nonjson 4 '{"Action":"pass",'
expect_detect build 1 '{"Action":"build-fail","ImportPath":"p"}
{"Action":"pass","Package":"q","Test":"TestA"}'
expect_detect package 1 '{"Action":"pass","Package":"p","Test":"TestA"}
{"Action":"fail","Package":"p"}'
PHASE10_REQUIRE="TestPhase10Coverage TestPhase10CSRF" expect_detect required 5 \
'{"Action":"pass","Package":"p","Test":"TestPhase10Coverage"}'
PHASE10_ALLOW="p:TestKnown" expect_detect allowed 0 '{"Action":"pass","Package":"p","Test":"TestA"}
{"Action":"fail","Package":"p","Test":"TestKnown"}
{"Action":"fail","Package":"p"}'
PHASE10_ALLOW="p:TestKnown" expect_detect allowed-other 1 '{"Action":"fail","Package":"p","Test":"TestKnown"}
{"Action":"fail","Package":"p","Test":"TestOther"}'
PHASE10_ALLOW="p:TestKnown" expect_detect allowed-wrong-package 1 '{"Action":"pass","Package":"p","Test":"TestA"}
{"Action":"fail","Package":"q","Test":"TestKnown"}'
PHASE10_ALLOW="p:TestKnown" expect_detect allowed-now-passes 6 '{"Action":"pass","Package":"p","Test":"TestKnown"}'
for flag in --self-test --go --security --postgres --spa --openapi --dist --hygiene --evidence --all; do
grep -q -- "^$flag)" "${BASH_SOURCE[0]}" || {
echo "refuse: missing mode $flag" >&2
exit 1
}
done
# The hygiene stage passes on a scratch copy of the tree and fails once a
# raw-HTML directive, a direct fetch or an app name is planted in it.
local scratch
scratch="$(mktemp -d)"
trap 'rm -rf "$scratch"' RETURN
local dir
for dir in "${HYGIENE_DIRS[@]}"; do
mkdir -p "$scratch/$(dirname "$dir")"
cp -R "$ROOT/$dir" "$scratch/$dir"
done
(hygiene_checks "$scratch" "$APP") >/dev/null 2>&1 || {
echo "refuse: self-test hygiene rejected the clean scratch copy" >&2
exit 1
}
# Each plant is imported by a scratch test, so only its own rule can fire;
# the refusal must name that rule.
local plant want out
for plant in vhtml fetch appname storage; do
rm -rf "$scratch/admin/src/__plant" "$scratch/admin/tests/__plant.test.ts"
mkdir -p "$scratch/admin/src/__plant"
case "$plant" in
vhtml)
printf '<template><div v-html="raw" /></template>\n' >"$scratch/admin/src/__plant/Plant.vue"
printf "import Plant from '../src/__plant/Plant.vue'\n" >"$scratch/admin/tests/__plant.test.ts"
want="raw-HTML directive"
;;
fetch)
printf 'export const load = () => fetch("/x")\n' >"$scratch/admin/src/__plant/plant.ts"
want="direct fetch"
;;
appname)
printf '// Fonoteka\nexport {}\n' >"$scratch/admin/src/__plant/plant.ts"
want="application names"
;;
storage)
printf 'export const keep = (v: string) => localStorage.setItem("token", v)\n' >"$scratch/admin/src/__plant/plant.ts"
want="browser storage"
;;
esac
[[ -f "$scratch/admin/tests/__plant.test.ts" ]] ||
printf "import '../src/__plant/plant'\n" >"$scratch/admin/tests/__plant.test.ts"
if out="$( (hygiene_checks "$scratch" "$APP") 2>&1)"; then
echo "refuse: self-test hygiene accepted a planted $plant" >&2
exit 1
fi
if ! grep -q "$want" <<<"$out" || grep -q "imported by no test" <<<"$out"; then
echo "refuse: self-test hygiene rejected the $plant plant for the wrong reason: $out" >&2
exit 1
fi
done
echo "phase10 self-test passed"
}
run_go() {
(cd "$ROOT" && go vet ./...)
phase10_go "$ROOT" ./...
(cd "$APP" && go vet ./... "${APP_PLUGINS[@]}")
PHASE10_ALLOW="$KNOWN_APP_FAILURES" PHASE10_EXPECT_SKIP="$APP_PENDING_SKIPS" PHASE10_SKIP_TEXT="$APP_PENDING_TEXT" \
phase10_go "$APP" ./... "${APP_PLUGINS[@]}"
echo "phase10 go passed"
}
run_security() {
phase10_tests "$ROOT" ./modules/cabana TestPhase10CookieAuth TestPhase10CSRF TestPhase10Prefix TestPhase10RelationForgedID TestPhase10Bundle TestPhase10Coverage
phase10_tests "$ROOT" ./modules/bouncer TestPhase10CookieGuard
phase10_tests "$ROOT" ./modules/surf TestPhase10AdminPrefixCollision
phase10_go "$ROOT" ./modules/boardwalk
phase10_tests "$APP" ./plugins/golem15/fonoteka TestPhase10CollectionOwnerReadOnly TestPhase10AdminAuth
"$ROOT/scripts/check-phase9.sh" --security
echo "phase10 security passed"
}
run_postgres() {
phase10_tests "$APP" ./plugins/golem15/fonoteka TestPhase10TracerSPA TestPhase10AdminAuth TestPhase10AlbumRelations \
TestPhase10Controllers TestPhase10AssembledAcceptance
phase10_tests "$ROOT" ./modules/cabana TestPhase10RelationOptions TestPhase10RelationSave TestPhase10OpenAPIConformance TestPhase10Coverage
echo "phase10 postgres passed"
}
run_spa() {
npm --prefix "$ROOT/admin" ci --no-audit --no-fund
npm --prefix "$ROOT/admin" run typecheck
local log
log="$(mktemp)"
set +e
npm --prefix "$ROOT/admin" test >"$log" 2>&1
local rc=$?
set -e
if [[ "$rc" -ne 0 ]] || grep -qE 'No test files found|Unhandled (Errors|Rejection)|FAIL ' "$log"; then
tail -n 60 "$log" >&2
rm -f "$log"
echo "refuse: admin Vitest run failed (exit $rc)" >&2
exit 1
fi
grep -E 'Test Files|Tests ' "$log" || true
rm -f "$log"
echo "phase10 spa passed"
}
run_openapi() {
"$ROOT/scripts/check-admin-openapi.sh" --check
phase10_tests "$ROOT" ./modules/cabana TestPhase10OpenAPIConformance TestPhase09ContractInventory
(cd "$APP" && bash scripts/check-openapi.sh)
if ! git -C "$APP" diff --quiet -- docs/openapi.json; then
git -C "$APP" diff --stat -- docs/openapi.json >&2
echo "refuse: fonoteka docs/openapi.json drifted from the committed document" >&2
exit 1
fi
echo "phase10 openapi passed"
}
run_dist() {
"$ROOT/scripts/check-admin-dist.sh"
echo "phase10 dist passed"
}
# hygiene_checks TREE APPTREE: the SC-4 and framework/app boundary rules.
hygiene_checks() {
local tree="$1" app="$2" bad=0
fail() {
echo "refuse: hygiene: $*" >&2
bad=1
}
local hits
# Application or Polish catalogue names in framework code, tests and build.
hits="$(cd "$tree" && grep -rniIE 'pl[yý]tarium|fonoteka|albumy|kolekcj|winyl|p[lł]yt[aęy]' \
admin/src admin/tests admin/openapi modules/boardwalk modules/cabana modules/phrasebook 2>/dev/null || true)"
[[ -z "$hits" ]] || fail "application names in the framework: $hits"
# Plugin and server strings are rendered as text only.
hits="$(cd "$tree" && grep -rnE 'v-html|innerHTML|outerHTML|insertAdjacentHTML' admin/src 2>/dev/null || true)"
[[ -z "$hits" ]] || fail "raw-HTML directive in admin/src: $hits"
# All HTTP goes through the typed openapi-fetch client.
hits="$(cd "$tree" && grep -rnE '(^|[^A-Za-z0-9_.])fetch\(|XMLHttpRequest\(|axios' admin/src --include='*.ts' --include='*.vue' 2>/dev/null |
grep -v '^admin/src/api/client.ts:' || true)"
[[ -z "$hits" ]] || fail "direct fetch outside admin/src/api/client.ts: $hits"
# admin/src/api holds the generated schema, the client and aliases only.
hits="$(cd "$tree" && find admin/src/api -type f ! -name schema.d.ts ! -name client.ts ! -name types.ts 2>/dev/null || true)"
[[ -z "$hits" ]] || fail "unexpected files in admin/src/api: $hits"
hits="$(cd "$tree" && grep -nE '\binterface\b|=\s*\{|:\s*\{' admin/src/api/types.ts 2>/dev/null || true)"
[[ -z "$hits" ]] || fail "admin/src/api/types.ts declares a shape instead of aliasing the generated schema: $hits"
hits="$(cd "$tree" && grep -nE '^export type [A-Za-z]+ = ' admin/src/api/types.ts | grep -vE "= (Schemas\['cabana\.[A-Za-z_-]+'\]|NonNullable<[A-Za-z]+Path\['get'\]\['parameters'\]\['query'\]>|[A-Za-z]+Path\['get'\]\['parameters'\]\['path'\])\$" || true)"
[[ -z "$hits" ]] || fail "admin/src/api/types.ts alias not onto the generated schema: $hits"
# Browser storage holds only the sidebar flag (T-10-22), never a token.
hits="$(cd "$tree" && grep -rnE 'localStorage|sessionStorage|indexedDB|document\.cookie' admin/src 2>/dev/null |
grep -v '^admin/src/state/useSidebar.ts:' || true)"
[[ -z "$hits" ]] || fail "browser storage outside admin/src/state/useSidebar.ts: $hits"
# The embedded build loads nothing from another origin.
hits="$(cd "$tree" && grep -noE '(src|href)="(https?:)?//[^"]*"|url\((["'"'"']?)(https?:)?//' modules/boardwalk/dist/index.html modules/boardwalk/dist/assets/*.css 2>/dev/null || true)"
[[ -z "$hits" ]] || fail "modules/boardwalk/dist references another origin: $hits"
hits="$(cd "$tree" && grep -ohE 'https?://[A-Za-z0-9.-]+' modules/boardwalk/dist/assets/*.js 2>/dev/null | sort -u |
grep -vxE 'http://www\.w3\.org|https://vuejs\.org|http://local' || true)"
[[ -z "$hits" ]] || fail "modules/boardwalk/dist JS names another origin: $hits"
# Icons: named imports only, and never the deprecated lucide package.
hits="$(cd "$tree" && grep -rnE "import \* as [A-Za-z_]+ from '@lucide/vue'|lucide-vue-next" admin/src admin/package.json 2>/dev/null || true)"
[[ -z "$hits" ]] || fail "lucide namespace import or deprecated package: $hits"
# The retired Phase 9 admin prefix appears only as a MintAudience issuer.
hits="$( (cd "$tree" && grep -rnE '/_admin/' --include='*.go' . 2>/dev/null; cd "$app" && grep -rnE '/_admin/' --include='*.go' . 2>/dev/null) |
grep -v 'MintAudience(' || true)"
[[ -z "$hits" ]] || fail "route literal for the retired /_admin prefix: $hits"
# Every SPA module is imported by at least one test.
local file spec
while IFS= read -r file; do
case "$file" in
main.ts | api/schema.d.ts) continue ;;
esac
spec="src/${file%.ts}"
grep -rqF --include='*.ts' "$spec'" "$tree/admin/tests" || fail "admin/src/$file is imported by no test"
done < <(cd "$tree/admin/src" && find . -type f \( -name '*.ts' -o -name '*.vue' \) | sed 's#^\./##' | sort)
return "$bad"
}
run_hygiene() {
hygiene_checks "$ROOT" "$APP"
echo "phase10 hygiene passed"
}
run_evidence() {
[[ -f "$REVIEW" && -f "$VALIDATION" ]] || {
echo "refuse: security review or validation file is missing" >&2
exit 1
}
python3 - "$REVIEW" "$VALIDATION" <<'PY'
import pathlib, re, sys
review = pathlib.Path(sys.argv[1]).read_text()
validation = pathlib.Path(sys.argv[2]).read_text()
required = [f"T-10-{i:02d}" for i in range(1, 26)] + ["T-10-SC"]
missing = [item for item in required if not re.search(re.escape(item) + r"\b", review)]
if missing:
print("refuse: review missing " + ", ".join(missing), file=sys.stderr)
sys.exit(1)
for item in required:
row = next((line for line in review.splitlines() if line.startswith("| " + item + " ")), None)
if row is None:
print(f"refuse: review has no table row for {item}", file=sys.stderr)
sys.exit(1)
if "high" in row.lower() and "mitigate" in row.lower() and not re.search(r"Test[A-Z][A-Za-z0-9]+|check-phase\d+\.sh|check-admin-|tests/", row):
print(f"refuse: high threat {item} names no failing-when-broken test or gate stage", file=sys.stderr)
sys.exit(1)
if not re.search(r"^nyquist_compliant: true$", validation, re.M):
print("refuse: validation is not nyquist_compliant", file=sys.stderr)
sys.exit(1)
for line in validation.splitlines():
if line.startswith("|") and "pending" in line.lower():
print("refuse: validation row still pending: " + line, file=sys.stderr)
sys.exit(1)
if "ADMIN-06" not in validation:
print("refuse: validation does not name ADMIN-06", file=sys.stderr)
sys.exit(1)
print("phase10 evidence files passed")
PY
run_security
run_postgres
run_openapi
echo "phase10 evidence passed"
}
case "${1:-}" in
--self-test) run_self_test ;;
--go) run_go ;;
--security) run_security ;;
--postgres) run_postgres ;;
--spa) run_spa ;;
--openapi) run_openapi ;;
--dist) run_dist ;;
--hygiene) run_hygiene ;;
--evidence) run_evidence ;;
--all)
run_self_test
run_go
run_security
run_postgres
run_spa
run_openapi
run_dist
run_hygiene
run_evidence
echo "phase10 all passed"
;;
*) usage ;;
esac