test(10-05): fail-closed Phase 10 gate script
- scripts/check-phase10.sh with --self-test, --go, --security, --postgres, --spa, --openapi, --dist, --hygiene, --evidence and --all - phase10_detect refuses failed, skipped, zero-test, non-JSON and build-failed go test runs and named tests that did not pass - the two known fonoteka parity failures are the only allow-listed ones and refuse the gate once they pass again - hygiene enforces the framework/app boundary, SC-4 alias-only API types, typed-client-only HTTP, no raw HTML, same-origin dist, named lucide imports, no retired admin prefix routes and a test import for every SPA module
This commit is contained in:
Jakub Zych
403
scripts/check-phase10.sh
Executable file
403
scripts/check-phase10.sh
Executable file
@@ -0,0 +1,403 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase 10 fail-closed gate (Admin Vue SPA). Every stage exits non-zero on a
|
||||
# failing command, a go test run that fails, skips or matches zero tests, a
|
||||
# named test that did not run, OpenAPI or dist drift, a hygiene violation or
|
||||
# an evidence gap. --self-test proves each detector fails closed.
|
||||
#
|
||||
# Known pre-existing failures: the fonoteka.go parity package fails
|
||||
# TestMigrateSeedsCanonicalGenres and TestSchemaMatchesPHPSnapshot since
|
||||
# Phase 9 (.planning/phases/10-admin-vue-spa/deferred-items.md). The --go
|
||||
# stage accepts exactly those two failures in exactly that package, prints
|
||||
# each one, and refuses as soon as either passes again so the list cannot go
|
||||
# stale. Nothing else is allow-listed.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
APP="$(cd "$ROOT/../fonoteka.go" && pwd)"
|
||||
PHASE_DIR="$ROOT/.planning/phases/10-admin-vue-spa"
|
||||
REVIEW="$PHASE_DIR/10-SECURITY-REVIEW.md"
|
||||
VALIDATION="$PHASE_DIR/10-VALIDATION.md"
|
||||
APP_PLUGINS=(./plugins/golem15/fonoteka/... ./plugins/golem15/user/...)
|
||||
KNOWN_APP_FAILURES="git.golem15.com/golem15/fonoteka/parity:TestMigrateSeedsCanonicalGenres git.golem15.com/golem15/fonoteka/parity:TestSchemaMatchesPHPSnapshot"
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
check-phase10.sh --self-test
|
||||
check-phase10.sh --go
|
||||
check-phase10.sh --security
|
||||
check-phase10.sh --postgres
|
||||
check-phase10.sh --spa
|
||||
check-phase10.sh --openapi
|
||||
check-phase10.sh --dist
|
||||
check-phase10.sh --hygiene
|
||||
check-phase10.sh --evidence
|
||||
check-phase10.sh --all
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
# phase10_detect reads go test -json. Exit 1 fail, 2 skip, 3 zero tests,
|
||||
# 4 non-JSON, 5 a required test did not pass, 6 an allow-listed failure now
|
||||
# passes. PHASE10_REQUIRE lists test names that must pass; PHASE10_ALLOW lists
|
||||
# "package:Test" failures that are accepted (and must still fail).
|
||||
phase10_detect() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json, os, sys
|
||||
path = sys.argv[1]
|
||||
allow = set(os.environ.get("PHASE10_ALLOW", "").split())
|
||||
require = set(os.environ.get("PHASE10_REQUIRE", "").split())
|
||||
passed = set()
|
||||
failed_tests = {}
|
||||
failed_pkgs = []
|
||||
build_failed = False
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
for raw in fh:
|
||||
line = raw.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
print("refuse: non-json test output", file=sys.stderr)
|
||||
sys.exit(4)
|
||||
action = ev.get("Action")
|
||||
test = ev.get("Test") or ""
|
||||
pkg = ev.get("Package") or ""
|
||||
if action == "build-fail":
|
||||
build_failed = True
|
||||
if action == "skip" and test:
|
||||
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if action == "fail":
|
||||
if ev.get("FailedBuild"):
|
||||
build_failed = True
|
||||
if test:
|
||||
failed_tests.setdefault(pkg, []).append(test)
|
||||
else:
|
||||
failed_pkgs.append(pkg)
|
||||
if action == "pass" and test:
|
||||
passed.add(test)
|
||||
top = test.split("/", 1)[0]
|
||||
if f"{pkg}:{top}" in allow and "/" not in test:
|
||||
print(f"refuse: allow-listed failure {pkg} {test} now passes; remove it from the gate", file=sys.stderr)
|
||||
sys.exit(6)
|
||||
if build_failed:
|
||||
print("refuse: build failed", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
accepted = []
|
||||
for pkg, tests in failed_tests.items():
|
||||
for test in tests:
|
||||
top = test.split("/", 1)[0]
|
||||
if f"{pkg}:{top}" in allow:
|
||||
accepted.append(f"{pkg} {test}")
|
||||
continue
|
||||
print(f"refuse: failed {pkg} {test}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for pkg in failed_pkgs:
|
||||
if not failed_tests.get(pkg):
|
||||
print(f"refuse: failed {pkg or 'unknown package'}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for item in sorted(set(accepted)):
|
||||
print(f"known pre-existing failure (deferred-items.md): {item}", file=sys.stderr)
|
||||
missing = sorted(name for name in require if name not in passed)
|
||||
if missing:
|
||||
print("refuse: required tests did not pass: " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(5)
|
||||
if not passed:
|
||||
print("refuse: zero tests", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
PY
|
||||
}
|
||||
|
||||
# phase10_go DIR PKGS... [-run REGEX] runs go test -json through the detector.
|
||||
# The go test exit status is trusted only when no failure was allow-listed.
|
||||
phase10_go() {
|
||||
local dir="$1"
|
||||
shift
|
||||
local log err
|
||||
log="$(mktemp)"
|
||||
err="$(mktemp)"
|
||||
set +e
|
||||
(cd "$dir" && go test -json -count=1 "$@") >"$log" 2>"$err"
|
||||
local rc=$?
|
||||
set -e
|
||||
local dc=0
|
||||
phase10_detect "$log" || dc=$?
|
||||
if [[ "$dc" -ne 0 || ("$rc" -ne 0 && -z "${PHASE10_ALLOW:-}") ]]; then
|
||||
cat "$err" >&2 || true
|
||||
tail -n 40 "$log" >&2 || true
|
||||
rm -f "$log" "$err"
|
||||
echo "refuse: go test $* in $dir (test=$rc detect=$dc)" >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$log" "$err"
|
||||
}
|
||||
|
||||
# phase10_tests DIR PKG TEST... requires every named test to run and pass.
|
||||
phase10_tests() {
|
||||
local dir="$1" pkg="$2"
|
||||
shift 2
|
||||
local names="$*"
|
||||
local regex="^($(tr ' ' '|' <<<"$names"))\$"
|
||||
PHASE10_REQUIRE="$names" phase10_go "$dir" "$pkg" -run "$regex"
|
||||
}
|
||||
|
||||
expect_detect() {
|
||||
local name="$1" want="$2" payload="$3"
|
||||
local log dc=0
|
||||
log="$(mktemp)"
|
||||
printf '%s\n' "$payload" >"$log"
|
||||
phase10_detect "$log" 2>/dev/null || dc=$?
|
||||
rm -f "$log"
|
||||
if [[ "$dc" -ne "$want" ]]; then
|
||||
echo "refuse: self-test $name: detector exit $dc, want $want" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# The directories the hygiene stage scans, copied for the self-test.
|
||||
HYGIENE_DIRS=(admin/src admin/tests admin/openapi admin/package.json boardwalk cabana phrasebook)
|
||||
|
||||
run_self_test() {
|
||||
bash -n "${BASH_SOURCE[0]}"
|
||||
expect_detect pass 0 '{"Action":"pass","Package":"p","Test":"TestPhase10Coverage"}'
|
||||
expect_detect fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p","Test":"TestPhase10CSRF"}'
|
||||
expect_detect skip 2 '{"Action":"skip","Package":"p","Test":"TestPhase10AssembledAcceptance"}'
|
||||
expect_detect zero 3 '{"Action":"pass","Package":"git.golem15.com/golem15/summercms/cabana"}'
|
||||
expect_detect nonjson 4 '{"Action":"pass",'
|
||||
expect_detect build 1 '{"Action":"build-fail","ImportPath":"p"}
|
||||
{"Action":"pass","Package":"q","Test":"TestA"}'
|
||||
expect_detect package 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p"}'
|
||||
PHASE10_REQUIRE="TestPhase10Coverage TestPhase10CSRF" expect_detect required 5 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestPhase10Coverage"}'
|
||||
PHASE10_ALLOW="p:TestKnown" expect_detect allowed 0 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p","Test":"TestKnown"}
|
||||
{"Action":"fail","Package":"p"}'
|
||||
PHASE10_ALLOW="p:TestKnown" expect_detect allowed-other 1 '{"Action":"fail","Package":"p","Test":"TestKnown"}
|
||||
{"Action":"fail","Package":"p","Test":"TestOther"}'
|
||||
PHASE10_ALLOW="p:TestKnown" expect_detect allowed-wrong-package 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"q","Test":"TestKnown"}'
|
||||
PHASE10_ALLOW="p:TestKnown" expect_detect allowed-now-passes 6 '{"Action":"pass","Package":"p","Test":"TestKnown"}'
|
||||
for flag in --self-test --go --security --postgres --spa --openapi --dist --hygiene --evidence --all; do
|
||||
grep -q -- "^$flag)" "${BASH_SOURCE[0]}" || {
|
||||
echo "refuse: missing mode $flag" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
# The hygiene stage passes on a scratch copy of the tree and fails once a
|
||||
# raw-HTML directive, a direct fetch or an app name is planted in it.
|
||||
local scratch
|
||||
scratch="$(mktemp -d)"
|
||||
trap 'rm -rf "$scratch"' RETURN
|
||||
local dir
|
||||
for dir in "${HYGIENE_DIRS[@]}"; do
|
||||
mkdir -p "$scratch/$(dirname "$dir")"
|
||||
cp -R "$ROOT/$dir" "$scratch/$dir"
|
||||
done
|
||||
(hygiene_checks "$scratch" "$APP") >/dev/null 2>&1 || {
|
||||
echo "refuse: self-test hygiene rejected the clean scratch copy" >&2
|
||||
exit 1
|
||||
}
|
||||
local plant
|
||||
for plant in vhtml fetch appname; do
|
||||
rm -rf "$scratch/admin/src/__plant"
|
||||
mkdir -p "$scratch/admin/src/__plant"
|
||||
case "$plant" in
|
||||
vhtml) printf '<template><div v-html="raw" /></template>\n' >"$scratch/admin/src/__plant/Plant.vue" ;;
|
||||
fetch) printf 'export const load = () => fetch("/x")\n' >"$scratch/admin/src/__plant/plant.ts" ;;
|
||||
appname) printf '// Fonoteka\nexport {}\n' >"$scratch/admin/src/__plant/plant.ts" ;;
|
||||
esac
|
||||
if (hygiene_checks "$scratch" "$APP") >/dev/null 2>&1; then
|
||||
echo "refuse: self-test hygiene accepted a planted $plant" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "phase10 self-test passed"
|
||||
}
|
||||
|
||||
run_go() {
|
||||
(cd "$ROOT" && go vet ./...)
|
||||
phase10_go "$ROOT" ./...
|
||||
(cd "$APP" && go vet ./... "${APP_PLUGINS[@]}")
|
||||
PHASE10_ALLOW="$KNOWN_APP_FAILURES" phase10_go "$APP" ./... "${APP_PLUGINS[@]}"
|
||||
echo "phase10 go passed"
|
||||
}
|
||||
|
||||
run_security() {
|
||||
phase10_tests "$ROOT" ./cabana TestPhase10CookieAuth TestPhase10CSRF TestPhase10Prefix TestPhase10RelationForgedID TestPhase10Bundle TestPhase10Coverage
|
||||
phase10_tests "$ROOT" ./bouncer TestPhase10CookieGuard
|
||||
phase10_tests "$ROOT" ./surf TestPhase10AdminPrefixCollision
|
||||
phase10_go "$ROOT" ./boardwalk
|
||||
phase10_tests "$APP" ./plugins/golem15/fonoteka TestPhase10CollectionOwnerReadOnly TestPhase10AdminAuth
|
||||
"$ROOT/scripts/check-phase9.sh" --security
|
||||
echo "phase10 security passed"
|
||||
}
|
||||
|
||||
run_postgres() {
|
||||
phase10_tests "$APP" ./plugins/golem15/fonoteka TestPhase10TracerSPA TestPhase10AdminAuth TestPhase10AlbumRelations \
|
||||
TestPhase10Controllers TestPhase10AssembledAcceptance
|
||||
phase10_tests "$ROOT" ./cabana TestPhase10RelationOptions TestPhase10RelationSave TestPhase10OpenAPIConformance TestPhase10Coverage
|
||||
echo "phase10 postgres passed"
|
||||
}
|
||||
|
||||
run_spa() {
|
||||
npm --prefix "$ROOT/admin" ci --no-audit --no-fund
|
||||
npm --prefix "$ROOT/admin" run typecheck
|
||||
local log
|
||||
log="$(mktemp)"
|
||||
set +e
|
||||
npm --prefix "$ROOT/admin" test >"$log" 2>&1
|
||||
local rc=$?
|
||||
set -e
|
||||
if [[ "$rc" -ne 0 ]] || grep -qE 'No test files found|Unhandled (Errors|Rejection)|FAIL ' "$log"; then
|
||||
tail -n 60 "$log" >&2
|
||||
rm -f "$log"
|
||||
echo "refuse: admin Vitest run failed (exit $rc)" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -E 'Test Files|Tests ' "$log" || true
|
||||
rm -f "$log"
|
||||
echo "phase10 spa passed"
|
||||
}
|
||||
|
||||
run_openapi() {
|
||||
"$ROOT/scripts/check-admin-openapi.sh" --check
|
||||
phase10_tests "$ROOT" ./cabana TestPhase10OpenAPIConformance TestPhase09ContractInventory
|
||||
(cd "$APP" && bash scripts/check-openapi.sh)
|
||||
if ! git -C "$APP" diff --quiet -- docs/openapi.json; then
|
||||
git -C "$APP" diff --stat -- docs/openapi.json >&2
|
||||
echo "refuse: fonoteka docs/openapi.json drifted from the committed document" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "phase10 openapi passed"
|
||||
}
|
||||
|
||||
run_dist() {
|
||||
"$ROOT/scripts/check-admin-dist.sh"
|
||||
echo "phase10 dist passed"
|
||||
}
|
||||
|
||||
# hygiene_checks TREE APPTREE: the SC-4 and framework/app boundary rules.
|
||||
hygiene_checks() {
|
||||
local tree="$1" app="$2" bad=0
|
||||
fail() {
|
||||
echo "refuse: hygiene: $*" >&2
|
||||
bad=1
|
||||
}
|
||||
local hits
|
||||
# Application or Polish catalogue names in framework code, tests and build.
|
||||
hits="$(cd "$tree" && grep -rniIE 'pl[yý]tarium|fonoteka|albumy|kolekcj|winyl|p[lł]yt[aęy]' \
|
||||
admin/src admin/tests admin/openapi boardwalk cabana phrasebook 2>/dev/null || true)"
|
||||
[[ -z "$hits" ]] || fail "application names in the framework: $hits"
|
||||
# Plugin and server strings are rendered as text only.
|
||||
hits="$(cd "$tree" && grep -rnE 'v-html|innerHTML|outerHTML|insertAdjacentHTML' admin/src 2>/dev/null || true)"
|
||||
[[ -z "$hits" ]] || fail "raw-HTML directive in admin/src: $hits"
|
||||
# All HTTP goes through the typed openapi-fetch client.
|
||||
hits="$(cd "$tree" && grep -rnE '(^|[^A-Za-z0-9_.])fetch\(|XMLHttpRequest\(|axios' admin/src --include='*.ts' --include='*.vue' 2>/dev/null |
|
||||
grep -v '^admin/src/api/client.ts:' || true)"
|
||||
[[ -z "$hits" ]] || fail "direct fetch outside admin/src/api/client.ts: $hits"
|
||||
# admin/src/api holds the generated schema, the client and aliases only.
|
||||
hits="$(cd "$tree" && find admin/src/api -type f ! -name schema.d.ts ! -name client.ts ! -name types.ts 2>/dev/null || true)"
|
||||
[[ -z "$hits" ]] || fail "unexpected files in admin/src/api: $hits"
|
||||
hits="$(cd "$tree" && grep -nE '\binterface\b|=\s*\{|:\s*\{' admin/src/api/types.ts 2>/dev/null || true)"
|
||||
[[ -z "$hits" ]] || fail "admin/src/api/types.ts declares a shape instead of aliasing the generated schema: $hits"
|
||||
hits="$(cd "$tree" && grep -nE '^export type [A-Za-z]+ = ' admin/src/api/types.ts | grep -vE "= (Schemas\['cabana\.[A-Za-z_-]+'\]|NonNullable<[A-Za-z]+Path\['get'\]\['parameters'\]\['query'\]>|[A-Za-z]+Path\['get'\]\['parameters'\]\['path'\])\$" || true)"
|
||||
[[ -z "$hits" ]] || fail "admin/src/api/types.ts alias not onto the generated schema: $hits"
|
||||
# The embedded build loads nothing from another origin.
|
||||
hits="$(cd "$tree" && grep -noE '(src|href)="(https?:)?//[^"]*"|url\((["'"'"']?)(https?:)?//' boardwalk/dist/index.html boardwalk/dist/assets/*.css 2>/dev/null || true)"
|
||||
[[ -z "$hits" ]] || fail "boardwalk/dist references another origin: $hits"
|
||||
hits="$(cd "$tree" && grep -ohE 'https?://[A-Za-z0-9.-]+' boardwalk/dist/assets/*.js 2>/dev/null | sort -u |
|
||||
grep -vxE 'http://www\.w3\.org|https://vuejs\.org|http://local' || true)"
|
||||
[[ -z "$hits" ]] || fail "boardwalk/dist JS names another origin: $hits"
|
||||
# Icons: named imports only, and never the deprecated lucide package.
|
||||
hits="$(cd "$tree" && grep -rnE "import \* as [A-Za-z_]+ from '@lucide/vue'|lucide-vue-next" admin/src admin/package.json 2>/dev/null || true)"
|
||||
[[ -z "$hits" ]] || fail "lucide namespace import or deprecated package: $hits"
|
||||
# The retired Phase 9 admin prefix appears only as a MintAudience issuer.
|
||||
hits="$( (cd "$tree" && grep -rnE '/_admin/' --include='*.go' . 2>/dev/null; cd "$app" && grep -rnE '/_admin/' --include='*.go' . 2>/dev/null) |
|
||||
grep -v 'MintAudience(' || true)"
|
||||
[[ -z "$hits" ]] || fail "route literal for the retired /_admin prefix: $hits"
|
||||
# Every SPA module is imported by at least one test.
|
||||
local file spec
|
||||
while IFS= read -r file; do
|
||||
case "$file" in
|
||||
main.ts | api/schema.d.ts) continue ;;
|
||||
esac
|
||||
spec="src/${file%.ts}"
|
||||
grep -rqF --include='*.ts' "$spec'" "$tree/admin/tests" || fail "admin/src/$file is imported by no test"
|
||||
done < <(cd "$tree/admin/src" && find . -type f \( -name '*.ts' -o -name '*.vue' \) | sed 's#^\./##' | sort)
|
||||
return "$bad"
|
||||
}
|
||||
|
||||
run_hygiene() {
|
||||
hygiene_checks "$ROOT" "$APP"
|
||||
echo "phase10 hygiene passed"
|
||||
}
|
||||
|
||||
run_evidence() {
|
||||
[[ -f "$REVIEW" && -f "$VALIDATION" ]] || {
|
||||
echo "refuse: security review or validation file is missing" >&2
|
||||
exit 1
|
||||
}
|
||||
python3 - "$REVIEW" "$VALIDATION" <<'PY'
|
||||
import pathlib, re, sys
|
||||
review = pathlib.Path(sys.argv[1]).read_text()
|
||||
validation = pathlib.Path(sys.argv[2]).read_text()
|
||||
required = [f"T-10-{i:02d}" for i in range(1, 26)] + ["T-10-SC"]
|
||||
missing = [item for item in required if not re.search(re.escape(item) + r"\b", review)]
|
||||
if missing:
|
||||
print("refuse: review missing " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for item in required:
|
||||
row = next((line for line in review.splitlines() if line.startswith("| " + item + " ")), None)
|
||||
if row is None:
|
||||
print(f"refuse: review has no table row for {item}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if "high" in row.lower() and "mitigate" in row.lower() and not re.search(r"Test[A-Z][A-Za-z0-9]+|check-phase\d+\.sh|check-admin-|tests/", row):
|
||||
print(f"refuse: high threat {item} names no failing-when-broken test or gate stage", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not re.search(r"^nyquist_compliant: true$", validation, re.M):
|
||||
print("refuse: validation is not nyquist_compliant", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for line in validation.splitlines():
|
||||
if line.startswith("|") and "pending" in line.lower():
|
||||
print("refuse: validation row still pending: " + line, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if "ADMIN-06" not in validation:
|
||||
print("refuse: validation does not name ADMIN-06", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("phase10 evidence files passed")
|
||||
PY
|
||||
run_security
|
||||
run_postgres
|
||||
run_openapi
|
||||
echo "phase10 evidence passed"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--self-test) run_self_test ;;
|
||||
--go) run_go ;;
|
||||
--security) run_security ;;
|
||||
--postgres) run_postgres ;;
|
||||
--spa) run_spa ;;
|
||||
--openapi) run_openapi ;;
|
||||
--dist) run_dist ;;
|
||||
--hygiene) run_hygiene ;;
|
||||
--evidence) run_evidence ;;
|
||||
--all)
|
||||
run_self_test
|
||||
run_go
|
||||
run_security
|
||||
run_postgres
|
||||
run_spa
|
||||
run_openapi
|
||||
run_dist
|
||||
run_hygiene
|
||||
run_evidence
|
||||
echo "phase10 all passed"
|
||||
;;
|
||||
*) usage ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user