test(16-05): Phase 16 gate
scripts/check-phase16.sh runs the layer unit tests, the layer playground SSR smoke, the site app install/build/tests/smokes, the host boot tests, the grzyby command tests with -race and check-deploy. Each stage fails on not ok, fail/skip/todo/cancelled counts, missing named tests, go test failures, skips, no tests to run, data races or a smoke without ok; it ends with "Phase 16 gate passed".
This commit is contained in:
402
scripts/check-phase16.sh
Executable file
402
scripts/check-phase16.sh
Executable file
@@ -0,0 +1,402 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase 16 fail-closed gate (vue-journal-layer, the grzybyfunkcjonalne.pl site
|
||||
# app and its SummerCMS host). Every stage exits non-zero on a failing
|
||||
# command; a node --test run that reports "not ok", a failed, skipped, todo or
|
||||
# cancelled test, no passing test or a missing named test; a go test run that
|
||||
# fails, skips, matches zero tests, prints "no tests to run", misses a named
|
||||
# required test or reports a data race; or an SSR smoke that does not end ok.
|
||||
# --self-test proves both detectors fail closed on planted inputs. --all runs
|
||||
# every stage and must end with "Phase 16 gate passed".
|
||||
#
|
||||
# The grzyby command tests need Postgres: GRZYBY_TEST_ADMIN_DSN (a
|
||||
# maintenance-database DSN whose role has CREATEDB) or Docker for
|
||||
# testcontainers. Without either they fail; -short is not final evidence.
|
||||
#
|
||||
# Environment overrides: PHASE16_LAYER (default ../vue-journal-layer),
|
||||
# PHASE16_HOST (default ../sm-grzybyfunkcjonalne-app), PHASE16_APP (default
|
||||
# $PHASE16_HOST/vue-grzybyfunkcjonalne-app).
|
||||
set -euo pipefail
|
||||
|
||||
unset FORCE_COLOR
|
||||
|
||||
ROOT="${PHASE16_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
LAYER="${PHASE16_LAYER:-$ROOT/../vue-journal-layer}"
|
||||
HOST="${PHASE16_HOST:-$ROOT/../sm-grzybyfunkcjonalne-app}"
|
||||
APP="${PHASE16_APP:-$HOST/vue-grzybyfunkcjonalne-app}"
|
||||
PLUGIN="$HOST/plugins/grzyby"
|
||||
|
||||
# One named test per layer test file.
|
||||
LAYER_REQUIRE=(
|
||||
"an override replaces one key and keeps the others"
|
||||
"a post path without :slug throws naming journal.routes.post"
|
||||
"listCategories always sends include_empty=1"
|
||||
"getPost returns null for status 404 and statusCode 404"
|
||||
"anything else is a 404 (null)"
|
||||
"Go case vbscript is unsafe"
|
||||
"Go case data in upper case is unsafe"
|
||||
"a custom root category route gives /cordyceps"
|
||||
"without an excerpt the description is the first 160 characters of the body text"
|
||||
"pl and en have identical leaf key sets"
|
||||
)
|
||||
APP_REQUIRE=(
|
||||
"D-15: every PHP URL form answers the expected status"
|
||||
"static pages win over the root category route (RESEARCH A2)"
|
||||
"D-16: Polish only, without a prefix"
|
||||
"T-16-01: an unsafe post body renders the blocked notice only"
|
||||
"Pitfall 5: SSR fetches forward the visitor X-Forwarded-For"
|
||||
"D-23: NUXT_PUBLIC_GA_ID loads gtag for its id"
|
||||
"D-23: a malformed NUXT_PUBLIC_GA_ID loads no script (T-16-07)"
|
||||
"malformed ids give no scripts"
|
||||
"the FAQ has the 16 live entries with question and answer"
|
||||
"every t('…') key used in app/**/*.vue exists in pl.json or the layer's journal strings"
|
||||
)
|
||||
HOST_REQUIRE=(
|
||||
TestBootUserTranslateJournal
|
||||
TestCORS
|
||||
TestBackendURIGfadmin
|
||||
TestGrzybyCommandsRegistered
|
||||
)
|
||||
PLUGIN_REQUIRE=(
|
||||
TestReadFeaturedImageMap
|
||||
TestValidDiskName
|
||||
TestFeaturedImageSourceEscape
|
||||
TestImportFeaturedImages
|
||||
TestImportFeaturedImagesDryRun
|
||||
TestReadCategoryMap
|
||||
TestImportCategories
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
check-phase16.sh --self-test
|
||||
check-phase16.sh --layer
|
||||
check-phase16.sh --layer-smoke
|
||||
check-phase16.sh --app
|
||||
check-phase16.sh --host
|
||||
check-phase16.sh --plugin
|
||||
check-phase16.sh --deploy
|
||||
check-phase16.sh --all
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
# detect_go reads go test -json. Exit 1 fail/build, 2 skip, 3 zero/no-tests,
|
||||
# 4 non-JSON, 5 missing required name, 6 data race.
|
||||
detect_go() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json, os, sys
|
||||
path = sys.argv[1]
|
||||
require = [n for n in os.environ.get("REQUIRE_TESTS", "").split() if n]
|
||||
passed = set()
|
||||
failed = []
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
for raw in fh:
|
||||
line = raw.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
print("refuse: non-json test output", file=sys.stderr)
|
||||
sys.exit(4)
|
||||
action = ev.get("Action")
|
||||
test = ev.get("Test") or ""
|
||||
pkg = ev.get("Package") or ev.get("ImportPath") or ""
|
||||
if action == "build-fail" or (action == "fail" and ev.get("FailedBuild")):
|
||||
print(f"refuse: build failed {pkg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
text = ev.get("Output") or ""
|
||||
if action == "output":
|
||||
if "no tests to run" in text:
|
||||
print(f"refuse: no tests to run in {pkg}", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
if "WARNING: DATA RACE" in text:
|
||||
print(f"refuse: data race in {pkg} {test}", file=sys.stderr)
|
||||
sys.exit(6)
|
||||
if action == "skip" and test:
|
||||
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if action == "fail":
|
||||
failed.append(f"{pkg} {test}".strip())
|
||||
if action == "pass" and test:
|
||||
passed.add(test)
|
||||
if failed:
|
||||
print("refuse: failed " + ", ".join(failed), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not passed:
|
||||
print("refuse: zero tests", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
missing = [n for n in require if n not in passed]
|
||||
if missing:
|
||||
print("refuse: required tests did not pass: " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(5)
|
||||
PY
|
||||
}
|
||||
|
||||
# detect_tap reads node --test --test-reporter=tap output. Exit 1 "not ok" or
|
||||
# a fail/cancelled count, 2 skip/todo, 3 no pass count or zero passes, 5
|
||||
# missing required name. REQUIRE_NAMES holds one test name per line.
|
||||
detect_tap() {
|
||||
python3 - "$1" <<'PY'
|
||||
import os, re, sys
|
||||
path = sys.argv[1]
|
||||
require = [n for n in os.environ.get("REQUIRE_NAMES", "").split("\n") if n.strip()]
|
||||
passed = set()
|
||||
counts = {}
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
for raw in fh:
|
||||
line = raw.rstrip("\n")
|
||||
body = line.lstrip()
|
||||
if body.startswith("not ok"):
|
||||
print(f"refuse: {body}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
m = re.match(r"ok \d+ - (.*?)(\s+#\s*(SKIP|TODO)\b.*)?$", body, re.I)
|
||||
if m:
|
||||
if m.group(2):
|
||||
print(f"refuse: {body}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if line == body:
|
||||
passed.add(m.group(1))
|
||||
continue
|
||||
m = re.match(r"# (tests|suites|pass|fail|cancelled|skipped|todo) (\d+)$", line)
|
||||
if m:
|
||||
counts[m.group(1)] = int(m.group(2))
|
||||
for key in ("fail", "cancelled"):
|
||||
if counts.get(key, 0) > 0:
|
||||
print(f"refuse: {counts[key]} {key}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for key in ("skipped", "todo"):
|
||||
if counts.get(key, 0) > 0:
|
||||
print(f"refuse: {counts[key]} {key}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if counts.get("pass", 0) < 1 or not passed:
|
||||
print("refuse: no passing tests", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
missing = [n for n in require if n not in passed]
|
||||
if missing:
|
||||
print("refuse: required tests did not pass: " + "; ".join(missing), file=sys.stderr)
|
||||
sys.exit(5)
|
||||
PY
|
||||
}
|
||||
|
||||
go_json() {
|
||||
local dir="$1"
|
||||
shift
|
||||
local log err rc=0 dc=0
|
||||
log="$(mktemp)"
|
||||
err="$(mktemp)"
|
||||
(cd "$dir" && go test -json "$@") >"$log" 2>"$err" || rc=$?
|
||||
detect_go "$log" || dc=$?
|
||||
if [[ "$rc" -ne 0 || "$dc" -ne 0 ]]; then
|
||||
cat "$err" >&2 || true
|
||||
grep -v '^{' "$log" | tail -n 40 >&2 || true
|
||||
rm -f "$log" "$err"
|
||||
echo "refuse: go test $* in $dir (test=$rc detect=$dc)" >&2
|
||||
return 1
|
||||
fi
|
||||
rm -f "$log" "$err"
|
||||
}
|
||||
|
||||
# node_tap runs `pnpm -C <dir> test` and checks its TAP output.
|
||||
node_tap() {
|
||||
local dir="$1" log rc=0 dc=0
|
||||
log="$(mktemp)"
|
||||
pnpm -C "$dir" test >"$log" 2>&1 || rc=$?
|
||||
detect_tap "$log" || dc=$?
|
||||
if [[ "$rc" -ne 0 || "$dc" -ne 0 ]]; then
|
||||
grep -E '^\s*(not ok|# )|Error' "$log" | tail -n 40 >&2 || true
|
||||
rm -f "$log"
|
||||
echo "refuse: pnpm test in $dir (test=$rc detect=$dc)" >&2
|
||||
return 1
|
||||
fi
|
||||
grep -E '^# (tests|pass) ' "$log"
|
||||
rm -f "$log"
|
||||
}
|
||||
|
||||
# smoke runs a command that must exit 0 and print "smoke-ssr: ok".
|
||||
smoke() {
|
||||
local log rc=0
|
||||
log="$(mktemp)"
|
||||
"$@" >"$log" 2>&1 || rc=$?
|
||||
if [[ "$rc" -ne 0 ]] || ! grep -q '^smoke-ssr: ok' "$log"; then
|
||||
tail -n 40 "$log" >&2 || true
|
||||
rm -f "$log"
|
||||
echo "refuse: $* (exit $rc)" >&2
|
||||
return 1
|
||||
fi
|
||||
grep '^smoke-ssr: ok' "$log"
|
||||
rm -f "$log"
|
||||
}
|
||||
|
||||
expect_go() {
|
||||
local name="$1" want="$2" payload="$3" log dc=0
|
||||
log="$(mktemp)"
|
||||
printf '%s\n' "$payload" >"$log"
|
||||
detect_go "$log" 2>/dev/null || dc=$?
|
||||
rm -f "$log"
|
||||
if [[ "$dc" -ne "$want" ]]; then
|
||||
echo "refuse: self-test go $name: detector exit $dc, want $want" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_tap() {
|
||||
local name="$1" want="$2" payload="$3" log dc=0
|
||||
log="$(mktemp)"
|
||||
printf '%s\n' "$payload" >"$log"
|
||||
detect_tap "$log" 2>/dev/null || dc=$?
|
||||
rm -f "$log"
|
||||
if [[ "$dc" -ne "$want" ]]; then
|
||||
echo "refuse: self-test tap $name: detector exit $dc, want $want" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_self_test() {
|
||||
bash -n "${BASH_SOURCE[0]}"
|
||||
expect_go pass 0 '{"Action":"pass","Package":"p","Test":"TestImportCategories"}'
|
||||
expect_go fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p","Test":"TestImportFeaturedImages"}'
|
||||
expect_go package-fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p"}'
|
||||
expect_go build 1 '{"Action":"build-fail","ImportPath":"p"}'
|
||||
expect_go skip 2 '{"Action":"skip","Package":"p","Test":"TestImportFeaturedImages"}'
|
||||
expect_go zero 3 '{"Action":"pass","Package":"p"}'
|
||||
expect_go no-tests 3 '{"Action":"output","Package":"p","Output":"testing: warning: no tests to run\n"}
|
||||
{"Action":"pass","Package":"p"}'
|
||||
expect_go nonjson 4 '{"Action":"pass",'
|
||||
expect_go race 6 '{"Action":"output","Package":"p","Test":"TestA","Output":"WARNING: DATA RACE\n"}
|
||||
{"Action":"pass","Package":"p","Test":"TestA"}'
|
||||
REQUIRE_TESTS="TestBackendURIGfadmin TestCORS" expect_go missing-named 5 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestCORS"}'
|
||||
REQUIRE_TESTS="TestImportFeaturedImages" expect_go subtest-is-not-the-test 5 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestImportFeaturedImagesDryRun"}'
|
||||
|
||||
local ok='TAP version 13
|
||||
ok 1 - routes ok
|
||||
1..1
|
||||
# tests 1
|
||||
# pass 1
|
||||
# fail 0
|
||||
# cancelled 0
|
||||
# skipped 0
|
||||
# todo 0'
|
||||
expect_tap pass 0 "$ok"
|
||||
expect_tap not-ok 1 'TAP version 13
|
||||
ok 1 - a
|
||||
not ok 2 - b
|
||||
# pass 1
|
||||
# fail 1'
|
||||
expect_tap nested-not-ok 1 'TAP version 13
|
||||
not ok 1 - sub
|
||||
ok 1 - a
|
||||
# pass 1
|
||||
# fail 0'
|
||||
expect_tap fail-count 1 'ok 1 - a
|
||||
# pass 1
|
||||
# fail 2'
|
||||
expect_tap cancelled 1 'ok 1 - a
|
||||
# pass 1
|
||||
# cancelled 1'
|
||||
expect_tap skip-directive 2 'ok 1 - a # SKIP needs db
|
||||
# pass 1'
|
||||
expect_tap skipped-count 2 'ok 1 - a
|
||||
# pass 1
|
||||
# skipped 1'
|
||||
expect_tap todo 2 'ok 1 - a # TODO later
|
||||
# pass 1'
|
||||
expect_tap no-pass-line 3 'ok 1 - a'
|
||||
expect_tap zero 3 '# tests 0
|
||||
# pass 0
|
||||
# fail 0'
|
||||
REQUIRE_NAMES=$'routes ok\nmissing name' expect_tap missing-named 5 "$ok"
|
||||
REQUIRE_NAMES='routes ok' expect_tap named 0 "$ok"
|
||||
|
||||
local flag
|
||||
for flag in --self-test --layer --layer-smoke --app --host --plugin --deploy --all; do
|
||||
grep -q -- "^ $flag)" "${BASH_SOURCE[0]}" || {
|
||||
echo "refuse: missing mode $flag" >&2
|
||||
return 1
|
||||
}
|
||||
done
|
||||
echo "phase16 self-test passed"
|
||||
}
|
||||
|
||||
require_dir() {
|
||||
[[ -d "$1" ]] || {
|
||||
echo "refuse: $2 $1 not found" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
run_layer() {
|
||||
require_dir "$LAYER" "layer repository"
|
||||
REQUIRE_NAMES="$(printf '%s\n' "${LAYER_REQUIRE[@]}")" node_tap "$LAYER"
|
||||
echo "phase16 layer passed"
|
||||
}
|
||||
|
||||
run_layer_smoke() {
|
||||
require_dir "$LAYER" "layer repository"
|
||||
pnpm -C "$LAYER" build:playground >/dev/null
|
||||
smoke node "$LAYER/testing/smoke-ssr.mjs" \
|
||||
--output "$LAYER/.playground/.output" \
|
||||
--fixtures "$LAYER/testing/fixtures/acme" \
|
||||
--expect "$LAYER/testing/expect-acme.txt"
|
||||
echo "phase16 layer-smoke passed"
|
||||
}
|
||||
|
||||
run_app() {
|
||||
require_dir "$APP" "site app"
|
||||
pnpm -C "$APP" install --frozen-lockfile >/dev/null
|
||||
pnpm -C "$APP" build >/dev/null
|
||||
REQUIRE_NAMES="$(printf '%s\n' "${APP_REQUIRE[@]}")" node_tap "$APP"
|
||||
smoke pnpm -C "$APP" smoke:ssr
|
||||
smoke pnpm -C "$APP" smoke:ga
|
||||
echo "phase16 app passed"
|
||||
}
|
||||
|
||||
run_host() {
|
||||
require_dir "$HOST" "host repository"
|
||||
go -C "$HOST" vet ./...
|
||||
REQUIRE_TESTS="${HOST_REQUIRE[*]}" go_json "$HOST" ./... -count=1 -v -timeout 5m
|
||||
echo "phase16 host passed"
|
||||
}
|
||||
|
||||
run_plugin() {
|
||||
require_dir "$PLUGIN" "grzyby plugin"
|
||||
go -C "$PLUGIN" vet ./...
|
||||
REQUIRE_TESTS="${PLUGIN_REQUIRE[*]}" go_json "$PLUGIN" ./... -count=1 -v -race -timeout 10m
|
||||
echo "phase16 plugin passed"
|
||||
}
|
||||
|
||||
run_deploy() {
|
||||
require_dir "$HOST" "host repository"
|
||||
bash "$HOST/scripts/check-deploy.sh"
|
||||
echo "phase16 deploy passed"
|
||||
}
|
||||
|
||||
run_all() {
|
||||
local stage
|
||||
for stage in self-test layer layer-smoke app host plugin deploy; do
|
||||
if bash "${BASH_SOURCE[0]}" "--$stage"; then
|
||||
echo "PASS $stage"
|
||||
else
|
||||
echo "FAIL $stage"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "Phase 16 gate passed"
|
||||
}
|
||||
|
||||
case "${1:---all}" in
|
||||
--self-test) run_self_test ;;
|
||||
--layer) run_layer ;;
|
||||
--layer-smoke) run_layer_smoke ;;
|
||||
--app) run_app ;;
|
||||
--host) run_host ;;
|
||||
--plugin) run_plugin ;;
|
||||
--deploy) run_deploy ;;
|
||||
--all) run_all ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user