chore(14.3-04): fail-closed phase 14.3 gate with --unit, --security and --all
- --unit: vet, full suite with named --- PASS lines for every required test, goldens unchanged, examples/hello, docs check, site plugin/app, landing page - --forbidden: no SUMMER_ name in new.sh, no new go.mod require since v0.1.12, no plugin/yaegi dependency, relative golden paths, goldens not ignored - --bootstrap and --terminal run the gated end-to-end checks - --security parses the threat register; --all runs every stage in its own process
This commit is contained in:
Jakub Zych
@@ -1,6 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase 14.3 gate (one-command app scaffolding).
|
||||
#
|
||||
# --all is the authoritative, fail-closed phase gate. It runs each stage
|
||||
# below in its own process and ends with "Phase 14.3 gate passed":
|
||||
# --unit, --generator, --bootstrap, --terminal, --acceptance, --security.
|
||||
#
|
||||
# --unit runs go vet ./..., the full framework test suite (-count=1 -v, never
|
||||
# -short or -update) and requires a "--- PASS:" line for every test in
|
||||
# REQUIRED_TESTS; a skipped required test, "no tests to run" or a golden file
|
||||
# changed by the run fails the stage. It also runs the examples/hello tests,
|
||||
# docs:build --check, the site plugin and app tests, the landing page
|
||||
# generate and tests, and --forbidden.
|
||||
#
|
||||
# --forbidden scans for surfaces the phase must not add: a framework config
|
||||
# variable (SUMMER_ prefix) in new.sh, a new require in go.mod since v0.1.12,
|
||||
# the stdlib plugin package or yaegi in the dependency graph, an absolute
|
||||
# path in a golden go.mod/go.work, or a golden ignored by .gitignore.
|
||||
#
|
||||
# --bootstrap runs the gated TestBootstrapEndToEnd; --terminal runs the
|
||||
# site's gated TestTerminalCommands against the committed new.sh and this
|
||||
# checkout at HEAD (SUMMERCMS_NEW_URL/REPO/REF overrides). Each requires its
|
||||
# named PASS.
|
||||
#
|
||||
# --security requires every threat T-14.3-01..26 and T-14.3-SC in
|
||||
# 14.3-SECURITY-REVIEW.md, no high row with status open, and a named test or
|
||||
# gate stage in the "Pinned by" column of every high row.
|
||||
#
|
||||
# --generator proves D-12 end to end: `summer new demo` run next to a
|
||||
# symlinked framework checkout, with no --framework flag and no git
|
||||
# identity, writes an app that builds, migrates against PostgreSQL 16 from
|
||||
@@ -34,12 +59,72 @@ unset FORCE_COLOR
|
||||
|
||||
ROOT="${PHASE143_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
SERVE_TIMEOUT="${PHASE143_TIMEOUT:-60}"
|
||||
SITE="${PHASE143_SITE:-$ROOT/../sm-summercmsio-app}"
|
||||
PHASE_DIR="${PHASE143_PHASE_DIR:-$ROOT/.planning/phases/14.3-one-command-app-scaffolding-summer-new-and-summercms-io-new}"
|
||||
REVIEW="$PHASE_DIR/14.3-SECURITY-REVIEW.md"
|
||||
|
||||
# Framework tests that must each print "--- PASS:" in the --unit run.
|
||||
REQUIRED_TESTS=(
|
||||
TestNewGolden
|
||||
TestNewGoldenNoPlugin
|
||||
TestNewAppSmoke
|
||||
TestNewNoPluginBuilds
|
||||
TestNewValidateAppName
|
||||
TestNewRefusesExistingTarget
|
||||
TestNewFrameworkResolution
|
||||
TestNewRefusesTargetInsideFramework
|
||||
TestNewCleansUpOnFailure
|
||||
TestNewRequiresValidAppKey
|
||||
TestNewEnvFileMode
|
||||
TestNewAgentsSymlink
|
||||
TestNewGitFirstCommit
|
||||
TestNewGitUsesConfiguredIdentity
|
||||
TestNewGitMissingIsWarning
|
||||
TestNewNoGit
|
||||
TestMakeAndAddPluginOnHelloCopy
|
||||
TestMakePluginRoutesUnchanged
|
||||
TestGenerateAppKey
|
||||
TestNewCommandViaCLI
|
||||
TestToolCommandNames
|
||||
TestScriptEmbedded
|
||||
TestScriptShape
|
||||
TestScriptTruncationRunsNothing
|
||||
TestScriptBehavior
|
||||
TestScriptShellcheck
|
||||
TestDocsTree
|
||||
TestDocsAIOutputsInSync
|
||||
)
|
||||
# Site tests that pin GET /new and the landing page commands.
|
||||
SITE_PLUGIN_TESTS=(TestNewScriptRoute TestRoutesAssemble)
|
||||
SITE_APP_TESTS=(TestLoadTerminal TestTerminalScript TestTerminalEnv TestTerminalRunScript)
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# require_named_pass LOG NAME...: the go test -v log must contain no
|
||||
# "no tests to run", no failure, no skip of a named test and a top-level
|
||||
# "--- PASS: NAME (" line for every name.
|
||||
require_named_pass() {
|
||||
local log="$1" name
|
||||
shift
|
||||
if grep -q 'no tests to run' "$log"; then
|
||||
fail "$log: a test pattern matched no tests"
|
||||
fi
|
||||
if grep -qE '^[[:space:]]*--- FAIL|^FAIL' "$log"; then
|
||||
grep -E '^[[:space:]]*--- FAIL|^FAIL' "$log" >&2
|
||||
fail "$log: a test failed"
|
||||
fi
|
||||
for name in "$@"; do
|
||||
if grep -qE "^[[:space:]]*--- SKIP: ${name}( |/|\$)" "$log"; then
|
||||
fail "required test $name was skipped"
|
||||
fi
|
||||
grep -qE "^--- PASS: ${name} \(" "$log" || fail "required test $name has no --- PASS line"
|
||||
echo "PASS $name"
|
||||
done
|
||||
}
|
||||
|
||||
port_free() {
|
||||
# bash /dev/tcp connects when something listens; free means it fails.
|
||||
if (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; then
|
||||
@@ -296,12 +381,186 @@ stage_acceptance() {
|
||||
fi
|
||||
}
|
||||
|
||||
stage_forbidden() {
|
||||
cd "$ROOT"
|
||||
echo "== forbidden surfaces"
|
||||
if grep -nE 'SUMMER_' modules/sunrise/new.sh; then
|
||||
fail "new.sh names a framework config variable (SUMMER_ prefix)"
|
||||
fi
|
||||
git rev-parse -q --verify 'refs/tags/v0.1.12^{commit}' >/dev/null || fail "tag v0.1.12 is missing; cannot compare go.mod"
|
||||
local added
|
||||
added="$(git diff v0.1.12 -- go.mod | grep -E '^\+[[:space:]]*(require[[:space:]]|[^+[:space:]]+[[:space:]]+v[0-9])' || true)"
|
||||
[ -z "$added" ] || fail "go.mod gained a requirement since v0.1.12: $added"
|
||||
local deps
|
||||
deps="$(go list -deps ./...)"
|
||||
if grep -xE 'plugin|.*yaegi.*' <<<"$deps"; then
|
||||
fail "the dependency graph contains runtime plugin loading (plugin or yaegi)"
|
||||
fi
|
||||
local golden
|
||||
for golden in internal/build/testdata/new/go.mod.golden internal/build/testdata/new/go.work.golden \
|
||||
internal/build/testdata/new-no-plugin/go.mod.golden internal/build/testdata/new-no-plugin/go.work.golden; do
|
||||
[ -f "$golden" ] || fail "$golden is missing"
|
||||
if grep -nE '^[[:space:]]*/|=> /' "$golden"; then
|
||||
fail "$golden contains an absolute path"
|
||||
fi
|
||||
done
|
||||
local ignored
|
||||
ignored="$(git ls-files internal/build/testdata | git check-ignore --no-index --stdin || true)"
|
||||
[ -z "$ignored" ] || fail "the framework .gitignore ignores goldens: $ignored"
|
||||
[ "$(git ls-files internal/build/testdata | wc -l)" -ge 23 ] || fail "fewer than 23 goldens are tracked"
|
||||
echo "Phase 14.3 forbidden-surface scan passed"
|
||||
}
|
||||
|
||||
stage_unit() {
|
||||
cd "$ROOT"
|
||||
for tool in go git pnpm python3 shellcheck; do
|
||||
command -v "$tool" >/dev/null || fail "$tool is not installed"
|
||||
done
|
||||
unit_tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$unit_tmp"' EXIT
|
||||
|
||||
echo "== go vet ./..."
|
||||
go vet ./...
|
||||
|
||||
echo "== go test ./... -count=1 -v"
|
||||
if ! go test ./... -count=1 -v >"$unit_tmp/framework.log" 2>&1; then
|
||||
grep -E '^[[:space:]]*--- FAIL|^FAIL|panic:' "$unit_tmp/framework.log" >&2 || tail -50 "$unit_tmp/framework.log" >&2
|
||||
fail "go test ./... failed"
|
||||
fi
|
||||
require_named_pass "$unit_tmp/framework.log" "${REQUIRED_TESTS[@]}"
|
||||
# Goldens change only with an explicit -update, never during the gate.
|
||||
git diff --quiet -- internal/build/testdata || fail "golden files differ from the index"
|
||||
[ -z "$(git status --porcelain -- internal/build/testdata)" ] || fail "untracked or changed golden files"
|
||||
|
||||
echo "== examples/hello"
|
||||
go -C examples/hello test ./... -count=1
|
||||
|
||||
echo "== docs:build --check"
|
||||
go run ./cmd/summer docs:build --check
|
||||
|
||||
echo "== site plugin"
|
||||
[ -d "$SITE" ] || fail "site checkout $SITE is missing (set PHASE143_SITE)"
|
||||
if ! go -C "$SITE/plugins/golem15/summercms" test ./... -count=1 -v >"$unit_tmp/site-plugin.log" 2>&1; then
|
||||
tail -50 "$unit_tmp/site-plugin.log" >&2
|
||||
fail "site plugin tests failed"
|
||||
fi
|
||||
require_named_pass "$unit_tmp/site-plugin.log" "${SITE_PLUGIN_TESTS[@]}"
|
||||
|
||||
echo "== site app"
|
||||
if ! go -C "$SITE" test ./... -count=1 -v >"$unit_tmp/site-app.log" 2>&1; then
|
||||
tail -50 "$unit_tmp/site-app.log" >&2
|
||||
fail "site app tests failed"
|
||||
fi
|
||||
require_named_pass "$unit_tmp/site-app.log" "${SITE_APP_TESTS[@]}"
|
||||
|
||||
echo "== landing page: pnpm generate, pnpm test"
|
||||
pnpm -C "$SITE/vue-summercmsio-app" generate >"$unit_tmp/generate.log" 2>&1 || {
|
||||
tail -50 "$unit_tmp/generate.log" >&2
|
||||
fail "pnpm generate failed"
|
||||
}
|
||||
pnpm -C "$SITE/vue-summercmsio-app" test
|
||||
|
||||
stage_forbidden
|
||||
echo "Phase 14.3 unit stage passed"
|
||||
}
|
||||
|
||||
stage_bootstrap() {
|
||||
cd "$ROOT"
|
||||
bootstrap_tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$bootstrap_tmp"' EXIT
|
||||
echo "== SUMMERCMS_BOOTSTRAP_CHECK=1 TestBootstrapEndToEnd"
|
||||
if ! SUMMERCMS_BOOTSTRAP_CHECK=1 go test ./modules/sunrise/ -run '^TestBootstrapEndToEnd$' -count=1 -v >"$bootstrap_tmp/log" 2>&1; then
|
||||
tail -60 "$bootstrap_tmp/log" >&2
|
||||
fail "TestBootstrapEndToEnd failed"
|
||||
fi
|
||||
require_named_pass "$bootstrap_tmp/log" TestBootstrapEndToEnd
|
||||
echo "Phase 14.3 bootstrap check passed"
|
||||
}
|
||||
|
||||
stage_terminal() {
|
||||
cd "$ROOT"
|
||||
command -v docker >/dev/null || fail "docker is not installed"
|
||||
port_free 54320 || fail "127.0.0.1:54320 is in use"
|
||||
port_free 8080 || fail "127.0.0.1:8080 is in use"
|
||||
git diff --quiet HEAD -- modules/sunrise/new.sh ||
|
||||
fail "modules/sunrise/new.sh differs from HEAD; commit it first (the check clones HEAD)"
|
||||
terminal_tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$terminal_tmp"' EXIT
|
||||
local head
|
||||
head="$(git rev-parse HEAD)"
|
||||
echo "== site TestTerminalCommands (local script, framework at $head)"
|
||||
if ! SUMMERCMS_TERMINAL_CHECK=1 SUMMERCMS_NEW_URL="file://$ROOT/modules/sunrise/new.sh" \
|
||||
SUMMERCMS_REPO="$ROOT" SUMMERCMS_REF="$head" \
|
||||
go -C "$SITE" test -run '^TestTerminalCommands$' -count=1 -v . >"$terminal_tmp/log" 2>&1; then
|
||||
tail -60 "$terminal_tmp/log" >&2
|
||||
fail "TestTerminalCommands failed"
|
||||
fi
|
||||
require_named_pass "$terminal_tmp/log" TestTerminalCommands
|
||||
echo "Phase 14.3 terminal check passed"
|
||||
}
|
||||
|
||||
stage_security() {
|
||||
[ -f "$REVIEW" ] || fail "missing $REVIEW"
|
||||
python3 - "$REVIEW" <<'PY' || fail "security review check failed"
|
||||
import re, sys
|
||||
text = open(sys.argv[1], encoding="utf-8").read()
|
||||
want = [f"T-14.3-{i:02d}" for i in range(1, 27)] + ["T-14.3-SC"]
|
||||
rows = {}
|
||||
for line in text.splitlines():
|
||||
if not line.startswith("| T-14.3-"):
|
||||
continue
|
||||
cells = [c.strip() for c in re.split(r"(?<!\\)\|", line)[1:-1]]
|
||||
rows[cells[0]] = cells
|
||||
bad = []
|
||||
for tid in want:
|
||||
if tid not in rows:
|
||||
bad.append(f"{tid}: missing")
|
||||
continue
|
||||
cells = rows[tid]
|
||||
if len(cells) < 9:
|
||||
bad.append(f"{tid}: expected 9 columns, found {len(cells)}")
|
||||
continue
|
||||
severity, status, pinned = cells[3].lower(), cells[7].lower(), cells[8]
|
||||
if status == "open":
|
||||
bad.append(f"{tid}: status open")
|
||||
if severity == "high":
|
||||
if status != "mitigated":
|
||||
bad.append(f"{tid}: high threat with status {status!r}")
|
||||
if not re.search(r"Test[A-Z]\w+|--(unit|generator|bootstrap|terminal|acceptance|security|forbidden)\b|gate", pinned):
|
||||
bad.append(f"{tid}: high threat pinned by no named test or gate stage ({pinned!r})")
|
||||
for b in bad:
|
||||
print("refuse: " + b, file=sys.stderr)
|
||||
sys.exit(1 if bad else 0)
|
||||
PY
|
||||
echo "Phase 14.3 security check passed"
|
||||
}
|
||||
|
||||
stage_all() {
|
||||
local stage
|
||||
for stage in unit generator bootstrap terminal acceptance security; do
|
||||
echo "=== stage --$stage"
|
||||
if bash "${BASH_SOURCE[0]}" "--$stage"; then
|
||||
echo "PASS $stage"
|
||||
else
|
||||
echo "FAIL $stage" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "Phase 14.3 gate passed"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--all) stage_all ;;
|
||||
--unit) stage_unit ;;
|
||||
--forbidden) stage_forbidden ;;
|
||||
--generator) stage_generator ;;
|
||||
--bootstrap) stage_bootstrap ;;
|
||||
--terminal) stage_terminal ;;
|
||||
--acceptance) stage_acceptance local ;;
|
||||
--acceptance-production) stage_acceptance production ;;
|
||||
--security) stage_security ;;
|
||||
*)
|
||||
echo "usage: $0 --generator | --acceptance | --acceptance-production" >&2
|
||||
echo "usage: $0 --all | --unit | --forbidden | --generator | --bootstrap | --terminal | --acceptance | --acceptance-production | --security" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user