From 89f1ab2c449af97a130f8c67a6a305a3381307f9 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Thu, 8 Oct 2026 12:25:24 +0200 Subject: [PATCH] chore(14.3-04): fail-closed phase 14.3 gate with --unit, --security and --all - --unit: vet, full suite with named --- PASS lines for every required test, goldens unchanged, examples/hello, docs check, site plugin/app, landing page - --forbidden: no SUMMER_ name in new.sh, no new go.mod require since v0.1.12, no plugin/yaegi dependency, relative golden paths, goldens not ignored - --bootstrap and --terminal run the gated end-to-end checks - --security parses the threat register; --all runs every stage in its own process --- scripts/check-phase14.3.sh | 261 ++++++++++++++++++++++++++++++++++++- 1 file changed, 260 insertions(+), 1 deletion(-) diff --git a/scripts/check-phase14.3.sh b/scripts/check-phase14.3.sh index 01f9c65..8ad56f2 100755 --- a/scripts/check-phase14.3.sh +++ b/scripts/check-phase14.3.sh @@ -1,6 +1,31 @@ #!/usr/bin/env bash # Phase 14.3 gate (one-command app scaffolding). # +# --all is the authoritative, fail-closed phase gate. It runs each stage +# below in its own process and ends with "Phase 14.3 gate passed": +# --unit, --generator, --bootstrap, --terminal, --acceptance, --security. +# +# --unit runs go vet ./..., the full framework test suite (-count=1 -v, never +# -short or -update) and requires a "--- PASS:" line for every test in +# REQUIRED_TESTS; a skipped required test, "no tests to run" or a golden file +# changed by the run fails the stage. It also runs the examples/hello tests, +# docs:build --check, the site plugin and app tests, the landing page +# generate and tests, and --forbidden. +# +# --forbidden scans for surfaces the phase must not add: a framework config +# variable (SUMMER_ prefix) in new.sh, a new require in go.mod since v0.1.12, +# the stdlib plugin package or yaegi in the dependency graph, an absolute +# path in a golden go.mod/go.work, or a golden ignored by .gitignore. +# +# --bootstrap runs the gated TestBootstrapEndToEnd; --terminal runs the +# site's gated TestTerminalCommands against the committed new.sh and this +# checkout at HEAD (SUMMERCMS_NEW_URL/REPO/REF overrides). Each requires its +# named PASS. +# +# --security requires every threat T-14.3-01..26 and T-14.3-SC in +# 14.3-SECURITY-REVIEW.md, no high row with status open, and a named test or +# gate stage in the "Pinned by" column of every high row. +# # --generator proves D-12 end to end: `summer new demo` run next to a # symlinked framework checkout, with no --framework flag and no git # identity, writes an app that builds, migrates against PostgreSQL 16 from @@ -34,12 +59,72 @@ unset FORCE_COLOR ROOT="${PHASE143_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" SERVE_TIMEOUT="${PHASE143_TIMEOUT:-60}" +SITE="${PHASE143_SITE:-$ROOT/../sm-summercmsio-app}" +PHASE_DIR="${PHASE143_PHASE_DIR:-$ROOT/.planning/phases/14.3-one-command-app-scaffolding-summer-new-and-summercms-io-new}" +REVIEW="$PHASE_DIR/14.3-SECURITY-REVIEW.md" + +# Framework tests that must each print "--- PASS:" in the --unit run. +REQUIRED_TESTS=( + TestNewGolden + TestNewGoldenNoPlugin + TestNewAppSmoke + TestNewNoPluginBuilds + TestNewValidateAppName + TestNewRefusesExistingTarget + TestNewFrameworkResolution + TestNewRefusesTargetInsideFramework + TestNewCleansUpOnFailure + TestNewRequiresValidAppKey + TestNewEnvFileMode + TestNewAgentsSymlink + TestNewGitFirstCommit + TestNewGitUsesConfiguredIdentity + TestNewGitMissingIsWarning + TestNewNoGit + TestMakeAndAddPluginOnHelloCopy + TestMakePluginRoutesUnchanged + TestGenerateAppKey + TestNewCommandViaCLI + TestToolCommandNames + TestScriptEmbedded + TestScriptShape + TestScriptTruncationRunsNothing + TestScriptBehavior + TestScriptShellcheck + TestDocsTree + TestDocsAIOutputsInSync +) +# Site tests that pin GET /new and the landing page commands. +SITE_PLUGIN_TESTS=(TestNewScriptRoute TestRoutesAssemble) +SITE_APP_TESTS=(TestLoadTerminal TestTerminalScript TestTerminalEnv TestTerminalRunScript) fail() { echo "FAIL: $*" >&2 exit 1 } +# require_named_pass LOG NAME...: the go test -v log must contain no +# "no tests to run", no failure, no skip of a named test and a top-level +# "--- PASS: NAME (" line for every name. +require_named_pass() { + local log="$1" name + shift + if grep -q 'no tests to run' "$log"; then + fail "$log: a test pattern matched no tests" + fi + if grep -qE '^[[:space:]]*--- FAIL|^FAIL' "$log"; then + grep -E '^[[:space:]]*--- FAIL|^FAIL' "$log" >&2 + fail "$log: a test failed" + fi + for name in "$@"; do + if grep -qE "^[[:space:]]*--- SKIP: ${name}( |/|\$)" "$log"; then + fail "required test $name was skipped" + fi + grep -qE "^--- PASS: ${name} \(" "$log" || fail "required test $name has no --- PASS line" + echo "PASS $name" + done +} + port_free() { # bash /dev/tcp connects when something listens; free means it fails. if (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; then @@ -296,12 +381,186 @@ stage_acceptance() { fi } +stage_forbidden() { + cd "$ROOT" + echo "== forbidden surfaces" + if grep -nE 'SUMMER_' modules/sunrise/new.sh; then + fail "new.sh names a framework config variable (SUMMER_ prefix)" + fi + git rev-parse -q --verify 'refs/tags/v0.1.12^{commit}' >/dev/null || fail "tag v0.1.12 is missing; cannot compare go.mod" + local added + added="$(git diff v0.1.12 -- go.mod | grep -E '^\+[[:space:]]*(require[[:space:]]|[^+[:space:]]+[[:space:]]+v[0-9])' || true)" + [ -z "$added" ] || fail "go.mod gained a requirement since v0.1.12: $added" + local deps + deps="$(go list -deps ./...)" + if grep -xE 'plugin|.*yaegi.*' <<<"$deps"; then + fail "the dependency graph contains runtime plugin loading (plugin or yaegi)" + fi + local golden + for golden in internal/build/testdata/new/go.mod.golden internal/build/testdata/new/go.work.golden \ + internal/build/testdata/new-no-plugin/go.mod.golden internal/build/testdata/new-no-plugin/go.work.golden; do + [ -f "$golden" ] || fail "$golden is missing" + if grep -nE '^[[:space:]]*/|=> /' "$golden"; then + fail "$golden contains an absolute path" + fi + done + local ignored + ignored="$(git ls-files internal/build/testdata | git check-ignore --no-index --stdin || true)" + [ -z "$ignored" ] || fail "the framework .gitignore ignores goldens: $ignored" + [ "$(git ls-files internal/build/testdata | wc -l)" -ge 23 ] || fail "fewer than 23 goldens are tracked" + echo "Phase 14.3 forbidden-surface scan passed" +} + +stage_unit() { + cd "$ROOT" + for tool in go git pnpm python3 shellcheck; do + command -v "$tool" >/dev/null || fail "$tool is not installed" + done + unit_tmp="$(mktemp -d)" + trap 'rm -rf "$unit_tmp"' EXIT + + echo "== go vet ./..." + go vet ./... + + echo "== go test ./... -count=1 -v" + if ! go test ./... -count=1 -v >"$unit_tmp/framework.log" 2>&1; then + grep -E '^[[:space:]]*--- FAIL|^FAIL|panic:' "$unit_tmp/framework.log" >&2 || tail -50 "$unit_tmp/framework.log" >&2 + fail "go test ./... failed" + fi + require_named_pass "$unit_tmp/framework.log" "${REQUIRED_TESTS[@]}" + # Goldens change only with an explicit -update, never during the gate. + git diff --quiet -- internal/build/testdata || fail "golden files differ from the index" + [ -z "$(git status --porcelain -- internal/build/testdata)" ] || fail "untracked or changed golden files" + + echo "== examples/hello" + go -C examples/hello test ./... -count=1 + + echo "== docs:build --check" + go run ./cmd/summer docs:build --check + + echo "== site plugin" + [ -d "$SITE" ] || fail "site checkout $SITE is missing (set PHASE143_SITE)" + if ! go -C "$SITE/plugins/golem15/summercms" test ./... -count=1 -v >"$unit_tmp/site-plugin.log" 2>&1; then + tail -50 "$unit_tmp/site-plugin.log" >&2 + fail "site plugin tests failed" + fi + require_named_pass "$unit_tmp/site-plugin.log" "${SITE_PLUGIN_TESTS[@]}" + + echo "== site app" + if ! go -C "$SITE" test ./... -count=1 -v >"$unit_tmp/site-app.log" 2>&1; then + tail -50 "$unit_tmp/site-app.log" >&2 + fail "site app tests failed" + fi + require_named_pass "$unit_tmp/site-app.log" "${SITE_APP_TESTS[@]}" + + echo "== landing page: pnpm generate, pnpm test" + pnpm -C "$SITE/vue-summercmsio-app" generate >"$unit_tmp/generate.log" 2>&1 || { + tail -50 "$unit_tmp/generate.log" >&2 + fail "pnpm generate failed" + } + pnpm -C "$SITE/vue-summercmsio-app" test + + stage_forbidden + echo "Phase 14.3 unit stage passed" +} + +stage_bootstrap() { + cd "$ROOT" + bootstrap_tmp="$(mktemp -d)" + trap 'rm -rf "$bootstrap_tmp"' EXIT + echo "== SUMMERCMS_BOOTSTRAP_CHECK=1 TestBootstrapEndToEnd" + if ! SUMMERCMS_BOOTSTRAP_CHECK=1 go test ./modules/sunrise/ -run '^TestBootstrapEndToEnd$' -count=1 -v >"$bootstrap_tmp/log" 2>&1; then + tail -60 "$bootstrap_tmp/log" >&2 + fail "TestBootstrapEndToEnd failed" + fi + require_named_pass "$bootstrap_tmp/log" TestBootstrapEndToEnd + echo "Phase 14.3 bootstrap check passed" +} + +stage_terminal() { + cd "$ROOT" + command -v docker >/dev/null || fail "docker is not installed" + port_free 54320 || fail "127.0.0.1:54320 is in use" + port_free 8080 || fail "127.0.0.1:8080 is in use" + git diff --quiet HEAD -- modules/sunrise/new.sh || + fail "modules/sunrise/new.sh differs from HEAD; commit it first (the check clones HEAD)" + terminal_tmp="$(mktemp -d)" + trap 'rm -rf "$terminal_tmp"' EXIT + local head + head="$(git rev-parse HEAD)" + echo "== site TestTerminalCommands (local script, framework at $head)" + if ! SUMMERCMS_TERMINAL_CHECK=1 SUMMERCMS_NEW_URL="file://$ROOT/modules/sunrise/new.sh" \ + SUMMERCMS_REPO="$ROOT" SUMMERCMS_REF="$head" \ + go -C "$SITE" test -run '^TestTerminalCommands$' -count=1 -v . >"$terminal_tmp/log" 2>&1; then + tail -60 "$terminal_tmp/log" >&2 + fail "TestTerminalCommands failed" + fi + require_named_pass "$terminal_tmp/log" TestTerminalCommands + echo "Phase 14.3 terminal check passed" +} + +stage_security() { + [ -f "$REVIEW" ] || fail "missing $REVIEW" + python3 - "$REVIEW" <<'PY' || fail "security review check failed" +import re, sys +text = open(sys.argv[1], encoding="utf-8").read() +want = [f"T-14.3-{i:02d}" for i in range(1, 27)] + ["T-14.3-SC"] +rows = {} +for line in text.splitlines(): + if not line.startswith("| T-14.3-"): + continue + cells = [c.strip() for c in re.split(r"(?&2 + exit 1 + fi + done + echo "Phase 14.3 gate passed" +} + case "${1:-}" in +--all) stage_all ;; +--unit) stage_unit ;; +--forbidden) stage_forbidden ;; --generator) stage_generator ;; +--bootstrap) stage_bootstrap ;; +--terminal) stage_terminal ;; --acceptance) stage_acceptance local ;; --acceptance-production) stage_acceptance production ;; +--security) stage_security ;; *) - echo "usage: $0 --generator | --acceptance | --acceptance-production" >&2 + echo "usage: $0 --all | --unit | --forbidden | --generator | --bootstrap | --terminal | --acceptance | --acceptance-production | --security" >&2 exit 2 ;; esac