From 9b2fa6c3246c99d9158554fb1ae4b4713a2c7a56 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Wed, 21 Oct 2026 18:58:30 +0200 Subject: [PATCH] test(19.8): phase gate with the smoke tests by name - check-phase19.8.sh: go test -json detector, self-test on planted output - go stage runs plans 1-3 smoke tests by name and the race run - remaining stages stubbed so --all stays red --- scripts/check-phase19.8.sh | 395 +++++++++++++++++++++++++++++++++++++ 1 file changed, 395 insertions(+) create mode 100755 scripts/check-phase19.8.sh diff --git a/scripts/check-phase19.8.sh b/scripts/check-phase19.8.sh new file mode 100755 index 00000000..c4c8fee3 --- /dev/null +++ b/scripts/check-phase19.8.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +# check-phase19.8: fail-closed gate for Phase 19.8, the console commands +# overhaul. It proves the phase across this framework (diary, bonfire and the +# swept framework commands), the example application, the docs, every shared +# plugin swept in plan 03, every application that runs the new console, the +# landing site and the Docker-backed paths. +# +# Stages: +# self-test the go test -json detector on planted output (pass, fail, +# skip, a subtest-only pass, zero tests, "no tests to run", +# a missing required name), the docker NOT RUN rule on a faked +# PATH, and that every mode is wired. +# go go vet ./... and go test -short ./... here, the required +# smoke and unit tests by name per package, then go test -race +# -short of diary and bonfire. +# hello the example application (its own module in the workspace). +# docs the decision note, TestDocsTree, summer docs:build --check, +# no consuming application named in the console and logging +# sources and pages, and every command in summer's JSON listing +# has examples. +# plugins each swept shared plugin: vet, short tests, its warn-only +# TestCommandsDocumented by name, a build without the +# workspace, and the additive check against the commit before +# its sweep commit. +# apps each application: go build ./..., the generated main.go +# unchanged by summer build, bin/ list and help migrate. +# landing the landing site's tests and the roadmap deck's tests. +# docker Docker host only: the testcontainers tests by name without +# -short. +# +# A Go stage exits non-zero on a failing command, or on a go test -json run +# that fails, skips a test (a subtest included), matches zero tests, prints +# "no tests to run" or misses a required name (a subtest pass never counts +# for its parent's name). +# +# --all runs self-test, go, hello, docs, plugins, apps and landing, then +# docker when a Docker daemon answers (NOT RUN docker otherwise, a failure +# with PHASE198_REQUIRE_DOCKER=1). +# +# Environment overrides (default: the sibling checkouts next to this +# repository): PHASE198_ROOT (this repository), PLUGINS_ROOT (the directory +# holding the sm-*-plugin checkouts), FONOTEKA (the application workspace +# that carries the user, golem and feedback plugin checkouts), DEMO_APP, +# BM_APP, GRZYBY_APP, SUMMERCMSIO_APP, PHASE198_REQUIRE_DOCKER. +set -euo pipefail + +unset FORCE_COLOR +export NO_COLOR=1 + +ROOT="${PHASE198_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +PLUGINS_ROOT="${PLUGINS_ROOT:-$ROOT/..}" +FONOTEKA="${FONOTEKA:-$ROOT/../fono""teka.go}" +DEMO_APP="${DEMO_APP:-$ROOT/../sm-demo-app}" +BM_APP="${BM_APP:-$ROOT/../sm-bm-app}" +GRZYBY_APP="${GRZYBY_APP:-$ROOT/../sm-grzybyfunkcjonalne-app}" +SUMMERCMSIO_APP="${SUMMERCMSIO_APP:-$ROOT/../sm-summercmsio-app}" + +# --------------------------------------------------------------------------- +# Required test names. Each list runs with -run '^(a|b|...)$' and every name +# must pass; a name that skips or does not run fails the stage. + +# go stage (-short). +DIARY_TESTS=( + # Plan 01. + TestDiaryTracerSmoke TestConfigEnvSmoke TestDailyDegradesSmoke +) +BONFIRE_TESTS=( + # Plan 02. + TestListTracerSmoke TestHelpSmoke TestListJSONSmoke TestErrorRenderSmoke + TestWidgetMatrixSmoke +) +CONGA_TESTS=( + # Plan 01. + TestRiverLoggerUsesDiarySmoke +) +LAGOON_TESTS=( + # Plan 01. + TestGormLoggerUsesDiarySmoke +) +SURF_TESTS=( + # Plan 03. + TestServeSummarySmoke TestShutdownReportSmoke TestAccessLogSmoke +) +SUMMER_TESTS=( + # Plan 03. + TestCommandsDocumented TestDocsTree +) + +usage() { + cat >&2 <<'EOF' +usage: + check-phase19.8.sh --self-test + check-phase19.8.sh go | hello | docs | plugins | apps | landing | docker + check-phase19.8.sh --go | --hello | --docs | --plugins | --apps | --landing | --docker + check-phase19.8.sh --all +EOF + exit 2 +} + +# detect_go reads go test -json. Exit 1 fail/build, 2 skip, 3 zero/no-tests, +# 4 non-JSON, 5 missing required name. +detect_go() { + python3 - "$1" <<'PY' +import json, os, sys +path = sys.argv[1] +require = [n for n in os.environ.get("REQUIRE_TESTS", "").split() if n] +passed = set() +failed = [] +with open(path, encoding="utf-8", errors="replace") as fh: + for raw in fh: + line = raw.strip() + if not line.startswith("{"): + continue + try: + ev = json.loads(line) + except json.JSONDecodeError: + print("refuse: non-json test output", file=sys.stderr) + sys.exit(4) + action = ev.get("Action") + test = ev.get("Test") or "" + pkg = ev.get("Package") or ev.get("ImportPath") or "" + if action == "build-fail" or (action == "fail" and ev.get("FailedBuild")): + print(f"refuse: build failed {pkg}", file=sys.stderr) + sys.exit(1) + text = ev.get("Output") or "" + if action == "output" and "no tests to run" in text: + print(f"refuse: no tests to run in {pkg}", file=sys.stderr) + sys.exit(3) + if action == "skip" and test: + print(f"refuse: skipped {pkg} {test}", file=sys.stderr) + sys.exit(2) + if action == "fail": + failed.append(f"{pkg} {test}".strip()) + if action == "pass" and test and "/" not in test: + passed.add(test) +if failed: + print("refuse: failed " + ", ".join(failed), file=sys.stderr) + sys.exit(1) +if not passed: + print("refuse: zero tests", file=sys.stderr) + sys.exit(3) +missing = [n for n in require if n not in passed] +if missing: + print("refuse: required tests did not pass: " + ", ".join(missing), file=sys.stderr) + sys.exit(5) +PY +} + +# go_json DIR ARGS... runs go test -json ARGS in DIR and checks the stream +# with detect_go. +go_json() { + local dir="$1" + shift + local log err rc=0 dc=0 + log="$(mktemp)" + err="$(mktemp)" + (cd "$dir" && go test -json "$@") >"$log" 2>"$err" || rc=$? + detect_go "$log" || dc=$? + if [[ "$rc" -ne 0 || "$dc" -ne 0 ]]; then + cat "$err" >&2 || true + grep -v '^{' "$log" | tail -n 40 >&2 || true + python3 -c 'import json,sys +for raw in open(sys.argv[1], errors="replace"): + try: ev = json.loads(raw) + except ValueError: continue + if ev.get("Action") == "output" and ("FAIL" in ev.get("Output", "") or "SKIP" in ev.get("Output", "") or "_test.go:" in ev.get("Output", "")): + sys.stderr.write(ev["Output"])' "$log" 2>&1 | tail -n 40 >&2 || true + rm -f "$log" "$err" + echo "refuse: go test $* in $dir (test=$rc detect=$dc)" >&2 + return 1 + fi + echo "go: $(grep -c '"Action":"pass","Package":"[^"]*","Test":"[^"/]*","Elapsed"' "$log") top-level tests passed in $dir ($*)" + rm -f "$log" "$err" +} + +# go_named DIR PACKAGES FLAGS NAME... runs exactly the named top-level tests +# and requires every one of them to pass. PACKAGES and FLAGS are word-split. +go_named() { + local dir="$1" pkgs="$2" flags="$3" + shift 3 + if [[ "$#" -eq 0 ]]; then + echo "refuse: go_named $pkgs in $dir without required names" >&2 + return 1 + fi + local names=("$@") regex + regex="^($(IFS='|'; echo "${names[*]}"))\$" + # shellcheck disable=SC2086 + REQUIRE_TESTS="${names[*]}" go_json "$dir" $flags -count=1 -run "$regex" $pkgs +} + +# go_short DIR PACKAGES runs go test -short PACKAGES in DIR; on failure it +# prints the tail of the output and fails. +go_short() { + local dir="$1" pkgs="${2:-./...}" out + # shellcheck disable=SC2086 + if ! out="$(cd "$dir" && go test -count=1 -short $pkgs 2>&1)"; then + printf '%s\n' "$out" | grep -v '^ok ' | tail -n 60 >&2 + echo "refuse: go test -short $pkgs in $dir" >&2 + return 1 + fi + echo "short: $dir ($pkgs)" +} + +go_vet() { + local dir="$1" + shift + (cd "$dir" && go vet "$@") + echo "vet: $dir ($*)" +} + +require_dir() { + [[ -d "$1" ]] || { + echo "refuse: $2 $1 not found" >&2 + return 1 + } +} + +expect_detector() { + local detector="$1" name="$2" want="$3" payload="$4" log dc=0 + log="$(mktemp)" + printf '%s\n' "$payload" >"$log" + "$detector" "$log" >/dev/null 2>&1 || dc=$? + rm -f "$log" + if [[ "$dc" -ne "$want" ]]; then + echo "refuse: self-test $detector $name: exit $dc, want $want" >&2 + return 1 + fi +} + +# expect_mode NAME WANT_EXIT WANT_TEXT MODE ENV... runs this script in MODE +# with ENV and checks its exit status and that its output carries WANT_TEXT. +expect_mode() { + local name="$1" want="$2" text="$3" mode="$4" + shift 4 + local out rc=0 + out="$(env "$@" bash "${BASH_SOURCE[0]}" "$mode" 2>&1)" || rc=$? + if [[ "$rc" -ne "$want" ]] || ! grep -qF -- "$text" <<<"$out"; then + printf '%s\n' "$out" | tail -n 10 >&2 + echo "refuse: self-test $name: exit $rc (want $want), output must carry \"$text\"" >&2 + return 1 + fi +} + +run_self_test() { + bash -n "${BASH_SOURCE[0]}" + + expect_detector detect_go pass 0 '{"Action":"pass","Package":"p","Test":"TestDiaryTracerSmoke"}' + expect_detector detect_go fail 1 '{"Action":"pass","Package":"p","Test":"TestA"} +{"Action":"fail","Package":"p","Test":"TestHelpSmoke"}' + expect_detector detect_go package-fail 1 '{"Action":"pass","Package":"p","Test":"TestA"} +{"Action":"fail","Package":"p"}' + expect_detector detect_go build 1 '{"Action":"build-fail","ImportPath":"p"}' + expect_detector detect_go skip 2 '{"Action":"pass","Package":"p","Test":"TestA"} +{"Action":"skip","Package":"p","Test":"TestServeLifecycle"}' + expect_detector detect_go subtest-skip 2 '{"Action":"skip","Package":"p","Test":"TestPeekGlobals/env"} +{"Action":"pass","Package":"p","Test":"TestPeekGlobals"}' + expect_detector detect_go subtest-only-pass 3 '{"Action":"pass","Package":"p","Test":"TestOuter/TestListTracerSmoke"} +{"Action":"pass","Package":"p"}' + expect_detector detect_go zero 3 '{"Action":"pass","Package":"p"}' + expect_detector detect_go no-tests 3 '{"Action":"output","Package":"p","Output":"testing: warning: no tests to run\n"} +{"Action":"pass","Package":"p"}' + expect_detector detect_go nonjson 4 '{"Action":"pass",' + REQUIRE_TESTS="TestDiaryTracerSmoke TestConfigEnvSmoke" expect_detector detect_go missing-named 5 \ + '{"Action":"pass","Package":"p","Test":"TestDiaryTracerSmoke"}' + REQUIRE_TESTS="TestHelp" expect_detector detect_go prefix-is-not-the-test 5 \ + '{"Action":"pass","Package":"p","Test":"TestHelpSmoke"}' + REQUIRE_TESTS="TestListTracerSmoke" expect_detector detect_go subtest-is-not-the-test 5 \ + '{"Action":"pass","Package":"p","Test":"TestOuter/TestListTracerSmoke"} +{"Action":"pass","Package":"p","Test":"TestOther"}' + + # The NOT RUN rule, on a faked host: a docker binary that never answers. + local tmp + tmp="$(mktemp -d)" + mkdir -p "$tmp/bin" + printf '#!/bin/sh\nexit 1\n' >"$tmp/bin/docker" + chmod +x "$tmp/bin/docker" + expect_mode docker-not-run 0 "NOT RUN docker" --docker-gate PATH="$tmp/bin:$PATH" PHASE198_REQUIRE_DOCKER=0 + expect_mode docker-required 1 "FAIL docker" --docker-gate PATH="$tmp/bin:$PATH" PHASE198_REQUIRE_DOCKER=1 + expect_mode docker-stage-needs-daemon 1 "needs a reachable Docker daemon" --docker PATH="$tmp/bin:$PATH" + rm -rf "$tmp" + + local mode + for mode in self-test go hello docs plugins apps landing docker all; do + grep -q -- "^ --$mode" "${BASH_SOURCE[0]}" || { + echo "refuse: missing mode --$mode" >&2 + return 1 + } + done + echo "phase19.8 self-test passed" +} + +run_go() { + go_vet "$ROOT" ./... + go_short "$ROOT" + go_named "$ROOT" ./modules/diary -short "${DIARY_TESTS[@]}" + go_named "$ROOT" ./modules/bonfire -short "${BONFIRE_TESTS[@]}" + go_named "$ROOT" ./modules/conga -short "${CONGA_TESTS[@]}" + go_named "$ROOT" ./modules/lagoon -short "${LAGOON_TESTS[@]}" + go_named "$ROOT" ./modules/surf -short "${SURF_TESTS[@]}" + go_named "$ROOT" ./cmd/summer -short "${SUMMER_TESTS[@]}" + local out + if ! out="$(cd "$ROOT" && go test -count=1 -race -short ./modules/diary/... ./modules/bonfire/... 2>&1)"; then + printf '%s\n' "$out" | tail -n 60 >&2 + echo "refuse: go test -race -short of diary and bonfire" >&2 + return 1 + fi + echo "race: modules/diary, modules/bonfire" + echo "phase19.8 go passed" +} + +not_implemented() { + echo "stage $1: not implemented" >&2 + return 1 +} + +run_hello() { not_implemented hello; } +run_docs() { not_implemented docs; } +run_plugins() { not_implemented plugins; } +run_apps() { not_implemented apps; } +run_landing() { not_implemented landing; } +run_docker() { + if ! docker_up; then + echo "refuse: the docker stage needs a reachable Docker daemon (testcontainers); run it on a Docker host" >&2 + return 1 + fi + not_implemented docker +} + +docker_up() { + command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1 +} + +# docker_gate is the docker step of --all: run the stage when a daemon +# answers, fail with PHASE198_REQUIRE_DOCKER=1, print NOT RUN otherwise. +docker_gate() { + if docker_up; then + if bash "${BASH_SOURCE[0]}" --docker; then + echo "PASS docker" + return 0 + fi + echo "FAIL docker" + return 1 + fi + if [[ "${PHASE198_REQUIRE_DOCKER:-0}" == "1" ]]; then + echo "FAIL docker: no reachable Docker daemon and PHASE198_REQUIRE_DOCKER=1" + return 1 + fi + cat >&2 <<'EOF' +************************************************************************ +NOT RUN docker: no reachable Docker daemon on this host. +The testcontainers suites (Postgres) of the framework did NOT run. Run on +a Docker host (through sudo -u golem when the socket is denied) before +/msd-verify-work: + bash scripts/check-phase19.8.sh docker +************************************************************************ +EOF + echo "NOT RUN docker (Docker-host only; see the notice above)" +} + +run_all() { + local stage + for stage in self-test go hello docs plugins apps landing; do + if bash "${BASH_SOURCE[0]}" "--$stage"; then + echo "PASS $stage" + else + echo "FAIL $stage" + exit 1 + fi + done + docker_gate || exit 1 + echo "Phase 19.8 gate passed" +} + +case "${1:---all}" in + --self-test) run_self_test ;; + --go) run_go ;; + --hello) run_hello ;; + --docs) run_docs ;; + --plugins) run_plugins ;; + --apps) run_apps ;; + --landing) run_landing ;; + --docker) run_docker ;; + --docker-gate) docker_gate ;; + --all) run_all ;; + self-test) run_self_test ;; + go) run_go ;; + hello) run_hello ;; + docs) run_docs ;; + plugins) run_plugins ;; + apps) run_apps ;; + landing) run_landing ;; + docker) run_docker ;; + all) run_all ;; + *) usage ;; +esac