test(19.8): phase gate with the smoke tests by name
- check-phase19.8.sh: go test -json detector, self-test on planted output - go stage runs plans 1-3 smoke tests by name and the race run - remaining stages stubbed so --all stays red
This commit is contained in:
Jakub Zych
395
scripts/check-phase19.8.sh
Executable file
395
scripts/check-phase19.8.sh
Executable file
@@ -0,0 +1,395 @@
|
||||
#!/usr/bin/env bash
|
||||
# check-phase19.8: fail-closed gate for Phase 19.8, the console commands
|
||||
# overhaul. It proves the phase across this framework (diary, bonfire and the
|
||||
# swept framework commands), the example application, the docs, every shared
|
||||
# plugin swept in plan 03, every application that runs the new console, the
|
||||
# landing site and the Docker-backed paths.
|
||||
#
|
||||
# Stages:
|
||||
# self-test the go test -json detector on planted output (pass, fail,
|
||||
# skip, a subtest-only pass, zero tests, "no tests to run",
|
||||
# a missing required name), the docker NOT RUN rule on a faked
|
||||
# PATH, and that every mode is wired.
|
||||
# go go vet ./... and go test -short ./... here, the required
|
||||
# smoke and unit tests by name per package, then go test -race
|
||||
# -short of diary and bonfire.
|
||||
# hello the example application (its own module in the workspace).
|
||||
# docs the decision note, TestDocsTree, summer docs:build --check,
|
||||
# no consuming application named in the console and logging
|
||||
# sources and pages, and every command in summer's JSON listing
|
||||
# has examples.
|
||||
# plugins each swept shared plugin: vet, short tests, its warn-only
|
||||
# TestCommandsDocumented by name, a build without the
|
||||
# workspace, and the additive check against the commit before
|
||||
# its sweep commit.
|
||||
# apps each application: go build ./..., the generated main.go
|
||||
# unchanged by summer build, bin/<app> list and help migrate.
|
||||
# landing the landing site's tests and the roadmap deck's tests.
|
||||
# docker Docker host only: the testcontainers tests by name without
|
||||
# -short.
|
||||
#
|
||||
# A Go stage exits non-zero on a failing command, or on a go test -json run
|
||||
# that fails, skips a test (a subtest included), matches zero tests, prints
|
||||
# "no tests to run" or misses a required name (a subtest pass never counts
|
||||
# for its parent's name).
|
||||
#
|
||||
# --all runs self-test, go, hello, docs, plugins, apps and landing, then
|
||||
# docker when a Docker daemon answers (NOT RUN docker otherwise, a failure
|
||||
# with PHASE198_REQUIRE_DOCKER=1).
|
||||
#
|
||||
# Environment overrides (default: the sibling checkouts next to this
|
||||
# repository): PHASE198_ROOT (this repository), PLUGINS_ROOT (the directory
|
||||
# holding the sm-*-plugin checkouts), FONOTEKA (the application workspace
|
||||
# that carries the user, golem and feedback plugin checkouts), DEMO_APP,
|
||||
# BM_APP, GRZYBY_APP, SUMMERCMSIO_APP, PHASE198_REQUIRE_DOCKER.
|
||||
set -euo pipefail
|
||||
|
||||
unset FORCE_COLOR
|
||||
export NO_COLOR=1
|
||||
|
||||
ROOT="${PHASE198_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
PLUGINS_ROOT="${PLUGINS_ROOT:-$ROOT/..}"
|
||||
FONOTEKA="${FONOTEKA:-$ROOT/../fono""teka.go}"
|
||||
DEMO_APP="${DEMO_APP:-$ROOT/../sm-demo-app}"
|
||||
BM_APP="${BM_APP:-$ROOT/../sm-bm-app}"
|
||||
GRZYBY_APP="${GRZYBY_APP:-$ROOT/../sm-grzybyfunkcjonalne-app}"
|
||||
SUMMERCMSIO_APP="${SUMMERCMSIO_APP:-$ROOT/../sm-summercmsio-app}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Required test names. Each list runs with -run '^(a|b|...)$' and every name
|
||||
# must pass; a name that skips or does not run fails the stage.
|
||||
|
||||
# go stage (-short).
|
||||
DIARY_TESTS=(
|
||||
# Plan 01.
|
||||
TestDiaryTracerSmoke TestConfigEnvSmoke TestDailyDegradesSmoke
|
||||
)
|
||||
BONFIRE_TESTS=(
|
||||
# Plan 02.
|
||||
TestListTracerSmoke TestHelpSmoke TestListJSONSmoke TestErrorRenderSmoke
|
||||
TestWidgetMatrixSmoke
|
||||
)
|
||||
CONGA_TESTS=(
|
||||
# Plan 01.
|
||||
TestRiverLoggerUsesDiarySmoke
|
||||
)
|
||||
LAGOON_TESTS=(
|
||||
# Plan 01.
|
||||
TestGormLoggerUsesDiarySmoke
|
||||
)
|
||||
SURF_TESTS=(
|
||||
# Plan 03.
|
||||
TestServeSummarySmoke TestShutdownReportSmoke TestAccessLogSmoke
|
||||
)
|
||||
SUMMER_TESTS=(
|
||||
# Plan 03.
|
||||
TestCommandsDocumented TestDocsTree
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
check-phase19.8.sh --self-test
|
||||
check-phase19.8.sh go | hello | docs | plugins | apps | landing | docker
|
||||
check-phase19.8.sh --go | --hello | --docs | --plugins | --apps | --landing | --docker
|
||||
check-phase19.8.sh --all
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
# detect_go reads go test -json. Exit 1 fail/build, 2 skip, 3 zero/no-tests,
|
||||
# 4 non-JSON, 5 missing required name.
|
||||
detect_go() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json, os, sys
|
||||
path = sys.argv[1]
|
||||
require = [n for n in os.environ.get("REQUIRE_TESTS", "").split() if n]
|
||||
passed = set()
|
||||
failed = []
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
for raw in fh:
|
||||
line = raw.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
print("refuse: non-json test output", file=sys.stderr)
|
||||
sys.exit(4)
|
||||
action = ev.get("Action")
|
||||
test = ev.get("Test") or ""
|
||||
pkg = ev.get("Package") or ev.get("ImportPath") or ""
|
||||
if action == "build-fail" or (action == "fail" and ev.get("FailedBuild")):
|
||||
print(f"refuse: build failed {pkg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
text = ev.get("Output") or ""
|
||||
if action == "output" and "no tests to run" in text:
|
||||
print(f"refuse: no tests to run in {pkg}", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
if action == "skip" and test:
|
||||
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if action == "fail":
|
||||
failed.append(f"{pkg} {test}".strip())
|
||||
if action == "pass" and test and "/" not in test:
|
||||
passed.add(test)
|
||||
if failed:
|
||||
print("refuse: failed " + ", ".join(failed), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not passed:
|
||||
print("refuse: zero tests", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
missing = [n for n in require if n not in passed]
|
||||
if missing:
|
||||
print("refuse: required tests did not pass: " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(5)
|
||||
PY
|
||||
}
|
||||
|
||||
# go_json DIR ARGS... runs go test -json ARGS in DIR and checks the stream
|
||||
# with detect_go.
|
||||
go_json() {
|
||||
local dir="$1"
|
||||
shift
|
||||
local log err rc=0 dc=0
|
||||
log="$(mktemp)"
|
||||
err="$(mktemp)"
|
||||
(cd "$dir" && go test -json "$@") >"$log" 2>"$err" || rc=$?
|
||||
detect_go "$log" || dc=$?
|
||||
if [[ "$rc" -ne 0 || "$dc" -ne 0 ]]; then
|
||||
cat "$err" >&2 || true
|
||||
grep -v '^{' "$log" | tail -n 40 >&2 || true
|
||||
python3 -c 'import json,sys
|
||||
for raw in open(sys.argv[1], errors="replace"):
|
||||
try: ev = json.loads(raw)
|
||||
except ValueError: continue
|
||||
if ev.get("Action") == "output" and ("FAIL" in ev.get("Output", "") or "SKIP" in ev.get("Output", "") or "_test.go:" in ev.get("Output", "")):
|
||||
sys.stderr.write(ev["Output"])' "$log" 2>&1 | tail -n 40 >&2 || true
|
||||
rm -f "$log" "$err"
|
||||
echo "refuse: go test $* in $dir (test=$rc detect=$dc)" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "go: $(grep -c '"Action":"pass","Package":"[^"]*","Test":"[^"/]*","Elapsed"' "$log") top-level tests passed in $dir ($*)"
|
||||
rm -f "$log" "$err"
|
||||
}
|
||||
|
||||
# go_named DIR PACKAGES FLAGS NAME... runs exactly the named top-level tests
|
||||
# and requires every one of them to pass. PACKAGES and FLAGS are word-split.
|
||||
go_named() {
|
||||
local dir="$1" pkgs="$2" flags="$3"
|
||||
shift 3
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
echo "refuse: go_named $pkgs in $dir without required names" >&2
|
||||
return 1
|
||||
fi
|
||||
local names=("$@") regex
|
||||
regex="^($(IFS='|'; echo "${names[*]}"))\$"
|
||||
# shellcheck disable=SC2086
|
||||
REQUIRE_TESTS="${names[*]}" go_json "$dir" $flags -count=1 -run "$regex" $pkgs
|
||||
}
|
||||
|
||||
# go_short DIR PACKAGES runs go test -short PACKAGES in DIR; on failure it
|
||||
# prints the tail of the output and fails.
|
||||
go_short() {
|
||||
local dir="$1" pkgs="${2:-./...}" out
|
||||
# shellcheck disable=SC2086
|
||||
if ! out="$(cd "$dir" && go test -count=1 -short $pkgs 2>&1)"; then
|
||||
printf '%s\n' "$out" | grep -v '^ok ' | tail -n 60 >&2
|
||||
echo "refuse: go test -short $pkgs in $dir" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "short: $dir ($pkgs)"
|
||||
}
|
||||
|
||||
go_vet() {
|
||||
local dir="$1"
|
||||
shift
|
||||
(cd "$dir" && go vet "$@")
|
||||
echo "vet: $dir ($*)"
|
||||
}
|
||||
|
||||
require_dir() {
|
||||
[[ -d "$1" ]] || {
|
||||
echo "refuse: $2 $1 not found" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
expect_detector() {
|
||||
local detector="$1" name="$2" want="$3" payload="$4" log dc=0
|
||||
log="$(mktemp)"
|
||||
printf '%s\n' "$payload" >"$log"
|
||||
"$detector" "$log" >/dev/null 2>&1 || dc=$?
|
||||
rm -f "$log"
|
||||
if [[ "$dc" -ne "$want" ]]; then
|
||||
echo "refuse: self-test $detector $name: exit $dc, want $want" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# expect_mode NAME WANT_EXIT WANT_TEXT MODE ENV... runs this script in MODE
|
||||
# with ENV and checks its exit status and that its output carries WANT_TEXT.
|
||||
expect_mode() {
|
||||
local name="$1" want="$2" text="$3" mode="$4"
|
||||
shift 4
|
||||
local out rc=0
|
||||
out="$(env "$@" bash "${BASH_SOURCE[0]}" "$mode" 2>&1)" || rc=$?
|
||||
if [[ "$rc" -ne "$want" ]] || ! grep -qF -- "$text" <<<"$out"; then
|
||||
printf '%s\n' "$out" | tail -n 10 >&2
|
||||
echo "refuse: self-test $name: exit $rc (want $want), output must carry \"$text\"" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_self_test() {
|
||||
bash -n "${BASH_SOURCE[0]}"
|
||||
|
||||
expect_detector detect_go pass 0 '{"Action":"pass","Package":"p","Test":"TestDiaryTracerSmoke"}'
|
||||
expect_detector detect_go fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p","Test":"TestHelpSmoke"}'
|
||||
expect_detector detect_go package-fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p"}'
|
||||
expect_detector detect_go build 1 '{"Action":"build-fail","ImportPath":"p"}'
|
||||
expect_detector detect_go skip 2 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"skip","Package":"p","Test":"TestServeLifecycle"}'
|
||||
expect_detector detect_go subtest-skip 2 '{"Action":"skip","Package":"p","Test":"TestPeekGlobals/env"}
|
||||
{"Action":"pass","Package":"p","Test":"TestPeekGlobals"}'
|
||||
expect_detector detect_go subtest-only-pass 3 '{"Action":"pass","Package":"p","Test":"TestOuter/TestListTracerSmoke"}
|
||||
{"Action":"pass","Package":"p"}'
|
||||
expect_detector detect_go zero 3 '{"Action":"pass","Package":"p"}'
|
||||
expect_detector detect_go no-tests 3 '{"Action":"output","Package":"p","Output":"testing: warning: no tests to run\n"}
|
||||
{"Action":"pass","Package":"p"}'
|
||||
expect_detector detect_go nonjson 4 '{"Action":"pass",'
|
||||
REQUIRE_TESTS="TestDiaryTracerSmoke TestConfigEnvSmoke" expect_detector detect_go missing-named 5 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestDiaryTracerSmoke"}'
|
||||
REQUIRE_TESTS="TestHelp" expect_detector detect_go prefix-is-not-the-test 5 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestHelpSmoke"}'
|
||||
REQUIRE_TESTS="TestListTracerSmoke" expect_detector detect_go subtest-is-not-the-test 5 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestOuter/TestListTracerSmoke"}
|
||||
{"Action":"pass","Package":"p","Test":"TestOther"}'
|
||||
|
||||
# The NOT RUN rule, on a faked host: a docker binary that never answers.
|
||||
local tmp
|
||||
tmp="$(mktemp -d)"
|
||||
mkdir -p "$tmp/bin"
|
||||
printf '#!/bin/sh\nexit 1\n' >"$tmp/bin/docker"
|
||||
chmod +x "$tmp/bin/docker"
|
||||
expect_mode docker-not-run 0 "NOT RUN docker" --docker-gate PATH="$tmp/bin:$PATH" PHASE198_REQUIRE_DOCKER=0
|
||||
expect_mode docker-required 1 "FAIL docker" --docker-gate PATH="$tmp/bin:$PATH" PHASE198_REQUIRE_DOCKER=1
|
||||
expect_mode docker-stage-needs-daemon 1 "needs a reachable Docker daemon" --docker PATH="$tmp/bin:$PATH"
|
||||
rm -rf "$tmp"
|
||||
|
||||
local mode
|
||||
for mode in self-test go hello docs plugins apps landing docker all; do
|
||||
grep -q -- "^ --$mode" "${BASH_SOURCE[0]}" || {
|
||||
echo "refuse: missing mode --$mode" >&2
|
||||
return 1
|
||||
}
|
||||
done
|
||||
echo "phase19.8 self-test passed"
|
||||
}
|
||||
|
||||
run_go() {
|
||||
go_vet "$ROOT" ./...
|
||||
go_short "$ROOT"
|
||||
go_named "$ROOT" ./modules/diary -short "${DIARY_TESTS[@]}"
|
||||
go_named "$ROOT" ./modules/bonfire -short "${BONFIRE_TESTS[@]}"
|
||||
go_named "$ROOT" ./modules/conga -short "${CONGA_TESTS[@]}"
|
||||
go_named "$ROOT" ./modules/lagoon -short "${LAGOON_TESTS[@]}"
|
||||
go_named "$ROOT" ./modules/surf -short "${SURF_TESTS[@]}"
|
||||
go_named "$ROOT" ./cmd/summer -short "${SUMMER_TESTS[@]}"
|
||||
local out
|
||||
if ! out="$(cd "$ROOT" && go test -count=1 -race -short ./modules/diary/... ./modules/bonfire/... 2>&1)"; then
|
||||
printf '%s\n' "$out" | tail -n 60 >&2
|
||||
echo "refuse: go test -race -short of diary and bonfire" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "race: modules/diary, modules/bonfire"
|
||||
echo "phase19.8 go passed"
|
||||
}
|
||||
|
||||
not_implemented() {
|
||||
echo "stage $1: not implemented" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
run_hello() { not_implemented hello; }
|
||||
run_docs() { not_implemented docs; }
|
||||
run_plugins() { not_implemented plugins; }
|
||||
run_apps() { not_implemented apps; }
|
||||
run_landing() { not_implemented landing; }
|
||||
run_docker() {
|
||||
if ! docker_up; then
|
||||
echo "refuse: the docker stage needs a reachable Docker daemon (testcontainers); run it on a Docker host" >&2
|
||||
return 1
|
||||
fi
|
||||
not_implemented docker
|
||||
}
|
||||
|
||||
docker_up() {
|
||||
command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# docker_gate is the docker step of --all: run the stage when a daemon
|
||||
# answers, fail with PHASE198_REQUIRE_DOCKER=1, print NOT RUN otherwise.
|
||||
docker_gate() {
|
||||
if docker_up; then
|
||||
if bash "${BASH_SOURCE[0]}" --docker; then
|
||||
echo "PASS docker"
|
||||
return 0
|
||||
fi
|
||||
echo "FAIL docker"
|
||||
return 1
|
||||
fi
|
||||
if [[ "${PHASE198_REQUIRE_DOCKER:-0}" == "1" ]]; then
|
||||
echo "FAIL docker: no reachable Docker daemon and PHASE198_REQUIRE_DOCKER=1"
|
||||
return 1
|
||||
fi
|
||||
cat >&2 <<'EOF'
|
||||
************************************************************************
|
||||
NOT RUN docker: no reachable Docker daemon on this host.
|
||||
The testcontainers suites (Postgres) of the framework did NOT run. Run on
|
||||
a Docker host (through sudo -u golem when the socket is denied) before
|
||||
/msd-verify-work:
|
||||
bash scripts/check-phase19.8.sh docker
|
||||
************************************************************************
|
||||
EOF
|
||||
echo "NOT RUN docker (Docker-host only; see the notice above)"
|
||||
}
|
||||
|
||||
run_all() {
|
||||
local stage
|
||||
for stage in self-test go hello docs plugins apps landing; do
|
||||
if bash "${BASH_SOURCE[0]}" "--$stage"; then
|
||||
echo "PASS $stage"
|
||||
else
|
||||
echo "FAIL $stage"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
docker_gate || exit 1
|
||||
echo "Phase 19.8 gate passed"
|
||||
}
|
||||
|
||||
case "${1:---all}" in
|
||||
--self-test) run_self_test ;;
|
||||
--go) run_go ;;
|
||||
--hello) run_hello ;;
|
||||
--docs) run_docs ;;
|
||||
--plugins) run_plugins ;;
|
||||
--apps) run_apps ;;
|
||||
--landing) run_landing ;;
|
||||
--docker) run_docker ;;
|
||||
--docker-gate) docker_gate ;;
|
||||
--all) run_all ;;
|
||||
self-test) run_self_test ;;
|
||||
go) run_go ;;
|
||||
hello) run_hello ;;
|
||||
docs) run_docs ;;
|
||||
plugins) run_plugins ;;
|
||||
apps) run_apps ;;
|
||||
landing) run_landing ;;
|
||||
docker) run_docker ;;
|
||||
all) run_all ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user