9b2fa6c324
- check-phase19.8.sh: go test -json detector, self-test on planted output - go stage runs plans 1-3 smoke tests by name and the race run - remaining stages stubbed so --all stays red
396 lines
14 KiB
Bash
Executable File
396 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check-phase19.8: fail-closed gate for Phase 19.8, the console commands
|
|
# overhaul. It proves the phase across this framework (diary, bonfire and the
|
|
# swept framework commands), the example application, the docs, every shared
|
|
# plugin swept in plan 03, every application that runs the new console, the
|
|
# landing site and the Docker-backed paths.
|
|
#
|
|
# Stages:
|
|
# self-test the go test -json detector on planted output (pass, fail,
|
|
# skip, a subtest-only pass, zero tests, "no tests to run",
|
|
# a missing required name), the docker NOT RUN rule on a faked
|
|
# PATH, and that every mode is wired.
|
|
# go go vet ./... and go test -short ./... here, the required
|
|
# smoke and unit tests by name per package, then go test -race
|
|
# -short of diary and bonfire.
|
|
# hello the example application (its own module in the workspace).
|
|
# docs the decision note, TestDocsTree, summer docs:build --check,
|
|
# no consuming application named in the console and logging
|
|
# sources and pages, and every command in summer's JSON listing
|
|
# has examples.
|
|
# plugins each swept shared plugin: vet, short tests, its warn-only
|
|
# TestCommandsDocumented by name, a build without the
|
|
# workspace, and the additive check against the commit before
|
|
# its sweep commit.
|
|
# apps each application: go build ./..., the generated main.go
|
|
# unchanged by summer build, bin/<app> list and help migrate.
|
|
# landing the landing site's tests and the roadmap deck's tests.
|
|
# docker Docker host only: the testcontainers tests by name without
|
|
# -short.
|
|
#
|
|
# A Go stage exits non-zero on a failing command, or on a go test -json run
|
|
# that fails, skips a test (a subtest included), matches zero tests, prints
|
|
# "no tests to run" or misses a required name (a subtest pass never counts
|
|
# for its parent's name).
|
|
#
|
|
# --all runs self-test, go, hello, docs, plugins, apps and landing, then
|
|
# docker when a Docker daemon answers (NOT RUN docker otherwise, a failure
|
|
# with PHASE198_REQUIRE_DOCKER=1).
|
|
#
|
|
# Environment overrides (default: the sibling checkouts next to this
|
|
# repository): PHASE198_ROOT (this repository), PLUGINS_ROOT (the directory
|
|
# holding the sm-*-plugin checkouts), FONOTEKA (the application workspace
|
|
# that carries the user, golem and feedback plugin checkouts), DEMO_APP,
|
|
# BM_APP, GRZYBY_APP, SUMMERCMSIO_APP, PHASE198_REQUIRE_DOCKER.
|
|
set -euo pipefail
|
|
|
|
unset FORCE_COLOR
|
|
export NO_COLOR=1
|
|
|
|
ROOT="${PHASE198_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
|
PLUGINS_ROOT="${PLUGINS_ROOT:-$ROOT/..}"
|
|
FONOTEKA="${FONOTEKA:-$ROOT/../fono""teka.go}"
|
|
DEMO_APP="${DEMO_APP:-$ROOT/../sm-demo-app}"
|
|
BM_APP="${BM_APP:-$ROOT/../sm-bm-app}"
|
|
GRZYBY_APP="${GRZYBY_APP:-$ROOT/../sm-grzybyfunkcjonalne-app}"
|
|
SUMMERCMSIO_APP="${SUMMERCMSIO_APP:-$ROOT/../sm-summercmsio-app}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Required test names. Each list runs with -run '^(a|b|...)$' and every name
|
|
# must pass; a name that skips or does not run fails the stage.
|
|
|
|
# go stage (-short).
|
|
DIARY_TESTS=(
|
|
# Plan 01.
|
|
TestDiaryTracerSmoke TestConfigEnvSmoke TestDailyDegradesSmoke
|
|
)
|
|
BONFIRE_TESTS=(
|
|
# Plan 02.
|
|
TestListTracerSmoke TestHelpSmoke TestListJSONSmoke TestErrorRenderSmoke
|
|
TestWidgetMatrixSmoke
|
|
)
|
|
CONGA_TESTS=(
|
|
# Plan 01.
|
|
TestRiverLoggerUsesDiarySmoke
|
|
)
|
|
LAGOON_TESTS=(
|
|
# Plan 01.
|
|
TestGormLoggerUsesDiarySmoke
|
|
)
|
|
SURF_TESTS=(
|
|
# Plan 03.
|
|
TestServeSummarySmoke TestShutdownReportSmoke TestAccessLogSmoke
|
|
)
|
|
SUMMER_TESTS=(
|
|
# Plan 03.
|
|
TestCommandsDocumented TestDocsTree
|
|
)
|
|
|
|
usage() {
|
|
cat >&2 <<'EOF'
|
|
usage:
|
|
check-phase19.8.sh --self-test
|
|
check-phase19.8.sh go | hello | docs | plugins | apps | landing | docker
|
|
check-phase19.8.sh --go | --hello | --docs | --plugins | --apps | --landing | --docker
|
|
check-phase19.8.sh --all
|
|
EOF
|
|
exit 2
|
|
}
|
|
|
|
# detect_go reads go test -json. Exit 1 fail/build, 2 skip, 3 zero/no-tests,
|
|
# 4 non-JSON, 5 missing required name.
|
|
detect_go() {
|
|
python3 - "$1" <<'PY'
|
|
import json, os, sys
|
|
path = sys.argv[1]
|
|
require = [n for n in os.environ.get("REQUIRE_TESTS", "").split() if n]
|
|
passed = set()
|
|
failed = []
|
|
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
for raw in fh:
|
|
line = raw.strip()
|
|
if not line.startswith("{"):
|
|
continue
|
|
try:
|
|
ev = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
print("refuse: non-json test output", file=sys.stderr)
|
|
sys.exit(4)
|
|
action = ev.get("Action")
|
|
test = ev.get("Test") or ""
|
|
pkg = ev.get("Package") or ev.get("ImportPath") or ""
|
|
if action == "build-fail" or (action == "fail" and ev.get("FailedBuild")):
|
|
print(f"refuse: build failed {pkg}", file=sys.stderr)
|
|
sys.exit(1)
|
|
text = ev.get("Output") or ""
|
|
if action == "output" and "no tests to run" in text:
|
|
print(f"refuse: no tests to run in {pkg}", file=sys.stderr)
|
|
sys.exit(3)
|
|
if action == "skip" and test:
|
|
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
|
|
sys.exit(2)
|
|
if action == "fail":
|
|
failed.append(f"{pkg} {test}".strip())
|
|
if action == "pass" and test and "/" not in test:
|
|
passed.add(test)
|
|
if failed:
|
|
print("refuse: failed " + ", ".join(failed), file=sys.stderr)
|
|
sys.exit(1)
|
|
if not passed:
|
|
print("refuse: zero tests", file=sys.stderr)
|
|
sys.exit(3)
|
|
missing = [n for n in require if n not in passed]
|
|
if missing:
|
|
print("refuse: required tests did not pass: " + ", ".join(missing), file=sys.stderr)
|
|
sys.exit(5)
|
|
PY
|
|
}
|
|
|
|
# go_json DIR ARGS... runs go test -json ARGS in DIR and checks the stream
|
|
# with detect_go.
|
|
go_json() {
|
|
local dir="$1"
|
|
shift
|
|
local log err rc=0 dc=0
|
|
log="$(mktemp)"
|
|
err="$(mktemp)"
|
|
(cd "$dir" && go test -json "$@") >"$log" 2>"$err" || rc=$?
|
|
detect_go "$log" || dc=$?
|
|
if [[ "$rc" -ne 0 || "$dc" -ne 0 ]]; then
|
|
cat "$err" >&2 || true
|
|
grep -v '^{' "$log" | tail -n 40 >&2 || true
|
|
python3 -c 'import json,sys
|
|
for raw in open(sys.argv[1], errors="replace"):
|
|
try: ev = json.loads(raw)
|
|
except ValueError: continue
|
|
if ev.get("Action") == "output" and ("FAIL" in ev.get("Output", "") or "SKIP" in ev.get("Output", "") or "_test.go:" in ev.get("Output", "")):
|
|
sys.stderr.write(ev["Output"])' "$log" 2>&1 | tail -n 40 >&2 || true
|
|
rm -f "$log" "$err"
|
|
echo "refuse: go test $* in $dir (test=$rc detect=$dc)" >&2
|
|
return 1
|
|
fi
|
|
echo "go: $(grep -c '"Action":"pass","Package":"[^"]*","Test":"[^"/]*","Elapsed"' "$log") top-level tests passed in $dir ($*)"
|
|
rm -f "$log" "$err"
|
|
}
|
|
|
|
# go_named DIR PACKAGES FLAGS NAME... runs exactly the named top-level tests
|
|
# and requires every one of them to pass. PACKAGES and FLAGS are word-split.
|
|
go_named() {
|
|
local dir="$1" pkgs="$2" flags="$3"
|
|
shift 3
|
|
if [[ "$#" -eq 0 ]]; then
|
|
echo "refuse: go_named $pkgs in $dir without required names" >&2
|
|
return 1
|
|
fi
|
|
local names=("$@") regex
|
|
regex="^($(IFS='|'; echo "${names[*]}"))\$"
|
|
# shellcheck disable=SC2086
|
|
REQUIRE_TESTS="${names[*]}" go_json "$dir" $flags -count=1 -run "$regex" $pkgs
|
|
}
|
|
|
|
# go_short DIR PACKAGES runs go test -short PACKAGES in DIR; on failure it
|
|
# prints the tail of the output and fails.
|
|
go_short() {
|
|
local dir="$1" pkgs="${2:-./...}" out
|
|
# shellcheck disable=SC2086
|
|
if ! out="$(cd "$dir" && go test -count=1 -short $pkgs 2>&1)"; then
|
|
printf '%s\n' "$out" | grep -v '^ok ' | tail -n 60 >&2
|
|
echo "refuse: go test -short $pkgs in $dir" >&2
|
|
return 1
|
|
fi
|
|
echo "short: $dir ($pkgs)"
|
|
}
|
|
|
|
go_vet() {
|
|
local dir="$1"
|
|
shift
|
|
(cd "$dir" && go vet "$@")
|
|
echo "vet: $dir ($*)"
|
|
}
|
|
|
|
require_dir() {
|
|
[[ -d "$1" ]] || {
|
|
echo "refuse: $2 $1 not found" >&2
|
|
return 1
|
|
}
|
|
}
|
|
|
|
expect_detector() {
|
|
local detector="$1" name="$2" want="$3" payload="$4" log dc=0
|
|
log="$(mktemp)"
|
|
printf '%s\n' "$payload" >"$log"
|
|
"$detector" "$log" >/dev/null 2>&1 || dc=$?
|
|
rm -f "$log"
|
|
if [[ "$dc" -ne "$want" ]]; then
|
|
echo "refuse: self-test $detector $name: exit $dc, want $want" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# expect_mode NAME WANT_EXIT WANT_TEXT MODE ENV... runs this script in MODE
|
|
# with ENV and checks its exit status and that its output carries WANT_TEXT.
|
|
expect_mode() {
|
|
local name="$1" want="$2" text="$3" mode="$4"
|
|
shift 4
|
|
local out rc=0
|
|
out="$(env "$@" bash "${BASH_SOURCE[0]}" "$mode" 2>&1)" || rc=$?
|
|
if [[ "$rc" -ne "$want" ]] || ! grep -qF -- "$text" <<<"$out"; then
|
|
printf '%s\n' "$out" | tail -n 10 >&2
|
|
echo "refuse: self-test $name: exit $rc (want $want), output must carry \"$text\"" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
run_self_test() {
|
|
bash -n "${BASH_SOURCE[0]}"
|
|
|
|
expect_detector detect_go pass 0 '{"Action":"pass","Package":"p","Test":"TestDiaryTracerSmoke"}'
|
|
expect_detector detect_go fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
|
{"Action":"fail","Package":"p","Test":"TestHelpSmoke"}'
|
|
expect_detector detect_go package-fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
|
{"Action":"fail","Package":"p"}'
|
|
expect_detector detect_go build 1 '{"Action":"build-fail","ImportPath":"p"}'
|
|
expect_detector detect_go skip 2 '{"Action":"pass","Package":"p","Test":"TestA"}
|
|
{"Action":"skip","Package":"p","Test":"TestServeLifecycle"}'
|
|
expect_detector detect_go subtest-skip 2 '{"Action":"skip","Package":"p","Test":"TestPeekGlobals/env"}
|
|
{"Action":"pass","Package":"p","Test":"TestPeekGlobals"}'
|
|
expect_detector detect_go subtest-only-pass 3 '{"Action":"pass","Package":"p","Test":"TestOuter/TestListTracerSmoke"}
|
|
{"Action":"pass","Package":"p"}'
|
|
expect_detector detect_go zero 3 '{"Action":"pass","Package":"p"}'
|
|
expect_detector detect_go no-tests 3 '{"Action":"output","Package":"p","Output":"testing: warning: no tests to run\n"}
|
|
{"Action":"pass","Package":"p"}'
|
|
expect_detector detect_go nonjson 4 '{"Action":"pass",'
|
|
REQUIRE_TESTS="TestDiaryTracerSmoke TestConfigEnvSmoke" expect_detector detect_go missing-named 5 \
|
|
'{"Action":"pass","Package":"p","Test":"TestDiaryTracerSmoke"}'
|
|
REQUIRE_TESTS="TestHelp" expect_detector detect_go prefix-is-not-the-test 5 \
|
|
'{"Action":"pass","Package":"p","Test":"TestHelpSmoke"}'
|
|
REQUIRE_TESTS="TestListTracerSmoke" expect_detector detect_go subtest-is-not-the-test 5 \
|
|
'{"Action":"pass","Package":"p","Test":"TestOuter/TestListTracerSmoke"}
|
|
{"Action":"pass","Package":"p","Test":"TestOther"}'
|
|
|
|
# The NOT RUN rule, on a faked host: a docker binary that never answers.
|
|
local tmp
|
|
tmp="$(mktemp -d)"
|
|
mkdir -p "$tmp/bin"
|
|
printf '#!/bin/sh\nexit 1\n' >"$tmp/bin/docker"
|
|
chmod +x "$tmp/bin/docker"
|
|
expect_mode docker-not-run 0 "NOT RUN docker" --docker-gate PATH="$tmp/bin:$PATH" PHASE198_REQUIRE_DOCKER=0
|
|
expect_mode docker-required 1 "FAIL docker" --docker-gate PATH="$tmp/bin:$PATH" PHASE198_REQUIRE_DOCKER=1
|
|
expect_mode docker-stage-needs-daemon 1 "needs a reachable Docker daemon" --docker PATH="$tmp/bin:$PATH"
|
|
rm -rf "$tmp"
|
|
|
|
local mode
|
|
for mode in self-test go hello docs plugins apps landing docker all; do
|
|
grep -q -- "^ --$mode" "${BASH_SOURCE[0]}" || {
|
|
echo "refuse: missing mode --$mode" >&2
|
|
return 1
|
|
}
|
|
done
|
|
echo "phase19.8 self-test passed"
|
|
}
|
|
|
|
run_go() {
|
|
go_vet "$ROOT" ./...
|
|
go_short "$ROOT"
|
|
go_named "$ROOT" ./modules/diary -short "${DIARY_TESTS[@]}"
|
|
go_named "$ROOT" ./modules/bonfire -short "${BONFIRE_TESTS[@]}"
|
|
go_named "$ROOT" ./modules/conga -short "${CONGA_TESTS[@]}"
|
|
go_named "$ROOT" ./modules/lagoon -short "${LAGOON_TESTS[@]}"
|
|
go_named "$ROOT" ./modules/surf -short "${SURF_TESTS[@]}"
|
|
go_named "$ROOT" ./cmd/summer -short "${SUMMER_TESTS[@]}"
|
|
local out
|
|
if ! out="$(cd "$ROOT" && go test -count=1 -race -short ./modules/diary/... ./modules/bonfire/... 2>&1)"; then
|
|
printf '%s\n' "$out" | tail -n 60 >&2
|
|
echo "refuse: go test -race -short of diary and bonfire" >&2
|
|
return 1
|
|
fi
|
|
echo "race: modules/diary, modules/bonfire"
|
|
echo "phase19.8 go passed"
|
|
}
|
|
|
|
not_implemented() {
|
|
echo "stage $1: not implemented" >&2
|
|
return 1
|
|
}
|
|
|
|
run_hello() { not_implemented hello; }
|
|
run_docs() { not_implemented docs; }
|
|
run_plugins() { not_implemented plugins; }
|
|
run_apps() { not_implemented apps; }
|
|
run_landing() { not_implemented landing; }
|
|
run_docker() {
|
|
if ! docker_up; then
|
|
echo "refuse: the docker stage needs a reachable Docker daemon (testcontainers); run it on a Docker host" >&2
|
|
return 1
|
|
fi
|
|
not_implemented docker
|
|
}
|
|
|
|
docker_up() {
|
|
command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1
|
|
}
|
|
|
|
# docker_gate is the docker step of --all: run the stage when a daemon
|
|
# answers, fail with PHASE198_REQUIRE_DOCKER=1, print NOT RUN otherwise.
|
|
docker_gate() {
|
|
if docker_up; then
|
|
if bash "${BASH_SOURCE[0]}" --docker; then
|
|
echo "PASS docker"
|
|
return 0
|
|
fi
|
|
echo "FAIL docker"
|
|
return 1
|
|
fi
|
|
if [[ "${PHASE198_REQUIRE_DOCKER:-0}" == "1" ]]; then
|
|
echo "FAIL docker: no reachable Docker daemon and PHASE198_REQUIRE_DOCKER=1"
|
|
return 1
|
|
fi
|
|
cat >&2 <<'EOF'
|
|
************************************************************************
|
|
NOT RUN docker: no reachable Docker daemon on this host.
|
|
The testcontainers suites (Postgres) of the framework did NOT run. Run on
|
|
a Docker host (through sudo -u golem when the socket is denied) before
|
|
/msd-verify-work:
|
|
bash scripts/check-phase19.8.sh docker
|
|
************************************************************************
|
|
EOF
|
|
echo "NOT RUN docker (Docker-host only; see the notice above)"
|
|
}
|
|
|
|
run_all() {
|
|
local stage
|
|
for stage in self-test go hello docs plugins apps landing; do
|
|
if bash "${BASH_SOURCE[0]}" "--$stage"; then
|
|
echo "PASS $stage"
|
|
else
|
|
echo "FAIL $stage"
|
|
exit 1
|
|
fi
|
|
done
|
|
docker_gate || exit 1
|
|
echo "Phase 19.8 gate passed"
|
|
}
|
|
|
|
case "${1:---all}" in
|
|
--self-test) run_self_test ;;
|
|
--go) run_go ;;
|
|
--hello) run_hello ;;
|
|
--docs) run_docs ;;
|
|
--plugins) run_plugins ;;
|
|
--apps) run_apps ;;
|
|
--landing) run_landing ;;
|
|
--docker) run_docker ;;
|
|
--docker-gate) docker_gate ;;
|
|
--all) run_all ;;
|
|
self-test) run_self_test ;;
|
|
go) run_go ;;
|
|
hello) run_hello ;;
|
|
docs) run_docs ;;
|
|
plugins) run_plugins ;;
|
|
apps) run_apps ;;
|
|
landing) run_landing ;;
|
|
docker) run_docker ;;
|
|
all) run_all ;;
|
|
*) usage ;;
|
|
esac
|