#!/usr/bin/env bash # check-phase19.8: fail-closed gate for Phase 19.8, the console commands # overhaul. It proves the phase across this framework (diary, bonfire and the # swept framework commands), the example application, the docs, every shared # plugin swept in plan 03, every application that runs the new console, the # landing site and the Docker-backed paths. # # Stages: # self-test the go test -json detector on planted output (pass, fail, # skip, a subtest-only pass, zero tests, "no tests to run", # a missing required name), the docker NOT RUN rule on a faked # PATH, and that every mode is wired. # go go vet ./... and go test -short ./... here, the required # smoke and unit tests by name per package, then go test -race # -short of diary and bonfire. # hello the example application (its own module in the workspace). # docs the decision note, TestDocsTree, summer docs:build --check, # no consuming application named in the console and logging # sources and pages, and every command in summer's JSON listing # has examples. # plugins each swept shared plugin: vet, short tests, its warn-only # TestCommandsDocumented by name, a build without the # workspace, and the additive check against the commit before # its sweep commit. # apps each application: go build ./..., the generated main.go # unchanged by summer build, bin/ list and help migrate. # landing the landing site's tests and the roadmap deck's tests. # docker Docker host only: the testcontainers tests by name without # -short. # # A Go stage exits non-zero on a failing command, or on a go test -json run # that fails, skips a test (a subtest included), matches zero tests, prints # "no tests to run" or misses a required name (a subtest pass never counts # for its parent's name). # # --all runs self-test, go, hello, docs, plugins, apps and landing, then # docker when a Docker daemon answers (NOT RUN docker otherwise, a failure # with PHASE198_REQUIRE_DOCKER=1). # # Environment overrides (default: the sibling checkouts next to this # repository): PHASE198_ROOT (this repository), PLUGINS_ROOT (the directory # holding the sm-*-plugin checkouts), FONOTEKA (the application workspace # that carries the user, golem and feedback plugin checkouts), DEMO_APP, # BM_APP, GRZYBY_APP, SUMMERCMSIO_APP, PHASE198_REQUIRE_DOCKER. set -euo pipefail unset FORCE_COLOR export NO_COLOR=1 ROOT="${PHASE198_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" PLUGINS_ROOT="${PLUGINS_ROOT:-$ROOT/..}" FONOTEKA="${FONOTEKA:-$ROOT/../fono""teka.go}" DEMO_APP="${DEMO_APP:-$ROOT/../sm-demo-app}" BM_APP="${BM_APP:-$ROOT/../sm-bm-app}" GRZYBY_APP="${GRZYBY_APP:-$ROOT/../sm-grzybyfunkcjonalne-app}" SUMMERCMSIO_APP="${SUMMERCMSIO_APP:-$ROOT/../sm-summercmsio-app}" # --------------------------------------------------------------------------- # Required test names. Each list runs with -run '^(a|b|...)$' and every name # must pass; a name that skips or does not run fails the stage. # go stage (-short). DIARY_TESTS=( # Plan 01. TestDiaryTracerSmoke TestConfigEnvSmoke TestDailyDegradesSmoke ) BONFIRE_TESTS=( # Plan 02. TestListTracerSmoke TestHelpSmoke TestListJSONSmoke TestErrorRenderSmoke TestWidgetMatrixSmoke ) CONGA_TESTS=( # Plan 01. TestRiverLoggerUsesDiarySmoke ) LAGOON_TESTS=( # Plan 01. TestGormLoggerUsesDiarySmoke ) SURF_TESTS=( # Plan 03. TestServeSummarySmoke TestShutdownReportSmoke TestAccessLogSmoke ) SUMMER_TESTS=( # Plan 03. TestCommandsDocumented TestDocsTree ) usage() { cat >&2 <<'EOF' usage: check-phase19.8.sh --self-test check-phase19.8.sh go | hello | docs | plugins | apps | landing | docker check-phase19.8.sh --go | --hello | --docs | --plugins | --apps | --landing | --docker check-phase19.8.sh --all EOF exit 2 } # detect_go reads go test -json. Exit 1 fail/build, 2 skip, 3 zero/no-tests, # 4 non-JSON, 5 missing required name. detect_go() { python3 - "$1" <<'PY' import json, os, sys path = sys.argv[1] require = [n for n in os.environ.get("REQUIRE_TESTS", "").split() if n] passed = set() failed = [] with open(path, encoding="utf-8", errors="replace") as fh: for raw in fh: line = raw.strip() if not line.startswith("{"): continue try: ev = json.loads(line) except json.JSONDecodeError: print("refuse: non-json test output", file=sys.stderr) sys.exit(4) action = ev.get("Action") test = ev.get("Test") or "" pkg = ev.get("Package") or ev.get("ImportPath") or "" if action == "build-fail" or (action == "fail" and ev.get("FailedBuild")): print(f"refuse: build failed {pkg}", file=sys.stderr) sys.exit(1) text = ev.get("Output") or "" if action == "output" and "no tests to run" in text: print(f"refuse: no tests to run in {pkg}", file=sys.stderr) sys.exit(3) if action == "skip" and test: print(f"refuse: skipped {pkg} {test}", file=sys.stderr) sys.exit(2) if action == "fail": failed.append(f"{pkg} {test}".strip()) if action == "pass" and test and "/" not in test: passed.add(test) if failed: print("refuse: failed " + ", ".join(failed), file=sys.stderr) sys.exit(1) if not passed: print("refuse: zero tests", file=sys.stderr) sys.exit(3) missing = [n for n in require if n not in passed] if missing: print("refuse: required tests did not pass: " + ", ".join(missing), file=sys.stderr) sys.exit(5) PY } # go_json DIR ARGS... runs go test -json ARGS in DIR and checks the stream # with detect_go. go_json() { local dir="$1" shift local log err rc=0 dc=0 log="$(mktemp)" err="$(mktemp)" (cd "$dir" && go test -json "$@") >"$log" 2>"$err" || rc=$? detect_go "$log" || dc=$? if [[ "$rc" -ne 0 || "$dc" -ne 0 ]]; then cat "$err" >&2 || true grep -v '^{' "$log" | tail -n 40 >&2 || true python3 -c 'import json,sys for raw in open(sys.argv[1], errors="replace"): try: ev = json.loads(raw) except ValueError: continue if ev.get("Action") == "output" and ("FAIL" in ev.get("Output", "") or "SKIP" in ev.get("Output", "") or "_test.go:" in ev.get("Output", "")): sys.stderr.write(ev["Output"])' "$log" 2>&1 | tail -n 40 >&2 || true rm -f "$log" "$err" echo "refuse: go test $* in $dir (test=$rc detect=$dc)" >&2 return 1 fi echo "go: $(grep -c '"Action":"pass","Package":"[^"]*","Test":"[^"/]*","Elapsed"' "$log") top-level tests passed in $dir ($*)" rm -f "$log" "$err" } # go_named DIR PACKAGES FLAGS NAME... runs exactly the named top-level tests # and requires every one of them to pass. PACKAGES and FLAGS are word-split. go_named() { local dir="$1" pkgs="$2" flags="$3" shift 3 if [[ "$#" -eq 0 ]]; then echo "refuse: go_named $pkgs in $dir without required names" >&2 return 1 fi local names=("$@") regex regex="^($(IFS='|'; echo "${names[*]}"))\$" # shellcheck disable=SC2086 REQUIRE_TESTS="${names[*]}" go_json "$dir" $flags -count=1 -run "$regex" $pkgs } # go_short DIR PACKAGES runs go test -short PACKAGES in DIR; on failure it # prints the tail of the output and fails. go_short() { local dir="$1" pkgs="${2:-./...}" out # shellcheck disable=SC2086 if ! out="$(cd "$dir" && go test -count=1 -short $pkgs 2>&1)"; then printf '%s\n' "$out" | grep -v '^ok ' | tail -n 60 >&2 echo "refuse: go test -short $pkgs in $dir" >&2 return 1 fi echo "short: $dir ($pkgs)" } go_vet() { local dir="$1" shift (cd "$dir" && go vet "$@") echo "vet: $dir ($*)" } require_dir() { [[ -d "$1" ]] || { echo "refuse: $2 $1 not found" >&2 return 1 } } expect_detector() { local detector="$1" name="$2" want="$3" payload="$4" log dc=0 log="$(mktemp)" printf '%s\n' "$payload" >"$log" "$detector" "$log" >/dev/null 2>&1 || dc=$? rm -f "$log" if [[ "$dc" -ne "$want" ]]; then echo "refuse: self-test $detector $name: exit $dc, want $want" >&2 return 1 fi } # expect_mode NAME WANT_EXIT WANT_TEXT MODE ENV... runs this script in MODE # with ENV and checks its exit status and that its output carries WANT_TEXT. expect_mode() { local name="$1" want="$2" text="$3" mode="$4" shift 4 local out rc=0 out="$(env "$@" bash "${BASH_SOURCE[0]}" "$mode" 2>&1)" || rc=$? if [[ "$rc" -ne "$want" ]] || ! grep -qF -- "$text" <<<"$out"; then printf '%s\n' "$out" | tail -n 10 >&2 echo "refuse: self-test $name: exit $rc (want $want), output must carry \"$text\"" >&2 return 1 fi } run_self_test() { bash -n "${BASH_SOURCE[0]}" expect_detector detect_go pass 0 '{"Action":"pass","Package":"p","Test":"TestDiaryTracerSmoke"}' expect_detector detect_go fail 1 '{"Action":"pass","Package":"p","Test":"TestA"} {"Action":"fail","Package":"p","Test":"TestHelpSmoke"}' expect_detector detect_go package-fail 1 '{"Action":"pass","Package":"p","Test":"TestA"} {"Action":"fail","Package":"p"}' expect_detector detect_go build 1 '{"Action":"build-fail","ImportPath":"p"}' expect_detector detect_go skip 2 '{"Action":"pass","Package":"p","Test":"TestA"} {"Action":"skip","Package":"p","Test":"TestServeLifecycle"}' expect_detector detect_go subtest-skip 2 '{"Action":"skip","Package":"p","Test":"TestPeekGlobals/env"} {"Action":"pass","Package":"p","Test":"TestPeekGlobals"}' expect_detector detect_go subtest-only-pass 3 '{"Action":"pass","Package":"p","Test":"TestOuter/TestListTracerSmoke"} {"Action":"pass","Package":"p"}' expect_detector detect_go zero 3 '{"Action":"pass","Package":"p"}' expect_detector detect_go no-tests 3 '{"Action":"output","Package":"p","Output":"testing: warning: no tests to run\n"} {"Action":"pass","Package":"p"}' expect_detector detect_go nonjson 4 '{"Action":"pass",' REQUIRE_TESTS="TestDiaryTracerSmoke TestConfigEnvSmoke" expect_detector detect_go missing-named 5 \ '{"Action":"pass","Package":"p","Test":"TestDiaryTracerSmoke"}' REQUIRE_TESTS="TestHelp" expect_detector detect_go prefix-is-not-the-test 5 \ '{"Action":"pass","Package":"p","Test":"TestHelpSmoke"}' REQUIRE_TESTS="TestListTracerSmoke" expect_detector detect_go subtest-is-not-the-test 5 \ '{"Action":"pass","Package":"p","Test":"TestOuter/TestListTracerSmoke"} {"Action":"pass","Package":"p","Test":"TestOther"}' # The NOT RUN rule, on a faked host: a docker binary that never answers. local tmp tmp="$(mktemp -d)" mkdir -p "$tmp/bin" printf '#!/bin/sh\nexit 1\n' >"$tmp/bin/docker" chmod +x "$tmp/bin/docker" expect_mode docker-not-run 0 "NOT RUN docker" --docker-gate PATH="$tmp/bin:$PATH" PHASE198_REQUIRE_DOCKER=0 expect_mode docker-required 1 "FAIL docker" --docker-gate PATH="$tmp/bin:$PATH" PHASE198_REQUIRE_DOCKER=1 expect_mode docker-stage-needs-daemon 1 "needs a reachable Docker daemon" --docker PATH="$tmp/bin:$PATH" rm -rf "$tmp" local mode for mode in self-test go hello docs plugins apps landing docker all; do grep -q -- "^ --$mode" "${BASH_SOURCE[0]}" || { echo "refuse: missing mode --$mode" >&2 return 1 } done echo "phase19.8 self-test passed" } run_go() { go_vet "$ROOT" ./... go_short "$ROOT" go_named "$ROOT" ./modules/diary -short "${DIARY_TESTS[@]}" go_named "$ROOT" ./modules/bonfire -short "${BONFIRE_TESTS[@]}" go_named "$ROOT" ./modules/conga -short "${CONGA_TESTS[@]}" go_named "$ROOT" ./modules/lagoon -short "${LAGOON_TESTS[@]}" go_named "$ROOT" ./modules/surf -short "${SURF_TESTS[@]}" go_named "$ROOT" ./cmd/summer -short "${SUMMER_TESTS[@]}" local out if ! out="$(cd "$ROOT" && go test -count=1 -race -short ./modules/diary/... ./modules/bonfire/... 2>&1)"; then printf '%s\n' "$out" | tail -n 60 >&2 echo "refuse: go test -race -short of diary and bonfire" >&2 return 1 fi echo "race: modules/diary, modules/bonfire" echo "phase19.8 go passed" } not_implemented() { echo "stage $1: not implemented" >&2 return 1 } run_hello() { not_implemented hello; } run_docs() { not_implemented docs; } run_plugins() { not_implemented plugins; } run_apps() { not_implemented apps; } run_landing() { not_implemented landing; } run_docker() { if ! docker_up; then echo "refuse: the docker stage needs a reachable Docker daemon (testcontainers); run it on a Docker host" >&2 return 1 fi not_implemented docker } docker_up() { command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1 } # docker_gate is the docker step of --all: run the stage when a daemon # answers, fail with PHASE198_REQUIRE_DOCKER=1, print NOT RUN otherwise. docker_gate() { if docker_up; then if bash "${BASH_SOURCE[0]}" --docker; then echo "PASS docker" return 0 fi echo "FAIL docker" return 1 fi if [[ "${PHASE198_REQUIRE_DOCKER:-0}" == "1" ]]; then echo "FAIL docker: no reachable Docker daemon and PHASE198_REQUIRE_DOCKER=1" return 1 fi cat >&2 <<'EOF' ************************************************************************ NOT RUN docker: no reachable Docker daemon on this host. The testcontainers suites (Postgres) of the framework did NOT run. Run on a Docker host (through sudo -u golem when the socket is denied) before /msd-verify-work: bash scripts/check-phase19.8.sh docker ************************************************************************ EOF echo "NOT RUN docker (Docker-host only; see the notice above)" } run_all() { local stage for stage in self-test go hello docs plugins apps landing; do if bash "${BASH_SOURCE[0]}" "--$stage"; then echo "PASS $stage" else echo "FAIL $stage" exit 1 fi done docker_gate || exit 1 echo "Phase 19.8 gate passed" } case "${1:---all}" in --self-test) run_self_test ;; --go) run_go ;; --hello) run_hello ;; --docs) run_docs ;; --plugins) run_plugins ;; --apps) run_apps ;; --landing) run_landing ;; --docker) run_docker ;; --docker-gate) docker_gate ;; --all) run_all ;; self-test) run_self_test ;; go) run_go ;; hello) run_hello ;; docs) run_docs ;; plugins) run_plugins ;; apps) run_apps ;; landing) run_landing ;; docker) run_docker ;; all) run_all ;; *) usage ;; esac