test(11.2): add the phase gate with the plugin stage
- scripts/check-phase11.2.sh --plugin: go vet, the named plugin tests through a go test -json detector (fail, skip, missing test, zero tests and no tests to run all refuse) and a 90% statement coverage floor - unknown modes print usage and exit 2
This commit is contained in:
Jakub Zych
144
scripts/check-phase11.2.sh
Executable file
144
scripts/check-phase11.2.sh
Executable file
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase 11.2 fail-closed gate (summercms.io Alpha 0.1 landing page).
|
||||
#
|
||||
# The phase spans four repositories: summercms.go (this one, the framework),
|
||||
# sm-summercmsio-app (the application, a sibling directory), its site plugin
|
||||
# submodule plugins/golem15/summercms and its Nuxt site submodule
|
||||
# vue-summercmsio-app. Each stage runs one repository's checks. Go tests run
|
||||
# with -json through a detector that requires every named test to PASS: a
|
||||
# failure, a skip, a missing or renamed test, zero matched tests and "no
|
||||
# tests to run" all refuse.
|
||||
set -euo pipefail
|
||||
|
||||
SCRATCH=()
|
||||
cleanup() {
|
||||
[ "${#SCRATCH[@]}" -eq 0 ] || rm -rf "${SCRATCH[@]}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
ROOT="${PHASE11_2_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
APP="${PHASE11_2_APP:-$ROOT/../sm-summercmsio-app}"
|
||||
PLUG="$APP/plugins/golem15/summercms"
|
||||
SITE="$APP/vue-summercmsio-app"
|
||||
# Statement coverage floors (SC5).
|
||||
PLUGIN_COVERAGE_MIN=90.0
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
check-phase11.2.sh --plugin
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
refuse() {
|
||||
echo "refuse: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
need_dir() {
|
||||
[ -d "$1" ] || refuse "$1 not found (the phase expects sm-summercmsio-app next to summercms.go)"
|
||||
}
|
||||
|
||||
# detect_json reads a go test -json log. It refuses a build failure, any
|
||||
# failed test or package, any skipped test, "no tests to run", a run with
|
||||
# zero passing tests, and any required "<package> <Test>" pair (from
|
||||
# PHASE11_2_REQUIRE, newline separated) that did not PASS.
|
||||
detect_json() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json, os, sys
|
||||
path = sys.argv[1]
|
||||
require = [r.strip() for r in os.environ.get("PHASE11_2_REQUIRE", "").splitlines() if r.strip()]
|
||||
passed = set()
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
for raw in fh:
|
||||
line = raw.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
print("refuse: non-json test output", file=sys.stderr)
|
||||
sys.exit(4)
|
||||
action, test, pkg = ev.get("Action"), ev.get("Test") or "", ev.get("Package") or ""
|
||||
if action == "build-fail" or (action == "fail" and ev.get("FailedBuild")):
|
||||
print(f"refuse: build failed in {pkg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if action == "output" and "no tests to run" in (ev.get("Output") or ""):
|
||||
print(f"refuse: no tests to run in {pkg}", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
if action == "skip" and test:
|
||||
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if action == "fail":
|
||||
print(f"refuse: failed {pkg} {test}".rstrip(), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if action == "pass" and test:
|
||||
passed.add(f"{pkg} {test}")
|
||||
if not passed:
|
||||
print("refuse: zero tests", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
missing = [r for r in require if r not in passed]
|
||||
if missing:
|
||||
print("refuse: named tests did not pass (missing, renamed or filtered out): " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(5)
|
||||
print(f"named tests passed: {len(require)} required, {len(passed)} passing (subtests included)")
|
||||
PY
|
||||
}
|
||||
|
||||
# named_tests DIR PKG NAME... runs `go -C DIR test -json -count=1 PKG -run
|
||||
# '^(NAME|...)$'` and requires every named test to PASS. Environment set on
|
||||
# the call (VAR=1 named_tests ...) reaches go test; run it in a subshell to
|
||||
# unset a variable for one call.
|
||||
named_tests() {
|
||||
local dir="$1" pkg="$2" log import names name require="" code=0
|
||||
shift 2
|
||||
import="$(go -C "$dir" list -f '{{.ImportPath}}' "$pkg")" || refuse "go list $pkg in $dir failed" || return 1
|
||||
names="$(IFS='|'; echo "$*")"
|
||||
for name in "$@"; do
|
||||
require+="$import $name"$'\n'
|
||||
done
|
||||
log="$(mktemp)"
|
||||
go -C "$dir" test -json -count=1 "$pkg" -run "^($names)\$" >"$log" 2>&1 || code=$?
|
||||
if ! PHASE11_2_REQUIRE="$require" detect_json "$log"; then
|
||||
rm -f "$log"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$log"
|
||||
[ "$code" -eq 0 ] || refuse "go test $pkg in $dir exited $code"
|
||||
}
|
||||
|
||||
# coverage_at_least DIR MIN PKG... prints the total statement coverage of
|
||||
# the packages and refuses when it is below MIN. Build-gated tests skip
|
||||
# here (their variables are cleared), so the figure comes from tests that
|
||||
# need no build output.
|
||||
coverage_at_least() {
|
||||
local dir="$1" min="$2" profile total
|
||||
shift 2
|
||||
profile="$(mktemp)"
|
||||
SCRATCH+=("$profile")
|
||||
env -u SUMMERCMS_REQUIRE_BUILD -u SUMMERCMS_TERMINAL_CHECK -u SUMMERCMS_CHECK_EXTERNAL \
|
||||
go -C "$dir" test -count=1 -coverprofile="$profile" "$@" >/dev/null || refuse "coverage run in $dir failed" || return 1
|
||||
total="$(go -C "$dir" tool cover -func="$profile" | awk '/^total:/ {sub("%", "", $NF); print $NF}')"
|
||||
[ -n "$total" ] || refuse "no coverage total in $dir" || return 1
|
||||
echo "coverage: $total% of statements in $dir (minimum $min%)"
|
||||
awk -v t="$total" -v m="$min" 'BEGIN { exit !(t + 0 >= m + 0) }' || refuse "coverage $total% is below $min% in $dir"
|
||||
}
|
||||
|
||||
PLUGIN_TESTS=(TestStaticSmoke TestStaticSite TestStaticDocs TestStaticConditionalAndRange TestStaticNoBlockingHeaders
|
||||
TestStaticRedirectLocations TestStaticMissing404Page TestNewHandlersMissingIndex TestContentType TestSiteImmutable
|
||||
TestRoutesAssemble TestRoutesFailClosed TestRoutesCoexistWithAdminPatterns TestPluginIdentity TestPluginEmbeddedTree
|
||||
TestPageLinks TestResolve)
|
||||
|
||||
run_plugin() {
|
||||
need_dir "$PLUG"
|
||||
go -C "$PLUG" vet ./...
|
||||
(unset SUMMERCMS_REQUIRE_BUILD && named_tests "$PLUG" . "${PLUGIN_TESTS[@]}") || refuse "plugin: named tests"
|
||||
coverage_at_least "$PLUG" "$PLUGIN_COVERAGE_MIN" ./...
|
||||
echo "phase11.2 plugin passed"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--plugin) run_plugin ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user