test(12.1-05): threat test for the Phase 12.1 framework contracts and the first gate stages
- TestPhase121Threats: one subtest per mitigated threat T-12.1-01 to T-12.1-15 - roster fixture: sentinel names and knobs for failing hooks and providers - scripts/check-phase12.1.sh: fail-closed go test detector, --self-test and --security
This commit is contained in:
Jakub Zych
269
scripts/check-phase12.1.sh
Executable file
269
scripts/check-phase12.1.sh
Executable file
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase 12.1 fail-closed gate (admin bulk and record actions, preview screen,
|
||||
# row state, permission editor, form seams, and the user plugin's admin
|
||||
# screens built on them).
|
||||
#
|
||||
# Every stage exits non-zero on a failing command, a go test run that fails,
|
||||
# skips, matches zero tests or does not build, and a named security test that
|
||||
# is missing, renamed or skipped. --self-test proves each detector fails
|
||||
# closed on planted input. A stage that is not implemented refuses.
|
||||
#
|
||||
# Framework commands run in this repository. The plugin's tests run inside
|
||||
# the application workspace named by PHASE121_APP (default: the sibling
|
||||
# checkout next to this repository). Output about the application workspace
|
||||
# has the application's name masked; set PHASE121_VERBOSE=1 to see it as it
|
||||
# is while debugging.
|
||||
# Run with FORCE_COLOR unset: bonfire's colour tests read it.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${PHASE121_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
APP="${PHASE121_APP:-$ROOT/../fonoteka.go}"
|
||||
# The user plugin inside the application workspace.
|
||||
PLUGIN="./plugins/golem15/user"
|
||||
APP_NAMES='fonoteka|p[lł]ytarium'
|
||||
|
||||
# The named tests of the security stage, by prefix. Each prefix must match
|
||||
# at least one top-level test that passes; any skip refuses.
|
||||
SECURITY_CABANA=(TestPhase121Threats TestBulkAction TestRecordAction TestRowState TestForbidden
|
||||
TestSoftDeletedRecord TestPreview TestPasswordField TestVirtualFields TestFormRules
|
||||
TestPermissionEditor TestRelationLock TestWritableForeignKey TestInvisibleColumn
|
||||
TestFilterOptionsController)
|
||||
SECURITY_PLUGIN=(TestPhase121Threats TestAdminPrivilegedGroups TestAdminPrivilegedMember
|
||||
TestAdminUserGroupsField TestAdminUserActions TestAdminUserForceDelete TestAdminUserPassword
|
||||
TestAdminUserInvite TestAdminAvatarSharedWithAPI TestAdminGroups TestAdminOrganisations
|
||||
TestAdminOrganisationMembers TestLastSeen)
|
||||
SECURITY_PLUGIN_CLASSES=(TestMergedPermissions TestPermissionSetScan)
|
||||
|
||||
STAGES=(self-test go security removal coverage spa openapi dist docs hygiene app evidence all)
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
check-phase12.1.sh --self-test
|
||||
check-phase12.1.sh --go
|
||||
check-phase12.1.sh --security
|
||||
check-phase12.1.sh --removal
|
||||
check-phase12.1.sh --coverage
|
||||
check-phase12.1.sh --spa
|
||||
check-phase12.1.sh --openapi
|
||||
check-phase12.1.sh --dist
|
||||
check-phase12.1.sh --docs
|
||||
check-phase12.1.sh --hygiene
|
||||
check-phase12.1.sh --app
|
||||
check-phase12.1.sh --evidence
|
||||
check-phase12.1.sh --all (every stage except --removal)
|
||||
|
||||
environment:
|
||||
PHASE121_APP the application workspace (default: the sibling checkout)
|
||||
PHASE121_VERBOSE 1 shows application output without masking its name
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
# mask hides the application's name in output about its workspace.
|
||||
mask() {
|
||||
if [[ "${PHASE121_VERBOSE:-}" == "1" ]]; then
|
||||
cat
|
||||
else
|
||||
sed -E "s/($APP_NAMES)(\.go)?/<app>/gI"
|
||||
fi
|
||||
}
|
||||
|
||||
# where DIR names a directory in output: the application workspace is never
|
||||
# printed by its path.
|
||||
where() {
|
||||
if [[ "$1" == "$APP" ]]; then
|
||||
echo "the application workspace"
|
||||
else
|
||||
echo "${1#"$ROOT"/}"
|
||||
fi
|
||||
}
|
||||
|
||||
# detect reads go test -json. Exit 1 fail or build failure, 2 skip, 3 zero
|
||||
# tests or "no tests to run", 4 non-JSON, 5 a required prefix has no passing
|
||||
# top-level test. REQUIRE_PREFIXES lists the prefixes.
|
||||
detect() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json, os, sys
|
||||
path = sys.argv[1]
|
||||
prefixes = os.environ.get("REQUIRE_PREFIXES", "").split()
|
||||
passed = set()
|
||||
failed = []
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
for raw in fh:
|
||||
line = raw.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
print("refuse: non-json test output", file=sys.stderr)
|
||||
sys.exit(4)
|
||||
action = ev.get("Action")
|
||||
test = ev.get("Test") or ""
|
||||
pkg = ev.get("Package") or ev.get("ImportPath") or ""
|
||||
if action == "build-fail" or (action == "fail" and ev.get("FailedBuild")):
|
||||
print(f"refuse: build failed {pkg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if action == "output" and "no tests to run" in (ev.get("Output") or ""):
|
||||
print(f"refuse: no tests to run in {pkg}", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
if action == "skip" and test:
|
||||
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if action == "fail":
|
||||
failed.append(f"{pkg} {test}".strip())
|
||||
if action == "pass" and test:
|
||||
passed.add(test)
|
||||
if failed:
|
||||
print("refuse: failed " + ", ".join(failed), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not passed:
|
||||
print("refuse: zero tests", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
top = {name for name in passed if "/" not in name}
|
||||
missing = [p for p in prefixes if not any(name.startswith(p) for name in top)]
|
||||
if missing:
|
||||
print("refuse: missing named test: no passing test for " + ", ".join(missing), file=sys.stderr)
|
||||
sys.exit(5)
|
||||
PY
|
||||
}
|
||||
|
||||
# go_json DIR [go test args...] runs go test -json -count=1 through detect.
|
||||
go_json() {
|
||||
local dir="$1"
|
||||
shift
|
||||
local log err out rc=0 dc=0
|
||||
log="$(mktemp)"
|
||||
err="$(mktemp)"
|
||||
out="$(mktemp)"
|
||||
(cd "$dir" && go test -json -count=1 "$@") >"$log" 2>"$err" || rc=$?
|
||||
detect "$log" 2>"$out" || dc=$?
|
||||
if [[ "$rc" -ne 0 || "$dc" -ne 0 ]]; then
|
||||
{
|
||||
cat "$err" || true
|
||||
grep -v '^{' "$log" | tail -n 20 || true
|
||||
python3 - "$log" <<'PY' || true
|
||||
import json, sys
|
||||
for raw in open(sys.argv[1], encoding="utf-8", errors="replace"):
|
||||
try:
|
||||
ev = json.loads(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
text = ev.get("Output") or ""
|
||||
if ev.get("Action") == "output" and ("--- FAIL" in text or "_test.go:" in text or "panic:" in text):
|
||||
sys.stdout.write(text)
|
||||
PY
|
||||
cat "$out" || true
|
||||
echo "refuse: go test $* in $(where "$dir") (test=$rc detect=$dc)"
|
||||
} 2>&1 | mask | tail -n 80 >&2
|
||||
rm -f "$log" "$err" "$out"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$log" "$err" "$out"
|
||||
}
|
||||
|
||||
# named DIR PKG PREFIX... runs the tests matching the prefixes verbosely and
|
||||
# requires a passing top-level test for each one.
|
||||
named() {
|
||||
local dir="$1" pkg="$2"
|
||||
shift 2
|
||||
local regex
|
||||
regex="^($(
|
||||
IFS='|'
|
||||
echo "$*"
|
||||
))"
|
||||
REQUIRE_PREFIXES="$*" go_json "$dir" "$pkg" -v -run "$regex"
|
||||
}
|
||||
|
||||
expect_detect() {
|
||||
local name="$1" want="$2" payload="$3" log dc=0
|
||||
log="$(mktemp)"
|
||||
printf '%s\n' "$payload" >"$log"
|
||||
detect "$log" 2>/dev/null || dc=$?
|
||||
rm -f "$log"
|
||||
if [[ "$dc" -ne "$want" ]]; then
|
||||
echo "refuse: self-test $name: detector exit $dc, want $want" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
need_app() {
|
||||
[[ -d "$APP" && -f "$APP/go.work" ]] || {
|
||||
echo "refuse: the application workspace was not found (set PHASE121_APP)" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
run_self_test() {
|
||||
bash -n "${BASH_SOURCE[0]}"
|
||||
expect_detect pass 0 '{"Action":"pass","Package":"p","Test":"TestPhase121Threats"}'
|
||||
expect_detect fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p","Test":"TestPhase121Threats/T-12.1-28"}'
|
||||
expect_detect package-fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p"}'
|
||||
expect_detect build 1 '{"Action":"build-fail","ImportPath":"p"}'
|
||||
expect_detect build-flag 1 '{"Action":"pass","Package":"q","Test":"TestA"}
|
||||
{"Action":"fail","Package":"p","FailedBuild":"p"}'
|
||||
expect_detect skip 2 '{"Action":"skip","Package":"p","Test":"TestPhase121Threats/T-12.1-38"}'
|
||||
expect_detect zero 3 '{"Action":"pass","Package":"p"}'
|
||||
expect_detect no-tests 3 '{"Action":"output","Package":"p","Output":"testing: warning: no tests to run\n"}
|
||||
{"Action":"pass","Package":"p"}'
|
||||
expect_detect nonjson 4 '{"Action":"pass",'
|
||||
REQUIRE_PREFIXES="TestPhase121Threats TestAdminPrivilegedMember" expect_detect missing-named 5 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestPhase121Threats"}'
|
||||
REQUIRE_PREFIXES="TestAdminPrivilegedMember" expect_detect subtest-only 5 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestOther/TestAdminPrivilegedMember"}'
|
||||
REQUIRE_PREFIXES="TestPhase121Threats TestAdminPrivilegedMember" expect_detect named 0 \
|
||||
'{"Action":"pass","Package":"p","Test":"TestPhase121Threats"}
|
||||
{"Action":"pass","Package":"p","Test":"TestAdminPrivilegedMember"}'
|
||||
local stage
|
||||
for stage in "${STAGES[@]}"; do
|
||||
grep -q -- "^ --$stage)" "${BASH_SOURCE[0]}" || {
|
||||
echo "refuse: missing mode --$stage" >&2
|
||||
return 1
|
||||
}
|
||||
grep -q -- "check-phase12.1.sh --$stage" "${BASH_SOURCE[0]}" || {
|
||||
echo "refuse: usage does not list --$stage" >&2
|
||||
return 1
|
||||
}
|
||||
done
|
||||
# The mask hides the application's name unless asked not to.
|
||||
local masked
|
||||
masked="$(printf 'ok \tgit.example.test/x/fonoteka.go/parity\n' | PHASE121_VERBOSE= mask)"
|
||||
if grep -qiE "$APP_NAMES" <<<"$masked"; then
|
||||
echo "refuse: self-test mask left the application's name: $masked" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "phase12.1 self-test passed"
|
||||
}
|
||||
|
||||
run_security() {
|
||||
need_app
|
||||
named "$ROOT" ./modules/cabana "${SECURITY_CABANA[@]}"
|
||||
named "$APP" "$PLUGIN" "${SECURITY_PLUGIN[@]}"
|
||||
named "$APP" "$PLUGIN/classes" "${SECURITY_PLUGIN_CLASSES[@]}"
|
||||
echo "phase12.1 security passed"
|
||||
}
|
||||
|
||||
not_implemented() {
|
||||
echo "refuse: stage --$1 is not implemented" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--self-test) run_self_test ;;
|
||||
--go) not_implemented go ;;
|
||||
--security) run_security ;;
|
||||
--removal) not_implemented removal ;;
|
||||
--coverage) not_implemented coverage ;;
|
||||
--spa) not_implemented spa ;;
|
||||
--openapi) not_implemented openapi ;;
|
||||
--dist) not_implemented dist ;;
|
||||
--docs) not_implemented docs ;;
|
||||
--hygiene) not_implemented hygiene ;;
|
||||
--app) not_implemented app ;;
|
||||
--evidence) not_implemented evidence ;;
|
||||
--all) not_implemented all ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user