test(08-01): add failing RFC 8414 metadata RED test and fail-closed verifier
- wristband.Server.Metadata is a compiling 501 stub; TestPhase8RedMetadata asserts the exact unwrapped PHP metadata document, headers and status and fails with the PHASE8_RED:metadata sentinel (D-06) - scripts/check-phase8-red.sh implements the shared go/shell RED contract for the rest of Phase 8: exact selected test/package failure plus sentinel, rejecting unrelated fail actions, compile/setup failures, panics, malformed JSON, missing/duplicate sentinels and zero selection (D-04/D-18)
This commit is contained in:
Jakub Zych
222
scripts/check-phase8-red.sh
Executable file
222
scripts/check-phase8-red.sh
Executable file
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fail-closed Phase 8 RED verifier (08-CONTEXT.md D-04/D-18; 08-01-PLAN.md
|
||||
# Task 1). Every later Phase 8 plan proves its RED test through this script
|
||||
# before implementing the matching GREEN, so its acceptance is intentionally
|
||||
# strict: it must accept exactly one deliberate, exact-sentinel behavior
|
||||
# failure and reject every other failure class (unrelated test/package,
|
||||
# compile/setup failure, panic, malformed output, or zero selection).
|
||||
#
|
||||
# D-01: standard library only. The `go` mode delegates JSON-event evaluation
|
||||
# to a small stdlib-only Go program (D-01 applies to the verifier too, not
|
||||
# just wristband) so this script never depends on jq or another JSON tool.
|
||||
#
|
||||
# Usage:
|
||||
# check-phase8-red.sh go <sentinel> <package> <test> -- <go test -json command...>
|
||||
# check-phase8-red.sh shell <sentinel> <stage> -- <command...>
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
check-phase8-red.sh go <sentinel> <package> <test> -- <go test -json command...>
|
||||
check-phase8-red.sh shell <sentinel> <stage> -- <command...>
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
refuse() {
|
||||
echo "REFUSE: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[[ $# -ge 1 ]] || usage
|
||||
MODE="$1"
|
||||
shift
|
||||
|
||||
case "$MODE" in
|
||||
go)
|
||||
[[ $# -ge 4 ]] || usage
|
||||
SENTINEL="$1"
|
||||
PKG="$2"
|
||||
TEST="$3"
|
||||
shift 3
|
||||
[[ "${1:-}" == "--" ]] || usage
|
||||
shift
|
||||
[[ $# -ge 1 ]] || usage
|
||||
|
||||
OUT_FILE="$(mktemp)"
|
||||
CHECKER_FILE="$(mktemp --suffix=.go)"
|
||||
cleanup_go() { rm -f "$OUT_FILE" "$CHECKER_FILE"; }
|
||||
trap cleanup_go EXIT
|
||||
|
||||
set +e
|
||||
"$@" >"$OUT_FILE"
|
||||
set -e
|
||||
|
||||
cat >"$CHECKER_FILE" <<'GOEOF'
|
||||
// Command check-phase8-red-checker evaluates one go test -json event stream
|
||||
// against the Phase 8 fail-closed RED contract (08-01-PLAN.md Task 1). It is
|
||||
// intentionally stdlib-only (D-01) and is invoked by check-phase8-red.sh via
|
||||
// `go run`, never built into the module.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type event struct {
|
||||
Action string `json:"Action"`
|
||||
Package string `json:"Package"`
|
||||
Test string `json:"Test"`
|
||||
Output string `json:"Output"`
|
||||
FailedBuild string `json:"FailedBuild"`
|
||||
}
|
||||
|
||||
func refuse(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, "REFUSE: "+format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 5 {
|
||||
refuse("usage: checker <sentinel> <package> <test> <output-file>")
|
||||
}
|
||||
sentinel, pkg, test, outPath := os.Args[1], os.Args[2], os.Args[3], os.Args[4]
|
||||
|
||||
f, err := os.Open(outPath)
|
||||
if err != nil {
|
||||
refuse("cannot open captured output: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
|
||||
var (
|
||||
lineCount int
|
||||
selectedRun bool
|
||||
selectedFail bool
|
||||
packageFail bool
|
||||
sentinelCount int
|
||||
)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
lineCount++
|
||||
|
||||
var e event
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
refuse("malformed JSON event (not go test -json output): %s", line)
|
||||
}
|
||||
|
||||
if strings.Contains(e.Output, "panic:") {
|
||||
refuse("panic detected in test output: %s", strings.TrimSpace(e.Output))
|
||||
}
|
||||
if strings.Contains(e.Output, "[build failed]") || strings.Contains(e.Output, "[setup failed]") {
|
||||
refuse("compile/setup failure detected: %s", strings.TrimSpace(e.Output))
|
||||
}
|
||||
if e.FailedBuild != "" {
|
||||
refuse("build failure detected (FailedBuild=%s)", e.FailedBuild)
|
||||
}
|
||||
if e.Action == "build-fail" {
|
||||
refuse("build failure (build-fail action) for %s", e.Package)
|
||||
}
|
||||
|
||||
sentinelCount += strings.Count(e.Output, sentinel)
|
||||
|
||||
switch e.Action {
|
||||
case "run":
|
||||
if e.Test == test && e.Package == pkg {
|
||||
selectedRun = true
|
||||
}
|
||||
case "fail":
|
||||
switch {
|
||||
case e.Test == test && e.Package == pkg:
|
||||
selectedFail = true
|
||||
case e.Test == "" && e.Package == pkg:
|
||||
packageFail = true
|
||||
default:
|
||||
refuse("unrelated failure: package=%q test=%q", e.Package, e.Test)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
refuse("reading captured output: %v", err)
|
||||
}
|
||||
|
||||
if lineCount == 0 {
|
||||
refuse("no JSON events observed (empty or non -json output)")
|
||||
}
|
||||
if !selectedRun {
|
||||
refuse("selected test %q in package %q never ran (zero selection or build/setup failure)", test, pkg)
|
||||
}
|
||||
if !selectedFail {
|
||||
refuse("selected test %q in package %q did not fail", test, pkg)
|
||||
}
|
||||
if !packageFail {
|
||||
refuse("package %q did not report a package-level failure", pkg)
|
||||
}
|
||||
if sentinelCount == 0 {
|
||||
refuse("sentinel %q was not observed in test output", sentinel)
|
||||
}
|
||||
if sentinelCount > 1 {
|
||||
refuse("sentinel %q observed %d times, expected exactly 1", sentinel, sentinelCount)
|
||||
}
|
||||
|
||||
fmt.Printf("PHASE8_RED_OK:%s\n", sentinel)
|
||||
}
|
||||
GOEOF
|
||||
|
||||
go run "$CHECKER_FILE" "$SENTINEL" "$PKG" "$TEST" "$OUT_FILE"
|
||||
;;
|
||||
|
||||
shell)
|
||||
[[ $# -ge 2 ]] || usage
|
||||
SENTINEL="$1"
|
||||
STAGE="$2"
|
||||
shift 2
|
||||
[[ "${1:-}" == "--" ]] || usage
|
||||
shift
|
||||
[[ $# -ge 1 ]] || usage
|
||||
|
||||
set +e
|
||||
OUT="$("$@" 2>&1)"
|
||||
STATUS=$?
|
||||
set -e
|
||||
|
||||
if [[ "$STATUS" -ne 86 ]]; then
|
||||
refuse "expected exit 86, got $STATUS"
|
||||
fi
|
||||
|
||||
EXPECTED_LINE="PHASE8_STAGE:${STAGE}:FAIL:${SENTINEL}"
|
||||
STAGE_LINE_COUNT=0
|
||||
MATCH_COUNT=0
|
||||
while IFS= read -r line; do
|
||||
[[ "$line" == PHASE8_STAGE:* ]] || continue
|
||||
STAGE_LINE_COUNT=$((STAGE_LINE_COUNT + 1))
|
||||
if [[ "$line" == "$EXPECTED_LINE" ]]; then
|
||||
MATCH_COUNT=$((MATCH_COUNT + 1))
|
||||
fi
|
||||
done <<<"$OUT"
|
||||
|
||||
if [[ "$MATCH_COUNT" -ne 1 ]]; then
|
||||
refuse "expected exactly one line '$EXPECTED_LINE', found $MATCH_COUNT"
|
||||
fi
|
||||
if [[ "$STAGE_LINE_COUNT" -ne 1 ]]; then
|
||||
refuse "unexpected additional PHASE8_STAGE lines (found $STAGE_LINE_COUNT total)"
|
||||
fi
|
||||
|
||||
echo "PHASE8_RED_OK:${SENTINEL}"
|
||||
;;
|
||||
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user