12b961b1c0
- scripts/check-phase13.sh with --self-test, --go, --parity, --named, --removal, --coverage, --evidence and --all, modelled on check-phase12 - Parity reads the replay's own coverage line: 157 ported and passing, 0 failing, 14 recorded and not ported; every TestFonotekaNuxtFlows subtest, the three wishlist goldens, docs checks and the corpus scan - Coverage floors of 80% for surf, conga, lagoon, tide and the four application packages, the user plugin's classes and every function of controllers/registration.go, and the controllers package's pre-phase value - 31 anchor-exact removal checks with cmp restore, dirty-file refusal and a signal-safe restore; FORCE_COLOR unset by the gate itself
1013 lines
49 KiB
Bash
Executable File
1013 lines
49 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Phase 13 fail-closed gate (wishlist, notifications, CSV import and export,
|
|
# credentials, onboarding and public routes: API-03 to API-07).
|
|
#
|
|
# Every stage exits non-zero on a failing command, a go test run that fails,
|
|
# skips, matches zero tests or prints "no tests to run", a named test that
|
|
# did not pass, a data race, a parity count other than the expected one, a
|
|
# coverage floor missed, a corpus secret or an evidence gap. --self-test
|
|
# proves each detector fails closed on planted inputs.
|
|
#
|
|
# --removal is the anchor-exact mutation harness behind the RC rows of
|
|
# 13-SECURITY-REVIEW.md: it removes one protection at a time, requires its
|
|
# named test to fail on an assertion, and restores the file byte for byte
|
|
# (checked with cmp). It refuses a file with uncommitted changes and edits
|
|
# tracked source while it runs, so it is not part of --all.
|
|
#
|
|
# Framework commands run in summercms.go; application commands run in the
|
|
# sibling repository named by PHASE13_APP (default ../fonoteka.go).
|
|
set -euo pipefail
|
|
|
|
# A colour-forcing shell variable changes the output some framework tests
|
|
# compare byte for byte; the gate runs without it.
|
|
unset FORCE_COLOR
|
|
|
|
ROOT="${PHASE13_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
|
APP="${PHASE13_APP:-$(cd "$ROOT/../fonoteka.go" && pwd)}"
|
|
PHASE_DIR="${PHASE13_PHASE_DIR:-$ROOT/.planning/phases/13-p-ytarium-api-wishlist-notifications-csv-credentials-public}"
|
|
REVIEW="$PHASE_DIR/13-SECURITY-REVIEW.md"
|
|
VALIDATION="$PHASE_DIR/13-VALIDATION.md"
|
|
APP_PLUGINS=(./plugins/golem15/fonoteka/... ./plugins/golem15/user/...)
|
|
EXPECTED_PORTED=157
|
|
EXPECTED_PENDING=14
|
|
COVERAGE_FLOOR=80
|
|
# The user plugin's controllers package before Phase 13 (measured at
|
|
# sm-user-plugin c258e9f); Phase 13 may not lower it.
|
|
USER_CONTROLLERS_PRE=68.8
|
|
XTEXT_VERSION="v0.42.0"
|
|
FUZZ_CORPUS="plugins/golem15/fonoteka/testdata/fuzz"
|
|
|
|
usage() {
|
|
cat >&2 <<'EOF'
|
|
usage:
|
|
check-phase13.sh --self-test
|
|
check-phase13.sh --go
|
|
check-phase13.sh --parity
|
|
check-phase13.sh --named
|
|
check-phase13.sh --removal
|
|
check-phase13.sh --coverage
|
|
check-phase13.sh --evidence
|
|
check-phase13.sh --all
|
|
EOF
|
|
exit 2
|
|
}
|
|
|
|
# phase13_detect reads go test -json. Exit 1 fail, 2 skip, 3 zero tests or
|
|
# "no tests to run", 4 non-JSON, 5 a required test did not pass, 6 a data
|
|
# race was reported. PHASE13_REQUIRE lists tests that must pass.
|
|
phase13_detect() {
|
|
python3 - "$1" <<'PY'
|
|
import json, os, sys
|
|
path = sys.argv[1]
|
|
require = set(os.environ.get("PHASE13_REQUIRE", "").split())
|
|
passed = set()
|
|
failed_tests, failed_pkgs = {}, []
|
|
build_failed = False
|
|
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
for raw in fh:
|
|
line = raw.strip()
|
|
if not line.startswith("{"):
|
|
continue
|
|
try:
|
|
ev = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
print("refuse: non-json test output", file=sys.stderr)
|
|
sys.exit(4)
|
|
action = ev.get("Action")
|
|
test = ev.get("Test") or ""
|
|
pkg = ev.get("Package") or ""
|
|
if action == "build-fail":
|
|
build_failed = True
|
|
if action == "output":
|
|
text = ev.get("Output") or ""
|
|
if "no tests to run" in text:
|
|
print(f"refuse: no tests to run in {pkg}", file=sys.stderr)
|
|
sys.exit(3)
|
|
if "WARNING: DATA RACE" in text:
|
|
print(f"refuse: data race in {pkg} {test}", file=sys.stderr)
|
|
sys.exit(6)
|
|
if action == "skip" and test:
|
|
print(f"refuse: skipped {pkg} {test}", file=sys.stderr)
|
|
sys.exit(2)
|
|
if action == "fail":
|
|
if ev.get("FailedBuild"):
|
|
build_failed = True
|
|
if test:
|
|
failed_tests.setdefault(pkg, []).append(test)
|
|
else:
|
|
failed_pkgs.append(pkg)
|
|
if action == "pass" and test:
|
|
passed.add(test)
|
|
if build_failed:
|
|
print("refuse: build failed", file=sys.stderr)
|
|
sys.exit(1)
|
|
for pkg, tests in failed_tests.items():
|
|
for test in tests:
|
|
print(f"refuse: failed {pkg} {test}", file=sys.stderr)
|
|
sys.exit(1)
|
|
for pkg in failed_pkgs:
|
|
print(f"refuse: failed {pkg or 'unknown package'}", file=sys.stderr)
|
|
sys.exit(1)
|
|
missing = sorted(name for name in require if name not in passed)
|
|
if missing:
|
|
print("refuse: required tests did not pass: " + ", ".join(missing), file=sys.stderr)
|
|
sys.exit(5)
|
|
if not passed:
|
|
print("refuse: zero tests", file=sys.stderr)
|
|
sys.exit(3)
|
|
PY
|
|
}
|
|
|
|
# phase13_go DIR ARGS... runs go test -json -count=1 ARGS through the
|
|
# detector. With PHASE13_KEEP set, the JSON log is copied there.
|
|
phase13_go() {
|
|
local dir="$1"
|
|
shift
|
|
local log err
|
|
log="$(mktemp)"
|
|
err="$(mktemp)"
|
|
set +e
|
|
(cd "$dir" && go test -json -count=1 "$@") >"$log" 2>"$err"
|
|
local rc=$?
|
|
set -e
|
|
local dc=0
|
|
phase13_detect "$log" || dc=$?
|
|
if [[ "$dc" -ne 0 || "$rc" -ne 0 ]]; then
|
|
cat "$err" >&2 || true
|
|
tail -n 40 "$log" >&2 || true
|
|
rm -f "$log" "$err"
|
|
echo "refuse: go test $* in $dir (test=$rc detect=$dc)" >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "${PHASE13_KEEP:-}" ]]; then
|
|
cp "$log" "$PHASE13_KEEP"
|
|
fi
|
|
rm -f "$log" "$err"
|
|
}
|
|
|
|
# phase13_tests DIR PKG [-race] TEST... requires every named test to run and
|
|
# pass, each matched by its exact name.
|
|
phase13_tests() {
|
|
local dir="$1" pkg="$2"
|
|
shift 2
|
|
local extra=()
|
|
if [[ "${1:-}" == "-race" ]]; then
|
|
extra=(-race)
|
|
shift
|
|
fi
|
|
local names="$*"
|
|
local regex="^($(tr ' ' '|' <<<"$names"))\$"
|
|
PHASE13_REQUIRE="$names" phase13_go "$dir" "$pkg" "${extra[@]}" -run "$regex"
|
|
}
|
|
|
|
expect_detect() {
|
|
local name="$1" want="$2" payload="$3"
|
|
local log dc=0
|
|
log="$(mktemp)"
|
|
printf '%s\n' "$payload" >"$log"
|
|
phase13_detect "$log" 2>/dev/null || dc=$?
|
|
rm -f "$log"
|
|
if [[ "$dc" -ne "$want" ]]; then
|
|
echo "refuse: self-test $name: detector exit $dc, want $want" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# The named tests: every test 13-VALIDATION.md names, by package. The
|
|
# evidence stage refuses a validation or review row naming a test missing
|
|
# here.
|
|
NAMED_ROOT_SURF="TestOverlappingConstrainedRoutes TestOverlapFamilyOfThree TestOverlapHeadAndAllow TestOverlapFamilyAcrossPlugins TestOverlapUnsupportedShapes"
|
|
NAMED_ROOT_CONGA="TestUnregisteredKindWithWorker TestUnregisteredKindWithoutWorker TestUnregisteredKindRefusalAndDelay"
|
|
NAMED_ROOT_LAGOON="TestValidateRequestProhibited TestValidateRequestProhibitedNested"
|
|
NAMED_ROOT_TIDE="TestNormalizeContentDispositionDate TestNormalizeNotificationPublication TestNormalizePhase13Edges"
|
|
NAMED_APP_USER="TestRegisterEventPayload TestRegisterUserExports"
|
|
NAMED_APP_FONOTEKA="TestWishlistOverlapPatternsDispatch TestJobContractDispatchWhileWorkerRuns TestNotificationsRoutes TestCredentialsCRUD TestCredentialSecretsNeverSerialized TestResolveAIConfigPrecedence TestDiscogsSharedMirror TestBootstrapConcurrent TestRegisterInvitationListener TestInspectInvitation TestWishlistOwnListAndShow TestWishlistItemAddedOncePerPath TestDigestCoalescing TestWishlistShareSettingsHousehold TestReserveConcurrent TestRevealIdempotent TestReservationMask TestWishlistSubscriptions TestPurchaseSideEffects TestPurchaseMailAfterCommit TestWishlistOverlapRoutesAssembled TestCsvExport TestCsvStoreAndShow TestCsvImportScope TestCsvCommitCAS TestCsvJobRows TestCsvCancel TestCsvRowPickSeam TestPublicResolve TestPubfailCounter TestPubfailCounterPerApp TestPublicBucketsPerRoute TestPublicAlbumFieldSet TestPublicAlbumsIndex TestPublicAlbumsEngine TestRouteTablePhase13 TestRouteTablePhase12 TestFullRouteTableAuthGroupMutualExclusivity FuzzWriteEndpoints TestPhase13Threats TestPhase13EmptyBodies TestPhase13Boundaries TestPhase13HandlersFailClosed TestPhase09SecurityRoutes"
|
|
NAMED_APP_CLASSES="TestJobContract TestPhase13ReservationMask TestPhase13PubfailWindow TestPhase13SmallHelpers TestPhase13NilHandles"
|
|
NAMED_APP_CSV="TestPHPFputcsv TestCsvParserTruthTable TestCsvDetectorTruthTable TestCsvPHPCasts TestCsvOrderedMap"
|
|
NAMED_APP_API="TestCsvMappingInput"
|
|
NAMED_APP_MIDDLEWARE="TestPublicShareHeadersRewrites429 TestPublicShareHeadersLeaves200Body TestPublicShareHeadersExactBytes"
|
|
NAMED_APP_PARITY="TestCheckCorpusPortedCaseStatus TestParityCorpus TestBroadcastGoldens TestFonotekaNuxtFlows TestUserAPINuxtFlows"
|
|
|
|
all_named() {
|
|
echo "$NAMED_ROOT_SURF $NAMED_ROOT_CONGA $NAMED_ROOT_LAGOON $NAMED_ROOT_TIDE $NAMED_APP_USER $NAMED_APP_FONOTEKA $NAMED_APP_CLASSES $NAMED_APP_CSV $NAMED_APP_API $NAMED_APP_MIDDLEWARE $NAMED_APP_PARITY"
|
|
}
|
|
|
|
# module_pin MODLIST: golang.org/x/text stays at the audited version
|
|
# (T-13-SC: the CSV decoder's charmap).
|
|
REASON_PIN="golang.org/x/text is not pinned at $XTEXT_VERSION"
|
|
module_pin() {
|
|
local modlist="$1" hits
|
|
hits="$(grep -E '^golang\.org/x/text ' "$modlist" | sort -u || true)"
|
|
if [[ -z "$hits" ]] || grep -vqE "^golang\.org/x/text $XTEXT_VERSION\$" <<<"$hits"; then
|
|
echo "refuse: hygiene: $REASON_PIN: ${hits:-<absent>}" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
run_go() {
|
|
(cd "$ROOT" && go vet ./...)
|
|
phase13_go "$ROOT" ./...
|
|
(cd "$APP" && go vet ./... "${APP_PLUGINS[@]}")
|
|
phase13_go "$APP" ./... "${APP_PLUGINS[@]}"
|
|
local modlist
|
|
modlist="$(mktemp)"
|
|
(cd "$ROOT" && go list -m all) >"$modlist"
|
|
(cd "$APP" && go list -m all) >>"$modlist"
|
|
if ! module_pin "$modlist"; then
|
|
rm -f "$modlist"
|
|
exit 1
|
|
fi
|
|
rm -f "$modlist"
|
|
echo "phase13 go passed"
|
|
}
|
|
|
|
# corpus_scan DIR: the fuzz seed corpus holds synthetic values only
|
|
# (T-13-36): no 64-hex token, inv_ personal token, JWT or bearer header.
|
|
corpus_scan() {
|
|
python3 - "$1" <<'PY'
|
|
import os, re, sys
|
|
root = sys.argv[1]
|
|
if not os.path.isdir(root):
|
|
print(f"refuse: fuzz corpus {root} is missing", file=sys.stderr)
|
|
sys.exit(1)
|
|
patterns = [
|
|
("64-hex value", re.compile(r"(?<![0-9A-Fa-f])[0-9A-Fa-f]{64}(?![0-9A-Fa-f])")),
|
|
("personal token", re.compile(r"inv_[A-Za-z0-9]{16,}")),
|
|
("JWT", re.compile(r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.")),
|
|
("bearer header", re.compile(r"(?i)bearer\s+[A-Za-z0-9._-]{12,}")),
|
|
]
|
|
files = 0
|
|
for dirpath, _, names in os.walk(root):
|
|
for name in names:
|
|
files += 1
|
|
text = open(os.path.join(dirpath, name), encoding="utf-8", errors="replace").read()
|
|
for label, rx in patterns:
|
|
if rx.search(text):
|
|
print(f"refuse: fuzz corpus {os.path.join(dirpath, name)} holds a {label}", file=sys.stderr)
|
|
sys.exit(1)
|
|
if files == 0:
|
|
print(f"refuse: fuzz corpus {root} is empty", file=sys.stderr)
|
|
sys.exit(1)
|
|
PY
|
|
}
|
|
|
|
# manifest_count MANIFEST STATUS: the number of routes with that status.
|
|
manifest_count() {
|
|
grep -cE "^[[:space:]]*status:[[:space:]]*$2[[:space:]]*\$" "$1" || true
|
|
}
|
|
|
|
# corpus_coverage LOG: the replay's own coverage line in a go test -json
|
|
# log ("recorded R/T passing P failing F unrecorded U pending Q") must say
|
|
# EXPECTED_PORTED passing, 0 failing, 0 unrecorded and EXPECTED_PENDING
|
|
# pending. Pending routes are never counted as passing.
|
|
corpus_coverage() {
|
|
python3 - "$1" "$EXPECTED_PORTED" "$EXPECTED_PENDING" <<'PY'
|
|
import json, re, sys
|
|
path, ported, pending = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
|
|
rx = re.compile(r"recorded (\d+)/(\d+) passing (\d+) failing (\d+) unrecorded (\d+) pending (\d+)")
|
|
found = None
|
|
for raw in open(path, encoding="utf-8", errors="replace"):
|
|
raw = raw.strip()
|
|
if not raw.startswith("{"):
|
|
continue
|
|
try:
|
|
ev = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
m = rx.search(ev.get("Output") or "")
|
|
if m:
|
|
found = [int(x) for x in m.groups()]
|
|
if found is None:
|
|
print("refuse: the corpus replay printed no coverage line", file=sys.stderr)
|
|
sys.exit(1)
|
|
recorded, total, passing, failing, unrecorded, pend = found
|
|
if passing != ported or failing != 0 or unrecorded != 0 or pend != pending or recorded != total or passing + pend != total:
|
|
print(f"refuse: corpus coverage recorded {recorded}/{total} passing {passing} failing {failing} unrecorded {unrecorded} pending {pend}, want {ported} passing, 0 failing, {pending} pending", file=sys.stderr)
|
|
sys.exit(1)
|
|
print(f"phase13 corpus: {passing} ported and passing, 0 failing, {pend} pending")
|
|
PY
|
|
}
|
|
|
|
PARITY_REQUIRE="TestParityCorpus TestParityCorpus/coverage TestBroadcastGoldens TestBroadcastGoldens/created TestBroadcastGoldens/updated TestBroadcastGoldens/deleted TestBroadcastGoldens/bulk TestBroadcastGoldens/wishlist-item-added TestBroadcastGoldens/reservation-revealed TestBroadcastGoldens/wishlist-purchased TestFonotekaNuxtFlows TestFonotekaNuxtFlows/nuxt-collections TestFonotekaNuxtFlows/nuxt-albums TestFonotekaNuxtFlows/onboarding TestFonotekaNuxtFlows/nuxt-wishlist TestFonotekaNuxtFlows/mcp-wishlist TestFonotekaNuxtFlows/nuxt-csv TestFonotekaNuxtFlows/public-anonymous TestFonotekaNuxtFlows/public-pubfail TestUserAPINuxtFlows TestCheckCorpusPortedCaseStatus"
|
|
|
|
run_parity() {
|
|
local n p
|
|
n="$(manifest_count "$APP/parity/manifest.yaml" ported)"
|
|
p="$(manifest_count "$APP/parity/manifest.yaml" pending)"
|
|
if [[ "$n" -ne "$EXPECTED_PORTED" || "$p" -ne "$EXPECTED_PENDING" ]]; then
|
|
echo "refuse: parity manifest has $n ported and $p pending routes, want $EXPECTED_PORTED and $EXPECTED_PENDING" >&2
|
|
exit 1
|
|
fi
|
|
local log
|
|
log="$(mktemp)"
|
|
PHASE13_KEEP="$log" PHASE13_REQUIRE="$PARITY_REQUIRE" \
|
|
phase13_go "$APP" ./parity -run '^(TestParityCorpus|TestBroadcastGoldens|TestFonotekaNuxtFlows|TestUserAPINuxtFlows|TestCheckCorpusPortedCaseStatus)$'
|
|
if ! corpus_coverage "$log"; then
|
|
rm -f "$log"
|
|
exit 1
|
|
fi
|
|
rm -f "$log"
|
|
(cd "$APP" && go run ./parity/check_corpus.go --manifest parity/manifest.yaml --require-recorded --check-secrets)
|
|
corpus_scan "$APP/$FUZZ_CORPUS"
|
|
PHASE13_REQUIRE="TestDocsTree" phase13_go "$ROOT" ./cmd/summer -run '^TestDocsTree$'
|
|
(cd "$ROOT" && go run ./cmd/summer docs:build --check)
|
|
echo "phase13 parity passed ($n ported, 0 failing, $p pending)"
|
|
}
|
|
|
|
run_named() {
|
|
phase13_tests "$ROOT" ./modules/surf $NAMED_ROOT_SURF
|
|
phase13_tests "$ROOT" ./modules/conga $NAMED_ROOT_CONGA
|
|
phase13_tests "$ROOT" ./modules/lagoon $NAMED_ROOT_LAGOON
|
|
phase13_tests "$ROOT" ./modules/tide $NAMED_ROOT_TIDE
|
|
phase13_tests "$APP" ./plugins/golem15/user $NAMED_APP_USER
|
|
phase13_tests "$APP" ./plugins/golem15/fonoteka -race $NAMED_APP_FONOTEKA
|
|
phase13_tests "$APP" ./plugins/golem15/fonoteka/classes $NAMED_APP_CLASSES
|
|
phase13_tests "$APP" ./plugins/golem15/fonoteka/classes/csv $NAMED_APP_CSV
|
|
phase13_tests "$APP" ./plugins/golem15/fonoteka/controllers/api $NAMED_APP_API
|
|
phase13_tests "$APP" ./plugins/golem15/fonoteka/middleware $NAMED_APP_MIDDLEWARE
|
|
phase13_tests "$APP" ./parity $NAMED_APP_PARITY
|
|
echo "phase13 named passed"
|
|
}
|
|
|
|
# coverage_report FLOOR PROFILE... prints one line per package of the merged
|
|
# profiles (a block counts as covered when any profile covered it) and
|
|
# refuses any package below FLOOR percent. COVERAGE_ONLY limits the report
|
|
# to packages whose import path ends with one of its words.
|
|
coverage_report() {
|
|
python3 - "$@" <<'PY'
|
|
import collections, os, sys
|
|
floor = float(sys.argv[1])
|
|
only = os.environ.get("COVERAGE_ONLY", "").split()
|
|
blocks = {}
|
|
for path in sys.argv[2:]:
|
|
for line in open(path):
|
|
if line.startswith("mode:") or not line.strip():
|
|
continue
|
|
loc, n, c = line.rsplit(" ", 2)
|
|
n, c = int(n), int(c)
|
|
prev = blocks.get(loc, (n, 0))
|
|
blocks[loc] = (n, max(prev[1], c))
|
|
total, covered = collections.Counter(), collections.Counter()
|
|
for loc, (n, c) in blocks.items():
|
|
pkg = loc.split(":")[0].rsplit("/", 1)[0]
|
|
if only and not any(pkg.endswith(o) for o in only):
|
|
continue
|
|
total[pkg] += n
|
|
if c:
|
|
covered[pkg] += n
|
|
if not total:
|
|
print("refuse: coverage profile is empty", file=sys.stderr)
|
|
sys.exit(1)
|
|
low = []
|
|
for pkg in sorted(total):
|
|
pct = 100.0 * covered[pkg] / total[pkg]
|
|
print(f"coverage {pkg} {pct:.1f}%")
|
|
if pct < floor:
|
|
low.append(f"{pkg} {pct:.1f}%")
|
|
if low:
|
|
print(f"refuse: below the {floor:.1f}% coverage floor: " + ", ".join(low), file=sys.stderr)
|
|
sys.exit(1)
|
|
PY
|
|
}
|
|
|
|
# func_floor FLOOR FILE: reads go tool cover -func output on stdin and
|
|
# refuses any function of FILE below FLOOR percent, or none at all.
|
|
func_floor() {
|
|
python3 -c '
|
|
import sys
|
|
floor, suffix = float(sys.argv[1]), sys.argv[2]
|
|
seen = 0
|
|
low = []
|
|
for line in sys.stdin:
|
|
parts = line.split()
|
|
if len(parts) < 3 or not parts[0].split(":")[0].endswith(suffix):
|
|
continue
|
|
seen += 1
|
|
pct = float(parts[-1].rstrip("%"))
|
|
print(f"coverage {parts[0]} {parts[1]} {pct:.1f}%")
|
|
if pct < floor:
|
|
low.append(f"{parts[1]} {pct:.1f}%")
|
|
if not seen:
|
|
print(f"refuse: no function of {suffix} in the profile", file=sys.stderr)
|
|
sys.exit(1)
|
|
if low:
|
|
print(f"refuse: below the {floor:.0f}% function floor in {suffix}: " + ", ".join(low), file=sys.stderr)
|
|
sys.exit(1)
|
|
' "$@"
|
|
}
|
|
|
|
# cover_profile DIR OUT ARGS... writes a coverage profile of go test ARGS.
|
|
cover_profile() {
|
|
local dir="$1" out="$2"
|
|
shift 2
|
|
local log
|
|
log="$(mktemp)"
|
|
if ! (cd "$dir" && go test -count=1 -coverprofile="$out" "$@") >"$log" 2>&1; then
|
|
tail -n 40 "$log" >&2
|
|
rm -f "$log"
|
|
echo "refuse: go test -coverprofile $* in $dir" >&2
|
|
exit 1
|
|
fi
|
|
rm -f "$log"
|
|
}
|
|
|
|
run_coverage() {
|
|
local dir
|
|
dir="$(mktemp -d)"
|
|
trap 'rm -rf "$dir"' RETURN
|
|
local pkg i=0
|
|
# Framework packages changed in Phase 13: each package's own tests.
|
|
for pkg in ./modules/surf ./modules/conga ./modules/lagoon ./modules/tide; do
|
|
i=$((i + 1))
|
|
cover_profile "$ROOT" "$dir/root$i.out" "$pkg"
|
|
done
|
|
coverage_report "$COVERAGE_FLOOR" "$dir"/root*.out
|
|
# Application packages: every test of the plugin that exercises them.
|
|
cover_profile "$APP" "$dir/app.out" ./plugins/golem15/fonoteka/... \
|
|
-coverpkg=./plugins/golem15/fonoteka/classes,./plugins/golem15/fonoteka/classes/csv,./plugins/golem15/fonoteka/controllers/api,./plugins/golem15/fonoteka/middleware
|
|
coverage_report "$COVERAGE_FLOOR" "$dir/app.out"
|
|
# The shared user plugin: classes at the floor, every registration
|
|
# export at the floor, the controllers package not below its pre-phase
|
|
# value.
|
|
cover_profile "$APP" "$dir/user.out" ./plugins/golem15/user/... \
|
|
-coverpkg=./plugins/golem15/user/classes,./plugins/golem15/user/controllers
|
|
COVERAGE_ONLY="/classes" coverage_report "$COVERAGE_FLOOR" "$dir/user.out"
|
|
COVERAGE_ONLY="/controllers" coverage_report "$USER_CONTROLLERS_PRE" "$dir/user.out"
|
|
(cd "$APP" && go tool cover -func="$dir/user.out") | func_floor "$COVERAGE_FLOOR" controllers/registration.go
|
|
echo "phase13 coverage passed"
|
|
}
|
|
|
|
# removal_table: the RC rows of 13-SECURITY-REVIEW.md. Fields: id, threat,
|
|
# repo (root|app|script, or rootapp for a framework file whose test runs in
|
|
# the application), file, anchor, replacement, package, test regex.
|
|
# Anchors must occur exactly once.
|
|
removal_table() {
|
|
cat <<'EOF'
|
|
[
|
|
["RC-01", "T-13-23", "root", "modules/surf/overlap.go",
|
|
"\t\tif !m.constraintsMatch(vals) {\n\t\t\tcontinue\n\t\t}\n", "", "./modules/surf", "^TestOverlapConstraintFallsThrough$"],
|
|
["RC-02", "T-13-23", "root", "modules/surf/overlap.go",
|
|
"\t\tif s.param == \"\" && s.lit != vals[i] {\n\t\t\treturn false\n\t\t}", "\t\tif false && s.param == \"\" && s.lit != vals[i] {\n\t\t\treturn false\n\t\t}", "./modules/surf", "^TestOverlappingConstrainedRoutes$"],
|
|
["RC-03", "T-13-23", "rootapp", "modules/surf/overlap.go",
|
|
"\t\tif s.param == \"\" && s.lit != vals[i] {\n\t\t\treturn false\n\t\t}", "\t\tif false && s.param == \"\" && s.lit != vals[i] {\n\t\t\treturn false\n\t\t}", "./plugins/golem15/fonoteka", "^TestRouteTablePhase13$"],
|
|
["RC-04", "T-13-22", "root", "modules/conga/conga.go",
|
|
"\t}\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\treturn m.insertOnlyLocked()\n}\n\n// insertClient", "\t}\n\treturn m.insertClient()\n}\n\n// insertClient", "./modules/conga", "^TestUnregisteredKindWithWorker$"],
|
|
["RC-05", "T-13-05", "app", "plugins/golem15/fonoteka/classes/reservations.go",
|
|
"if rc.IsOwner && !revealed {", "if false && rc.IsOwner && !revealed {", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-05$"],
|
|
["RC-06", "T-13-03", "app", "plugins/golem15/fonoteka/classes/share_service.go",
|
|
"Where(\"LOWER(public_token) = ? AND public_enabled = ? AND kind = ?\", strings.ToLower(token), true, kind)",
|
|
"Where(\"LOWER(public_token) = ? AND ? AND kind = ?\", strings.ToLower(token), true, kind)", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-03$"],
|
|
["RC-07", "T-13-29", "app", "plugins/golem15/fonoteka/classes/share_service.go",
|
|
"subtle.ConstantTimeCompare([]byte(*stored), []byte(token)) == 1",
|
|
"subtle.ConstantTimeCompare([]byte(strings.ToLower(*stored)), []byte(strings.ToLower(token))) == 1", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-29$"],
|
|
["RC-08", "T-13-01", "app", "plugins/golem15/fonoteka/classes/public_share.go",
|
|
"\tif w.hits+w.inflight >= c.limit {\n\t\tc.release(key, w)\n\t\treturn nil, false\n\t}",
|
|
"\tif false && w.hits+w.inflight >= c.limit {\n\t\tc.release(key, w)\n\t\treturn nil, false\n\t}", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-01$"],
|
|
["RC-09", "T-13-01", "app", "plugins/golem15/fonoteka/classes/public_share.go",
|
|
"too := w.hits+w.inflight >= c.limit", "too := false && w.hits+w.inflight >= c.limit", "./plugins/golem15/fonoteka", "^TestPubfailCounter$"],
|
|
["RC-10", "T-13-20", "app", "plugins/golem15/user/controllers/registration.go",
|
|
"\tdelete(payload, \"password_confirmation\")\n", "", "./plugins/golem15/user", "^TestRegisterEventPayload$"],
|
|
["RC-11", "T-13-20", "app", "plugins/golem15/user/controllers/registration.go",
|
|
"\tdelete(payload, \"password_confirmation\")\n", "", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-20$"],
|
|
["RC-12", "T-13-18", "app", "plugins/golem15/fonoteka/classes/onboarding.go",
|
|
"\t\tn, err := countLiveUsers(tx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif n != 0 {\n\t\t\treturn ErrOnboardingCompleted\n\t\t}\n", "", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-18$"],
|
|
["RC-13", "T-13-12", "app", "plugins/golem15/fonoteka/classes/csv_import_service.go",
|
|
"\tif n == 0 {\n\t\treturn nil, nil\n\t}\n\treturn &imps[0], nil", "\t_ = n\n\treturn &imps[0], nil", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-12$"],
|
|
["RC-14", "T-13-17", "app", "plugins/golem15/fonoteka/classes/csv_import_service.go",
|
|
"WHERE id = ? AND status = ?`", "WHERE id = ? AND ? <> ''`", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-17$"],
|
|
["RC-15", "T-13-10", "app", "plugins/golem15/fonoteka/classes/credential_write_service.go",
|
|
"var CredentialFillFields = []string{\"provider\", \"model\", \"base_url\"}",
|
|
"var CredentialFillFields = []string{\"provider\", \"model\", \"base_url\", \"user_id\"}", "./plugins/golem15/fonoteka", "^FuzzWriteEndpoints$"],
|
|
["RC-16", "T-13-10", "app", "plugins/golem15/fonoteka/classes/credential_write_service.go",
|
|
"var CredentialFillFields = []string{\"provider\", \"model\", \"base_url\"}",
|
|
"var CredentialFillFields = []string{\"provider\", \"model\", \"base_url\", \"user_id\"}", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-10$"],
|
|
["RC-17", "T-13-21", "app", "plugins/golem15/fonoteka/classes/notifications.go",
|
|
"WHERE user_id = ? AND id = ?`, userID, id)", "WHERE ? > 0 AND id = ?`, userID, id)", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-21$"],
|
|
["RC-18", "T-13-16", "app", "plugins/golem15/fonoteka/classes/csv_import_service.go",
|
|
"func (unavailableReleaseFetcher) FetchRelease(context.Context, *usermodels.User, string) (map[string]any, error) {\n\treturn nil, ErrDiscogsUnavailable",
|
|
"func (unavailableReleaseFetcher) FetchRelease(context.Context, *usermodels.User, string) (map[string]any, error) {\n\treturn map[string]any{\"name\": \"Forged\"}, nil", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-16$"],
|
|
["RC-19", "T-13-28", "app", "plugins/golem15/fonoteka/classes/wishlist_notifications.go",
|
|
"if len(inserted) != 1 || !inserted[0] {", "if len(inserted) != 1 {", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-28$"],
|
|
["RC-20", "T-13-02", "app", "plugins/golem15/fonoteka/classes/serialize_public_album.go",
|
|
"\tCreatedAt *wire.Time `json:\"created_at\"`\n}",
|
|
"\tCreatedAt *wire.Time `json:\"created_at\"`\n\tShelf *string `json:\"shelf\"`\n}", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-02$"],
|
|
["RC-21", "T-13-08", "app", "plugins/golem15/fonoteka/controllers/api/credentials_controller.go",
|
|
"err := gdb.WithContext(r.Context()).Select(\"id\", \"provider\", \"model\", \"base_url\").Where(\"user_id = ?\", user.ID).Take(&cred).Error\n\t\twriteAICredentialStatus(w, cred.Provider, cred.Model, cred.BaseURL, err)",
|
|
"err := gdb.WithContext(r.Context()).Where(\"user_id = ?\", user.ID).Take(&cred).Error\n\t\t_ = err\n\t\twriteJSON(w, http.StatusOK, map[string]any{\"configured\": true, \"provider\": cred.Provider, \"api_key\": cred.APIKey.Reveal()})", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-08$"],
|
|
["RC-22", "T-13-07", "app", "plugins/golem15/fonoteka/routes.go",
|
|
"g.Get(\"/wishlist/albums\", wishlistIndex, \"inv.scope:read\")", "g.Get(\"/wishlist/albums\", wishlistIndex, \"inv.scope:read\", \"inv.scope:read\")", "./plugins/golem15/fonoteka", "^TestRouteTablePhase13$"],
|
|
["RC-23", "T-13-04", "app", "plugins/golem15/fonoteka/classes/reservations.go",
|
|
"WHERE album_id = ? AND user_id = ?`, albumID, userID)", "WHERE album_id = ? AND ? > 0`, albumID, userID)", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-04$"],
|
|
["RC-24", "T-13-06", "app", "plugins/golem15/fonoteka/controllers/api/wishlist_subscriptions_controller.go",
|
|
"Model(&models.Collection{}).Scopes(classes.WishlistsVisibleTo(user.ID)).\n", "Model(&models.Collection{}).Where(\"kind = 'wishlist' AND ? > 0\", user.ID).\n", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-06$"],
|
|
["RC-25", "T-13-09", "app", "plugins/golem15/fonoteka/controllers/api/credentials_controller.go",
|
|
"\t\tif !mayManageOrg(w, r, gdb, user) {\n\t\t\treturn\n\t\t}\n\t\tfields, apiKey, ok := aiCredentialInput(w, r, app, gdb)",
|
|
"\t\tfields, apiKey, ok := aiCredentialInput(w, r, app, gdb)", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-09$"],
|
|
["RC-26", "T-13-13", "app", "plugins/golem15/fonoteka/controllers/api/csv_import_controller.go",
|
|
"\t\tbucket, err := csvBucket(app)\n\t\tif err != nil {\n\t\t\twriteOpaque500(w)\n\t\t\treturn\n\t\t}\n\t\timp, err := classes.StoreCsvImport(",
|
|
"\t\tbucket, err := uploadBucket(app), error(nil)\n\t\tif err != nil {\n\t\t\twriteOpaque500(w)\n\t\t\treturn\n\t\t}\n\t\timp, err := classes.StoreCsvImport(", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-13$"],
|
|
["RC-27", "T-13-19", "app", "plugins/golem15/fonoteka/classes/onboarding.go",
|
|
"expires_at > NOW() AND LOWER(email) = ?", "expires_at > NOW() AND (LOWER(email) = ? OR TRUE)", "./plugins/golem15/fonoteka", "^TestPhase13Threats$/^T-13-19$"],
|
|
["RC-28", "T-13-SC", "script", "scripts/check-phase13.sh",
|
|
"hits=\"$(grep -E '^golang\\.org/x/text ' \"$modlist\" | sort -u || true)\"", "hits=\"golang.org/x/text $XTEXT_VERSION\"", "", "--self-test"],
|
|
["RC-29", "T-13-36", "script", "scripts/check-phase13.sh",
|
|
"holds a {label}\", file=sys.stderr)\n sys.exit(1)", "holds a {label}\", file=sys.stderr)\n pass", "", "--self-test"],
|
|
["RC-30", "T-13-35", "script", "scripts/check-phase13.sh",
|
|
" if not any(re.match(r\"^\\| RC-\\d+ \\| \" + re.escape(tid) + r\" \\|\", l) for l in removal):", " if False:", "", "--self-test"],
|
|
["RC-31", "T-13-34", "script", "scripts/check-phase13.sh",
|
|
" if dirty:\n", " if False:\n", "", "--self-test"]
|
|
]
|
|
EOF
|
|
}
|
|
|
|
# removal_harness TABLE_FILE: for each row, refuse a file with uncommitted
|
|
# changes, save it, apply the anchor-exact mutation, run the named test (or,
|
|
# for the gate script, its --self-test on a mutated copy) and require it to
|
|
# fail on an assertion, then restore the file and require cmp to match.
|
|
removal_harness() {
|
|
python3 - "$1" "$ROOT" "$APP" <<'PY'
|
|
import json, os, shutil, signal, subprocess, sys, tempfile
|
|
table = json.load(open(sys.argv[1]))
|
|
root, app = sys.argv[2], sys.argv[3]
|
|
only = set(os.environ.get("PHASE13_RC", "").split())
|
|
current = {}
|
|
|
|
def restore(*_):
|
|
# A signal mid-run still puts the file back.
|
|
if current:
|
|
with open(current["path"], "wb") as fh:
|
|
fh.write(current["original"])
|
|
sys.exit(1)
|
|
|
|
signal.signal(signal.SIGINT, restore)
|
|
signal.signal(signal.SIGTERM, restore)
|
|
failures = 0
|
|
for rc, threat, repo, rel, anchor, repl, pkg, run in table:
|
|
if only and rc not in only:
|
|
continue
|
|
base = {"root": root, "app": app, "script": root, "rootapp": root}[repo]
|
|
run_dir = {"root": root, "app": app, "script": root, "rootapp": app}[repo]
|
|
path = os.path.join(base, rel)
|
|
tracked = subprocess.run(["git", "-C", os.path.dirname(path), "rev-parse", "--is-inside-work-tree"], capture_output=True, text=True).returncode == 0
|
|
if tracked and repo != "script":
|
|
dirty = subprocess.run(["git", "-C", os.path.dirname(path), "status", "--porcelain", "--", os.path.basename(path)], capture_output=True, text=True).stdout.strip()
|
|
if dirty:
|
|
print(f"refuse: {rc}: {rel} is dirty; commit or restore it first", file=sys.stderr)
|
|
sys.exit(1)
|
|
original = open(path, "rb").read()
|
|
text = original.decode()
|
|
n = text.count(anchor)
|
|
if n != 1:
|
|
print(f"refuse: {rc} {threat}: anchor occurs {n} times in {rel}", file=sys.stderr)
|
|
sys.exit(1)
|
|
mutated = text.replace(anchor, repl, 1)
|
|
scratch = tempfile.mkdtemp(prefix="phase13-rc-")
|
|
saved = os.path.join(scratch, "saved")
|
|
shutil.copyfile(path, saved)
|
|
try:
|
|
if repo == "script":
|
|
copy = os.path.join(scratch, os.path.basename(rel))
|
|
open(copy, "w").write(mutated)
|
|
env = dict(os.environ, PHASE13_ROOT=root, PHASE13_APP=app)
|
|
proc = subprocess.run(["bash", copy, run], cwd=root, env=env, capture_output=True, text=True, timeout=900)
|
|
out = proc.stdout + proc.stderr
|
|
ok = proc.returncode != 0 and "refuse:" in out
|
|
evidence = next((l for l in out.splitlines() if l.startswith("refuse:")), "")
|
|
else:
|
|
current.update(path=path, original=original)
|
|
with open(path, "w") as fh:
|
|
fh.write(mutated)
|
|
proc = subprocess.run(["go", "test", pkg, "-run", run, "-count=1"], cwd=run_dir, capture_output=True, text=True, timeout=1800)
|
|
out = proc.stdout + proc.stderr
|
|
build = "[build failed]" in out or "[setup failed]" in out
|
|
ok = proc.returncode != 0 and "--- FAIL" in out and not build
|
|
fails = [l.strip() for l in out.splitlines() if l.strip().startswith("--- FAIL")]
|
|
names = [l.split()[2] for l in fails if len(l.split()) > 2]
|
|
evidence = ", ".join(names[:5]) + (f" (+{len(names) - 5} more)" if len(names) > 5 else "") if names else ("build failed" if build else "no failure")
|
|
finally:
|
|
with open(path, "wb") as fh:
|
|
fh.write(original)
|
|
current.clear()
|
|
same = subprocess.run(["cmp", "-s", saved, path]).returncode == 0
|
|
shutil.rmtree(scratch, ignore_errors=True)
|
|
if not same:
|
|
print(f"refuse: {rc}: {rel} was not restored byte for byte", file=sys.stderr)
|
|
sys.exit(1)
|
|
status = "fails as required" if ok else "SURVIVED"
|
|
print(f"{rc} {threat} {rel}: {status}: {evidence}", flush=True)
|
|
if not ok:
|
|
failures += 1
|
|
if failures:
|
|
print(f"refuse: {failures} removal check(s) survived", file=sys.stderr)
|
|
sys.exit(1)
|
|
PY
|
|
}
|
|
|
|
run_removal() {
|
|
local table
|
|
table="$(mktemp)"
|
|
removal_table >"$table"
|
|
if ! removal_harness "$table"; then
|
|
rm -f "$table"
|
|
exit 1
|
|
fi
|
|
rm -f "$table"
|
|
echo "phase13 removal passed"
|
|
}
|
|
|
|
# removal_harness_in ROOT TABLE runs the harness against another root.
|
|
removal_harness_in() {
|
|
local root="$1" table="$2"
|
|
(
|
|
ROOT="$root"
|
|
APP="$root"
|
|
export GOWORK=off GOFLAGS=-mod=mod
|
|
removal_harness "$table"
|
|
)
|
|
}
|
|
|
|
# evidence_check PHASE_DIR REVIEW VALIDATION NAMED: every T-13 threat the
|
|
# plans declare has exactly one review row copying its strictest severity
|
|
# and disposition (a threat several plans declare takes the strictest);
|
|
# a mitigated threat names a test the --named stage runs or a gate stage;
|
|
# a high mitigated threat has a removal row; the validation file is
|
|
# validated, Nyquist-compliant, Wave 0 complete, without a pending or TBD
|
|
# row, names API-03 to API-07, and every test it names is run by --named.
|
|
evidence_check() {
|
|
python3 - "$@" <<'PY'
|
|
import glob, os, re, sys
|
|
phase_dir, review_path, validation_path, named = sys.argv[1], sys.argv[2], sys.argv[3], set(sys.argv[4].split())
|
|
for p in (review_path, validation_path):
|
|
if not os.path.isfile(p):
|
|
print(f"refuse: {p} is missing", file=sys.stderr)
|
|
sys.exit(1)
|
|
review = open(review_path).read()
|
|
validation = open(validation_path).read()
|
|
sev_rank = {"low": 0, "medium": 1, "high": 2}
|
|
declared = {}
|
|
for plan in sorted(glob.glob(os.path.join(phase_dir, "13-0*-PLAN.md"))):
|
|
for line in open(plan):
|
|
m = re.match(r"^\| (T-13-(?:\d\d|SC)) \|", line)
|
|
if not m:
|
|
continue
|
|
cells = [c.strip().lower() for c in line.strip().strip("|").split("|")]
|
|
prev = declared.get(m.group(1))
|
|
if prev is None:
|
|
declared[m.group(1)] = cells
|
|
continue
|
|
sev = max(prev[3], cells[3], key=lambda s: sev_rank.get(s, -1))
|
|
disp = "mitigate" if "mitigate" in (prev[4], cells[4]) else prev[4]
|
|
declared[m.group(1)] = prev[:3] + [sev, disp] + prev[5:]
|
|
if not declared:
|
|
print("refuse: no plan declares a T-13 threat", file=sys.stderr)
|
|
sys.exit(1)
|
|
lines = review.splitlines()
|
|
removal = [l for l in lines if re.match(r"^\| RC-\d+ \| T-13-", l)]
|
|
for tid, cells in sorted(declared.items()):
|
|
rows = [l for l in lines if l.startswith("| " + tid + " |")]
|
|
if len(rows) != 1:
|
|
print(f"refuse: review has {len(rows)} threat rows for {tid}, want 1", file=sys.stderr)
|
|
sys.exit(1)
|
|
row = [c.strip().lower() for c in rows[0].strip().strip("|").split("|")]
|
|
severity, disposition = cells[3], cells[4]
|
|
if severity not in row or disposition not in row:
|
|
print(f"refuse: review row {tid} does not copy severity {severity!r} and disposition {disposition!r}", file=sys.stderr)
|
|
sys.exit(1)
|
|
if disposition == "mitigate":
|
|
tests = set(re.findall(r"\b(?:Test|Fuzz)[A-Z][A-Za-z0-9_]*", rows[0]))
|
|
if not tests and "check-phase13.sh" not in rows[0]:
|
|
print(f"refuse: mitigated threat {tid} names no test or gate stage", file=sys.stderr)
|
|
sys.exit(1)
|
|
unrun = sorted(t for t in tests if t not in named)
|
|
if unrun:
|
|
print(f"refuse: threat {tid} names {', '.join(unrun)}, which the --named stage does not run", file=sys.stderr)
|
|
sys.exit(1)
|
|
if severity == "high" and disposition == "mitigate":
|
|
if not any(re.match(r"^\| RC-\d+ \| " + re.escape(tid) + r" \|", l) for l in removal):
|
|
print(f"refuse: high threat {tid} has no removal check row", file=sys.stderr)
|
|
sys.exit(1)
|
|
for flag in ("nyquist_compliant: true", "wave_0_complete: true", "status: validated"):
|
|
if not re.search(r"^" + re.escape(flag) + r"$", validation, re.M):
|
|
print(f"refuse: validation lacks {flag!r}", file=sys.stderr)
|
|
sys.exit(1)
|
|
status_word = re.compile(r"(?<![A-Za-z])pending(?![A-Za-z])|\u2b1c|\| TBD \|", re.I)
|
|
for line in validation.splitlines():
|
|
if line.startswith("|") and status_word.search(line):
|
|
print("refuse: validation row still pending: " + line, file=sys.stderr)
|
|
sys.exit(1)
|
|
for req in ["API-03", "API-04", "API-05", "API-06", "API-07"]:
|
|
if req not in validation:
|
|
print(f"refuse: validation does not name {req}", file=sys.stderr)
|
|
sys.exit(1)
|
|
task_rows = "\n".join(l for l in validation.splitlines() if re.match(r"^\| 13-\d\d-T\d", l))
|
|
if not task_rows:
|
|
print("refuse: validation has no per-task verification rows", file=sys.stderr)
|
|
sys.exit(1)
|
|
for name in sorted(set(re.findall(r"\b(?:Test|Fuzz)[A-Z][A-Za-z0-9_]*", task_rows))):
|
|
if name not in named:
|
|
print(f"refuse: validation names {name}, which the --named stage does not run", file=sys.stderr)
|
|
sys.exit(1)
|
|
print("phase13 evidence passed")
|
|
PY
|
|
}
|
|
|
|
run_evidence() {
|
|
evidence_check "$PHASE_DIR" "$REVIEW" "$VALIDATION" "$(all_named)"
|
|
}
|
|
|
|
run_self_test() {
|
|
bash -n "${BASH_SOURCE[0]}"
|
|
expect_detect pass 0 '{"Action":"pass","Package":"p","Test":"TestPhase13Threats"}'
|
|
expect_detect fail 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
|
{"Action":"fail","Package":"p","Test":"TestPhase13Threats/T-13-01"}'
|
|
expect_detect skip 2 '{"Action":"skip","Package":"p","Test":"TestPhase13Threats"}'
|
|
expect_detect zero 3 '{"Action":"pass","Package":"p"}'
|
|
expect_detect no-tests-to-run 3 '{"Action":"pass","Package":"p","Test":"TestA"}
|
|
{"Action":"output","Package":"q","Output":"testing: warning: no tests to run\n"}'
|
|
expect_detect nonjson 4 '{"Action":"pass",'
|
|
expect_detect build 1 '{"Action":"build-fail","ImportPath":"p"}
|
|
{"Action":"pass","Package":"q","Test":"TestA"}'
|
|
expect_detect build-flag 1 '{"Action":"pass","Package":"q","Test":"TestA"}
|
|
{"Action":"fail","Package":"p","FailedBuild":"p"}'
|
|
expect_detect package 1 '{"Action":"pass","Package":"p","Test":"TestA"}
|
|
{"Action":"fail","Package":"p"}'
|
|
expect_detect race 6 '{"Action":"output","Package":"p","Test":"TestA","Output":"WARNING: DATA RACE\n"}
|
|
{"Action":"pass","Package":"p","Test":"TestA"}'
|
|
PHASE13_REQUIRE="TestRouteTablePhase13 TestPhase13Threats" expect_detect required 5 \
|
|
'{"Action":"pass","Package":"p","Test":"TestRouteTablePhase13"}'
|
|
local flag
|
|
for flag in --self-test --go --parity --named --removal --coverage --evidence --all; do
|
|
grep -q -- "^$flag)" "${BASH_SOURCE[0]}" || {
|
|
echo "refuse: missing mode $flag" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
if [[ -n "${FORCE_COLOR+x}" ]]; then
|
|
echo "refuse: self-test FORCE_COLOR is still set" >&2
|
|
exit 1
|
|
fi
|
|
|
|
local scratch
|
|
scratch="$(mktemp -d)"
|
|
trap 'rm -rf "$scratch"' RETURN
|
|
|
|
# The module pin refuses a changed or missing x/text and accepts the
|
|
# audited line.
|
|
printf 'golang.org/x/text %s\n' "$XTEXT_VERSION" >"$scratch/mods"
|
|
module_pin "$scratch/mods" 2>/dev/null || {
|
|
echo "refuse: self-test module_pin rejected the audited version" >&2
|
|
exit 1
|
|
}
|
|
for plant in 'golang.org/x/text v0.41.0' ''; do
|
|
printf '%s\n' "$plant" >"$scratch/mods"
|
|
if module_pin "$scratch/mods" 2>/dev/null; then
|
|
echo "refuse: self-test module_pin accepted ${plant:-a missing x/text}" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# The corpus scan refuses each planted secret shape and accepts
|
|
# synthetic values.
|
|
mkdir -p "$scratch/corpus"
|
|
printf 'go test fuzz v1\nstring("POST /x")\nstring("{\\"status\\":\\"imported\\",\\"pad\\":\\"QQQQ\\"}")\n' >"$scratch/corpus/seed"
|
|
corpus_scan "$scratch/corpus" 2>/dev/null || {
|
|
echo "refuse: self-test corpus_scan rejected a synthetic seed" >&2
|
|
exit 1
|
|
}
|
|
local secret
|
|
for secret in "$(printf 'a%.0s' $(seq 64))" "inv_ABCDEFGHIJKLMNOPQRST" "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.sig" "Bearer abcdefghijklmnop"; do
|
|
printf 'string("%s")\n' "$secret" >"$scratch/corpus/planted"
|
|
if corpus_scan "$scratch/corpus" 2>/dev/null; then
|
|
echo "refuse: self-test corpus_scan accepted a planted secret ${secret:0:12}" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
rm -f "$scratch/corpus/planted" "$scratch/corpus/seed"
|
|
if corpus_scan "$scratch/corpus" 2>/dev/null; then
|
|
echo "refuse: self-test corpus_scan accepted an empty corpus" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The manifest counter reads only status lines.
|
|
printf 'routes:\n - id: a\n status: ported\n - id: b\n status: pending\n - id: c\n status: ported\n# status: ported\n' >"$scratch/manifest.yaml"
|
|
if [[ "$(manifest_count "$scratch/manifest.yaml" ported)" -ne 2 || "$(manifest_count "$scratch/manifest.yaml" pending)" -ne 1 ]]; then
|
|
echo "refuse: self-test manifest_count miscounted" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The corpus coverage line must show the expected counts.
|
|
local line="recorded $((EXPECTED_PORTED + EXPECTED_PENDING))/$((EXPECTED_PORTED + EXPECTED_PENDING)) passing $EXPECTED_PORTED failing 0 unrecorded 0 pending $EXPECTED_PENDING"
|
|
printf '{"Action":"output","Package":"p","Test":"TestParityCorpus/coverage","Output":"%s\\n"}\n' "$line" >"$scratch/cov.json"
|
|
corpus_coverage "$scratch/cov.json" >/dev/null 2>&1 || {
|
|
echo "refuse: self-test corpus_coverage rejected the expected counts" >&2
|
|
exit 1
|
|
}
|
|
local total=$((EXPECTED_PORTED + EXPECTED_PENDING)) planted
|
|
for planted in \
|
|
"recorded $total/$total passing $((EXPECTED_PORTED - 1)) failing 1 unrecorded 0 pending $EXPECTED_PENDING" \
|
|
"recorded $((total + 1))/$((total + 1)) passing $EXPECTED_PORTED failing 0 unrecorded 0 pending $((EXPECTED_PENDING + 1))" \
|
|
"recorded $((total - 1))/$total passing $EXPECTED_PORTED failing 0 unrecorded 1 pending $((EXPECTED_PENDING - 1))" \
|
|
"recorded $total/$total passing $((EXPECTED_PORTED + 1)) failing 0 unrecorded 0 pending $((EXPECTED_PENDING - 1))"; do
|
|
printf '{"Action":"output","Package":"p","Output":"%s\\n"}\n' "$planted" >"$scratch/cov.json"
|
|
if corpus_coverage "$scratch/cov.json" >/dev/null 2>&1; then
|
|
echo "refuse: self-test corpus_coverage accepted: $planted" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
printf '{"Action":"pass","Package":"p","Test":"TestParityCorpus"}\n' >"$scratch/cov.json"
|
|
if corpus_coverage "$scratch/cov.json" >/dev/null 2>&1; then
|
|
echo "refuse: self-test corpus_coverage accepted a log without a coverage line" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The coverage report refuses a package under the floor and accepts one
|
|
# over it; a block covered by any profile counts once; COVERAGE_ONLY
|
|
# narrows the report.
|
|
printf 'mode: set\nexample.test/a/x.go:1.1,2.2 8 1\nexample.test/a/x.go:3.1,4.2 2 0\n' >"$scratch/p1"
|
|
printf 'mode: set\nexample.test/a/x.go:3.1,4.2 2 1\nexample.test/b/y.go:1.1,2.2 5 0\nexample.test/b/y.go:3.1,4.2 5 1\n' >"$scratch/p2"
|
|
local out
|
|
out="$(coverage_report 80 "$scratch/p1" 2>&1)" || {
|
|
echo "refuse: self-test coverage_report refused 80% at an 80% floor: $out" >&2
|
|
exit 1
|
|
}
|
|
if out="$(coverage_report 80 "$scratch/p1" "$scratch/p2" 2>&1)"; then
|
|
echo "refuse: self-test coverage_report accepted a 50% package" >&2
|
|
exit 1
|
|
fi
|
|
grep -q "coverage example.test/a 100.0%" <<<"$out" || {
|
|
echo "refuse: self-test coverage_report did not merge profiles: $out" >&2
|
|
exit 1
|
|
}
|
|
COVERAGE_ONLY="/a" coverage_report 80 "$scratch/p1" "$scratch/p2" >/dev/null 2>&1 || {
|
|
echo "refuse: self-test COVERAGE_ONLY did not narrow the report" >&2
|
|
exit 1
|
|
}
|
|
printf 'mode: set\n' >"$scratch/empty"
|
|
if coverage_report 80 "$scratch/empty" 2>/dev/null; then
|
|
echo "refuse: self-test coverage_report accepted an empty profile" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The function floor refuses a function below it and a file with no
|
|
# functions in the profile.
|
|
printf 'example.test/c/registration.go:10:\tRegisterUser\t84.4%%\nexample.test/c/registration.go:40:\tIssueToken\t100.0%%\ntotal:\t(statements)\t90.0%%\n' >"$scratch/func"
|
|
func_floor 80 c/registration.go <"$scratch/func" >/dev/null 2>&1 || {
|
|
echo "refuse: self-test func_floor rejected functions over the floor" >&2
|
|
exit 1
|
|
}
|
|
printf 'example.test/c/registration.go:10:\tRegisterUser\t79.9%%\n' >"$scratch/func"
|
|
if func_floor 80 c/registration.go <"$scratch/func" >/dev/null 2>&1; then
|
|
echo "refuse: self-test func_floor accepted a 79.9% function" >&2
|
|
exit 1
|
|
fi
|
|
if func_floor 80 c/other.go <"$scratch/func" >/dev/null 2>&1; then
|
|
echo "refuse: self-test func_floor accepted a file without functions" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The evidence check refuses a missing threat row, a wrong disposition,
|
|
# a high threat without a removal row, a test the named stage does not
|
|
# run, a pending validation row, a missing Wave 0 flag and a validation
|
|
# test the named stage does not run; a threat two plans declare takes
|
|
# the stricter severity and disposition.
|
|
mkdir -p "$scratch/phase"
|
|
printf '| T-13-90 | Spoofing | x | high | mitigate | y |\n| T-13-91 | Tampering | x | low | accept | y |\n' >"$scratch/phase/13-01-PLAN.md"
|
|
printf '| T-13-91 | Tampering | x | medium | mitigate | y |\n' >"$scratch/phase/13-02-PLAN.md"
|
|
cat >"$scratch/review.md" <<'EOR'
|
|
| T-13-90 | Spoofing | x | high | mitigate | y | TestAlpha | pass | none |
|
|
| T-13-91 | Tampering | x | medium | mitigate | y | TestAlpha | pass | none |
|
|
| RC-90 | T-13-90 | f | a | b | c | fails |
|
|
EOR
|
|
cat >"$scratch/validation.md" <<'EOV'
|
|
status: validated
|
|
nyquist_compliant: true
|
|
wave_0_complete: true
|
|
| 13-01-T1 | API-03, API-04, API-05, API-06, API-07 | `go test -run '^TestAlpha$'` | ✅ green |
|
|
EOV
|
|
evidence_check "$scratch/phase" "$scratch/review.md" "$scratch/validation.md" "TestAlpha" >/dev/null 2>&1 || {
|
|
echo "refuse: self-test evidence_check rejected a complete record" >&2
|
|
exit 1
|
|
}
|
|
local case
|
|
for case in missing-row disposition removal unrun pending wave0 unnamed; do
|
|
cp "$scratch/review.md" "$scratch/review.case"
|
|
cp "$scratch/validation.md" "$scratch/validation.case"
|
|
local named="TestAlpha"
|
|
case "$case" in
|
|
missing-row) sed -i '/^| T-13-91 /d' "$scratch/review.case" ;;
|
|
disposition) sed -i 's/| medium | mitigate |/| low | accept |/' "$scratch/review.case" ;;
|
|
removal) sed -i '/^| RC-90 /d' "$scratch/review.case" ;;
|
|
unrun) sed -i 's/| TestAlpha | pass | none |$/| TestGamma | pass | none |/' "$scratch/review.case" ;;
|
|
pending) printf '| 13-02-T1 | API-03 | x | ⬜ pending |\n' >>"$scratch/validation.case" ;;
|
|
wave0) sed -i '/^wave_0_complete: true$/d' "$scratch/validation.case" ;;
|
|
unnamed) printf '| 13-02-T1 | API-03 | `go test -run TestBeta` | ✅ green |\n' >>"$scratch/validation.case" ;;
|
|
esac
|
|
if evidence_check "$scratch/phase" "$scratch/review.case" "$scratch/validation.case" "$named" >/dev/null 2>&1; then
|
|
echo "refuse: self-test evidence_check accepted the $case plant" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# The removal harness refuses an anchor that is not unique and a dirty
|
|
# tracked file, restores the file byte for byte, and reports a mutation
|
|
# whose test passes.
|
|
local fake="$scratch/fake"
|
|
mkdir -p "$fake/modules/acme"
|
|
printf 'module example.test/acme\n\ngo 1.27\n' >"$fake/go.mod"
|
|
printf 'package acme\n\nfunc Guard(n int) bool {\n\tif n > 3 {\n\t\treturn false\n\t}\n\treturn true\n}\n' >"$fake/modules/acme/acme.go"
|
|
printf 'package acme\n\nimport "testing"\n\nfunc TestGuard(t *testing.T) {\n\tif Guard(4) {\n\t\tt.Fatal("guard removed")\n\t}\n}\n\nfunc TestOther(t *testing.T) {}\n' >"$fake/modules/acme/acme_test.go"
|
|
cp "$fake/modules/acme/acme.go" "$scratch/acme.go.saved"
|
|
local table="$scratch/table.json"
|
|
printf '[["RC-T1","T-X","root","modules/acme/acme.go","if n > 3 {","if false {","./modules/acme","^TestGuard$"]]' >"$table"
|
|
out="$(removal_harness_in "$fake" "$table" 2>&1)" || {
|
|
echo "refuse: self-test removal harness did not catch a guarded mutation: $out" >&2
|
|
exit 1
|
|
}
|
|
grep -q "RC-T1 T-X modules/acme/acme.go: fails as required" <<<"$out" || {
|
|
echo "refuse: self-test removal harness output: $out" >&2
|
|
exit 1
|
|
}
|
|
cmp -s "$fake/modules/acme/acme.go" "$scratch/acme.go.saved" || {
|
|
echo "refuse: self-test removal harness did not restore the file" >&2
|
|
exit 1
|
|
}
|
|
printf '[["RC-T2","T-X","root","modules/acme/acme.go","if n > 3 {","if false {","./modules/acme","^TestOther$"]]' >"$table"
|
|
if out="$(removal_harness_in "$fake" "$table" 2>&1)"; then
|
|
echo "refuse: self-test removal harness accepted a mutation whose test passes: $out" >&2
|
|
exit 1
|
|
fi
|
|
grep -q "RC-T2 T-X modules/acme/acme.go: SURVIVED" <<<"$out" || {
|
|
echo "refuse: self-test removal harness refused a surviving mutation for the wrong reason: $out" >&2
|
|
exit 1
|
|
}
|
|
printf '[["RC-T3","T-X","root","modules/acme/acme.go","return","x","./modules/acme","^TestGuard$"]]' >"$table"
|
|
if out="$(removal_harness_in "$fake" "$table" 2>&1)"; then
|
|
echo "refuse: self-test removal harness accepted a non-unique anchor" >&2
|
|
exit 1
|
|
fi
|
|
grep -q "anchor occurs 2 times" <<<"$out" || {
|
|
echo "refuse: self-test removal harness refused a non-unique anchor for the wrong reason: $out" >&2
|
|
exit 1
|
|
}
|
|
printf '[["RC-T5","T-X","root","modules/acme/acme.go","if n > 3 {","if n > 3 {\\n\\tundefinedCall()","./modules/acme","^TestGuard$"]]' >"$table"
|
|
if out="$(removal_harness_in "$fake" "$table" 2>&1)"; then
|
|
echo "refuse: self-test removal harness counted a build failure as a failing test" >&2
|
|
exit 1
|
|
fi
|
|
cmp -s "$fake/modules/acme/acme.go" "$scratch/acme.go.saved" || {
|
|
echo "refuse: self-test removal harness did not restore after a build failure" >&2
|
|
exit 1
|
|
}
|
|
(cd "$fake" && git init -q && git add -A && git -c user.email=gate@example.test -c user.name=gate commit -qm init) >/dev/null
|
|
printf '// local edit\n' >>"$fake/modules/acme/acme.go"
|
|
printf '[["RC-T4","T-X","root","modules/acme/acme.go","if n > 3 {","if false {","./modules/acme","^TestGuard$"]]' >"$table"
|
|
if out="$(removal_harness_in "$fake" "$table" 2>&1)"; then
|
|
echo "refuse: self-test removal harness mutated a dirty file" >&2
|
|
exit 1
|
|
fi
|
|
grep -q "is dirty" <<<"$out" || {
|
|
echo "refuse: self-test removal harness refused a dirty file for the wrong reason: $out" >&2
|
|
exit 1
|
|
}
|
|
|
|
# Every row of the real removal table names a unique anchor in the
|
|
# current tree (the --removal stage would refuse it otherwise).
|
|
removal_table >"$table"
|
|
python3 - "$table" "$ROOT" "$APP" <<'PY' || exit 1
|
|
import json, os, sys
|
|
table, root, app = json.load(open(sys.argv[1])), sys.argv[2], sys.argv[3]
|
|
for rc, threat, repo, rel, anchor, repl, pkg, run in table:
|
|
base = {"root": root, "app": app, "script": root, "rootapp": root}[repo]
|
|
path = os.path.join(base, rel)
|
|
if not os.path.isfile(path):
|
|
print(f"refuse: self-test {rc}: {rel} is missing", file=sys.stderr)
|
|
sys.exit(1)
|
|
n = open(path).read().count(anchor)
|
|
if n != 1:
|
|
print(f"refuse: self-test {rc}: anchor occurs {n} times in {rel}", file=sys.stderr)
|
|
sys.exit(1)
|
|
PY
|
|
echo "phase13 self-test passed"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
--self-test) run_self_test ;;
|
|
--go) run_go ;;
|
|
--parity) run_parity ;;
|
|
--named) run_named ;;
|
|
--removal) run_removal ;;
|
|
--coverage) run_coverage ;;
|
|
--evidence) run_evidence ;;
|
|
--all)
|
|
run_self_test
|
|
run_go
|
|
run_parity
|
|
run_named
|
|
run_coverage
|
|
run_evidence
|
|
echo "phase13 all passed"
|
|
;;
|
|
*) usage ;;
|
|
esac
|