diff --git a/scripts/check-phase14.sh b/scripts/check-phase14.sh index 4ba607f..f805db3 100755 --- a/scripts/check-phase14.sh +++ b/scripts/check-phase14.sh @@ -49,6 +49,7 @@ usage: check-phase14.sh --named check-phase14.sh --removal check-phase14.sh --coverage + check-phase14.sh --evidence check-phase14.sh --all EOF exit 2 @@ -722,6 +723,162 @@ removal_harness_in() { ) } +# evidence_check PHASE_DIR REVIEW VALIDATION COVERAGE REQUIREMENTS ROADMAP +# NAMED: every T-14 threat the plans declare has exactly one review row +# copying its strictest severity and disposition; a mitigated threat names +# a test the --named stage runs (or a gate stage) and has a removal row; the +# validation file is validated, Nyquist-compliant, Wave 0 complete, without +# a pending or TBD row, names the seven requirements, and every test it +# names is run by --named; every COVERAGE.md INTEGRATE row names a test the +# --named stage runs and every OPT-OUT row gives a reason; REQUIREMENTS.md +# no longer carries the SDK wording for INTG-02 or the sitemap output for +# API-08 (D-06, D-14); the ROADMAP Phase 14 repos line and success criteria +# name the album Discogs and recognize routes and both shared plugin repos +# (D-07, D-13). +evidence_check() { + python3 - "$@" <<'PY' +import glob, os, re, sys +phase_dir, review_path, validation_path, coverage_path, req_path, roadmap_path, named = sys.argv[1:8] +named = set(named.split()) +for p in (review_path, validation_path, coverage_path, req_path, roadmap_path): + if not os.path.isfile(p): + print(f"refuse: {p} is missing", file=sys.stderr) + sys.exit(1) +review = open(review_path).read() +validation = open(validation_path).read() +coverage = open(coverage_path).read() +requirements = open(req_path).read() +roadmap = open(roadmap_path).read() +test_re = re.compile(r"\b(?:Test|Fuzz)[A-Z][A-Za-z0-9_]*") + +# Threats. +sev_rank = {"low": 0, "medium": 1, "high": 2} +declared = {} +for plan in sorted(glob.glob(os.path.join(phase_dir, "14-0*-PLAN.md"))): + for line in open(plan): + m = re.match(r"^\| (T-14-(?:\d\d|SC)) \|", line) + if not m: + continue + cells = [c.strip().lower() for c in line.strip().strip("|").split("|")] + prev = declared.get(m.group(1)) + if prev is None: + declared[m.group(1)] = cells + continue + sev = max(prev[3], cells[3], key=lambda s: sev_rank.get(s, -1)) + disp = "mitigate" if "mitigate" in (prev[4], cells[4]) else prev[4] + declared[m.group(1)] = prev[:3] + [sev, disp] + prev[5:] +if not declared: + print("refuse: no plan declares a T-14 threat", file=sys.stderr) + sys.exit(1) +lines = review.splitlines() +removal = [l for l in lines if re.match(r"^\| RC-\d+ \| T-14-", l)] +for tid, cells in sorted(declared.items()): + rows = [l for l in lines if l.startswith("| " + tid + " |")] + if len(rows) != 1: + print(f"refuse: review has {len(rows)} threat rows for {tid}, want 1", file=sys.stderr) + sys.exit(1) + row = [c.strip().lower() for c in rows[0].strip().strip("|").split("|")] + severity, disposition = cells[3], cells[4] + if severity not in row or disposition not in row: + print(f"refuse: review row {tid} does not copy severity {severity!r} and disposition {disposition!r}", file=sys.stderr) + sys.exit(1) + if disposition == "mitigate": + tests = set(test_re.findall(rows[0])) + if not tests and "check-phase14.sh" not in rows[0]: + print(f"refuse: mitigated threat {tid} names no test or gate stage", file=sys.stderr) + sys.exit(1) + unrun = sorted(t for t in tests if t not in named) + if unrun: + print(f"refuse: threat {tid} names {', '.join(unrun)}, which the --named stage does not run", file=sys.stderr) + sys.exit(1) + if not any(re.match(r"^\| RC-\d+ \| " + re.escape(tid) + r" \|", l) for l in removal): + print(f"refuse: mitigated threat {tid} has no removal check row", file=sys.stderr) + sys.exit(1) + +# Validation. +for flag in ("nyquist_compliant: true", "wave_0_complete: true", "status: validated"): + if not re.search(r"^" + re.escape(flag) + r"$", validation, re.M): + print(f"refuse: validation lacks {flag!r}", file=sys.stderr) + sys.exit(1) +status_word = re.compile(r"(?"$scratch/phase/14-01-PLAN.md" + printf '| T-14-91 | Tampering | x | medium | mitigate | y |\n' >"$scratch/phase/14-02-PLAN.md" + cat >"$scratch/review.md" <<'EOR' +| T-14-90 | Spoofing | x | high | mitigate | y | TestAlpha | pass | none | +| T-14-91 | Tampering | x | medium | mitigate | y | TestAlpha | pass | none | +| RC-90 | T-14-90 | f | a | b | c | fails | +| RC-91 | T-14-91 | f | a | b | c | fails | +EOR + cat >"$scratch/validation.md" <<'EOV' +status: validated +nyquist_compliant: true +wave_0_complete: true +| 14-01-T1 | JOBS-02, JOBS-03, SRCH-02, INTG-01, INTG-02, API-08, CLI-05 | `go test -run '^TestAlpha$'` | ✅ green | +EOV + cat >"$scratch/coverage.md" <<'EOC' +| capability | decision | reason | +|---|---|---| +| vendor: thing one | INTEGRATE | test: TestAlpha (14-02) | +| vendor: thing two | OPT-OUT | not used by the PHP reference — parity port | +EOC + cat >"$scratch/req.md" <<'EOQ' +- [x] **INTG-02**: AI cover recognition through adapters over the guarded client +- [x] **API-08**: Feedback submissions (sitemap dropped for this application, D-14) +EOQ + cat >"$scratch/roadmap.md" <<'EOM' +### Phase 14: Domain jobs +**Repos:** fonoteka.go; sm-golem-plugin and sm-feedback-plugin +**Success Criteria** (what must be TRUE): + 4. albums/import/discogs passes. + 5. albums/recognize passes. +**Plans:** 6/6 +### Phase 15: Next +EOM + local ev=("$scratch/phase" "$scratch/review.case" "$scratch/validation.case" "$scratch/coverage.case" "$scratch/req.case" "$scratch/roadmap.case" "TestAlpha") + local case + for case in complete missing-row disposition removal unrun pending wave0 unnamed integrate optout sdk sitemap roadmap; do + cp "$scratch/review.md" "$scratch/review.case" + cp "$scratch/validation.md" "$scratch/validation.case" + cp "$scratch/coverage.md" "$scratch/coverage.case" + cp "$scratch/req.md" "$scratch/req.case" + cp "$scratch/roadmap.md" "$scratch/roadmap.case" + case "$case" in + missing-row) sed -i '/^| T-14-91 /d' "$scratch/review.case" ;; + disposition) sed -i 's/| medium | mitigate |/| low | accept |/' "$scratch/review.case" ;; + removal) sed -i '/^| RC-91 /d' "$scratch/review.case" ;; + unrun) sed -i 's/| TestAlpha | pass | none |$/| TestGamma | pass | none |/' "$scratch/review.case" ;; + pending) printf '| 14-02-T1 | API-08 | x | ⬜ pending |\n' >>"$scratch/validation.case" ;; + wave0) sed -i '/^wave_0_complete: true$/d' "$scratch/validation.case" ;; + unnamed) printf '| 14-02-T1 | API-08 | `go test -run TestBeta` | ✅ green |\n' >>"$scratch/validation.case" ;; + integrate) sed -i 's/test: TestAlpha (14-02)/covered by the replay/' "$scratch/coverage.case" ;; + optout) sed -i 's/not used by the PHP reference — parity port//' "$scratch/coverage.case" ;; + sdk) sed -i 's/through adapters/through the Anthropic Go SDK/' "$scratch/req.case" ;; + sitemap) sed -i 's/Feedback submissions/Feedback submissions and sitemap output/' "$scratch/req.case" ;; + roadmap) sed -i 's/ and sm-feedback-plugin//' "$scratch/roadmap.case" ;; + esac + if [[ "$case" == complete ]]; then + evidence_check "${ev[@]}" >/dev/null 2>&1 || { + echo "refuse: self-test evidence_check rejected a complete record" >&2 + exit 1 + } + continue + fi + if evidence_check "${ev[@]}" >/dev/null 2>&1; then + echo "refuse: self-test evidence_check accepted the $case plant" >&2 + exit 1 + fi + done echo "phase14 self-test passed" } @@ -1017,6 +1250,7 @@ case "${1:-}" in --named) run_named ;; --removal) run_removal ;; --coverage) run_coverage ;; +--evidence) run_evidence ;; --all) # --removal edits tracked source while it runs, so it runs on its own. run_self_test @@ -1024,6 +1258,7 @@ case "${1:-}" in run_parity run_named run_coverage + run_evidence echo "phase14 all passed" ;; *) usage ;;