test(14-06): check-phase14.sh --evidence refuses an incomplete phase record
- every T-14 threat has one review row copying its plan's severity and disposition; a mitigated one names a test --named runs and a removal row - the validation map is validated, Nyquist-compliant and Wave 0 complete, with no open row and only tests --named runs - each COVERAGE.md INTEGRATE row names a run test, each OPT-OUT a reason - REQUIREMENTS.md keeps no SDK wording for INTG-02 and no sitemap output for API-08; the ROADMAP Phase 14 repos and criteria name the album Discogs and recognize routes and both shared plugin repos - --all runs it after the coverage stage
This commit is contained in:
Jakub Zych
@@ -49,6 +49,7 @@ usage:
|
||||
check-phase14.sh --named
|
||||
check-phase14.sh --removal
|
||||
check-phase14.sh --coverage
|
||||
check-phase14.sh --evidence
|
||||
check-phase14.sh --all
|
||||
EOF
|
||||
exit 2
|
||||
@@ -722,6 +723,162 @@ removal_harness_in() {
|
||||
)
|
||||
}
|
||||
|
||||
# evidence_check PHASE_DIR REVIEW VALIDATION COVERAGE REQUIREMENTS ROADMAP
|
||||
# NAMED: every T-14 threat the plans declare has exactly one review row
|
||||
# copying its strictest severity and disposition; a mitigated threat names
|
||||
# a test the --named stage runs (or a gate stage) and has a removal row; the
|
||||
# validation file is validated, Nyquist-compliant, Wave 0 complete, without
|
||||
# a pending or TBD row, names the seven requirements, and every test it
|
||||
# names is run by --named; every COVERAGE.md INTEGRATE row names a test the
|
||||
# --named stage runs and every OPT-OUT row gives a reason; REQUIREMENTS.md
|
||||
# no longer carries the SDK wording for INTG-02 or the sitemap output for
|
||||
# API-08 (D-06, D-14); the ROADMAP Phase 14 repos line and success criteria
|
||||
# name the album Discogs and recognize routes and both shared plugin repos
|
||||
# (D-07, D-13).
|
||||
evidence_check() {
|
||||
python3 - "$@" <<'PY'
|
||||
import glob, os, re, sys
|
||||
phase_dir, review_path, validation_path, coverage_path, req_path, roadmap_path, named = sys.argv[1:8]
|
||||
named = set(named.split())
|
||||
for p in (review_path, validation_path, coverage_path, req_path, roadmap_path):
|
||||
if not os.path.isfile(p):
|
||||
print(f"refuse: {p} is missing", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
review = open(review_path).read()
|
||||
validation = open(validation_path).read()
|
||||
coverage = open(coverage_path).read()
|
||||
requirements = open(req_path).read()
|
||||
roadmap = open(roadmap_path).read()
|
||||
test_re = re.compile(r"\b(?:Test|Fuzz)[A-Z][A-Za-z0-9_]*")
|
||||
|
||||
# Threats.
|
||||
sev_rank = {"low": 0, "medium": 1, "high": 2}
|
||||
declared = {}
|
||||
for plan in sorted(glob.glob(os.path.join(phase_dir, "14-0*-PLAN.md"))):
|
||||
for line in open(plan):
|
||||
m = re.match(r"^\| (T-14-(?:\d\d|SC)) \|", line)
|
||||
if not m:
|
||||
continue
|
||||
cells = [c.strip().lower() for c in line.strip().strip("|").split("|")]
|
||||
prev = declared.get(m.group(1))
|
||||
if prev is None:
|
||||
declared[m.group(1)] = cells
|
||||
continue
|
||||
sev = max(prev[3], cells[3], key=lambda s: sev_rank.get(s, -1))
|
||||
disp = "mitigate" if "mitigate" in (prev[4], cells[4]) else prev[4]
|
||||
declared[m.group(1)] = prev[:3] + [sev, disp] + prev[5:]
|
||||
if not declared:
|
||||
print("refuse: no plan declares a T-14 threat", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
lines = review.splitlines()
|
||||
removal = [l for l in lines if re.match(r"^\| RC-\d+ \| T-14-", l)]
|
||||
for tid, cells in sorted(declared.items()):
|
||||
rows = [l for l in lines if l.startswith("| " + tid + " |")]
|
||||
if len(rows) != 1:
|
||||
print(f"refuse: review has {len(rows)} threat rows for {tid}, want 1", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
row = [c.strip().lower() for c in rows[0].strip().strip("|").split("|")]
|
||||
severity, disposition = cells[3], cells[4]
|
||||
if severity not in row or disposition not in row:
|
||||
print(f"refuse: review row {tid} does not copy severity {severity!r} and disposition {disposition!r}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if disposition == "mitigate":
|
||||
tests = set(test_re.findall(rows[0]))
|
||||
if not tests and "check-phase14.sh" not in rows[0]:
|
||||
print(f"refuse: mitigated threat {tid} names no test or gate stage", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
unrun = sorted(t for t in tests if t not in named)
|
||||
if unrun:
|
||||
print(f"refuse: threat {tid} names {', '.join(unrun)}, which the --named stage does not run", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not any(re.match(r"^\| RC-\d+ \| " + re.escape(tid) + r" \|", l) for l in removal):
|
||||
print(f"refuse: mitigated threat {tid} has no removal check row", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Validation.
|
||||
for flag in ("nyquist_compliant: true", "wave_0_complete: true", "status: validated"):
|
||||
if not re.search(r"^" + re.escape(flag) + r"$", validation, re.M):
|
||||
print(f"refuse: validation lacks {flag!r}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
status_word = re.compile(r"(?<![A-Za-z])pending(?![A-Za-z])|⬜|\| TBD \|", re.I)
|
||||
for line in validation.splitlines():
|
||||
if line.startswith("|") and status_word.search(line):
|
||||
print("refuse: validation row still pending: " + line, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for req in ["JOBS-02", "JOBS-03", "SRCH-02", "INTG-01", "INTG-02", "API-08", "CLI-05"]:
|
||||
if req not in validation:
|
||||
print(f"refuse: validation does not name {req}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
task_rows = "\n".join(l for l in validation.splitlines() if re.match(r"^\| 14-\d\d-T\d", l))
|
||||
if not task_rows:
|
||||
print("refuse: validation has no per-task verification rows", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for name in sorted(set(test_re.findall(task_rows))):
|
||||
if name not in named:
|
||||
print(f"refuse: validation names {name}, which the --named stage does not run", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# API coverage.
|
||||
rows = [l for l in coverage.splitlines() if re.match(r"^\| [a-z0-9]+: ", l)]
|
||||
if not rows:
|
||||
print("refuse: COVERAGE.md has no capability rows", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for l in rows:
|
||||
cells = [c.strip() for c in l.strip().strip("|").split("|")]
|
||||
if len(cells) < 3:
|
||||
print("refuse: malformed coverage row: " + l, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
decision, reason = cells[1], cells[2]
|
||||
if decision == "INTEGRATE":
|
||||
tests = set(test_re.findall(reason))
|
||||
if not tests:
|
||||
print("refuse: INTEGRATE row names no test: " + l, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
unrun = sorted(t for t in tests if t not in named)
|
||||
if unrun:
|
||||
print(f"refuse: INTEGRATE row names {', '.join(unrun)}, which the --named stage does not run: " + l, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
elif decision == "OPT-OUT":
|
||||
if len(reason) < 10:
|
||||
print("refuse: OPT-OUT row without a reason: " + l, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"refuse: coverage row decision {decision!r}: " + l, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Requirement wording (D-06, D-14).
|
||||
for line in requirements.splitlines():
|
||||
if "**INTG-02**" in line and re.search(r"\bSDK\b", line):
|
||||
print("refuse: REQUIREMENTS.md INTG-02 still names an SDK: " + line, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if "**API-08**" in line and re.search(r"sitemap output", line, re.I):
|
||||
print("refuse: REQUIREMENTS.md API-08 still names the sitemap output: " + line, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# ROADMAP Phase 14 (D-07, D-13): the repos line and the success criteria.
|
||||
m = re.search(r"^### Phase 14:.*?(?=^### Phase )", roadmap, re.M | re.S)
|
||||
if not m:
|
||||
print("refuse: ROADMAP.md has no Phase 14 section", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
section = m.group(0)
|
||||
repos = "\n".join(l for l in section.splitlines() if l.startswith("**Repos:**"))
|
||||
crit = re.search(r"\*\*Success Criteria\*\*(.*?)\*\*Plans:\*\*", section, re.S)
|
||||
if not repos or not crit:
|
||||
print("refuse: ROADMAP Phase 14 lacks its Repos line or success criteria", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
scope = repos + "\n" + crit.group(1)
|
||||
for needle in ("albums/import/discogs", "recognize", "sm-golem-plugin", "sm-feedback-plugin"):
|
||||
if needle not in scope:
|
||||
print(f"refuse: ROADMAP Phase 14 repos and success criteria do not name {needle}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("phase14 evidence passed")
|
||||
PY
|
||||
}
|
||||
|
||||
run_evidence() {
|
||||
evidence_check "$PHASE_DIR" "$REVIEW" "$VALIDATION" "$PHASE_DIR/COVERAGE.md" "$ROOT/.planning/REQUIREMENTS.md" "$ROOT/.planning/ROADMAP.md" "$(all_named)"
|
||||
}
|
||||
|
||||
run_self_test() {
|
||||
bash -n "${BASH_SOURCE[0]}"
|
||||
expect_detect pass 0 '{"Action":"pass","Package":"p","Test":"TestPhase14Threats"}'
|
||||
@@ -1007,6 +1164,82 @@ for rc, threat, repo, rel, anchor, repl, pkg, run in table:
|
||||
print(f"refuse: self-test {rc}: anchor occurs {n} times in {rel}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
|
||||
# The evidence check refuses a missing threat row, a wrong disposition,
|
||||
# a mitigated threat without a removal row, a test the named stage does
|
||||
# not run, a pending validation row, a missing Wave 0 flag, an unnamed
|
||||
# validation test, an INTEGRATE row without a run test, an OPT-OUT row
|
||||
# without a reason, the SDK and sitemap wording, and a roadmap without
|
||||
# the shared plugin repos; a threat two plans declare takes the stricter
|
||||
# severity and disposition.
|
||||
mkdir -p "$scratch/phase"
|
||||
printf '| T-14-90 | Spoofing | x | high | mitigate | y |\n| T-14-91 | Tampering | x | low | accept | y |\n' >"$scratch/phase/14-01-PLAN.md"
|
||||
printf '| T-14-91 | Tampering | x | medium | mitigate | y |\n' >"$scratch/phase/14-02-PLAN.md"
|
||||
cat >"$scratch/review.md" <<'EOR'
|
||||
| T-14-90 | Spoofing | x | high | mitigate | y | TestAlpha | pass | none |
|
||||
| T-14-91 | Tampering | x | medium | mitigate | y | TestAlpha | pass | none |
|
||||
| RC-90 | T-14-90 | f | a | b | c | fails |
|
||||
| RC-91 | T-14-91 | f | a | b | c | fails |
|
||||
EOR
|
||||
cat >"$scratch/validation.md" <<'EOV'
|
||||
status: validated
|
||||
nyquist_compliant: true
|
||||
wave_0_complete: true
|
||||
| 14-01-T1 | JOBS-02, JOBS-03, SRCH-02, INTG-01, INTG-02, API-08, CLI-05 | `go test -run '^TestAlpha$'` | ✅ green |
|
||||
EOV
|
||||
cat >"$scratch/coverage.md" <<'EOC'
|
||||
| capability | decision | reason |
|
||||
|---|---|---|
|
||||
| vendor: thing one | INTEGRATE | test: TestAlpha (14-02) |
|
||||
| vendor: thing two | OPT-OUT | not used by the PHP reference — parity port |
|
||||
EOC
|
||||
cat >"$scratch/req.md" <<'EOQ'
|
||||
- [x] **INTG-02**: AI cover recognition through adapters over the guarded client
|
||||
- [x] **API-08**: Feedback submissions (sitemap dropped for this application, D-14)
|
||||
EOQ
|
||||
cat >"$scratch/roadmap.md" <<'EOM'
|
||||
### Phase 14: Domain jobs
|
||||
**Repos:** fonoteka.go; sm-golem-plugin and sm-feedback-plugin
|
||||
**Success Criteria** (what must be TRUE):
|
||||
4. albums/import/discogs passes.
|
||||
5. albums/recognize passes.
|
||||
**Plans:** 6/6
|
||||
### Phase 15: Next
|
||||
EOM
|
||||
local ev=("$scratch/phase" "$scratch/review.case" "$scratch/validation.case" "$scratch/coverage.case" "$scratch/req.case" "$scratch/roadmap.case" "TestAlpha")
|
||||
local case
|
||||
for case in complete missing-row disposition removal unrun pending wave0 unnamed integrate optout sdk sitemap roadmap; do
|
||||
cp "$scratch/review.md" "$scratch/review.case"
|
||||
cp "$scratch/validation.md" "$scratch/validation.case"
|
||||
cp "$scratch/coverage.md" "$scratch/coverage.case"
|
||||
cp "$scratch/req.md" "$scratch/req.case"
|
||||
cp "$scratch/roadmap.md" "$scratch/roadmap.case"
|
||||
case "$case" in
|
||||
missing-row) sed -i '/^| T-14-91 /d' "$scratch/review.case" ;;
|
||||
disposition) sed -i 's/| medium | mitigate |/| low | accept |/' "$scratch/review.case" ;;
|
||||
removal) sed -i '/^| RC-91 /d' "$scratch/review.case" ;;
|
||||
unrun) sed -i 's/| TestAlpha | pass | none |$/| TestGamma | pass | none |/' "$scratch/review.case" ;;
|
||||
pending) printf '| 14-02-T1 | API-08 | x | ⬜ pending |\n' >>"$scratch/validation.case" ;;
|
||||
wave0) sed -i '/^wave_0_complete: true$/d' "$scratch/validation.case" ;;
|
||||
unnamed) printf '| 14-02-T1 | API-08 | `go test -run TestBeta` | ✅ green |\n' >>"$scratch/validation.case" ;;
|
||||
integrate) sed -i 's/test: TestAlpha (14-02)/covered by the replay/' "$scratch/coverage.case" ;;
|
||||
optout) sed -i 's/not used by the PHP reference — parity port//' "$scratch/coverage.case" ;;
|
||||
sdk) sed -i 's/through adapters/through the Anthropic Go SDK/' "$scratch/req.case" ;;
|
||||
sitemap) sed -i 's/Feedback submissions/Feedback submissions and sitemap output/' "$scratch/req.case" ;;
|
||||
roadmap) sed -i 's/ and sm-feedback-plugin//' "$scratch/roadmap.case" ;;
|
||||
esac
|
||||
if [[ "$case" == complete ]]; then
|
||||
evidence_check "${ev[@]}" >/dev/null 2>&1 || {
|
||||
echo "refuse: self-test evidence_check rejected a complete record" >&2
|
||||
exit 1
|
||||
}
|
||||
continue
|
||||
fi
|
||||
if evidence_check "${ev[@]}" >/dev/null 2>&1; then
|
||||
echo "refuse: self-test evidence_check accepted the $case plant" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "phase14 self-test passed"
|
||||
}
|
||||
|
||||
@@ -1017,6 +1250,7 @@ case "${1:-}" in
|
||||
--named) run_named ;;
|
||||
--removal) run_removal ;;
|
||||
--coverage) run_coverage ;;
|
||||
--evidence) run_evidence ;;
|
||||
--all)
|
||||
# --removal edits tracked source while it runs, so it runs on its own.
|
||||
run_self_test
|
||||
@@ -1024,6 +1258,7 @@ case "${1:-}" in
|
||||
run_parity
|
||||
run_named
|
||||
run_coverage
|
||||
run_evidence
|
||||
echo "phase14 all passed"
|
||||
;;
|
||||
*) usage ;;
|
||||
|
||||
Reference in New Issue
Block a user