From 6beaa66b2587e9d7352414a50746b05a96122b77 Mon Sep 17 00:00:00 2001 From: Tom Boucher Date: Sun, 30 Aug 2026 16:51:52 -0400 Subject: [PATCH] enhance(#3304): gate re-verification blockers on deterministic evidence (#4085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(#3304): add failing-first suite for the convergence evidence gate Content-assertion suite for the Step 7 re-verification evidence gate (agents/gsd-verifier.md / gsd-core/references/verifier-evidence-gate.md). Committed before the implementation to prove RED via gsd-test. * enhance(#3304): gate re-verification blockers on deterministic evidence Step 7's anti-pattern scan re-runs at full, unbounded scope on every re-verification pass, independent of the must-haves established in Step 2. A blocker it finds — other than the self-evidencing debt-marker check — previously reverted a completed gap-closure round and started another --gaps cycle on nothing more than the verifier's own new judgment call, with no bound on how many times that could repeat. A Step 7 blocker now blocks unconditionally in re-verification mode only if it is a carried-forward gap (present in the prior VERIFICATION.md's gaps: list) or the flagged file was git-modified since the prior pass (a regression; fails closed toward blocking when history is unresolvable). Otherwise it predates the gap-closure round unflagged and needs deterministic evidence — a named test run red, or another concrete reproducible artifact — to stay blocking. Unevidenced, it downgrades to a new advisory: frontmatter list and report section instead of setting status: gaps_found, and never reverts a completed must-have. Maintainer approval was narrowed to this evidence condition only, explicitly rejecting the broader "advisory whenever untraceable to a requirement/decision/prior-gap" proposal — implemented and pinned by tests/verifier-evidence-gate.test.cjs and documented as rejected in gsd-core/references/verifier-evidence-gate.md so it can't silently re-expand. Closes #3304 * fix(#3304): correct window-truncation and indentation bugs in evidence-gate tests gsd-test's GREEN checkpoint caught 3 real bugs in the test file itself (not the production prose): a {0,600} match window was shorter than the 724-char paragraph it was scanning (the "exclude from Step 9 Rule 1" phrase starts at offset 662), and two regexes assumed no indentation after a markdown list-continuation line break. All three phrases are confirmed unique across agents/gsd-verifier.md, so the windowed submatches are replaced with direct whole-string assertions instead of just widening the window. Also acknowledges the deliberate byte growth in agents/gsd-verifier.md that the differential-attribution check (ADR-2719) correctly flagged. Emitted-Drift-Ack-Growth: gsd-verifier.md — adds the #3304 re-verification evidence gate (Step 7 rule, Advisory bucket, advisory: frontmatter, report section); 1488 bytes, still within the LARGE-tier 48 KiB cap (48751/49152). * docs(#3304): backfill changeset PR number --------- Co-authored-by: sim --- .changeset/daring-tunas-travel.md | 5 + agents/gsd-verifier.md | 19 ++- docs/AGENTS.md | 1 + docs/INVENTORY-MANIFEST.json | 1 + docs/INVENTORY.md | 1 + docs/reference/planning-artifacts.md | 2 +- gsd-core/references/verifier-evidence-gate.md | 160 ++++++++++++++++++ tests/fixtures/install-tree/antigravity.json | 1 + tests/fixtures/install-tree/augment.json | 1 + tests/fixtures/install-tree/claude-local.json | 1 + tests/fixtures/install-tree/claude.json | 1 + tests/fixtures/install-tree/cline.json | 1 + tests/fixtures/install-tree/codebuddy.json | 1 + tests/fixtures/install-tree/codex.json | 1 + tests/fixtures/install-tree/copilot.json | 1 + tests/fixtures/install-tree/cursor.json | 1 + tests/fixtures/install-tree/hermes.json | 1 + tests/fixtures/install-tree/kilo.json | 1 + tests/fixtures/install-tree/kimi-code.json | 1 + tests/fixtures/install-tree/kimi.json | 1 + tests/fixtures/install-tree/opencode.json | 1 + tests/fixtures/install-tree/pi.json | 1 + tests/fixtures/install-tree/qwen.json | 1 + tests/fixtures/install-tree/trae.json | 1 + tests/fixtures/install-tree/windsurf.json | 1 + tests/fixtures/install-tree/zcode.json | 1 + tests/verifier-evidence-gate.test.cjs | 154 +++++++++++++++++ 27 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 .changeset/daring-tunas-travel.md create mode 100644 gsd-core/references/verifier-evidence-gate.md create mode 100644 tests/verifier-evidence-gate.test.cjs diff --git a/.changeset/daring-tunas-travel.md b/.changeset/daring-tunas-travel.md new file mode 100644 index 000000000..be6d2f7c5 --- /dev/null +++ b/.changeset/daring-tunas-travel.md @@ -0,0 +1,5 @@ +--- +type: Changed +pr: 4085 +--- +**`/gsd-verify-work` re-verification no longer reopens a closed gap-closure round on an unevidenced new finding** — a Step 7 anti-pattern blocker that isn't a carried-forward gap or a regression on a file touched since the prior pass now needs a red-capable test or another concrete artifact to stay blocking; without one it's recorded as advisory instead of reverting completed work and starting another `--gaps` cycle. (#3304) diff --git a/agents/gsd-verifier.md b/agents/gsd-verifier.md index c5e491252..0806fe639 100644 --- a/agents/gsd-verifier.md +++ b/agents/gsd-verifier.md @@ -42,6 +42,7 @@ Every truth must resolve to VERIFIED, FAILED (BLOCKER), or UNCERTAIN (WARNING wi @~/.claude/gsd-core/references/verification-overrides.md @~/.claude/gsd-core/references/gates.md @~/.claude/gsd-core/references/verifier-phase-gates.md +@~/.claude/gsd-core/references/verifier-evidence-gate.md This agent implements the **Escalation Gate** pattern (surfaces unresolvable gaps to the developer for decision). @@ -410,7 +411,9 @@ grep -n -B 2 -A 2 "console\.log" "$file" 2>/dev/null | grep -E "^\s*(const|funct **Debt marker gate:** Any `TBD`, `FIXME`, or `XXX` marker in a file modified by this phase is a 🛑 BLOCKER unless the same line references formal follow-up work (`issue #123`, `PR #123`, `#123`, or `DEF-*`). Unreferenced markers mean completion is not auditable; set `status: gaps_found` and list each marker under `gaps`. -Categorize: 🛑 Blocker (prevents goal or unresolved debt marker) | ⚠️ Warning (incomplete) | ℹ️ Info (notable) +**Re-verification evidence gate (#3304):** in re-verification mode, a 🛑 Blocker other than an unresolved debt marker (always self-evidencing) blocks unconditionally only if it is a carried-forward gap (Step 0's `gaps:`) or the flagged file was git-modified since the prior `verified:` timestamp (fail closed: unresolvable history counts as modified). Otherwise it predates the gap-closure round unflagged and needs deterministic evidence — a named test run red, or another concrete reproducible artifact — to stay blocking. Full algorithm: @gsd-core/references/verifier-evidence-gate.md. Unevidenced → 📋 Advisory: record in `advisory:` frontmatter, exclude from Step 9 Rule 1, never revert a completed must-have. + +Categorize: 🛑 Blocker (prevents goal or unresolved debt marker) | ⚠️ Warning (incomplete) | ℹ️ Info (notable) | 📋 Advisory (re-verification only — new-scope, unevidenced; see above) ## Step 7b: Behavioral Spot-Checks @@ -700,6 +703,11 @@ deferred: # Only if deferred items exist (Step 9b) - truth: "Observable truth addressed in a later phase" addressed_in: "Phase N" evidence: "Matching goal or success criteria text" +advisory: # Only if unevidenced new-scope findings exist (Step 7, re-verification only) + - finding: "Short description of the new-scope concern" + category: architectural | security | other + reason: "Why raised; what would resolve it" + evidence_status: "none provided" behavior_unverified_items: # Only if behavior_unverified > 0 — emitted regardless of overall status, so these survive a gaps_found phase - truth: "Observable truth whose state transition or cancellation/cleanup/ordering invariant no test exercises" test: "What to trigger" @@ -744,6 +752,15 @@ Only include this section if deferred items exist (from Step 9b). |---|------|-------------|----------| | 1 | {truth} | Phase {N} | {matching goal or success criteria} | +### Advisory (New Scope, Unevidenced) + +New-scope findings from Step 7 with no deterministic evidence — reported, +not blocking. Include this section (even "None") whenever re-verification ran. + +| # | Finding | Category | Why Advisory | +|---|---------|----------|--------------| +| 1 | {finding} | {category} | new-scope, no deterministic evidence | + ### Required Artifacts | Artifact | Expected | Status | Details | diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 6afd0915b..bad44b868 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -340,6 +340,7 @@ Three further dimensions carry no number: **Verify Command Format Sanity**, - **Coincidental-reliance advisory (#1955):** a truth that reaches `✓ VERIFIED` is additionally asked *why* it holds. When the recorded evidence shows the truth holding for an incidental reason — `undeclared-precondition`, `incidental-ordering`, or `fixture-only` — the verdict is qualified as `✓ VERIFIED (coincidental-reliance)` and the truth is listed in the `coincidental_reliance_items` frontmatter field with what to harden. This is **advisory**: the base `✓ VERIFIED` token is unchanged, the truth still counts toward `verified_truths`, the overall `status` is unaffected, and no human-verification item is emitted — a passing phase still passes. It classifies evidence the verifier already gathered rather than asking it to rate its own confidence — but it is honestly an **endogenous** check, and `gsd-core/references/honest-verifier.md` records that endogenous gates are measurably weaker than the exogenous `backstop` tag it routes on. Advisory status is the consequence, not a coincidence: a miss costs exactly today's behaviour (a plain `✓ VERIFIED`) and a false positive costs one line of prose, never a failed phase, so a weaker mechanism is affordable here in a way it would not be on a pass/fail axis. Its precision is unmeasured. It complements the two existing axes: `PRESENT_BEHAVIOR_UNVERIFIED` is *no* behavioral evidence, `insufficient_spec` is an under-specified truth, and this is evidence that exists and passes for the wrong reason. The advisory is carried by two surfaces. `agents/gsd-verifier.md` (Step 3, sub-step 5c) holds the detection rule, and the verifier's eagerly-imported `gsd-core/references/verifier-phase-gates.md` points at the canonical report template `@~/.claude/gsd-core/templates/verification-report.md`, whose `## Guidelines` carry the same instruction. (The former third surface, the retired `verify-phase` workflow, was deleted as an orphan in #1892 — every verification path is subagent-shaped today.) +- **Convergence evidence gate (#3304):** during re-verification (after a `/gsd-plan-phase --gaps` cycle), Step 7's anti-pattern scan re-runs at full, unbounded scope — by design — but a 🛑 Blocker it finds no longer auto-reverts a completed gap-closure round on its own judgment alone. A blocker other than the self-evidencing debt-marker check (`TBD`/`FIXME`/`XXX`) blocks unconditionally only if it is a carried-forward gap from the prior `VERIFICATION.md` or its file was git-modified since the prior pass (a regression, fail-closed toward blocking when history is unresolvable); otherwise it predates the gap-closure round unflagged and needs deterministic evidence — a named test run red, or another concrete reproducible artifact — to stay blocking. Unevidenced, it downgrades to the `advisory:` frontmatter list and the report's "Advisory (New Scope, Unevidenced)" section instead of setting `status: gaps_found`. Full algorithm in `gsd-core/references/verifier-evidence-gate.md`. --- diff --git a/docs/INVENTORY-MANIFEST.json b/docs/INVENTORY-MANIFEST.json index 6838a373c..1752f2bdf 100644 --- a/docs/INVENTORY-MANIFEST.json +++ b/docs/INVENTORY-MANIFEST.json @@ -301,6 +301,7 @@ "user-story-template.md", "verification-overrides.md", "verification-patterns.md", + "verifier-evidence-gate.md", "verifier-phase-gates.md", "verifier-wiring-patterns.md", "verify-command-path-resolvability.md", diff --git a/docs/INVENTORY.md b/docs/INVENTORY.md index 4fa4a36ca..c16fb471d 100644 --- a/docs/INVENTORY.md +++ b/docs/INVENTORY.md @@ -306,6 +306,7 @@ Full roster at `gsd-core/references/*.md`. References are shared knowledge docum | `verification-patterns.md` | How to verify different artifact types. | | `verification-overrides.md` | Per-artifact verification override rules. | | `verifier-phase-gates.md` | Verifier-time gates eagerly imported by `gsd-verifier` (migrated from the retired `verify-phase` workflow, #1892): decision-coverage validation (#2492), test-quality audit, and infrastructure-phase human-verification scoping (#2504). | +| `verifier-evidence-gate.md` | Re-verification convergence gate loaded by `gsd-verifier` (#3304): a Step 7 anti-pattern blocker that is neither a carried-forward gap nor a regression needs deterministic evidence to stay blocking, else it downgrades to advisory. | | `planning-config.md` | Full config schema and behavior. | | `security-asvs-levels.md` | OWASP ASVS level definitions for GSD threat modeling — per-level planner disposition rigor and auditor verification depth (L1 opportunistic, L2 standard, L3 comprehensive). | | `git-integration.md` | Git commit, branching, and history patterns. | diff --git a/docs/reference/planning-artifacts.md b/docs/reference/planning-artifacts.md index fe97ee046..b8b6802d4 100644 --- a/docs/reference/planning-artifacts.md +++ b/docs/reference/planning-artifacts.md @@ -204,7 +204,7 @@ actuals: | | | |---|---| -| **Purpose** | Phase goal verification report. Checks `must_haves.truths`, `must_haves.artifacts`, and `must_haves.key_links` from all plans against the actual codebase after execution. Records `status: passed \| gaps_found \| human_needed`. A truth whose correctness depends on runtime behaviour — a state transition or a cancellation/cleanup/ordering invariant — is marked `⚠️ PRESENT_BEHAVIOR_UNVERIFIED` (not `VERIFIED`) when no test exercises it: it is excluded from `score`, counted in the `behavior_unverified` frontmatter field, and routed to `human_needed`, so a behaviour-dependent gap can no longer count toward a clean N/N. A truth that *does* reach `✓ VERIFIED` but holds for an incidental reason (`undeclared-precondition`, `incidental-ordering`, `fixture-only`) is qualified `✓ VERIFIED (coincidental-reliance)` and listed in `coincidental_reliance_items` — **advisory only**: it still counts toward `score`, does not change `status`, and emits no human-verification item (#1955). | +| **Purpose** | Phase goal verification report. Checks `must_haves.truths`, `must_haves.artifacts`, and `must_haves.key_links` from all plans against the actual codebase after execution. Records `status: passed \| gaps_found \| human_needed`. A truth whose correctness depends on runtime behaviour — a state transition or a cancellation/cleanup/ordering invariant — is marked `⚠️ PRESENT_BEHAVIOR_UNVERIFIED` (not `VERIFIED`) when no test exercises it: it is excluded from `score`, counted in the `behavior_unverified` frontmatter field, and routed to `human_needed`, so a behaviour-dependent gap can no longer count toward a clean N/N. A truth that *does* reach `✓ VERIFIED` but holds for an incidental reason (`undeclared-precondition`, `incidental-ordering`, `fixture-only`) is qualified `✓ VERIFIED (coincidental-reliance)` and listed in `coincidental_reliance_items` — **advisory only**: it still counts toward `score`, does not change `status`, and emits no human-verification item (#1955). During re-verification (after a `/gsd-plan-phase --gaps` cycle), an anti-pattern blocker that is neither a carried-forward gap nor found in a file modified since the prior pass, and carries no deterministic evidence (a test run red, or another concrete reproducible artifact), is downgraded to the `advisory:` frontmatter list instead of reopening `status: gaps_found` — it does not revert a completed must-have (#3304). | | **Produced by** | `/gsd-verify-work` (or the verify step within `/gsd-execute-phase`). | | **Consumed by** | `plan-phase` closed-phase gate (a `status: passed` VERIFICATION.md marks the phase `Complete` and blocks replanning without `--force`); `/gsd-progress`; human review. | diff --git a/gsd-core/references/verifier-evidence-gate.md b/gsd-core/references/verifier-evidence-gate.md new file mode 100644 index 000000000..9478e59cb --- /dev/null +++ b/gsd-core/references/verifier-evidence-gate.md @@ -0,0 +1,160 @@ +# Convergence Evidence Gate (#3304) + +Bounds Step 7's anti-pattern scan so an approved gap-closure contract can +actually close. Applies **only** when `is_re_verification = true` (Step 0) — +a first pass has no prior contract to be out-of-contract from, so this gate +is a pure no-op there. + +## The problem this closes + +Steps 4-7c re-verify at full, unbounded scope on every re-verification pass — +that is documented, intended design, not the bug. The bug is narrower: Step +7's `Categorize:` line lets the verifier's own free-form judgment label +*anything* it believes "prevents goal" a 🛑 Blocker, and Step 9 Rule 1 +promotes any 🛑 Blocker straight into `status: gaps_found` — with no +distinction between a blocker tied to what the gap-closure round was actually +supposed to fix and a blocker that is simply a new opinion formed on this +pass. Reported real-world instance: a re-verification cycle promoted four +"architectural and security observations" to blockers, none backed by a +failing test, none traceable to a requirement/decision/prior gap, reverting a +completed, all-green gap-closure round and recommending another `--gaps` +cycle — with no bound on how many times that could repeat. + +Truths, artifacts, and key links (Steps 3-6) can **never** produce this +failure mode: Step 0 re-verification mode reuses the must-haves extracted in +Step 2 verbatim ("Skip to Step 3") rather than re-establishing them, so +whatever a truth/artifact/link *is* was fixed before this re-verification +round started. Only Step 7's blanket per-file scan is unbounded by that +must-haves contract — which is exactly the mechanism the issue's diagnosis +names. This gate therefore touches Step 7 only. + +## Definitions + +**Self-evidencing blocker (unaffected by this gate).** The debt-marker gate +(`TBD`/`FIXME`/`XXX` with no `issue #123`/`PR #123`/`#123`/`DEF-*` reference +on the same line) is the *only* Step 7 category with zero judgment +component — a regex match plus the absence of a follow-up reference, nothing +inferred. Its own textual presence in the file is the deterministic evidence. +It keeps blocking unconditionally, exactly as before. Do **not** extend this +carve-out to any other Step 7 category (stub classification, hollow props, +empty implementations, console-log-only): every one of those already +requires judgment per Step 7's own "Stub classification" paragraph ("a grep +match is a STUB only when the value flows to rendering... and no other code +path populates it with real data") — that judgment is exactly what this gate +exists to check. + +**New-scope finding.** Any Step 7 🛑 Blocker other than a self-evidencing one +(above) is a new-scope finding **unless** either of the following holds, in +which case it is in-contract and blocks unconditionally, evidence or not: + +1. **Carried-forward gap** — it matches an item in the previous + VERIFICATION.md's `gaps:` list, using the same 80%-token-overlap matching + algorithm Step 3b already uses for override matching (normalize to + lowercase, strip punctuation, collapse whitespace, tokenize, intersect). +2. **Regression** — the flagged file was modified since the previous + VERIFICATION.md's `verified:` timestamp. Check file-level, not + line-level — an LLM agent re-deriving precise line provenance mid-pass is + unreliable; file-level modification is a single, robust command: + + ```bash + git log --since="$PREV_VERIFIED_TS" --oneline -- "$file" + ``` + + A non-empty result means the file changed since the prior pass — the + gap-closure round could plausibly have introduced this finding, so it's + self-evidencing as a regression and blocks. **Fail closed**: if git + history is unavailable, ambiguous, or the timestamp can't be parsed, + treat the file as modified (blocks). The imprecision this trades away + (a big file with one unrelated hunk touched treats every pattern in it as + "new") only ever makes *more* things block, never fewer — consistent with + ``. + +A finding that is neither a carried-forward gap nor on a file modified since +the prior pass predates the gap-closure round entirely and was never flagged +as a gap then — this is the literal "some findings predated the gap +implementation and had previously been explicitly treated as non-blocking" +case from the issue. + +**Deterministic evidence** — required for a new-scope finding to stay +blocking. One of: + +- A **named test that FAILS when actually run** (red). Run exactly one test, + the same discipline Step 7b already uses for behavioral spot-checks — + never the full suite. Record the exact command and the failing output. +- **Another concrete, reproducible artifact** — a command + output that + demonstrates the defect (a crash, a probe failure, a reproducible bad + response). An assertion, opinion, or architectural preference with no test + and no reproducible command output is not evidence, however well-reasoned. + +## The gate + +- New-scope finding **with** deterministic evidence → 🛑 Blocker, unchanged. + This includes evidenced security findings — they are preserved and still + block. +- New-scope finding **without** deterministic evidence → downgrade out of + the blocker set. Record it in the `advisory:` frontmatter list (parallel to + the existing Step 9b `deferred:` list) with its reasoning intact. It does + **not** count toward Step 9 Rule 1's `gaps_found` trigger and does **not** + revert a completed must-have or, on its own, justify another + `/gsd:plan-phase --gaps` cycle. + +This changes nothing else: a carried-forward gap or a regression still +blocks with or without a pre-existing requirement to point at, and every +non-Step-7 trigger (FAILED truth, MISSING/STUB artifact, NOT_WIRED link) is +untouched, since those can never be new-scope in the first place. + +## What this deliberately does NOT implement + +The issue as filed proposed a broader rule: a finding is advisory whenever +it is untraceable to a requirement/decision/prior-gap (conditions A and B), +regardless of evidence. The maintainer approved **condition C only** — +evidence, not contract-traceability, is the bar. A finding with no +pre-existing requirement to point at but with a real failing test still +blocks. Do not implement A/B: that would demote a genuine, reproducible +defect to advisory purely for being newly discovered, which is exactly the +deferral this project's no-defer rule forbids. This gate narrows *when a +blocker needs proof*, not *what counts as in scope*. + +## Advisory frontmatter + +```yaml +advisory: # Only if new-scope findings lack deterministic evidence (Step 7) + - finding: "Short description of the new-scope concern" + category: architectural | security | other + reason: "Why this was raised; what would resolve it" + evidence_status: "none provided" # or cite what was attempted but inconclusive +``` + +## Report section + +```markdown +### Advisory (New Scope, Unevidenced) + +New-scope findings from Step 7 with no deterministic evidence — reported, +not blocking, do not revert a completed must-have. + +| # | Finding | Category | Why Advisory | +|---|---------|----------|--------------| +| 1 | {finding} | {category} | new-scope, no deterministic evidence | +``` + +Include this section (even if empty, stating "None") whenever +`is_re_verification = true` ran — an omitted section reads as "not +checked," not "checked and clean." + +## Worked example (from the issue's reported incident) + +Prior pass: `gaps_found`, 4 items — all closed by approved gap-closure plans, +re-verification begins. + +- Finding: "the retry loop's backoff strategy is architecturally fragile + under sustained load." Not in the prior `gaps:` list. The flagged file was + last modified 3 weeks before this verification pass (before the + gap-closure plans even started) — not a regression. No test run, no + reproducible command demonstrating a failure. → **advisory**, does not + block, does not revert the 4 closed gaps. +- Finding: `TBD: handle the timeout case` left in a file the gap-closure plan + edited this pass. → self-evidencing debt marker, unaffected by this gate, + blocks exactly as it always has. +- Finding: a previously-closed gap's file now fails the SAME named test that + originally proved it broken. → carried-forward gap, blocks. diff --git a/tests/fixtures/install-tree/antigravity.json b/tests/fixtures/install-tree/antigravity.json index 8f19fbc31..01f4035db 100644 --- a/tests/fixtures/install-tree/antigravity.json +++ b/tests/fixtures/install-tree/antigravity.json @@ -169,6 +169,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/augment.json b/tests/fixtures/install-tree/augment.json index f8cfb7028..de966f742 100644 --- a/tests/fixtures/install-tree/augment.json +++ b/tests/fixtures/install-tree/augment.json @@ -240,6 +240,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/claude-local.json b/tests/fixtures/install-tree/claude-local.json index 4b6039988..179179c50 100644 --- a/tests/fixtures/install-tree/claude-local.json +++ b/tests/fixtures/install-tree/claude-local.json @@ -240,6 +240,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/claude.json b/tests/fixtures/install-tree/claude.json index b3dbf6e38..ded29d413 100644 --- a/tests/fixtures/install-tree/claude.json +++ b/tests/fixtures/install-tree/claude.json @@ -169,6 +169,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/cline.json b/tests/fixtures/install-tree/cline.json index 987afd5b6..3de9525b9 100644 --- a/tests/fixtures/install-tree/cline.json +++ b/tests/fixtures/install-tree/cline.json @@ -171,6 +171,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/codebuddy.json b/tests/fixtures/install-tree/codebuddy.json index 43311ed72..02c72ecc7 100644 --- a/tests/fixtures/install-tree/codebuddy.json +++ b/tests/fixtures/install-tree/codebuddy.json @@ -240,6 +240,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/codex.json b/tests/fixtures/install-tree/codex.json index 279c8acce..07977dd3e 100644 --- a/tests/fixtures/install-tree/codex.json +++ b/tests/fixtures/install-tree/codex.json @@ -205,6 +205,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/copilot.json b/tests/fixtures/install-tree/copilot.json index e7d4ef5b4..bc8d458ca 100644 --- a/tests/fixtures/install-tree/copilot.json +++ b/tests/fixtures/install-tree/copilot.json @@ -170,6 +170,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/cursor.json b/tests/fixtures/install-tree/cursor.json index 86b63695c..8325d813f 100644 --- a/tests/fixtures/install-tree/cursor.json +++ b/tests/fixtures/install-tree/cursor.json @@ -169,6 +169,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/hermes.json b/tests/fixtures/install-tree/hermes.json index 09cd73956..d9aca8eac 100644 --- a/tests/fixtures/install-tree/hermes.json +++ b/tests/fixtures/install-tree/hermes.json @@ -169,6 +169,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/kilo.json b/tests/fixtures/install-tree/kilo.json index 635aa8da5..23757fe3b 100644 --- a/tests/fixtures/install-tree/kilo.json +++ b/tests/fixtures/install-tree/kilo.json @@ -240,6 +240,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/kimi-code.json b/tests/fixtures/install-tree/kimi-code.json index e2ca89fb0..c82a846b8 100644 --- a/tests/fixtures/install-tree/kimi-code.json +++ b/tests/fixtures/install-tree/kimi-code.json @@ -170,6 +170,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/kimi.json b/tests/fixtures/install-tree/kimi.json index 67b57e72b..be44e4430 100644 --- a/tests/fixtures/install-tree/kimi.json +++ b/tests/fixtures/install-tree/kimi.json @@ -206,6 +206,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/opencode.json b/tests/fixtures/install-tree/opencode.json index 04ab1c3ba..4d5ec1b0a 100644 --- a/tests/fixtures/install-tree/opencode.json +++ b/tests/fixtures/install-tree/opencode.json @@ -240,6 +240,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/pi.json b/tests/fixtures/install-tree/pi.json index 4da5691df..fee540b30 100644 --- a/tests/fixtures/install-tree/pi.json +++ b/tests/fixtures/install-tree/pi.json @@ -136,6 +136,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/qwen.json b/tests/fixtures/install-tree/qwen.json index 251cdbd27..42dee880a 100644 --- a/tests/fixtures/install-tree/qwen.json +++ b/tests/fixtures/install-tree/qwen.json @@ -169,6 +169,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/trae.json b/tests/fixtures/install-tree/trae.json index 6a6c93d1e..033684f00 100644 --- a/tests/fixtures/install-tree/trae.json +++ b/tests/fixtures/install-tree/trae.json @@ -169,6 +169,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/windsurf.json b/tests/fixtures/install-tree/windsurf.json index d07825a4e..ecd88676b 100644 --- a/tests/fixtures/install-tree/windsurf.json +++ b/tests/fixtures/install-tree/windsurf.json @@ -169,6 +169,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/fixtures/install-tree/zcode.json b/tests/fixtures/install-tree/zcode.json index 851c1dce1..f88c8b06f 100644 --- a/tests/fixtures/install-tree/zcode.json +++ b/tests/fixtures/install-tree/zcode.json @@ -240,6 +240,7 @@ "gsd-core/references/user-story-template.md", "gsd-core/references/verification-overrides.md", "gsd-core/references/verification-patterns.md", + "gsd-core/references/verifier-evidence-gate.md", "gsd-core/references/verifier-phase-gates.md", "gsd-core/references/verifier-wiring-patterns.md", "gsd-core/references/verify-command-path-resolvability.md", diff --git a/tests/verifier-evidence-gate.test.cjs b/tests/verifier-evidence-gate.test.cjs new file mode 100644 index 000000000..fba697f95 --- /dev/null +++ b/tests/verifier-evidence-gate.test.cjs @@ -0,0 +1,154 @@ +'use strict'; + +// Issue #3304 — re-verification convergence: an out-of-contract, unevidenced +// blocker must not revert a completed gap-closure round or trigger another +// --gaps cycle. Content-assertion contract for the gsd-verifier agent (same +// basis as tests/verifier-behavior-unverified.test.cjs and +// tests/verifier-deferred-items.test.cjs — the agent .md file IS the deployed +// product; testing its text tests the shipped contract). +// +// Maintainer approval was narrowed to condition C only (deterministic +// evidence gates a new-scope blocker), rejecting the reporter's broader A/B +// (advisory whenever untraceable to a requirement/decision/prior-gap, +// regardless of evidence) — several tests below pin that narrowing so a +// future edit can't silently widen it back out. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const ROOT = path.join(__dirname, '..'); +const verifierPath = path.join(ROOT, 'agents', 'gsd-verifier.md'); +const verifier = fs.readFileSync(verifierPath, 'utf-8'); +const gatePath = path.join(ROOT, 'gsd-core', 'references', 'verifier-evidence-gate.md'); +const gate = fs.readFileSync(gatePath, 'utf-8'); + +test('gsd-verifier.md required_reading loads the evidence-gate reference', () => { + assert.match( + verifier, + /[\s\S]*?@~\/\.claude\/gsd-core\/references\/verifier-evidence-gate\.md[\s\S]*?<\/required_reading>/, + ); +}); + +test('verifier-evidence-gate.md reference file exists and is non-trivial', () => { + assert.ok(fs.existsSync(gatePath), 'gsd-core/references/verifier-evidence-gate.md should exist'); + assert.ok(gate.length > 500, 'reference file should contain the full algorithm, not a stub'); +}); + +test('Step 7 gates on re-verification mode only', () => { + assert.match(verifier, /Re-verification evidence gate[\s\S]{0,80}#3304/); + assert.match(verifier, /in re-verification mode[\s\S]{0,400}?blocks unconditionally only if/); +}); + +test('debt marker gate stays self-evidencing and unconditional (not routed through the new gate)', () => { + assert.match(verifier, /Re-verification evidence gate/); + assert.match(verifier, /other than an unresolved debt marker \(always self-evidencing\)/); +}); + +test('carried-forward gaps and regressions still block unconditionally, evidence or not', () => { + assert.match(verifier, /carried-forward gap \(Step 0's `gaps:`\)/); + assert.match(verifier, /git-modified since the prior `verified:` timestamp/); +}); + +test('unresolvable git history fails closed toward blocking', () => { + assert.match(verifier, /fail closed: unresolvable history counts as modified/); + assert.match(gate, /\*\*Fail closed\*\*/); +}); + +test('unevidenced new-scope findings route to advisory, not gaps_found', () => { + assert.match(verifier, /Unevidenced → 📋 Advisory/); + assert.match(verifier, /exclude from Step 9 Rule 1/); + assert.match(verifier, /never revert a completed must-have/); +}); + +test('Categorize line adds the Advisory bucket alongside Blocker/Warning/Info', () => { + const line = verifier.match(/^Categorize:.*$/m); + assert.ok(line, 'Categorize line must exist'); + assert.match(line[0], /🛑 Blocker/); + assert.match(line[0], /⚠️ Warning/); + assert.match(line[0], /ℹ️ Info/); + assert.match(line[0], /📋 Advisory/); +}); + +test('VERIFICATION.md frontmatter template carries the advisory: list', () => { + assert.match( + verifier, + /advisory: # Only if unevidenced new-scope findings exist \(Step 7, re-verification only\)/, + ); + assert.match(verifier, /advisory:[\s\S]{0,200}?category: architectural \| security \| other/); + assert.match(verifier, /advisory:[\s\S]{0,300}?evidence_status:/); +}); + +test('report body template includes an Advisory section, positioned after Deferred Items', () => { + const deferredIdx = verifier.indexOf('### Deferred Items'); + const advisoryIdx = verifier.indexOf('### Advisory (New Scope, Unevidenced)'); + assert.notEqual(deferredIdx, -1, 'Deferred Items section must exist'); + assert.notEqual(advisoryIdx, -1, 'Advisory section must exist'); + assert.ok(advisoryIdx > deferredIdx, 'Advisory section should follow Deferred Items'); +}); + +test('Advisory section instructs inclusion even when empty', () => { + const sectionStart = verifier.indexOf('### Advisory (New Scope, Unevidenced)'); + const nextSection = verifier.indexOf('### Required Artifacts', sectionStart); + assert.notEqual(sectionStart, -1); + assert.notEqual(nextSection, -1); + const section = verifier.slice(sectionStart, nextSection); + assert.match(section, /even "None"/); +}); + +// ─── Reference file: definitions and rejected-scope pins ────────────────── + +test('reference file scopes the gate to Step 7 only — truths/artifacts/key-links excluded', () => { + assert.match( + gate, + /Truths, artifacts, and key links[\s\S]{0,200}?can \*\*never\*\* produce this\s*\nfailure mode/, + ); +}); + +test('reference file explicitly rejects conditions A/B from the issue (contract-traceability alone)', () => { + const m = gate.match(/## What this deliberately does NOT implement[\s\S]{0,700}/); + assert.ok(m, 'the "does NOT implement" section must exist'); + assert.match(m[0], /conditions A and B/); + assert.match(m[0], /maintainer approved \*\*condition C only\*\*/); + assert.match(m[0], /Do not implement A\/B/); +}); + +test('reference file defines deterministic evidence as a run-red test or a reproducible artifact', () => { + assert.match(gate, /named test that FAILS when actually run/); + assert.match(gate, /Another concrete, reproducible artifact/); + assert.match(gate, /is not evidence, however well-reasoned/); +}); + +test('reference file uses file-level git check, not line-level (reliability tradeoff is disclosed)', () => { + assert.match(gate, /git log --since="\$PREV_VERIFIED_TS" --oneline -- "\$file"/); + assert.match(gate, /Check file-level, not\s*\n\s*line-level/); +}); + +test('reference file carries a worked example grounded in the reported incident', () => { + assert.match(gate, /## Worked example \(from the issue's reported incident\)/); + assert.match(gate, /→ \*\*advisory\*\*, does not\s*\n\s*block/); +}); + +// ─── Parity: advisory is a per-finding annotation, never an overall status ─ + +test('PARITY: advisory never leaks into the overall-status vocabulary', () => { + const unionLines = verifier.match(/^status:\s+[a-z_]+(?:\s*\|\s*[a-z_]+)+\s*$/gm) || []; + assert.ok(unionLines.length > 0, 'expected at least one status union line'); + for (const line of unionLines) { + assert.doesNotMatch(line, /advisory/i); + } +}); + +test('PARITY: overall-status enum in verification.cts does not gain an "advisory" status', () => { + const verificationLib = require(path.join(ROOT, 'gsd-core', 'bin', 'lib', 'verification.cjs')); + const { VERIFIER_STATUSES } = verificationLib; + assert.ok(Array.isArray(VERIFIER_STATUSES), 'VERIFIER_STATUSES array must be present'); + assert.ok( + !VERIFIER_STATUSES.includes('advisory'), + 'VERIFIER_STATUSES must not gain a per-finding "advisory" state — it stays a per-finding annotation', + ); + for (const s of ['passed', 'gaps_found', 'human_needed']) { + assert.ok(VERIFIER_STATUSES.includes(s), `VERIFIER_STATUSES must still contain ${s}`); + } +});