From 285cd41be09e3bc3ae2ac32d79a1b7bfa160a0df Mon Sep 17 00:00:00 2001 From: 0xdhx Date: Sun, 16 Aug 2026 13:14:27 -0500 Subject: [PATCH] fix(#3206): define "explicit evidence" inline at verifier 5b; repair stale honest-verifier cites (#3435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(#3206): define "explicit evidence" inline at verifier 5b; repair stale honest-verifier cites Step 3 item 5b abstained on non-inferable (backstop) truths "unless confirmed by explicit evidence" with the term undefined — its definition lived only in the non-included gsd-core/references/honest-verifier.md, behind a stale bare `references/` cite that 404s. Undefined, the term falls back to presence + wiring, the exact false-pass the #1154 abstention protocol refuses. - 5b: inline the compressed definition (a passing wired held-out/property-based test or directly observed behavior; presence + wiring never qualifies) and fix the cite. +84 B on the rewritten line; file lands at 49,151 of the 49,152 LARGE cap. - verifier-phase-gates.md (already ): gains the backstop-abstention reporting contract — AFK completion line ("complete with N unverified non-inferable checks", never silent, never a halt) and reason-distinctness (insufficient_spec vs manual-UAT human_needed). New content, no relocation of measured prose. - 5c (line 204) and MVP-mode (line 644) bare cites repaired to gsd-core/references/ (+9 B each). - Drift acks per ADR-2719 §4; two entries merge-appended into existing fragments (two ack sources may never name the same path). - Changeset fragment with the sanctioned pr: 0 placeholder (post-create backfill). Sibling census at next@7976b1ca0: 7 bare-cite instances in 4 agent files; the 3 in gsd-verifier.md are fixed here, gsd-executor.md:429,439 and gsd-doc-synthesizer.md:20,176 stay with the epic #1891 follow-up. Refs #1891 * chore(#3206): set changeset fragment pr to 3435 * fix(#3206): drop stale emitted-drift-ack entries that trip the ADR-2719 ratchet The round's ack bookkeeping explained ripples that were already self-attributed, so `tests/emitted-attribution.test.cjs` failed deterministically on the PR head with 5 stale acknowledgments. `agents/gsd-verifier.md` and `gsd-core/references/verifier-phase-gates.md` appear directly in `git diff --name-only`, so PROVENANCE_RULES attributes their emitted deltas without an ack; `agents/gsd-verifier.agent.md`, `agents/gsd-verifier.toml` and `agents/subagents/gsd-verifier.md` are derived emissions of a changed source and are attributed the same way. None of the five entries could ever be consumed, so all five were stale. Removed: the whole `3206-verifier-explicit-evidence.json` fragment (all four entries) and the `#3206 append` to `0000-legacy-migration.json`. Deliberately KEPT: the `#3206 append` to `1955-verifier-coincidental-reliance.json`. Its `gsd-verifier.md` entry is consumed by the size-growth ratchet, not the hash pass — the agent grew 49049 -> 49151 bytes, and `diffEmitted` treats a base-identical ack as spent and excludes it from `ackEntries`. Reverting that append as well turns the stale-ack failure into `1 file(s) grew without an acknowledgment` (verified both ways locally). * fix(#3206): compress 5b and re-acknowledge growth after rebase onto next The rebase onto next (159145435, PR #3558) invalidated two things at once: - #3558 grew agents/gsd-verifier.md to 49,098 bytes, leaving 54 bytes of LARGE-cap headroom where this PR's +102 no longer fits. Compressed the 5b rewrite to +34 net by dropping the trailing honest-verifier cite — superseded by the now-inline definition; honest-verifier.md stays cited at the adjacent 5c line. File lands at 49,150 (2 under the cap). - #3558 deleted tests/emitted-drift-acks/1955-verifier-coincidental- reliance.json, which carried this PR's growth acknowledgment, and its own 3409 fragment now names gsd-verifier.md. Re-homed the #3206 growth ack as an append to that entry (two ack sources may never name the same path). The changeset is updated to match: two bare references/ cites repaired (5c honest-verifier.md, MVP-mode verify-mvp-mode.md), the third (5b's) superseded by the inline definition rather than repaired. Reversion controls: restoring the uncompressed 5b fails agent-size-budget.test.cjs (LARGE hard cap); reverting the ack append fails emitted-attribution.test.cjs (differential attribution over the real tree, GSD_EMITTED_BASE=upstream/next). Both re-verified green at this tree: 213/213 (size + attribution), 992/992 across the 14 suites reading the touched files. Refs #3206 * test(#3206): pin the 5b explicit-evidence definition and cite resolution * fix(#3206): pin regression tests to the shipped contract text --------- Co-authored-by: sim --- .changeset/3206-verifier-explicit-evidence.md | 5 +++ agents/gsd-verifier.md | 6 +-- gsd-core/references/verifier-phase-gates.md | 23 ++++++++-- .../3409-unreachable-guard-arms.json | 2 +- tests/verifier-behavior-unverified.test.cjs | 45 +++++++++++++++++++ 5 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 .changeset/3206-verifier-explicit-evidence.md diff --git a/.changeset/3206-verifier-explicit-evidence.md b/.changeset/3206-verifier-explicit-evidence.md new file mode 100644 index 000000000..fa686f67f --- /dev/null +++ b/.changeset/3206-verifier-explicit-evidence.md @@ -0,0 +1,5 @@ +--- +type: Fixed +pr: 3435 +--- +**The verifier's non-inferable (`backstop`) abstention rule now defines "explicit evidence" where the verifier is guaranteed to read it.** Step 3 item 5b used the term undefined — its definition was stranded in `gsd-core/references/honest-verifier.md` behind a stale `references/` cite that does not resolve, so the term fell back to the verifier's default notion of evidence (symbol presence + wiring), the exact false-pass the #1154 abstention protocol exists to refuse. 5b now carries the definition inline (a passing wired held-out/property-based test or directly observed behavior; presence + wiring never qualifies), the AFK never-silent/never-halt completion line and the `insufficient_spec`-vs-manual-UAT distinction ship in the eagerly-loaded `verifier-phase-gates.md` reference, and the agent file's three stale bare `references/` cites are gone: the two at 5c and the MVP-mode section now resolve under the `gsd-core/` prefix, and 5b's is superseded by the inline definition itself. (#3206) diff --git a/agents/gsd-verifier.md b/agents/gsd-verifier.md index 9d4d998a7..0051edb4f 100644 --- a/agents/gsd-verifier.md +++ b/agents/gsd-verifier.md @@ -201,8 +201,8 @@ For each truth: - A pre-existing test exercises the transition/invariant and passes (confirm via Step 7b's single-named-test path) → ✓ VERIFIED. - No such test exists, or it can't run without a server/state mutation → ⚠️ PRESENT_BEHAVIOR_UNVERIFIED. Emit a human-verification item (Step 8) and do not count it toward the verified score (Step 9). - An accepted override (Step 3b) carries the truth as PASSED (override), exactly as it does for a FAILED truth. -5b. **Non-inferable (`backstop`) truths:** a `verification: backstop` truth (via `truthVerification()`) abstains unless confirmed by explicit evidence — mark `insufficient_spec` -> a human-verification item -> `human_needed`. See `references/honest-verifier.md`. -5c. **Reliance check (advisory, #1955).** Before finalizing a ✓ VERIFIED truth, ask *why* it holds. Classify the evidence already recorded, not your confidence in it. Endogenous, and so weaker than the exogenous `backstop` tag (`references/honest-verifier.md`) — advisory for exactly that reason. Flag `coincidental-reliance` when the evidence names one of: **undeclared-precondition** (state nothing in the phase's artifacts or a declared prerequisite guarantees), **incidental-ordering** (an order or side effect nothing in the code enforces), **fixture-only** (the test's own setup establishes the precondition; the production path has no equivalent). **Do NOT flag:** a precondition the code establishes or explicitly defaults; ordering the code enforces (await, explicit sequencing); a fixture merely supplying input the real caller also supplies; unease naming no specific state, ordering, or fixture. Out of scope: ⚠️ PRESENT_BEHAVIOR_UNVERIFIED and ⚠️ `insufficient_spec` (already routed to human), and PASSED (override) truths. Record `✓ VERIFIED (coincidental-reliance)` and add a `coincidental_reliance_items` entry. **Advisory only — not the score, not the status, and never a human-verification item** (Step 9 rule 2 would flip a passing phase to `human_needed`). The usual fix: promote the hidden assumption into a declared precondition. +5b. **Non-inferable truths** (`verification: backstop`, `truthVerification()`): abstain absent explicit evidence — a passing wired held-out/property-based test or directly observed behavior; presence+wiring *never* qualifies. Mark `insufficient_spec` -> human-verification item -> `human_needed`. +5c. **Reliance check (advisory, #1955).** Before finalizing a ✓ VERIFIED truth, ask *why* it holds. Classify the evidence already recorded, not your confidence in it. Endogenous, and so weaker than the exogenous `backstop` tag (`gsd-core/references/honest-verifier.md`) — advisory for exactly that reason. Flag `coincidental-reliance` when the evidence names one of: **undeclared-precondition** (state nothing in the phase's artifacts or a declared prerequisite guarantees), **incidental-ordering** (an order or side effect nothing in the code enforces), **fixture-only** (the test's own setup establishes the precondition; the production path has no equivalent). **Do NOT flag:** a precondition the code establishes or explicitly defaults; ordering the code enforces (await, explicit sequencing); a fixture merely supplying input the real caller also supplies; unease naming no specific state, ordering, or fixture. Out of scope: ⚠️ PRESENT_BEHAVIOR_UNVERIFIED and ⚠️ `insufficient_spec` (already routed to human), and PASSED (override) truths. Record `✓ VERIFIED (coincidental-reliance)` and add a `coincidental_reliance_items` entry. **Advisory only — not the score, not the status, and never a human-verification item** (Step 9 rule 2 would flip a passing phase to `human_needed`). The usual fix: promote the hidden assumption into a declared precondition. 6. Determine truth status ## Step 3b: Check Verification Overrides @@ -642,7 +642,7 @@ Deferred items are informational only — they do not require closure plans. **VERIFICATION.md output structure under MVP mode:** -1. Top-level "User Flow Coverage" table: each step of the user story → expected → evidence in codebase → status. (Format defined in `references/verify-mvp-mode.md`.) +1. Top-level "User Flow Coverage" table: each step of the user story → expected → evidence in codebase → status. (Format defined in `gsd-core/references/verify-mvp-mode.md`.) 2. Standard technical-check sections (API verification, error handling, etc.) follow below — only if the user flow coverage is complete. **User Story format guard:** Apply via the centralized verb instead of inlining the regex: diff --git a/gsd-core/references/verifier-phase-gates.md b/gsd-core/references/verifier-phase-gates.md index ef36459e2..b36739174 100644 --- a/gsd-core/references/verifier-phase-gates.md +++ b/gsd-core/references/verifier-phase-gates.md @@ -3,7 +3,8 @@ > Loaded eagerly by `agents/gsd-verifier.md` (``). Carries the three > verification-time gates that lived in the retired `verify-phase` workflow > (#1892 / epic #1891 F7): decision-coverage validation (#2492), the test-quality audit, -> and infrastructure-phase human-verification scoping (#2504). Run each at its named +> and infrastructure-phase human-verification scoping (#2504) — plus the backstop-abstention +> reporting contract (#3206). Run each gate at its named > agent step; `gsd_run` is the launcher shim defined in the agent's own Step 1 block. ## verify_decisions — Decision Coverage Gate (run after Step 6, requirements coverage) @@ -151,14 +152,14 @@ Infrastructure and foundation phases — code foundations, database schema, inte - Mark human verification as **N/A** with rationale: "Infrastructure/foundation phase — no user-facing elements to test manually." - Set `human_verification: []` and do **not** produce a `human_needed` status solely due to lack of user-facing features. - Only add human verification items if the phase goal or success criteria explicitly describe something a user would interact with (UI, CLI command output visible to end users, external service UX). -- **Exception — behavior-unverified truths still count.** A truth marked ⚠️ PRESENT_BEHAVIOR_UNVERIFIED (a state transition or a cancellation/cleanup/ordering invariant with no test exercising it) is a behavioral-evidence gap, not an artificial user-facing step. Record it in `behavior_unverified_items` and emit a human-verification item for it **even on an infrastructure/foundation phase** — these invariants are exactly where infra phases hide runtime state leaks. Such a truth drives `human_needed`; the auto-pass-UAT shortcut applies only to the absence of user-facing UX, never to a behavior-unverified invariant. +- **Exception — behavior-unverified truths still count.** A truth marked ⚠️ PRESENT_BEHAVIOR_UNVERIFIED (a state transition or a cancellation/cleanup/ordering invariant with no test exercising it) is a behavioral-evidence gap, not an artificial user-facing step. Record it in `behavior_unverified_items` and emit a human-verification item for it **even on an infrastructure/foundation phase** — these invariants are exactly where infra phases hide runtime state leaks. Such a truth drives `human_needed`; the auto-pass-UAT shortcut applies only to the absence of user-facing UX, never to a behavior-unverified invariant. The same carve-out covers an **abstained non-inferable truth** (⚠️ `insufficient_spec`, § Backstop abstention below) — an insufficient-spec gap is an evidence gap, not a user-facing step, so it too still emits its human-verification item and drives `human_needed` on an infrastructure phase. **How to determine if a phase is infrastructure/foundation:** - Phase goal or name contains: "foundation", "infrastructure", "schema", "database", "internal API", "data model", "scaffolding", "pipeline", "tooling", "CI", "migrations", "service layer", "backend", "core library" - Phase success criteria describe only technical artifacts (files exist, tests pass, schema is valid) with no user interaction required - There is no UI, CLI output visible to end users, or real-time behavior to observe -**If the phase IS infrastructure/foundation:** auto-pass UAT — skip the human verification items list entirely, **except any ⚠️ PRESENT_BEHAVIOR_UNVERIFIED truth (see exception above), which still emits a human-verification item and drives `human_needed`.** Log: +**If the phase IS infrastructure/foundation:** auto-pass UAT — skip the human verification items list entirely, **except any ⚠️ PRESENT_BEHAVIOR_UNVERIFIED or abstained ⚠️ `insufficient_spec` truth (see exception above), which still emits a human-verification item and drives `human_needed`.** Only when no such excepted truth exists, log: ```markdown ## Human Verification @@ -169,6 +170,22 @@ All acceptance criteria are verifiable programmatically. **If the phase IS user-facing:** only flag items that genuinely require a human — per the Step 8 always/uncertain lists already in the agent. Do not invent steps. +## Backstop abstention — reporting contract (#3206, companion to agent Step 3 item 5b) + +When a non-inferable (`verification: backstop`) truth abstains for lack of explicit evidence: + +- **Never silent, never a hard halt.** *Interactive:* the abstained item routes to the end-of-phase + human checkpoint. *Autonomous (AFK):* it produces a prominent `unverified — held-out test + recommended` flag and the completion line reads "complete with N unverified non-inferable checks"; + the run neither silently passes the blind spot nor hard-halts. +- **Distinguishable reason.** The abstain disposition carries `reason: insufficient_spec` so its + `human_needed` outcome is never conflated with an ordinary manual-UAT `human_needed`. +- **Infrastructure phases included.** This rides the same carve-out as ⚠️ PRESENT_BEHAVIOR_UNVERIFIED + in the infrastructure-phase gate above: an abstention is an evidence gap, not a user-facing step, + so the infra auto-pass-UAT shortcut never absorbs it. + +Full protocol and rationale: `gsd-core/references/honest-verifier.md`. + ## Lazy references - **Per-stack verification patterns:** before Step 4 (artifact verification) on an unfamiliar stack, Read `~/.claude/gsd-core/references/verification-patterns.md` — the grep catalog for React/Next.js components, API routes, database schema, and the universal stub patterns. Read it lazily (only the sections for the stack under verification); it is too large to load wholesale on every run. diff --git a/tests/emitted-drift-acks/3409-unreachable-guard-arms.json b/tests/emitted-drift-acks/3409-unreachable-guard-arms.json index a2b4bfc78..3e1d5e360 100644 --- a/tests/emitted-drift-acks/3409-unreachable-guard-arms.json +++ b/tests/emitted-drift-acks/3409-unreachable-guard-arms.json @@ -2,7 +2,7 @@ "version": 1, "paths": { "gsd-phase-researcher.md": "#3409: guarded `cat \"$phase_dir\"/*-CONTEXT.md` against nullglob wiping the pattern to zero operands when no CONTEXT.md exists — a bare `cat` with no operands blocks reading stdin (hangs the agent) instead of the `2>/dev/null` guard ever firing, since a stalled read is not a failing exit. Now checks `${_CTX[0]}` is a real path before invoking cat. Growth is the array-guard idiom itself (+43 bytes).", - "gsd-verifier.md": "#3409: same nullglob-hang fix as gsd-phase-researcher.md, applied to `cat \"$PHASE_DIR\"/*-VERIFICATION.md` in Step 0 — an absent VERIFICATION.md previously left a zero-operand `cat` blocking on stdin instead of falling through to first-verification mode. Growth is the array-guard idiom (+49 bytes).", + "gsd-verifier.md": "#3409: same nullglob-hang fix as gsd-phase-researcher.md, applied to `cat \"$PHASE_DIR\"/*-VERIFICATION.md` in Step 0 — an absent VERIFICATION.md previously left a zero-operand `cat` blocking on stdin instead of falling through to first-verification mode. Growth is the array-guard idiom (+49 bytes). — #3206 append (merged into this fragment because two ack sources may never name the same path): +52 bytes, 49098 -> 49150 (2 under the LARGE cap). The growth is the literal fix for the term 5b used undefined: the compressed explicit-evidence definition inlined at 5b (+34 net on the rewritten line — the trailing honest-verifier cite there is dropped as superseded by the inline definition; honest-verifier.md stays cited at 5c) plus gsd-core/ path-prefix repairs on the two 404ing bare references/ cites at 5c (honest-verifier.md) and the MVP-mode section (verify-mvp-mode.md) (+9 each). Lazy extraction remains untakeable in this change: the large extractable blocks are content-pinned by tests that read the agent file directly (tests/verifier-behavior-unverified.test.cjs, tests/verification-overrides.test.cjs), so extraction is its own coordinated change.", "complete-milestone.md": "#3409: guarded `cat .planning/phases/*-*/*-SUMMARY.md` — with `shopt -s nullglob` active in this block's preamble (#2962), zero matching phase summaries collapses the glob to nothing and a bare `cat` blocks reading stdin rather than producing empty output, wedging the milestone-completion review. Growth is the array-existence-check idiom (+73 bytes, two glob segments makes this longer than the single-glob sites).", "discuss-phase-assumptions.md": "#3409: replaced the unreachable `AUTO_MODE=$(gsd_run query check auto-mode --pick active 2>/dev/null || echo \"false\")` — `||` never fires because the query exits 0 with empty stdout when the field is absent, not a failure, so AUTO_MODE silently ended up empty rather than \"false\" — with a two-line capture-then-default (`AUTO_MODE=\"${AUTO_MODE:-false}\"`) that actually reaches the fallback. Growth is the extra default-assignment line (+19 bytes).", "plan-phase.md": "#3409: three sites. `AUTO_CHAIN` and `PHASE_REQ_IDS` get the same unreachable-`||`-fallback fix as discuss-phase-assumptions.md (empty-but-successful `gsd_run query` output never triggered `|| echo`, now uses `${VAR:-default}`); `PRIOR_SUMMARIES` additionally swapped `gsd_run query phases.list --pick summaries_total` for `--type summaries --pick count` since the old pick key produced the same unreachable-fallback failure mode for the walking-skeleton check. Net growth across the three sites is +39 bytes.", diff --git a/tests/verifier-behavior-unverified.test.cjs b/tests/verifier-behavior-unverified.test.cjs index 9bddf4333..2cafa65c7 100644 --- a/tests/verifier-behavior-unverified.test.cjs +++ b/tests/verifier-behavior-unverified.test.cjs @@ -201,3 +201,48 @@ describe('bug #3321: gsd-verifier runs probes instead of trusting SUMMARY claims }); }); } + +// ─── #3206: explicit-evidence definition inline + cite resolution ──────────── +// The agent file IS the deployed product; content assertions test the shipped +// contract (same basis as every test above). + +const verifierPhaseGatesPath = path.join(ROOT, 'gsd-core', 'references', 'verifier-phase-gates.md'); +const verifierPhaseGates = fs.readFileSync(verifierPhaseGatesPath, 'utf-8'); + +test('#3206: step 5b defines explicit evidence inline — presence+wiring never qualifies', () => { + const m = verifier.match(/5b\.\s+\*\*Non-inferable[^\r\n]{0,400}/); + assert.ok(m, 'step 5b line must exist'); + assert.match(m[0], /held-out\/property-based test/i); + assert.match(m[0], /directly observed/i); + assert.match(m[0], /presence\+wirting|presence\+wiring/i); + assert.match(m[0], /\*never\* qualifies/, 'presence+wiring must be excluded in the 5b line itself'); +}); + +test('#3206: every references-tree cite in gsd-verifier.md resolves on disk as written', () => { + // Pre-fix, this file cited `references/honest-verifier.md` — a bare path that + // 404s from repo root after the reference-tree reorg. Every cite must now + // resolve exactly as written. + const citeRe = /`((?:gsd-core\/)?references\/[A-Za-z0-9._-]+\.md)`/g; + const cites = [...verifier.matchAll(citeRe)].map((m) => m[1]); + assert.ok(cites.includes('gsd-core/references/honest-verifier.md'), 'the 5c honest-verifier cite'); + assert.ok(cites.includes('gsd-core/references/verify-mvp-mode.md'), 'the MVP-mode cite'); + assert.ok(cites.length >= 2, `expected the two repaired cites, found ${cites.length}`); + for (const cited of cites) { + assert.ok( + fs.existsSync(path.join(ROOT, cited)), + `gsd-verifier.md cites \`${cited}\` but no such file exists — the agent is handed a path that does not resolve (#3206)`, + ); + } +}); + +test('#3206: backstop reporting contract is in the eagerly-included verifier-phase-gates.md', () => { + // The AFK-projection and insufficient_spec-distinctness clauses must be + // reachable from the agent's guaranteed reading path: verifier-phase-gates.md + // is @~/-included by gsd-verifier.md, so pinning their presence there pins + // their reachability. + assert.match(verifier, /@~\/\.claude\/gsd-core\/references\/verifier-phase-gates\.md/); + assert.match(verifierPhaseGates, /Never silent, never a hard halt/); + assert.match(verifierPhaseGates, /complete with N unverified non-inferable checks/); + assert.match(verifierPhaseGates, /Distinguishable reason/); + assert.match(verifierPhaseGates, /reason: insufficient_spec/); +});