From 4645328e2ed8ddd090e015f982758734a44194a5 Mon Sep 17 00:00:00 2001 From: Tibsfox Date: Sat, 4 Apr 2026 04:23:36 -0700 Subject: [PATCH] fix(verifier): filter gaps addressed in later milestone phases (#1624) (#1643) The gsd-verifier was reporting false-positive gaps for items explicitly scheduled in later phases of the milestone (e.g., reporting a Phase 5 item as a gap during Phase 1 verification). This adds Step 9b to cross-reference gaps against later phases using `roadmap analyze` and move matched items to a `deferred` list that does not affect status. Co-authored-by: Claude Opus 4.6 (1M context) --- agents/gsd-verifier.md | 53 +++++ .../references/planner-gap-closure.md | 2 + get-shit-done/workflows/verify-phase.md | 29 ++- sdk/prompts/workflows/verify-phase.md | 17 +- tests/verifier-deferred-items.test.cjs | 219 ++++++++++++++++++ 5 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 tests/verifier-deferred-items.test.cjs diff --git a/agents/gsd-verifier.md b/agents/gsd-verifier.md index 2bf1c1aed..1d37ddfc2 100644 --- a/agents/gsd-verifier.md +++ b/agents/gsd-verifier.md @@ -461,6 +461,33 @@ Classify status using this decision tree IN ORDER (most restrictive first): **Score:** `verified_truths / total_truths` +## Step 9b: Filter Deferred Items + +Before reporting gaps, check if any identified gaps are explicitly addressed in later phases of the current milestone. This prevents false-positive gap reports for items intentionally scheduled for future work. + +**Load the full milestone roadmap:** + +```bash +ROADMAP_DATA=$(node "$HOME/.claude/get-shit-done/bin/gsd-tools.cjs" roadmap analyze --raw) +``` + +Parse the JSON to extract all phases. Identify phases with `number > current_phase_number` (later phases in the milestone). For each later phase, extract its `goal` and `success_criteria`. + +**For each potential gap identified in Step 9:** + +1. Check if the gap's failed truth or missing item is covered by a later phase's goal or success criteria +2. **Match criteria:** The gap's concern appears in a later phase's goal text, success criteria text, or the later phase's name clearly suggests it covers this area of work +3. If a match is found → move the gap to the `deferred` list, recording which phase addresses it and the matching evidence (goal text or success criterion) +4. If the gap does not match any later phase → keep it as a real `gap` + +**Important:** Be conservative when matching. Only defer a gap when there is clear, specific evidence in a later phase's roadmap section. Vague or tangential matches should NOT cause a gap to be deferred — when in doubt, keep it as a real gap. + +**Deferred items do NOT affect the status determination.** After filtering, recalculate: + +- If the gaps list is now empty and no human verification items exist → `passed` +- If the gaps list is now empty but human verification items exist → `human_needed` +- If the gaps list still has items → `gaps_found` + ## Step 10: Structure Gap Output (If Gaps Found) Before writing VERIFICATION.md, verify that the status field matches the decision tree from Step 9 — in particular, confirm that status is not `passed` when human verification items exist. @@ -485,6 +512,17 @@ gaps: - `artifacts`: Files with issues - `missing`: Specific things to add/fix +If Step 9b identified deferred items, add a `deferred` section after `gaps`: + +```yaml +deferred: # Items addressed in later phases — not actionable gaps + - truth: "Observable truth not yet met" + addressed_in: "Phase 5" + evidence: "Phase 5 success criteria: 'Implement RuntimeConfigC FFI bindings'" +``` + +Deferred items are informational only — they do not require closure plans. + **Group related gaps by concern** — if multiple truths fail from the same root cause, note this to help the planner create focused plans. @@ -519,6 +557,10 @@ gaps: # Only if status: gaps_found issue: "What's wrong" missing: - "Specific thing to add/fix" +deferred: # Only if deferred items exist (Step 9b) + - truth: "Observable truth addressed in a later phase" + addressed_in: "Phase N" + evidence: "Matching goal or success criteria text" human_verification: # Only if status: human_needed - test: "What to do" expected: "What should happen" @@ -543,6 +585,15 @@ human_verification: # Only if status: human_needed **Score:** {N}/{M} truths verified +### Deferred Items + +Items not yet met but explicitly addressed in later milestone phases. +Only include this section if deferred items exist (from Step 9b). + +| # | Item | Addressed In | Evidence | +|---|------|-------------|----------| +| 1 | {truth} | Phase {N} | {matching goal or success criteria} | + ### Required Artifacts | Artifact | Expected | Status | Details | @@ -706,7 +757,9 @@ return
No messages
// Always shows "no messages" - [ ] Behavioral spot-checks run on runnable code (or skipped with reason) - [ ] Human verification items identified - [ ] Overall status determined +- [ ] Deferred items filtered against later milestone phases (Step 9b) - [ ] Gaps structured in YAML frontmatter (if gaps_found) +- [ ] Deferred items structured in YAML frontmatter (if deferred items exist) - [ ] Re-verification metadata included (if previous existed) - [ ] VERIFICATION.md created with complete report - [ ] Results returned to orchestrator (NOT committed) diff --git a/get-shit-done/references/planner-gap-closure.md b/get-shit-done/references/planner-gap-closure.md index 48f34d18e..b1f9f2432 100644 --- a/get-shit-done/references/planner-gap-closure.md +++ b/get-shit-done/references/planner-gap-closure.md @@ -2,6 +2,8 @@ Triggered by `--gaps` flag. Creates plans to address verification or UAT failures. +**Important: Skip deferred items.** When reading VERIFICATION.md, only the `gaps:` section contains actionable items that need closure plans. The `deferred:` section (if present) lists items explicitly addressed in later milestone phases — these are NOT gaps and must be ignored during gap closure planning. Creating plans for deferred items wastes effort on work already scheduled for future phases. + **1. Find gap sources:** Use init context (from load_project_state) which provides `phase_dir`: diff --git a/get-shit-done/workflows/verify-phase.md b/get-shit-done/workflows/verify-phase.md index 9ecf5913f..31e953dff 100644 --- a/get-shit-done/workflows/verify-phase.md +++ b/get-shit-done/workflows/verify-phase.md @@ -42,7 +42,12 @@ grep -E "^| ${phase_number}" .planning/REQUIREMENTS.md 2>/dev/null || true ls "$phase_dir"/*-SUMMARY.md "$phase_dir"/*-PLAN.md 2>/dev/null || true ``` -Extract **phase goal** from ROADMAP.md (the outcome to verify, not tasks) and **requirements** from REQUIREMENTS.md if it exists. +Load full milestone phases for deferred-item filtering (Step 9b): +```bash +node "$HOME/.claude/get-shit-done/bin/gsd-tools.cjs" roadmap analyze +``` + +Extract **phase goal** from ROADMAP.md (the outcome to verify, not tasks), **requirements** from REQUIREMENTS.md if it exists, and **all milestone phases** from roadmap analyze (for cross-referencing gaps against later phases). @@ -303,6 +308,25 @@ Classify status using this decision tree IN ORDER (most restrictive first): **Score:** `verified_truths / total_truths` + +Before reporting gaps, cross-reference each gap against later phases in the milestone using the full roadmap data loaded in load_context (from `roadmap analyze`). + +For each potential gap identified in determine_status: +1. Check if the gap's failed truth or missing item is covered by a later phase's goal or success criteria +2. **Match criteria:** The gap's concern appears in a later phase's goal text, success criteria text, or the later phase's name clearly suggests it covers this area +3. If a clear match is found → move the gap to a `deferred` list with the matching phase reference and evidence text +4. If no match in any later phase → keep as a real `gap` + +**Important:** Be conservative. Only defer a gap when there is clear, specific evidence in a later phase. Vague or tangential matches should NOT cause deferral — when in doubt, keep it as a real gap. + +**Deferred items do NOT affect the status determination.** Recalculate after filtering: +- If gaps list is now empty and no human items exist → `passed` +- If gaps list is now empty but human items exist → `human_needed` +- If gaps list still has items → `gaps_found` + +Include deferred items in VERIFICATION.md frontmatter (`deferred:` section) and body (Deferred Items table) for transparency. If no deferred items exist, omit these sections. + + If gaps_found: @@ -344,7 +368,8 @@ Orchestrator routes: `passed` → update_roadmap | `gaps_found` → create/execu - [ ] Test quality audited (disabled tests, circular patterns, assertion strength, provenance) - [ ] Human verification items identified - [ ] Overall status determined -- [ ] Fix plans generated (if gaps_found) +- [ ] Deferred items filtered against later milestone phases (if gaps found) +- [ ] Fix plans generated (if gaps_found after filtering) - [ ] VERIFICATION.md created with complete report - [ ] Results returned to orchestrator diff --git a/sdk/prompts/workflows/verify-phase.md b/sdk/prompts/workflows/verify-phase.md index ac6cd5074..72186a312 100644 --- a/sdk/prompts/workflows/verify-phase.md +++ b/sdk/prompts/workflows/verify-phase.md @@ -21,7 +21,9 @@ Then verify each level against the actual codebase. Load phase operation context from injected context files. Extract: phase directory, phase number, phase name, plan count. -Load phase details, plans, and summaries. Extract the **phase goal** from the roadmap (the outcome to verify, not tasks) and **requirements** if they exist. +Load phase details, plans, and summaries. Also load the full milestone roadmap via `roadmap analyze` so the verifier can cross-reference gaps against later phases (for deferred-item filtering). + +Extract the **phase goal** from the roadmap (the outcome to verify, not tasks), **requirements** if they exist, and **all milestone phases** for deferred-item filtering. @@ -94,6 +96,19 @@ Categorize: Blocker (prevents goal) | Warning (incomplete) | Info (notable). **Score:** verified_truths / total_truths + +Before reporting gaps, cross-reference each gap against later phases in the milestone (from the `roadmap analyze` data loaded in load_context). + +For each potential gap: check if a later phase's goal or success criteria explicitly covers the concern. If there is a clear match, move the gap to a `deferred` list with the matching phase reference and evidence. Only defer when there is specific evidence -- vague matches should remain as real gaps. + +Deferred items do not affect status. Recalculate after filtering: +- Gaps list empty, no human items -> passed +- Gaps list empty, human items exist -> human_needed (not applicable in SDK headless mode) +- Gaps list still has items -> gaps_found + +Include deferred items in VERIFICATION.md frontmatter and body for transparency. + + If gaps_found: 1. Cluster related gaps by concern diff --git a/tests/verifier-deferred-items.test.cjs b/tests/verifier-deferred-items.test.cjs new file mode 100644 index 000000000..c94b9b51e --- /dev/null +++ b/tests/verifier-deferred-items.test.cjs @@ -0,0 +1,219 @@ +/** + * Tests for verifier deferred-items filtering (#1624) + * + * Verifies that the gsd-verifier agent filters gaps addressed in later + * milestone phases, preventing false-positive gap reports. + */ + +const { test, describe } = require('node:test'); +const assert = require('node:assert'); +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); + +describe('verifier deferred-items filtering (#1624)', () => { + + // ── gsd-verifier.md ──────────────────────────────────────────────────────── + + describe('agents/gsd-verifier.md', () => { + const verifierPath = path.join(ROOT, 'agents', 'gsd-verifier.md'); + let verifierContent; + + test('file exists', () => { + assert.ok(fs.existsSync(verifierPath), 'gsd-verifier.md should exist'); + verifierContent = fs.readFileSync(verifierPath, 'utf-8'); + }); + + test('contains Step 9b for filtering deferred items', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('Step 9b') || verifierContent.includes('Filter Deferred'), + 'gsd-verifier.md should contain Step 9b or "Filter Deferred" section' + ); + }); + + test('Step 9b references roadmap analyze for cross-referencing', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('roadmap analyze'), + 'Step 9b should reference "roadmap analyze" command for loading full milestone data' + ); + }); + + test('VERIFICATION.md frontmatter template includes deferred section', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('deferred:'), + 'VERIFICATION.md template should include a deferred: section in frontmatter' + ); + }); + + test('deferred section includes addressed_in field', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('addressed_in'), + 'deferred items should include an addressed_in field referencing the later phase' + ); + }); + + test('deferred section includes evidence field', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('evidence'), + 'deferred items should include an evidence field with matching goal/criteria' + ); + }); + + test('deferred section is conditional (only if deferred items exist)', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('if deferred items exist') || + verifierContent.includes('If deferred items exist') || + verifierContent.includes('Only if deferred'), + 'deferred section should be conditional — only included when deferred items exist' + ); + }); + + test('deferred items do not affect status determination', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('do NOT affect the status') || + verifierContent.includes('do not affect status') || + verifierContent.includes('Deferred items do NOT affect'), + 'should explicitly state that deferred items do not affect status' + ); + }); + + test('includes conservative matching guidance', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('conservative') || verifierContent.includes('when in doubt'), + 'should include guidance to be conservative when matching gaps to later phases' + ); + }); + + test('report body template includes Deferred Items table', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('### Deferred Items'), + 'report body template should include a Deferred Items section' + ); + }); + + test('success criteria mentions deferred filtering', () => { + verifierContent = verifierContent || fs.readFileSync(verifierPath, 'utf-8'); + assert.ok( + verifierContent.includes('Deferred items filtered') || + verifierContent.includes('deferred items filtered') || + verifierContent.includes('Deferred items structured'), + 'success criteria should reference deferred item filtering' + ); + }); + }); + + // ── verify-phase.md (workflow) ───────────────────────────────────────────── + + describe('get-shit-done/workflows/verify-phase.md', () => { + const workflowPath = path.join(ROOT, 'get-shit-done', 'workflows', 'verify-phase.md'); + let workflowContent; + + test('file exists', () => { + assert.ok(fs.existsSync(workflowPath), 'verify-phase.md should exist'); + workflowContent = fs.readFileSync(workflowPath, 'utf-8'); + }); + + test('loads roadmap analyze in context step', () => { + workflowContent = workflowContent || fs.readFileSync(workflowPath, 'utf-8'); + assert.ok( + workflowContent.includes('roadmap analyze'), + 'verify-phase.md should load roadmap analyze in its context step' + ); + }); + + test('contains filter_deferred_items step', () => { + workflowContent = workflowContent || fs.readFileSync(workflowPath, 'utf-8'); + assert.ok( + workflowContent.includes('filter_deferred_items') || + workflowContent.includes('Filter Deferred'), + 'verify-phase.md should contain a deferred-item filtering step' + ); + }); + + test('success criteria mentions deferred filtering', () => { + workflowContent = workflowContent || fs.readFileSync(workflowPath, 'utf-8'); + assert.ok( + workflowContent.includes('Deferred items filtered') || + workflowContent.includes('deferred items filtered'), + 'success criteria should mention deferred item filtering' + ); + }); + }); + + // ── verify-phase.md (SDK variant) ────────────────────────────────────────── + + describe('sdk/prompts/workflows/verify-phase.md', () => { + const sdkPath = path.join(ROOT, 'sdk', 'prompts', 'workflows', 'verify-phase.md'); + let sdkContent; + + test('file exists', () => { + assert.ok(fs.existsSync(sdkPath), 'SDK verify-phase.md should exist'); + sdkContent = fs.readFileSync(sdkPath, 'utf-8'); + }); + + test('loads roadmap analyze in context step', () => { + sdkContent = sdkContent || fs.readFileSync(sdkPath, 'utf-8'); + assert.ok( + sdkContent.includes('roadmap analyze'), + 'SDK verify-phase.md should reference roadmap analyze for deferred-item filtering' + ); + }); + + test('contains deferred-item filtering step', () => { + sdkContent = sdkContent || fs.readFileSync(sdkPath, 'utf-8'); + assert.ok( + sdkContent.includes('filter_deferred_items') || + sdkContent.includes('deferred'), + 'SDK verify-phase.md should contain deferred-item filtering logic' + ); + }); + }); + + // ── planner-gap-closure.md ───────────────────────────────────────────────── + + describe('get-shit-done/references/planner-gap-closure.md', () => { + const closurePath = path.join(ROOT, 'get-shit-done', 'references', 'planner-gap-closure.md'); + let closureContent; + + test('file exists', () => { + assert.ok(fs.existsSync(closurePath), 'planner-gap-closure.md should exist'); + closureContent = fs.readFileSync(closurePath, 'utf-8'); + }); + + test('mentions skipping deferred items', () => { + closureContent = closureContent || fs.readFileSync(closurePath, 'utf-8'); + const lower = closureContent.toLowerCase(); + assert.ok( + lower.includes('deferred') && lower.includes('skip'), + 'planner-gap-closure.md should mention skipping deferred items' + ); + }); + + test('distinguishes gaps from deferred sections', () => { + closureContent = closureContent || fs.readFileSync(closurePath, 'utf-8'); + assert.ok( + closureContent.includes('gaps:') && closureContent.includes('deferred:'), + 'should reference both gaps: and deferred: sections to distinguish them' + ); + }); + + test('explains that deferred items are not actionable', () => { + closureContent = closureContent || fs.readFileSync(closurePath, 'utf-8'); + assert.ok( + closureContent.includes('NOT gaps') || closureContent.includes('not gaps') || + closureContent.includes('must be ignored'), + 'should explain that deferred items are not actionable gaps' + ); + }); + }); +});