diff --git a/.changeset/daring-ravens-wake.md b/.changeset/daring-ravens-wake.md new file mode 100644 index 000000000..556ce613b --- /dev/null +++ b/.changeset/daring-ravens-wake.md @@ -0,0 +1,5 @@ +--- +type: Changed +pr: 1421 +--- +**`/gsd-review` now asks external reviewers to verify plan claims against the source** — the reviewer prompt requires opening the referenced files, citing `file:line` evidence + mechanism, and tracing asserted behavior, with a graceful-degradation clause for reviewers that have no file access. This turns every capable agentic reviewer into a real second source instead of a plan-text paraphraser. (#1318) diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 5616a8341..ad872b62e 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -1370,6 +1370,8 @@ Execute a trivial task inline — no subagents, no planning overhead. For typo f Cross-AI peer review of phase plans from external AI CLIs. +Reviewers are prompted to verify the plan's claims against the actual repository source — opening the referenced files and citing `file:line` evidence with the mechanism — rather than reviewing the plan text in isolation. A reviewer that has no file access flags what it cannot verify instead of asserting it, and `file:line`-grounded findings are weighted more heavily during consensus synthesis. + | Argument | Required | Description | |----------|----------|-------------| | `--phase N` | **Yes** | Phase number to review | diff --git a/gsd-core/workflows/review.md b/gsd-core/workflows/review.md index 488f52da2..fb5658415 100644 --- a/gsd-core/workflows/review.md +++ b/gsd-core/workflows/review.md @@ -157,6 +157,14 @@ Provide structured feedback on plan quality, completeness, and risks. ## Review Instructions +**Verify against source — do not review the plan text in isolation.** You are running inside the project's git working tree (the current directory). The plans reference real files, migrations, routes, and tests that exist in this repo now. +1. Open the referenced files and check each claim against the actual code. +2. For every strength or concern, cite concrete `path/to/file:line` evidence plus the mechanism. +3. When a plan asserts a mechanism works (a guard, a query filter, a test that exercises a path), trace whether it actually does what is claimed — do not take the plan's word for it. +4. If you cannot read the repo (no file access), say so and downgrade that finding to an open question rather than asserting it. + +Findings citing `file:line` evidence are weighted far more heavily than impressionistic ones; a review that only restates the plan's own claims has low value. + Analyze each plan and provide: 1. **Summary** — One-paragraph assessment @@ -273,7 +281,7 @@ fi **CodeRabbit:** -Note: CodeRabbit reviews the current git diff/working tree — it does not accept a prompt or model flag. It may take up to 5 minutes. Use `timeout: 360000` on the Bash tool call. +Note: CodeRabbit reviews the current git diff/working tree — it does not accept a prompt or model flag. It may take up to 5 minutes. Use `timeout: 360000` on the Bash tool call. The source-grounding requirement in the build_prompt Review Instructions applies only to the prompt-fed reviewers above; CodeRabbit is a diff-only reviewer and never receives it. Treat its output as a diff observation, not a grounded plan-level verdict. ```bash coderabbit review --prompt-only 2>/dev/null > /tmp/gsd-review-coderabbit-{phase}.md @@ -714,7 +722,7 @@ trimmed_reviewers: # only present if at least one reviewer was trimmed ## Consensus Summary -{synthesize common concerns across all reviewers} +{synthesize common concerns across all reviewers. CodeRabbit is a diff-only reviewer (it never received the source-grounding prompt), so do not weight its verdict as a grounded plan review — fold in its diff findings, but base plan-level consensus on the prompt-fed reviewers.} ### Agreed Strengths {strengths mentioned by 2+ reviewers} diff --git a/tests/review-default-reviewers-workflow.test.cjs b/tests/review-default-reviewers-workflow.test.cjs index c2f524103..00cb3cda0 100644 --- a/tests/review-default-reviewers-workflow.test.cjs +++ b/tests/review-default-reviewers-workflow.test.cjs @@ -46,3 +46,107 @@ describe('review workflow default reviewer selection contract (#3079)', () => { ); }); }); + +describe('review workflow source-grounding requirement in build_prompt (#1318)', () => { + const workflow = fs.readFileSync( + path.join(process.cwd(), 'gsd-core', 'workflows', 'review.md'), + 'utf8' + ); + + // Extract ONLY the build_prompt Review Instructions region — the slice of the + // assembled prompt that is actually piped to the prompt-fed reviewers. The + // grounding instruction is worthless unless it lives HERE (#1318): asserting + // against the whole file would still pass if the text drifted into a note, + // the consensus step, or a comment that never reaches a reviewer's stdin. + // + // The region is the fenced prompt's `## Review Instructions` section, from + // that heading up to the next `## ` heading inside the same fenced block. + function buildPromptReviewInstructions(src) { + // Locate the build_prompt step, then its first fenced ```markdown block. + // NOTE: '' is a literal anchor — update it if the + // step is ever renamed or gains/reorders attributes. + const stepIdx = src.indexOf(''); + assert.ok(stepIdx !== -1, 'build_prompt step must exist'); + + // Fence-run-aware extraction (CommonMark): a naive `indexOf('\n```')` would + // terminate at the FIRST triple-backtick line, truncating the prompt if its + // body embeds a fenced code example. Mirror the close rule used by + // src/markdown-sectionizer.cts stripFencedCode: the closing fence is a line + // of the SAME char and >= the opener's run length, with no trailing content, + // so a shorter nested fence inside the block is treated as content (#1318). + // Backtick-fenced only by design — the build_prompt block is ```markdown. + const lines = src.slice(stepIdx).split('\n'); + const openRe = /^ {0,3}(`{3,})markdown\s*$/; + let openLen = 0; + let bodyStart = -1; + for (let i = 0; i < lines.length; i++) { + const m = openRe.exec(lines[i].replace(/\r$/, '')); + if (m) { openLen = m[1].length; bodyStart = i + 1; break; } + } + assert.ok(bodyStart !== -1, 'build_prompt must contain a ```markdown prompt block'); + const closeRe = new RegExp(`^ {0,3}\`{${openLen},}\\s*$`); + let bodyEnd = -1; + for (let i = bodyStart; i < lines.length; i++) { + if (closeRe.test(lines[i].replace(/\r$/, ''))) { bodyEnd = i; break; } + } + assert.ok(bodyEnd !== -1, 'build_prompt markdown fence must be closed'); + const fenced = lines.slice(bodyStart, bodyEnd).join('\n'); + + const hdr = fenced.indexOf('## Review Instructions'); + assert.ok(hdr !== -1, 'fenced prompt must contain a ## Review Instructions section'); + // Next top-level `## ` heading after the Review Instructions heading. + const after = fenced.indexOf('\n## ', hdr + 1); + return after === -1 ? fenced.slice(hdr) : fenced.slice(hdr, after); + } + + const reviewInstructions = buildPromptReviewInstructions(workflow); + + test('instructs reviewers to verify plan claims against source and cite file:line', () => { + // The cross-AI prompt assembled from plan text must push agentic reviewers + // to open the referenced source and ground findings in evidence, instead of + // paraphrasing plan text (the false-LOW failure mode in #1318). Assert the + // instruction lives INSIDE the prompt region, not merely somewhere in file. + assert.ok( + reviewInstructions.includes('Verify against source') && + reviewInstructions.includes('check each claim against the actual code') && + reviewInstructions.includes('`path/to/file:line`'), + 'build_prompt Review Instructions region must require source verification + file:line evidence' + ); + }); + + test('includes a graceful-degradation clause for reviewers without file access', () => { + // Prompt-only reviewers (ollama / lm_studio / llama.cpp) must flag that they + // could not verify rather than asserting an unverified finding — and this + // clause must sit WITHIN the prompt region so reviewers actually receive it. + assert.ok( + reviewInstructions.includes('If you cannot read the repo (no file access)') && + reviewInstructions.includes('downgrade that finding to an open question'), + 'build_prompt Review Instructions region must degrade gracefully for prompt-only reviewers' + ); + }); + + test('#1318: prompt extraction is fence-run-aware — a nested code fence does not truncate it', () => { + // Regression guard for the fenceClose hardening. The feature feeds source/plan + // content (which routinely contains code fences) into the prompt; a naive + // first-`\n```` close scan would stop at a nested fence and drop everything + // after it — including the `## Review Instructions` section — yielding a + // spurious failure or false pass. A 4-backtick outer fence must extract in + // full past a nested 3-backtick block. + const synthetic = [ + '', + '````markdown', + '# Prompt', + 'Example for reviewers:', + '```bash', + 'echo hi', + '```', + '## Review Instructions', + '- Verify against source and cite `path/to/file:line`.', + '````', + '', + ].join('\n'); + const extracted = buildPromptReviewInstructions(synthetic); + assert.match(extracted, /## Review Instructions/); + assert.match(extracted, /cite `path\/to\/file:line`/); + }); +}); diff --git a/tests/workflow-size-baseline.json b/tests/workflow-size-baseline.json index 0845b12b9..ac2e2c304 100644 --- a/tests/workflow-size-baseline.json +++ b/tests/workflow-size-baseline.json @@ -62,7 +62,7 @@ "remove-phase.md": 8469, "remove-workspace.md": 7507, "resume-project.md": 17226, - "review.md": 38031, + "review.md": 39404, "scan.md": 7688, "secure-phase.md": 12282, "session-report.md": 4044,