fix(#3194): verify source-grounded lane evidence from review output (#3436)

* fix(#3194): verify source-grounded lane evidence from review output

* chore(#3194): fill changeset pr number

---------

Co-authored-by: sim <sim@local>

src/review-lane-invocation.cts

@@ -24,6 +24,7 @@
 
 import type {
   EmptyOutputPolicy,
+  EvidenceClass,
   LaneHandler,
   LaneProbe,
   ReviewerLane,
@@ -92,6 +93,19 @@ export interface SpawnPlan {
   errPath: string;
   timeoutMs: number;
   emptyOutput: EmptyOutputPolicy;
+  /**
+   * The lane's declared evidence class, carried onto the plan so the runner can VERIFY the
+   * declaration against the review's actual output (#3194): a `source-grounded` lane whose
+   * review cites no `file:line` evidence is stamped and down-weighted in the Consensus
+   * Summary, while `diff-only` lanes are exempt (their verdict is already folded in as a
+   * diff observation).
+   *
+   * NORMALIZED, not trusted: this module is the overlay-manifest trust boundary and a
+   * third-party body can declare any value. Anything that is not exactly `'diff-only'`
+   * resolves as `'source-grounded'` — the fail-toward-verification direction, since the
+   * only behavioral consequence is whether the lane's OWN review gets down-weighted.
+   */
+  evidenceClass: EvidenceClass;
   handler: LaneHandler;
   requiresBinaries: readonly string[];
   probe: LaneProbe;
@@ -121,6 +135,8 @@ export interface HttpPlan {
   errPath: string;
   timeoutMs: number;
   emptyOutput: EmptyOutputPolicy;
+  /** Declared evidence class, carried for run-time verification — see `SpawnPlan`. */
+  evidenceClass: EvidenceClass;
   handler: LaneHandler;
   requiresBinaries: readonly string[];
   probe: LaneProbe;
@@ -319,6 +335,10 @@ export function resolveLanePlan(input: ResolveInput): ResolveResult {
       ? lane.timeoutFloorMs
       : 900_000;
   const emptyOutput: EmptyOutputPolicy = lane.emptyOutput === 'handler-owned' ? 'handler-owned' : 'stub-with-stderr';
+  // #3194: only an EXACT 'diff-only' declaration exempts a lane from evidence verification.
+  // Anything else — including a missing or garbage value on a third-party overlay body —
+  // resolves as 'source-grounded', so the runner verifies rather than trusts it.
+  const evidenceClass: EvidenceClass = lane.evidenceClass === 'diff-only' ? 'diff-only' : 'source-grounded';
   const requiresBinaries = Array.isArray(lane.requiresBinaries)
     ? lane.requiresBinaries.filter((b): b is string => typeof b === 'string')
     : [];
@@ -378,6 +398,7 @@ export function resolveLanePlan(input: ResolveInput): ResolveResult {
         errPath,
         timeoutMs,
         emptyOutput,
+        evidenceClass,
         handler,
         requiresBinaries,
         probe: lane.probe,
@@ -496,6 +517,7 @@ export function resolveLanePlan(input: ResolveInput): ResolveResult {
       errPath,
       timeoutMs,
       emptyOutput,
+      evidenceClass,
       handler,
       requiresBinaries,
       probe: lane.probe,

src/review-lane-runner.cts

@@ -559,6 +559,52 @@ export function stampBlindReview(review: string): string {
   );
 }
 
+/**
+ * A `path/to/file:line`-shaped source citation (#3194).
+ *
+ * The Review Instructions (review.md) require every reviewer to "cite concrete
+ * `path/to/file:line` evidence"; this recognizes that shape in review output. Two anchors
+ * make the match a source citation rather than any `colon-digits`:
+ *   - the token before the colon must contain a path separator (`/` or `\`) or end in a
+ *     `.extension`, so a bare PLAN-line reference — "see line 42", "L12-L18", the invented
+ *     references measured in #3194 — does not match;
+ *   - the token may not itself contain `:` and may not start immediately after `/` or `:`,
+ *     so a URL (`http://localhost:8080`) and its host:port do not match either.
+ *
+ * KNOWN LIMIT (deliberate, #3194 scope): presence is checked, not resolution. A citation to
+ * a line that does not exist still counts — catching invented references that look impeccable
+ * requires repo access at stamp time and is follow-up material, not part of this fix.
+ */
+const SOURCE_CITATION_RE = /(?<![/:])(?:[^\s:]*[/\\][^\s:]*|[^\s:]*\.[A-Za-z0-9]{1,16}):[0-9]+/;
+
+/** The marker the Consensus Summary step recognizes and down-weights (#3194). */
+const UNGROUNDED_MARKER = '[reviewed-without-source-citations]';
+
+/**
+ * Stamp a machine-readable marker when a source-grounded lane's review cites no `file:line`
+ * evidence (#3194).
+ *
+ * The sibling of `stampBlindReview`, for a different failure mode: that one fires when a
+ * reviewer REPORTS it had no repo access; this one fires when a lane that DECLARES
+ * `source-grounded` evidence delivered none — the review restates the plan's own claims with
+ * at most invented plan-line references. Either way the Consensus Summary must not count the
+ * verdict at full weight, which is why both prepend a marker the consensus step recognizes.
+ *
+ * Idempotent and empty-safe: an already-stamped review passes through unchanged, and an empty
+ * review is left to the empty-output policy's diagnostic stub.
+ */
+export function stampUngroundedReview(review: string): string {
+  if (isEmptyReview(review)) return review;
+  if (review.startsWith(`> ${UNGROUNDED_MARKER}`)) return review;
+  if (SOURCE_CITATION_RE.test(review)) return review;
+  return (
+    `> ${UNGROUNDED_MARKER} This reviewer declared source-grounded evidence but cited no ` +
+    'file:line source evidence, so it reviewed the pasted plan text only — down-weight its ' +
+    'verdict in the Consensus Summary.\n\n' +
+    review
+  );
+}
+
 /**
  * `openai-compatible` — model discovery, the chat-completions round trip, and the served-model
  * mismatch warning.
@@ -714,6 +760,13 @@ function runSpawnLane(plan: SpawnPlan, deps: RunnerDeps, repoRoot: string): Lane
     }
   }
 
+  // #3194: verify the declared evidence class against the review's actual output. A
+  // source-grounded lane whose review cites no file:line evidence reviewed the plan text
+  // only; stamp it so the Consensus Summary down-weights the verdict instead of silently
+  // trusting the lane's declaration. diff-only lanes are exempt — their verdict is already
+  // folded in as a diff observation, and the citation check must not change that surface.
+  if (plan.evidenceClass !== 'diff-only') review = stampUngroundedReview(review);
+
   const { stubbed } = writeReviewOrStub(plan, review, deps, extra);
   return { slug: plan.slug, ok: true, stubbed };
 }
@@ -721,7 +774,9 @@ function runSpawnLane(plan: SpawnPlan, deps: RunnerDeps, repoRoot: string): Lane
 async function runHttpLane(plan: HttpPlan, deps: RunnerDeps): Promise<LaneRunResult> {
   const promptText = deps.exists(plan.promptPath) ? deps.readFile(plan.promptPath) : '';
   const { review, rawBody } = await runOpenAiCompatible(plan, promptText, deps);
+  // #3194: same verification on the http path — see runSpawnLane.
+  const stamped = plan.evidenceClass !== 'diff-only' ? stampUngroundedReview(review) : review;
   deps.writeFile(plan.errPath, '');
-  const { stubbed } = writeReviewOrStub(plan, review, deps, rawBody);
+  const { stubbed } = writeReviewOrStub(plan, stamped, deps, rawBody);
   return { slug: plan.slug, ok: true, stubbed };
 }