From a4bc04a5ffac204c01f19588cec9a87984cede68 Mon Sep 17 00:00:00 2001 From: Rezolv Date: Thu, 2 Jul 2026 11:55:49 -0400 Subject: [PATCH] fix(#1905): normalize hand-authored backstop marker so it can't degrade to green (#1909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(#1905): normalize hand-authored backstop marker so it can't degrade to green A hand-authored non-inferable `backstop` truth with a stray trailing space (or surrounding quotes) silently graded {status:green} instead of abstaining — the exact #1154 false-pass. `truthVerification` returned null for any value != 'backstop'/'explicit', and the frontmatter continuation-KV parser preserved the stray whitespace captured inside the quotes. Normalize the marker before comparison (Postel) AND trim the continuation-KV value at the parser (durable root cause; also cleans the sibling check_target path). Regression: a trailing-space/quoted backstop truth now abstains (insufficient_spec), end-to-end from a hand-authored must_haves.truths block (#1820 rail). Refs #1905, epic #1904. * chore(changeset): Fixed fragment for #1909 (backstop marker normalize) --- .changeset/tidy-pumas-munch.md | 5 ++++ src/frontmatter.cts | 6 ++++- src/probe-core.cts | 15 +++++++++--- tests/frontmatter.test.cjs | 16 +++++++++++++ tests/probe-core.test.cjs | 42 ++++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 .changeset/tidy-pumas-munch.md diff --git a/.changeset/tidy-pumas-munch.md b/.changeset/tidy-pumas-munch.md new file mode 100644 index 000000000..5fa63d5a3 --- /dev/null +++ b/.changeset/tidy-pumas-munch.md @@ -0,0 +1,5 @@ +--- +type: Fixed +pr: 1909 +--- +Fixed: a hand-authored non-inferable backstop truth with a stray trailing space or surrounding quotes no longer silently grades green — it correctly abstains (insufficient_spec), restoring the #1154 honest-verifier guarantee. diff --git a/src/frontmatter.cts b/src/frontmatter.cts index d9c06782b..0ea2d453c 100644 --- a/src/frontmatter.cts +++ b/src/frontmatter.cts @@ -482,7 +482,11 @@ function parseMustHavesBlock(content: string, blockName: string): unknown[] { } else { const kvMatch = trimmed.match(/^(\w+):\s*"?([^"]*)"?\s*$/); if (kvMatch) { - const val = kvMatch[2]; + // Trim: a quoted value like `"backstop "` captures the inner trailing space in group 2. + // Left untrimmed, a hand-authored `must_haves` marker degrades (a `backstop` truth silently + // grades green instead of abstaining — #1905, the #1154 false-pass; also the sibling + // check_target/violationFixture path). Whitespace is never semantic in a scalar KV value. + const val = kvMatch[2].trim(); // Try to parse as number (current)[kvMatch[1]] = /^\d+$/.test(val) ? parseInt(val, 10) : val; } diff --git a/src/probe-core.cts b/src/probe-core.cts index 0297f9eaa..2ebd8a641 100644 --- a/src/probe-core.cts +++ b/src/probe-core.cts @@ -527,12 +527,21 @@ export function truthStatement(truth: unknown): string { /** * Extract a truth's verification tier, or `null` when it carries none (a plain string, or an object * with no/garbled marker). Failing toward `null` is the Postel-safe direction: an unrecognized marker - * grades NORMALLY (never a spurious abstention — the over-abstention guard, AC#3), and the marker is - * machine-emitted from validated edge data so garbling is not a live input path. + * grades NORMALLY (never a spurious abstention — the over-abstention guard, AC#3). + * + * The marker is NOT only machine-emitted: `must_haves` markers can be authored BY HAND (#1820's + * spec-optional predicate rail), and the frontmatter continuation-KV parser preserves stray + * surrounding whitespace/quotes on a hand-authored value. So we normalize before comparison + * (Postel: be liberal in what you accept) — `'backstop '`, `' backstop'`, `'"backstop"'` all + * recognize as the tier. Without this, a hand-authored non-inferable `backstop` truth with a stray + * trailing space silently grades green instead of abstaining — the exact #1154 false-pass (#1905). + * An unrecoverably-corrupted marker (e.g. an embedded quote) stays unrecognized → null → graded + * normally (AC#3): we cannot know its intent, and abstaining on it would be a spurious abstention. */ export function truthVerification(truth: unknown): TruthVerification | null { if (truth == null || typeof truth !== 'object') return null; - const v = (truth as { verification?: unknown }).verification; + const raw = (truth as { verification?: unknown }).verification; + const v = typeof raw === 'string' ? raw.trim().replace(/^["']|["']$/g, '').trim() : raw; return v === 'explicit' || v === 'backstop' ? v : null; } diff --git a/tests/frontmatter.test.cjs b/tests/frontmatter.test.cjs index 6acb7cae0..8e6ff2d17 100644 --- a/tests/frontmatter.test.cjs +++ b/tests/frontmatter.test.cjs @@ -366,6 +366,22 @@ Body content.`; assert.strictEqual(result[1], 'Coverage exceeds 80%'); }); + test('trims a continuation-KV value so a quoted trailing space does not survive (#1905, root cause of the #1154 false-pass)', () => { + // A quoted value like `"backstop "` captures the inner trailing space in group 2; left untrimmed, + // a hand-authored non-inferable `backstop` marker (#1820 spec-optional rail) degrades to `'backstop '`, + // which `truthVerification` no longer recognizes → the truth silently grades green instead of abstaining. + // Whitespace is never semantic in a scalar KV value, so the parser must trim it. + const content = `--- +must_haves: + truths: + - statement: user data is never logged + verification: "backstop " +--- +Body.`; + const result = parseMustHavesBlock(content, 'truths'); + assert.strictEqual(result[0].verification, 'backstop', 'the captured value is trimmed, not left as "backstop "'); + }); + test('extracts artifacts as object array', () => { const content = `--- phase: 01 diff --git a/tests/probe-core.test.cjs b/tests/probe-core.test.cjs index 15e65a8dc..75f6a2ebb 100644 --- a/tests/probe-core.test.cjs +++ b/tests/probe-core.test.cjs @@ -554,6 +554,19 @@ describe('probe-core: truthStatement / truthVerification normalizers (#1154, Hyr assert.equal(pc.truthVerification('Overlapping intervals are merged'), null); assert.equal(pc.truthVerification({ statement: 'x', verification: 'explicit' }), 'explicit'); }); + + test('normalizes a hand-authored marker with stray whitespace/surrounding quotes (#1905, the #1154 false-pass)', () => { + // A `must_haves` marker can be authored BY HAND (#1820 spec-optional predicate rail), and the + // frontmatter continuation-KV parser preserves stray surrounding whitespace/quotes. Failing to + // normalize means `'backstop '` → null → the non-inferable truth silently grades green (Postel: + // be liberal in what you accept). + assert.equal(pc.truthVerification({ statement: 'x', verification: 'backstop ' }), 'backstop', 'trailing space'); + assert.equal(pc.truthVerification({ statement: 'x', verification: ' backstop' }), 'backstop', 'leading space'); + assert.equal(pc.truthVerification({ statement: 'x', verification: '"backstop"' }), 'backstop', 'surrounding quotes'); + assert.equal(pc.truthVerification({ statement: 'x', verification: 'explicit ' }), 'explicit', 'trailing space, explicit tier'); + // An unrecoverably-corrupted marker stays unrecognized → null → graded normally (AC#3 over-abstention guard). + assert.equal(pc.truthVerification({ statement: 'x', verification: 'back"stop' }), null, 'an embedded quote is unrecoverable — no spurious tier'); + }); }); describe('probe-core: dispositionForUnverifiableTruth (#1154, ADR-550 D4 truth-axis mirror)', () => { @@ -588,6 +601,35 @@ describe('probe-core: dispositionForUnverifiableTruth (#1154, ADR-550 D4 truth-a assert.equal(d.flagged, false); }); + test('a whitespace-mangled backstop truth still ABSTAINS, never silently greens (#1905 — the #1154 false-pass)', () => { + const d = pc.dispositionForUnverifiableTruth( + { statement: 'user data is never logged', verification: 'backstop ' }, // stray trailing space + { evidence: [] }, + ); + assert.equal(d.status, 'unverified', 'a mangled backstop marker must not degrade to a silent green'); + assert.equal(d.flagged, true); + assert.equal(d.tier, 'backstop'); + assert.equal(d.reason, 'insufficient_spec'); + }); + + test('END-TO-END (#1905, #1820 hand-authoring path): a HAND-AUTHORED must_haves.truths trailing-space backstop marker parses to the tier and abstains, never greens', () => { + // A human authors the marker directly (the #1820 spec-optional predicate rail), NOT projectTruths. + // The frontmatter continuation-KV parser used to preserve the stray trailing space inside the quotes, + // so truthVerification saw 'backstop ' → null → the non-inferable truth graded green. It must parse + // clean and abstain — the exact honesty regression #1154 exists to eliminate (ADR-550 D4 truth axis). + const doc = [ + '---', 'must_haves:', ' truths:', + ' - statement: user data is never logged', + ' verification: "backstop "', + '---', 'body', + ].join('\n'); + const parsed = fm.parseMustHavesBlock(doc, 'truths'); + assert.equal(pc.truthVerification(parsed[0]), 'backstop', 'the mangled marker normalizes to the tier'); + const d = pc.dispositionForUnverifiableTruth(parsed[0], { evidence: [] }); + assert.equal(d.status, 'unverified', 'never a silent green (ADR-550 D4 truth-axis, #1154)'); + assert.equal(d.reason, 'insufficient_spec'); + }); + test('over-abstention guard (AC#3): an explicit-tier truth NEVER abstains even with no evidence (only backstop triggers it)', () => { const d = pc.dispositionForUnverifiableTruth({ statement: 'Symbol X is wired', verification: 'explicit' }, { evidence: [] }); assert.equal(d.status, 'green', 'an explicit (inferable) truth never abstains');