From 9223f2f4c851290e26a3f5563c4a65ca66a82c86 Mon Sep 17 00:00:00 2001 From: Tom Boucher Date: Thu, 11 Jun 2026 16:37:17 -0400 Subject: [PATCH] feat(#247): runtime-neutral phase uat-passed predicate from HUMAN-UAT results (#1063) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(#247): runtime-neutral phase uat-passed predicate from HUMAN-UAT results Wire the already-reserved `phase.uat-passed` alias (subcommand `uat-passed`, mutation:false) into the phase command router with a new markdown-aware predicate that evaluates HUMAN-UAT results and reports pass only when every required check passes. Post-SDK-retirement (ADR-0174/#174) successor to the SDK-framed #70, with no SDK-specific API surface. New pure module src/uat-predicate.cts: - stripFalsePositiveContexts: frontmatter -> HTML-comment -> CommonMark-style fenced-block state machine (tracks delimiter char+length) -> blockquote, each a small composable step, so a `result: passed` inside frontmatter, a fenced/~~~ block (incl. ~~~ nested in a ``` fence), a comment, or a blockquote is never counted. - parseUatResultItems: heading-block parser, column-0-anchored same-line result; a heading with no result -> `missing` (fail-closed). - analyzeMarkdown: unterminated fence/comment detection (malformed -> blocker). - evaluateUatPassed: allowlist pass/verification semantics; passed = no blockers && >=1 check && all passing; no_uat_artifacts discriminator (no vacuous pass); optional requireVerification policy hook. Thin cmdPhaseUatPassed handler in phase.cts; router closure rejects unknown flags via makeInvalidArgs. Hardened across two Codex adversarial passes (vacuous pass, dropped failing tests, permissive verification status, nested-fence escape, cross-line result value, masked unterminated comment) — all fixed fail-closed. New unit + CLI-integration suites incl. a fast-check property test; docs, CONTEXT glossary, inventory, and changeset updated. Co-Authored-By: Claude Opus 4.8 * chore(#247): backfill changeset PR number (#1063) * fix(#247): indexOf paired-scan for unterminated-comment detection CodeQL js/incomplete-multi-character-sanitization (high) flagged the `raw.replace(//g,'')`-then-`.includes('|$)/g, ''); + + // Step (c): remove fenced code blocks via CommonMark-style state machine (handles CRLF + indented fences) + stripped = _stripFencedBlocks(stripped).text; + + // Step (d): remove blockquote lines + stripped = stripped + .split('\n') + .filter(line => !/^\s*>/.test(line)) + .join('\n'); + + return stripped; +} + +interface FenceState { + char: '`' | '~'; + len: number; +} + +interface StripFencedResult { + text: string; + unterminatedFence: boolean; +} + +/** + * CommonMark-style fenced-code-block stripper. + * Tracks the opening delimiter char and length so that a ~~~ line inside a + * ``` fence is correctly treated as fence content, not a closing delimiter. + * + * Opening rule: first delimiter line with char+len sets openFence. + * Closing rule: delimiter line with SAME char, run length >= openFence.len, + * and NO trailing non-whitespace text closes the fence. + * All delimiter and content lines are dropped; non-fence lines are kept. + * Returns the kept text plus unterminatedFence:true if EOF inside a fence. + */ +function _stripFencedBlocks(content: string): StripFencedResult { + const lines = content.split('\n'); + const kept: string[] = []; + let openFence: FenceState | null = null; + const delimRe = /^(\s*)(`{3,}|~{3,})(.*)$/; + + for (const rawLine of lines) { + // Tolerate CRLF: strip trailing \r for matching, but we work on split-by-\n lines + // (the outer caller joined by \n already; we just handle a stray \r in the last char) + const line = rawLine.replace(/\r$/, ''); + const m = delimRe.exec(line); + if (m) { + const char = m[2][0] as '`' | '~'; + const len = m[2].length; + const trailing = m[3]; + if (openFence === null) { + // Opening delimiter — drop this line and record the fence + openFence = { char, len }; + } else if (char === openFence.char && len >= openFence.len && /^\s*$/.test(trailing)) { + // Closing delimiter (same char, sufficient length, no trailing text) — drop and close + openFence = null; + } + // else: mismatched delimiter inside fence (e.g. ~~~ inside ```) — drop as content + continue; // delimiter lines are always dropped + } + if (openFence === null) { + kept.push(rawLine); + } + // Lines inside fence are dropped + } + + return { text: kept.join('\n'), unterminatedFence: openFence !== null }; +} + +/** + * Analyse raw markdown for structural anomalies (unterminated fence / comment). + * Exported for unit-testability and used by evaluateUatPassed for per-file malformed detection. + * + * FIX C: properly balanced comments are stripped before checking for a dangling `. Using indexOf (not a regex .replace of the comment + // token) avoids the js/incomplete-multi-character-sanitization pattern — and + // is exact: a closed earlier comment never masks a later unterminated one. + let unterminatedComment = false; + for (let i = 0; ; ) { + const open = raw.indexOf('', open + 4); + if (close === -1) { unterminatedComment = true; break; } + i = close + 3; + } + + // Fence state machine gives the accurate unterminated-fence signal. + const { unterminatedFence } = _stripFencedBlocks(raw); + + return { unterminatedFence, unterminatedComment }; +} + +// ─── parseUatResultItems ────────────────────────────────────────────────────── + +/** + * HEADING-BLOCK parser: scan the CLEANED body (after stripFalsePositiveContexts) + * for UAT test blocks. + * + * For each ### N. Name heading, the block spans until the next ### heading or EOF. + * Within each block, find a column-0 anchored result line (rejects indented YAML + * block-scalar bodies and inline/quoted fakes). + * + * - If a heading block has NO column-0 result line → emit result:'missing' (blocker). + * - Support bracketed [passed] and bare passed (#2273). + * - Returns ALL items (both passing and non-passing). + */ +function parseUatResultItems(cleanContent: string): Array<{ test: number; name: string; result: string }> { + const items: Array<{ test: number; name: string; result: string }> = []; + + // Find all ### N. Name headings (line-anchored) + const headingPattern = /^###\s*(\d+)\.\s*(.+)$/gm; + const headings: Array<{ index: number; test: number; name: string }> = []; + let hMatch: RegExpExecArray | null; + while ((hMatch = headingPattern.exec(cleanContent)) !== null) { + headings.push({ + index: hMatch.index + hMatch[0].length, + test: parseInt(hMatch[1], 10), + name: hMatch[2].trim(), + }); + } + + for (let i = 0; i < headings.length; i++) { + const h = headings[i]; + const blockStart = h.index; + // More precise: find next heading's position in the original string + // We'll slice from current heading end to the position just before next heading's "###" + const nextHeadingMatch = i + 1 < headings.length + ? cleanContent.lastIndexOf('\n###', headings[i + 1].index) + : -1; + const blockContent = nextHeadingMatch >= blockStart + ? cleanContent.slice(blockStart, nextHeadingMatch) + : cleanContent.slice(blockStart); + + // Column-0 anchored result line: /^result:[ \t]*\[?([\w-]+)\]?/mi + // Uses [ \t]* (not \s*) so the captured value must sit on the SAME line as result:. + // A result: key with the value on a subsequent line yields no match → 'missing' (blocker). + const resultMatch = /^result:[ \t]*\[?([\w-]+)\]?/mi.exec(blockContent); + if (resultMatch) { + items.push({ + test: h.test, + name: h.name, + result: resultMatch[1].toLowerCase(), + }); + } else { + // No column-0 result line → emit 'missing' (a non-passing state) + items.push({ + test: h.test, + name: h.name, + result: 'missing', + }); + } + } + + return items; +} + +// ─── evaluateUatPassed ──────────────────────────────────────────────────────── + +/** + * Evaluate all UAT/VERIFICATION files in a phase directory. + * Returns a UatPassedReport with the locked, stable shape defined by the interface. + * + * FAIL-CLOSED: any absence/ambiguity/malformed input → NOT passed. + * Pass requires at least one real passing check AND no blockers. + */ +function evaluateUatPassed( + phaseFullDir: string, + opts?: { policy?: { requireVerification?: boolean } }, +): UatPassedReport { + const requireVerification = opts?.policy?.requireVerification === true; + + const blockers: string[] = []; + const checks: UatCheckItem[] = []; + const uatFiles: string[] = []; + const verificationFiles: string[] = []; + + // Read the directory — if unreadable, treat as no files (fail-closed: no artifacts → not passed) + let dirEntries: string[] = []; + try { + dirEntries = fs.readdirSync(phaseFullDir); + } catch { + // Unreadable dir — no_uat_artifacts:true, passed:false + const no_uat_artifacts = true; + if (requireVerification) { + blockers.push('policy: verification required but no passing *-VERIFICATION.md found'); + } + return { + passed: false, + uat_files: [], + verification_files: [], + checks: [], + blockers, + no_uat_artifacts, + policy: { require_verification: requireVerification }, + }; + } + + // Filter UAT and VERIFICATION files using the same filter as cmdPhaseComplete + const uatFileNames = dirEntries.filter(f => f.includes('-UAT') && f.endsWith('.md')); + const verFileNames = dirEntries.filter(f => f.includes('-VERIFICATION') && f.endsWith('.md')); + + // ── Process UAT files ────────────────────────────────────────────────────── + for (const file of uatFileNames) { + uatFiles.push(file); + let raw = ''; + try { + raw = fs.readFileSync(path.join(phaseFullDir, file), 'utf-8'); + } catch { + blockers.push(`${file}: could not read file`); + continue; + } + + // ── Per-file malformed markdown guard ────────────────────────────────── + // FIX D: use accurate signals from analyzeMarkdown instead of heuristics. + // unterminatedFence: CommonMark state machine detects a genuinely unclosed fence. + // unterminatedComment: strips balanced comments first, then checks for leftover \nAfter'; + const out = stripFalsePositiveContexts(input); + assert.ok(!out.includes('result: pending'), 'HTML comment content must be stripped'); + assert.ok(out.includes('Before'), 'content before comment must survive'); + assert.ok(out.includes('After'), 'content after comment must survive'); + }); + + test('removes multi-line HTML comment', () => { + const input = 'A\n\nB'; + const out = stripFalsePositiveContexts(input); + assert.ok(!out.includes('result: pending'), 'multi-line HTML comment content must be stripped'); + assert.ok(out.includes('A'), 'content before comment must survive'); + assert.ok(out.includes('B'), 'content after comment must survive'); + }); + + test('unterminated HTML comment swallows to EOF (fail-closed)', () => { + const input = 'Before\n', + ].join('\n'); + const clean = stripFalsePositiveContexts(rawContent); + const items = parseUatResultItems(clean); + assert.strictEqual(items.length, 0, 'Fake result inside HTML comment must produce no items'); + + writeFile(tmpDir, 'phase-UAT.md', rawContent); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false); + assert.strictEqual(report.no_uat_artifacts, true); + }); + + test('#247: result:passed inside frontmatter: parseUatResultItems returns [] + evaluateUatPassed → passed:false + no_uat_artifacts:true', () => { + const rawContent = [ + '---', + 'example_result: passed', + '---', + '', + 'No real test blocks here.', + ].join('\n'); + const clean = stripFalsePositiveContexts(rawContent); + const items = parseUatResultItems(clean); + assert.strictEqual(items.length, 0, 'Fake result inside frontmatter must produce no items'); + + writeFile(tmpDir, 'phase-UAT.md', rawContent); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false); + assert.strictEqual(report.no_uat_artifacts, true); + }); + + test('#247: result:passed inside fenced block is NOT treated as passing test (with real failing test)', () => { + const content = [ + '---', + 'status: partial', + '---', + '', + '# Example', + '', + '```', + '### 1. Test', + 'expected: Example output', + 'result: passed', + '```', + '', + '### 1. Real Test', + 'expected: Something', + 'result: pending', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false, + 'result:passed inside a fenced block must not flip passed to true'); + assert.ok(report.checks.some(c => c.result === 'pending' && !c.passing), + 'Real pending test must be captured'); + }); + + test('#247: result:passed inside blockquote is NOT treated as passing test', () => { + const content = [ + '---', + 'status: partial', + '---', + '', + '> ### 1. Test', + '> expected: Example', + '> result: passed', + '', + '### 1. Real Test', + 'expected: Something', + 'result: pending', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false, + 'result:passed inside a blockquote must not flip passed to true'); + }); + + test('#247: result:passed inside HTML comment is NOT treated as passing test', () => { + const content = [ + '---', + 'status: partial', + '---', + '', + '', + '', + '### 1. Real Test', + 'expected: Something', + 'result: pending', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false, + 'result:passed inside an HTML comment must not flip passed to true'); + }); + + test('#247: result:passed inside frontmatter example is NOT treated as passing test', () => { + const content = [ + '---', + 'status: partial', + 'example_result: passed', + '---', + '', + '### 1. Real Test', + 'expected: Something', + 'result: pending', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false, + 'result:passed in frontmatter must not flip passed to true'); + }); + + test('#247: real passing test block outside all contexts → passed:true', () => { + const content = [ + '---', + 'status: passed', + '---', + '', + '# UAT Results', + '', + '### 1. Login works', + 'expected: User logs in', + 'result: passed', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, true, + 'A real passing test outside false-positive contexts must pass'); + }); + + test('block-scalar expected: followed by blank line + result: pending → parsed as blocker (not dropped)', () => { + const content = [ + '### 1. Test A', + 'expected: |', + ' multi', + ' line expected output', + '', + 'result: pending', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false, + 'Block-scalar expected: with result: pending must be captured as a blocker'); + assert.ok(report.checks.some(c => c.result === 'pending' && !c.passing), + `Expected pending check, got: ${JSON.stringify(report.checks)}`); + }); +}); + +// ─── evaluateUatPassed — output shape contract ─────────────────────────────── + +describe('evaluateUatPassed — output shape (Hyrum\'s Law contract)', () => { + let tmpDir; + + beforeEach(() => { + tmpDir = makeTmpDir(); + }); + + afterEach(() => { + rmDir(tmpDir); + }); + + test('returns all required fields in the locked shape including no_uat_artifacts', () => { + writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); + const report = evaluateUatPassed(tmpDir); + + // Locked field names + assert.ok('passed' in report, 'report.passed must exist'); + assert.ok('uat_files' in report, 'report.uat_files must exist'); + assert.ok('verification_files' in report, 'report.verification_files must exist'); + assert.ok('checks' in report, 'report.checks must exist'); + assert.ok('blockers' in report, 'report.blockers must exist'); + assert.ok('no_uat_artifacts' in report, 'report.no_uat_artifacts must exist'); + assert.ok('policy' in report, 'report.policy must exist'); + assert.ok('require_verification' in report.policy, 'report.policy.require_verification must exist'); + + // checks item shape + if (report.checks.length > 0) { + const c = report.checks[0]; + assert.ok('file' in c, 'check.file must exist'); + assert.ok('test' in c, 'check.test must exist'); + assert.ok('name' in c, 'check.name must exist'); + assert.ok('result' in c, 'check.result must exist'); + assert.ok('passing' in c, 'check.passing must exist'); + } + }); + + test('no_uat_artifacts is false when real checks exist', () => { + writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.no_uat_artifacts, false); + }); + + test('no_uat_artifacts is true when no checks exist', () => { + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.no_uat_artifacts, true); + }); + + test('uat_files contains the filename', () => { + writeFile(tmpDir, 'my-UAT.md', makePassingUat(1)); + const report = evaluateUatPassed(tmpDir); + assert.ok(report.uat_files.includes('my-UAT.md'), + `uat_files should include 'my-UAT.md', got: ${JSON.stringify(report.uat_files)}`); + }); + + test('verification_files contains the filename', () => { + writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); + writeFile(tmpDir, 'phase-VERIFICATION.md', '---\nstatus: passed\n---\n'); + const report = evaluateUatPassed(tmpDir); + assert.ok(report.verification_files.includes('phase-VERIFICATION.md'), + `verification_files should include 'phase-VERIFICATION.md', got: ${JSON.stringify(report.verification_files)}`); + }); +}); + +// ─── FIX A regression: nested-fence (~~~ inside ```) ───────────────────────── + +describe('FIX A — nested fence: ~~~ inside ``` does not prematurely close outer fence', () => { + let tmpDir; + + beforeEach(() => { tmpDir = makeTmpDir(); }); + afterEach(() => { rmDir(tmpDir); }); + + test('parseUatResultItems sees [] for fake inside ``` that encloses ~~~', () => { + // A backtick fence that contains an inner ~~~ fence with a fake test block. + // The ~~~ must NOT close the ``` fence — the whole interior is content and is dropped. + const raw = [ + '```', + '~~~', + '### 1. Fake', + 'expected: X', + 'result: passed', + '~~~', + '```', + ].join('\n'); + const clean = stripFalsePositiveContexts(raw); + const items = parseUatResultItems(clean); + assert.strictEqual(items.length, 0, 'fake inside nested fence must not leak through'); + }); + + test('evaluateUatPassed → passed:false + no_uat_artifacts:true for nested-fence-only file', () => { + const raw = [ + '```', + '~~~', + '### 1. Fake', + 'expected: X', + 'result: passed', + '~~~', + '```', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', raw); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false, 'nested-fence fake must not flip passed'); + assert.strictEqual(report.no_uat_artifacts, true, 'no real items → no_uat_artifacts:true'); + assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake must not appear in checks'); + }); + + test('balanced nested fence (``` inside ~~~) is NOT flagged as malformed', () => { + // ~~~ outer, ``` inner — properly closed — should not trigger unterminatedFence + const raw = [ + '~~~', + '```', + 'code', + '```', + '~~~', + '', + '### 1. Real Test', + 'expected: Works', + 'result: passed', + ].join('\n'); + const { unterminatedFence } = analyzeMarkdown(raw); + assert.strictEqual(unterminatedFence, false, 'balanced nested fence must not be flagged'); + const clean = stripFalsePositiveContexts(raw); + const items = parseUatResultItems(clean); + assert.strictEqual(items.length, 1, 'real test outside fence must still be found'); + assert.strictEqual(items[0].result, 'passed'); + }); + + test('real result:passed outside ``` that encloses ~~~ → passed:true (no false-block)', () => { + const raw = [ + '---', + 'status: passed', + '---', + '', + '```', + '~~~', + '### 1. Fake', + 'result: passed', + '~~~', + '```', + '', + '### 1. Real Test', + 'expected: Works', + 'result: passed', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', raw); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, true, 'real result outside nested fence must still pass'); + assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake must not appear in checks'); + }); +}); + +// ─── FIX B regression: cross-line result value ──────────────────────────────── + +describe('FIX B — cross-line result: value must be on the same line', () => { + test('result: with value on next line → result:missing (not passed)', () => { + const content = [ + '### 1. Cross-line Test', + 'expected: Y', + 'result:', + '', + 'passed', + ].join('\n'); + const items = parseUatResultItems(content); + assert.strictEqual(items.length, 1); + assert.notStrictEqual(items[0].result, 'passed', + 'result value on a subsequent line must not be captured as passed'); + assert.strictEqual(items[0].result, 'missing', + 'cross-line result must yield missing (blocker)'); + }); + + test('evaluateUatPassed → passed:false for cross-line result:passed', () => { + const tmpDir = makeTmpDir(); + try { + const content = [ + '### 1. Cross-line Test', + 'expected: Y', + 'result:', + '', + 'passed', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false); + assert.ok(!report.checks.some(c => c.result === 'passed'), + 'cross-line result must not produce a passing check'); + } finally { + rmDir(tmpDir); + } + }); +}); + +// ─── FIX C regression: masked-comment (earlier closed comment + later unterminated) ── + +describe('FIX C — dangling comment survives earlier balanced comment', () => { + test('analyzeMarkdown detects unterminated comment after a properly closed one', () => { + const raw = [ + '', + 'Some text', + '', + '', + '', + 'Some text', + '', + '', + '### 1. Real Test', + 'result: passed', + ].join('\n'); + const { unterminatedComment } = analyzeMarkdown(raw); + assert.strictEqual(unterminatedComment, false, + 'only balanced comments must not be flagged as unterminated'); + }); +}); + +// ─── FIX D regression: balanced mixed fences are NOT flagged ───────────────── + +describe('FIX D — odd-fence-count heuristic replaced: balanced mixed fences not flagged', () => { + test('analyzeMarkdown: ``` followed by ~~~ (both balanced) → unterminatedFence:false', () => { + const raw = [ + '```', + 'code', + '```', + '', + '~~~', + 'more code', + '~~~', + ].join('\n'); + const { unterminatedFence } = analyzeMarkdown(raw); + assert.strictEqual(unterminatedFence, false, + 'two separate balanced fences must not trigger unterminatedFence'); + }); + + test('evaluateUatPassed: multiple balanced fences + real passing test → passed:true, no malformed blocker', () => { + const tmpDir = makeTmpDir(); + try { + const raw = [ + '---', + 'status: passed', + '---', + '', + '```', + 'result: fake', + '```', + '', + '~~~', + 'result: also fake', + '~~~', + '', + '### 1. Real Test', + 'expected: Works', + 'result: passed', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', raw); + const report = evaluateUatPassed(tmpDir); + assert.ok(!report.blockers.some(b => /malformed/i.test(b)), + `Balanced mixed fences must not produce malformed blocker, got: ${JSON.stringify(report.blockers)}`); + assert.strictEqual(report.passed, true, + 'real test outside balanced fences must still pass'); + } finally { + rmDir(tmpDir); + } + }); +}); + +// ─── FIX E extra: fake-not-in-checks assertions for existing mixed tests ────── + +describe('FIX E — fake items must NOT appear in checks (absence assertions)', () => { + let tmpDir; + + beforeEach(() => { tmpDir = makeTmpDir(); }); + afterEach(() => { rmDir(tmpDir); }); + + test('fake inside fence + real pending: fake NOT in checks', () => { + const content = [ + '---', 'status: partial', '---', '', + '```', + '### 10. Fake', + 'expected: Fake', + 'result: passed', + '```', + '', + '### 1. Real Test', + 'expected: Something', + 'result: pending', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false); + assert.ok(!report.checks.some(c => c.name === 'Fake'), + 'fake test inside fence must not appear in checks'); + }); + + test('fake inside blockquote + real pending: fake NOT in checks', () => { + const content = [ + '---', 'status: partial', '---', '', + '> ### 10. Fake', + '> expected: Fake', + '> result: passed', + '', + '### 1. Real Test', + 'expected: Something', + 'result: pending', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false); + assert.ok(!report.checks.some(c => c.name === 'Fake'), + 'fake test inside blockquote must not appear in checks'); + }); + + test('fake inside HTML comment + real pending: fake NOT in checks', () => { + const content = [ + '---', 'status: partial', '---', '', + '', + '', + '### 1. Real Test', + 'expected: Something', + 'result: pending', + '', + ].join('\n'); + writeFile(tmpDir, 'phase-UAT.md', content); + const report = evaluateUatPassed(tmpDir); + assert.strictEqual(report.passed, false); + assert.ok(!report.checks.some(c => c.name === 'Fake'), + 'fake test inside HTML comment must not appear in checks'); + }); +}); + +// ─── Property-based test (fast-check) ───────────────────────────────────────── + +describe('evaluateUatPassed — property: wrapping in false-positive context never flips to passed', () => { + let tmpDir; + + beforeEach(() => { + tmpDir = makeTmpDir(); + }); + + afterEach(() => { + rmDir(tmpDir); + }); + + test('fc: inserting result:passed inside wrapper context never flips a failing UAT to passed', () => { + // A baseline UAT file that has a pending item — it must always evaluate to passed:false + // regardless of how many "result: passed" lines we inject inside fenced/blockquote/comment wrappers. + const baseFailingBody = [ + '### 1. Real Test', + 'expected: It works', + 'result: pending', + '', + ].join('\n'); + + const wrappers = fc.constantFrom( + // backtick fence + (inner) => '```\n' + inner + '\n```', + // tilde fence + (inner) => '~~~\n' + inner + '\n~~~', + // HTML comment + (inner) => '', + // blockquote — prefix each line + (inner) => inner.split('\n').map(l => '> ' + l).join('\n'), + ); + + fc.assert( + fc.property(wrappers, fc.nat(3), (wrap, extraCount) => { + // Build a "fake passing block" that would fool a naive regex + const fakePassingLines = Array.from({ length: extraCount + 1 }, (_, i) => + `### ${i + 10}. Fake Test ${i + 10}\nexpected: Fake\nresult: passed` + ).join('\n'); + + const fullContent = [ + '---', + 'status: partial', + '---', + '', + wrap(fakePassingLines), + '', + baseFailingBody, + ].join('\n'); + + // Write to a unique tmp file to avoid cross-test state + const fcDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-fc-uat-')); + try { + fs.writeFileSync(path.join(fcDir, 'feature-UAT.md'), fullContent, 'utf-8'); + const report = evaluateUatPassed(fcDir); + // The pending item must always keep passed:false + // AND fake items injected via wrappers must never appear in checks + const hasFakeInChecks = report.checks.some(c => c.name.startsWith('Fake Test')); + return report.passed === false && !hasFakeInChecks; + } finally { + cleanup(fcDir); + } + }), + { numRuns: 50 } + ); + }); +});