'use strict'; /** * Unit tests for uat-predicate.cjs * Tests the pure-computation module: stripFalsePositiveContexts, * parseUatResultItems, evaluateUatPassed. * * Issue #247 — phase uat-passed predicate */ const { test, describe, beforeEach, afterEach } = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const os = require('node:os'); const fc = require('fast-check'); const { stripFalsePositiveContexts, parseUatResultItems, analyzeMarkdown, evaluateUatPassed, } = require('../msd-core/bin/lib/uat-predicate.cjs'); const { parseUatItemsWithStats } = require('../msd-core/bin/lib/uat.cjs'); const { cleanup } = require('./helpers.cjs'); // ─── Helpers ────────────────────────────────────────────────────────────────── function makeTmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'msd-uat-pred-test-')); } function rmDir(dir) { cleanup(dir); } function writeFile(dir, name, content) { fs.writeFileSync(path.join(dir, name), content, 'utf-8'); } function makePassingUat(n = 1) { const tests = Array.from({ length: n }, (_, i) => [ `### ${i + 1}. Test ${i + 1}`, `expected: It works`, `result: passed`, '', ].join('\n')).join('\n'); return `---\nstatus: passed\n---\n\n# UAT\n\n${tests}`; } // ─── stripFalsePositiveContexts ──────────────────────────────────────────────── describe('stripFalsePositiveContexts — frontmatter', () => { test('removes leading frontmatter block', () => { const input = '---\nstatus: pending\nresult: pending\n---\n\nReal content here.'; const out = stripFalsePositiveContexts(input); assert.ok(!out.includes('result: pending'), 'frontmatter result: pending should be stripped'); assert.ok(out.includes('Real content here.'), 'body content must be preserved'); }); test('does not strip non-frontmatter --- dividers later in document', () => { const input = '---\nstatus: ok\n---\n\n# Section\n\n---\n\nMore content.'; const out = stripFalsePositiveContexts(input); assert.ok(out.includes('More content.'), 'content after a non-frontmatter divider must survive'); }); test('handles CRLF frontmatter', () => { const input = '---\r\nstatus: partial\r\nresult: pending\r\n---\r\n\r\nBody text.'; const out = stripFalsePositiveContexts(input); assert.ok(!out.includes('result: pending'), 'CRLF frontmatter must be stripped'); assert.ok(out.includes('Body text.'), 'body after CRLF frontmatter must survive'); }); }); describe('stripFalsePositiveContexts — HTML comments', () => { test('removes single-line HTML comment', () => { const input = 'Before\n\nAfter'; const out = stripFalsePositiveContexts(input); assert.ok(!out.includes('result: pending'), 'HTML comment content must be stripped'); assert.ok(out.includes('Before'), 'content before comment must survive'); assert.ok(out.includes('After'), 'content after comment must survive'); }); test('removes multi-line HTML comment', () => { const input = 'A\n\nB'; const out = stripFalsePositiveContexts(input); assert.ok(!out.includes('result: pending'), 'multi-line HTML comment content must be stripped'); assert.ok(out.includes('A'), 'content before comment must survive'); assert.ok(out.includes('B'), 'content after comment must survive'); }); test('unterminated HTML comment swallows to EOF (fail-closed)', () => { const input = 'Before\n', ].join('\n'); const clean = stripFalsePositiveContexts(rawContent); const items = parseUatResultItems(clean); assert.strictEqual(items.length, 0, 'Fake result inside HTML comment must produce no items'); writeFile(tmpDir, 'phase-UAT.md', rawContent); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.strictEqual(report.no_uat_artifacts, true); }); test('#247: result:passed inside frontmatter: parseUatResultItems returns [] + evaluateUatPassed → passed:false + no_uat_artifacts:true', () => { const rawContent = [ '---', 'example_result: passed', '---', '', 'No real test blocks here.', ].join('\n'); const clean = stripFalsePositiveContexts(rawContent); const items = parseUatResultItems(clean); assert.strictEqual(items.length, 0, 'Fake result inside frontmatter must produce no items'); writeFile(tmpDir, 'phase-UAT.md', rawContent); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.strictEqual(report.no_uat_artifacts, true); }); test('#247: result:passed inside fenced block is NOT treated as passing test (with real failing test)', () => { const content = [ '---', 'status: partial', '---', '', '# Example', '', '```', '### 1. Test', 'expected: Example output', 'result: passed', '```', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'result:passed inside a fenced block must not flip passed to true'); assert.ok(report.checks.some(c => c.result === 'pending' && !c.passing), 'Real pending test must be captured'); }); test('#247: result:passed inside blockquote is NOT treated as passing test', () => { const content = [ '---', 'status: partial', '---', '', '> ### 1. Test', '> expected: Example', '> result: passed', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'result:passed inside a blockquote must not flip passed to true'); }); test('#247: result:passed inside HTML comment is NOT treated as passing test', () => { const content = [ '---', 'status: partial', '---', '', '', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'result:passed inside an HTML comment must not flip passed to true'); }); test('#247: result:passed inside frontmatter example is NOT treated as passing test', () => { const content = [ '---', 'status: partial', 'example_result: passed', '---', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'result:passed in frontmatter must not flip passed to true'); }); test('#247: real passing test block outside all contexts → passed:true', () => { const content = [ '---', 'status: passed', '---', '', '# UAT Results', '', '### 1. Login works', 'expected: User logs in', 'result: passed', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, true, 'A real passing test outside false-positive contexts must pass'); }); test('block-scalar expected: followed by blank line + result: pending → parsed as blocker (not dropped)', () => { const content = [ '### 1. Test A', 'expected: |', ' multi', ' line expected output', '', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'Block-scalar expected: with result: pending must be captured as a blocker'); assert.ok(report.checks.some(c => c.result === 'pending' && !c.passing), `Expected pending check, got: ${JSON.stringify(report.checks)}`); }); }); // ─── evaluateUatPassed — output shape contract ─────────────────────────────── describe('evaluateUatPassed — output shape (Hyrum\'s Law contract)', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('returns all required fields in the locked shape including no_uat_artifacts', () => { writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); const report = evaluateUatPassed(tmpDir); // Locked field names assert.ok('passed' in report, 'report.passed must exist'); assert.ok('uat_files' in report, 'report.uat_files must exist'); assert.ok('verification_files' in report, 'report.verification_files must exist'); assert.ok('checks' in report, 'report.checks must exist'); assert.ok('blockers' in report, 'report.blockers must exist'); assert.ok('no_uat_artifacts' in report, 'report.no_uat_artifacts must exist'); assert.ok('policy' in report, 'report.policy must exist'); assert.ok('require_verification' in report.policy, 'report.policy.require_verification must exist'); // checks item shape if (report.checks.length > 0) { const c = report.checks[0]; assert.ok('file' in c, 'check.file must exist'); assert.ok('test' in c, 'check.test must exist'); assert.ok('name' in c, 'check.name must exist'); assert.ok('result' in c, 'check.result must exist'); assert.ok('passing' in c, 'check.passing must exist'); } }); test('no_uat_artifacts is false when real checks exist', () => { writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.no_uat_artifacts, false); }); test('no_uat_artifacts is true when no checks exist', () => { const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.no_uat_artifacts, true); }); test('uat_files contains the filename', () => { writeFile(tmpDir, 'my-UAT.md', makePassingUat(1)); const report = evaluateUatPassed(tmpDir); assert.ok(report.uat_files.includes('my-UAT.md'), `uat_files should include 'my-UAT.md', got: ${JSON.stringify(report.uat_files)}`); }); test('verification_files contains the filename', () => { writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); writeFile(tmpDir, 'phase-VERIFICATION.md', '---\nstatus: passed\n---\n'); const report = evaluateUatPassed(tmpDir); assert.ok(report.verification_files.includes('phase-VERIFICATION.md'), `verification_files should include 'phase-VERIFICATION.md', got: ${JSON.stringify(report.verification_files)}`); }); }); // ─── #3511: phase-scoped UAT/VERIFICATION scanning — the transition gate ───── // // evaluateUatPassed is the CRITICAL anchor for #3511: its `passed`/`blockers` // fields directly gate a phase transition. A cross-phase stray file sitting // in a phase directory must never contribute a blocker to a phase it does not // belong to, and the phase's own artifacts must keep behaving exactly as // before. describe('#3511: evaluateUatPassed — cross-phase stray files do not contribute blockers', () => { let baseDir; let phaseDir; beforeEach(() => { baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'msd-3511-uat-pred-')); // Phase-shaped basename ("03-…") so isPhaseArtifact actually scopes — // extractPhaseToken('03-uat-predicate') derives token "03". phaseDir = path.join(baseDir, '03-uat-predicate'); fs.mkdirSync(phaseDir); }); afterEach(() => { rmDir(baseDir); }); test('passed:true with a passing own UAT + own VERIFICATION, despite a blocking cross-phase stray VERIFICATION', () => { writeFile(phaseDir, '03-UAT.md', makePassingUat(1)); writeFile(phaseDir, '03-VERIFICATION.md', '---\nstatus: passed\n---\n\nOK.'); // Cross-phase stray: a "04" VERIFICATION file sitting in phase 03's // directory, with a BLOCKING status. Pre-#3511, this unscoped scan would // have picked it up and blocked phase 03's transition. writeFile(phaseDir, '04-VERIFICATION.md', '---\nstatus: human_needed\n---\n\nNeeds human check.'); const report = evaluateUatPassed(phaseDir); assert.strictEqual(report.passed, true, 'a cross-phase stray VERIFICATION file must not block this phase\'s transition'); assert.ok( !report.blockers.some(b => /04-VERIFICATION\.md/.test(b) || /human_needed/i.test(b)), `blockers must not name the stray file; got: ${JSON.stringify(report.blockers)}`, ); assert.strictEqual(report.verification_files.includes('04-VERIFICATION.md'), false, 'the stray must not even be counted as a verification_files entry for this phase'); }); test('passed:false with own report still failing, unaffected by an unrelated passing cross-phase stray (non-stray case unchanged)', () => { writeFile(phaseDir, '03-UAT.md', makePassingUat(1)); // This phase's own VERIFICATION is blocking. writeFile(phaseDir, '03-VERIFICATION.md', '---\nstatus: gaps_found\n---\n\nHas gaps.'); // A cross-phase stray that is itself passing must not paper over the // phase's own real failure either — over-exclusion is as dangerous as // under-exclusion here. writeFile(phaseDir, '99-VERIFICATION.md', '---\nstatus: passed\n---\n\nOK.'); const report = evaluateUatPassed(phaseDir); assert.strictEqual(report.passed, false, 'the phase\'s own gaps_found VERIFICATION must still block, exactly as before #3511'); assert.ok(report.blockers.some(b => /gaps_found/i.test(b)), `blockers must still name this phase's own gaps_found status; got: ${JSON.stringify(report.blockers)}`); }); test('#3511 follow-up: passed:true from a NON-canonical dir shape "1-unpadded" (over-exclusion / no_uat_artifacts check)', () => { // "1-unpadded" tokenizes to literal "1"; scaffold writes the PADDED // "01-…" form (normalizePhaseName). A literal token compare excluded the // phase's own artifacts here, flipping `no_uat_artifacts: true` and // false-blocking the transition gate this predicate feeds. const unpaddedDir = path.join(baseDir, '1-unpadded'); fs.mkdirSync(unpaddedDir); writeFile(unpaddedDir, '01-UAT.md', makePassingUat(1)); writeFile(unpaddedDir, '01-VERIFICATION.md', '---\nstatus: passed\n---\n\nOK.'); const report = evaluateUatPassed(unpaddedDir); assert.strictEqual(report.no_uat_artifacts, false, `own UAT/VERIFICATION files in an unpadded-dir phase must be found; got: ${JSON.stringify(report)}`); assert.strictEqual(report.passed, true, `the phase's own passing files in a non-canonical dir must pass the gate; got: ${JSON.stringify(report)}`); }); }); // ─── FIX A regression: nested-fence (~~~ inside ```) ───────────────────────── describe('FIX A — nested fence: ~~~ inside ``` does not prematurely close outer fence', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('parseUatResultItems sees [] for fake inside ``` that encloses ~~~', () => { // A backtick fence that contains an inner ~~~ fence with a fake test block. // The ~~~ must NOT close the ``` fence — the whole interior is content and is dropped. const raw = [ '```', '~~~', '### 1. Fake', 'expected: X', 'result: passed', '~~~', '```', ].join('\n'); const clean = stripFalsePositiveContexts(raw); const items = parseUatResultItems(clean); assert.strictEqual(items.length, 0, 'fake inside nested fence must not leak through'); }); test('evaluateUatPassed → passed:false + no_uat_artifacts:true for nested-fence-only file', () => { const raw = [ '```', '~~~', '### 1. Fake', 'expected: X', 'result: passed', '~~~', '```', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', raw); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'nested-fence fake must not flip passed'); assert.strictEqual(report.no_uat_artifacts, true, 'no real items → no_uat_artifacts:true'); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake must not appear in checks'); }); test('balanced nested fence (``` inside ~~~) is NOT flagged as malformed', () => { // ~~~ outer, ``` inner — properly closed — should not trigger unterminatedFence const raw = [ '~~~', '```', 'code', '```', '~~~', '', '### 1. Real Test', 'expected: Works', 'result: passed', ].join('\n'); const { unterminatedFence } = analyzeMarkdown(raw); assert.strictEqual(unterminatedFence, false, 'balanced nested fence must not be flagged'); const clean = stripFalsePositiveContexts(raw); const items = parseUatResultItems(clean); assert.strictEqual(items.length, 1, 'real test outside fence must still be found'); assert.strictEqual(items[0].result, 'passed'); }); test('real result:passed outside ``` that encloses ~~~ → passed:true (no false-block)', () => { const raw = [ '---', 'status: passed', '---', '', '```', '~~~', '### 1. Fake', 'result: passed', '~~~', '```', '', '### 1. Real Test', 'expected: Works', 'result: passed', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', raw); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, true, 'real result outside nested fence must still pass'); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake must not appear in checks'); }); }); // ─── FIX B regression: cross-line result value ──────────────────────────────── describe('FIX B — cross-line result: value must be on the same line', () => { test('result: with value on next line → result:missing (not passed)', () => { const content = [ '### 1. Cross-line Test', 'expected: Y', 'result:', '', 'passed', ].join('\n'); const items = parseUatResultItems(content); assert.strictEqual(items.length, 1); assert.notStrictEqual(items[0].result, 'passed', 'result value on a subsequent line must not be captured as passed'); assert.strictEqual(items[0].result, 'missing', 'cross-line result must yield missing (blocker)'); }); test('result: whose value sits on a following INDENTED line → missing (pinned #3078-CR divergence)', () => { // #3078-CR (security review follow-up): on origin/next, the old // `/^result:\s*\[?(\w+)\]?.*$/im` regex's `\s*` is greedy and matches // ACROSS a newline, so `result:\n blocked` parsed as `blocked` — a real // row this shape reported 1/blocked. The split-then-match rewrite tests // `result:` against a SINGLE already-split line, per the documented // "value must sit on the SAME line as result:" contract (see the comment // above RESULT_LINE_RE), so this now yields 'missing' (a parse gap, with // the percentage withheld) instead of silently crossing the newline. // This is a DELIBERATE, FAIL-SAFE divergence from the old cross-newline // `\s*` behavior — pinned here so it is never "fixed" back by accident. const content = [ '### 1. Indented-continuation Test', 'expected: Y', 'result:', ' blocked', '', ].join('\n'); const items = parseUatResultItems(content); assert.strictEqual(items.length, 1); assert.notStrictEqual(items[0].result, 'blocked', 'a value on a following indented line must not be captured across the newline'); assert.strictEqual(items[0].result, 'missing', 'result: with its value on the next (even indented) line must yield missing, not the old cross-newline capture'); }); test('evaluateUatPassed → passed:false for cross-line result:passed', () => { const tmpDir = makeTmpDir(); try { const content = [ '### 1. Cross-line Test', 'expected: Y', 'result:', '', 'passed', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.ok(!report.checks.some(c => c.result === 'passed'), 'cross-line result must not produce a passing check'); } finally { rmDir(tmpDir); } }); }); // ─── FIX C regression: masked-comment (earlier closed comment + later unterminated) ── describe('FIX C — dangling comment survives earlier balanced comment', () => { test('analyzeMarkdown detects unterminated comment after a properly closed one', () => { const raw = [ '', 'Some text', '', '', '', 'Some text', '', '', '### 1. Real Test', 'result: passed', ].join('\n'); const { unterminatedComment } = analyzeMarkdown(raw); assert.strictEqual(unterminatedComment, false, 'only balanced comments must not be flagged as unterminated'); }); }); // ─── FIX D regression: balanced mixed fences are NOT flagged ───────────────── describe('FIX D — odd-fence-count heuristic replaced: balanced mixed fences not flagged', () => { test('analyzeMarkdown: ``` followed by ~~~ (both balanced) → unterminatedFence:false', () => { const raw = [ '```', 'code', '```', '', '~~~', 'more code', '~~~', ].join('\n'); const { unterminatedFence } = analyzeMarkdown(raw); assert.strictEqual(unterminatedFence, false, 'two separate balanced fences must not trigger unterminatedFence'); }); test('evaluateUatPassed: multiple balanced fences + real passing test → passed:true, no malformed blocker', () => { const tmpDir = makeTmpDir(); try { const raw = [ '---', 'status: passed', '---', '', '```', 'result: fake', '```', '', '~~~', 'result: also fake', '~~~', '', '### 1. Real Test', 'expected: Works', 'result: passed', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', raw); const report = evaluateUatPassed(tmpDir); assert.ok(!report.blockers.some(b => /malformed/i.test(b)), `Balanced mixed fences must not produce malformed blocker, got: ${JSON.stringify(report.blockers)}`); assert.strictEqual(report.passed, true, 'real test outside balanced fences must still pass'); } finally { rmDir(tmpDir); } }); }); // ─── FIX E extra: fake-not-in-checks assertions for existing mixed tests ────── describe('FIX E — fake items must NOT appear in checks (absence assertions)', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('fake inside fence + real pending: fake NOT in checks', () => { const content = [ '---', 'status: partial', '---', '', '```', '### 10. Fake', 'expected: Fake', 'result: passed', '```', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake test inside fence must not appear in checks'); }); test('fake inside blockquote + real pending: fake NOT in checks', () => { const content = [ '---', 'status: partial', '---', '', '> ### 10. Fake', '> expected: Fake', '> result: passed', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake test inside blockquote must not appear in checks'); }); test('fake inside HTML comment + real pending: fake NOT in checks', () => { const content = [ '---', 'status: partial', '---', '', '', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake test inside HTML comment must not appear in checks'); }); }); // ─── Property-based test (fast-check) ───────────────────────────────────────── describe('evaluateUatPassed — property: wrapping in false-positive context never flips to passed', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('fc: inserting result:passed inside wrapper context never flips a failing UAT to passed', () => { // A baseline UAT file that has a pending item — it must always evaluate to passed:false // regardless of how many "result: passed" lines we inject inside fenced/blockquote/comment wrappers. const baseFailingBody = [ '### 1. Real Test', 'expected: It works', 'result: pending', '', ].join('\n'); const wrappers = fc.constantFrom( // backtick fence (inner) => '```\n' + inner + '\n```', // tilde fence (inner) => '~~~\n' + inner + '\n~~~', // HTML comment (inner) => '', // blockquote — prefix each line (inner) => inner.split('\n').map(l => '> ' + l).join('\n'), ); fc.assert( fc.property(wrappers, fc.nat(3), (wrap, extraCount) => { // Build a "fake passing block" that would fool a naive regex const fakePassingLines = Array.from({ length: extraCount + 1 }, (_, i) => `### ${i + 10}. Fake Test ${i + 10}\nexpected: Fake\nresult: passed` ).join('\n'); const fullContent = [ '---', 'status: partial', '---', '', wrap(fakePassingLines), '', baseFailingBody, ].join('\n'); // Write to a unique tmp file to avoid cross-test state const fcDir = fs.mkdtempSync(path.join(os.tmpdir(), 'msd-fc-uat-')); try { fs.writeFileSync(path.join(fcDir, 'feature-UAT.md'), fullContent, 'utf-8'); const report = evaluateUatPassed(fcDir); // The pending item must always keep passed:false // AND fake items injected via wrappers must never appear in checks const hasFakeInChecks = report.checks.some(c => c.name.startsWith('Fake Test')); return report.passed === false && !hasFakeInChecks; } finally { cleanup(fcDir); } }), { numRuns: 50 } ); }); }); // ─── #3078-CR MEDIUM: acceptance gate (uat-predicate.cjs) must AGREE with the ── // ─── audit surface (uat.cjs's parseUatItemsWithStats) on the same bytes ─────── describe('#3078-CR: evaluateUatPassed agrees with the audit surface (parseUatItemsWithStats)', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('U+2028 scalar-injection: gate blocks, audit surface reports the outstanding row — they AGREE', () => { // A `result:` line reachable only via a U+2028 LINE SEPARATOR sitting inside // an `expected: |` block-scalar body must not be read as a genuine // column-0 match by EITHER surface. The real, later `result: blocked` line // is the one that must win. const LS = String.fromCharCode(0x2028); // never a raw separator in source: a formatter that normalizes line separators would silently turn this fixture into an ordinary-character control that still passes const body = [ '---', 'status: passed', '---', '', '# UAT', '', '### 1. Alpha', 'expected: |', ' x' + LS + 'result: pass', 'result: blocked', '', ].join('\n'); writeFile(tmpDir, '01-alpha-UAT.md', body); const gateReport = evaluateUatPassed(tmpDir); const auditReport = parseUatItemsWithStats(body); // AGREEMENT, asserted explicitly (not each surface independently): both // surfaces must consider this phase NOT clean, on the same test row. assert.equal(gateReport.passed, false, 'gate: must not accept a blocked test as passed'); assert.equal(auditReport.items.length, 1, 'audit: the blocked row must surface as outstanding'); assert.equal(auditReport.items[0].result, 'blocked', 'audit: must read the real result, not the injected one'); const gateCheck = gateReport.checks.find((c) => c.test === 1); assert.ok(gateCheck, 'gate: must record the test-1 check'); assert.equal(gateCheck.result, 'blocked', 'gate: must read the real result, not the injected one'); assert.equal(gateCheck.passing, false); // Cross-surface identity: same test number, same result token. assert.equal(gateCheck.result, auditReport.items[0].result, 'gate and audit surface must agree on the result token'); }); test('H-U28 restored: a heading delimited by U+2028 (not \n) is still found and blocks', () => { // #3078-CR MEDIUM 1 (security review follow-up): origin/next found this // heading via an /m-anchored scan whose LineTerminator set includes // U+2028/U+2029; a naive split('\n')-only port of that scan silently // stopped finding it, making the gate MORE PERMISSIVE than origin/next // (measured: HEAD passed:true/0 blockers, origin/next passed:false/1 // blocker, for this exact shape). The heading scan now splits on // \n/U+2028/U+2029 (a STRUCTURE frame) while the result: scan below it // stays \n-only (an ATTRIBUTION frame, unchanged) -- so the heading is // found, but its result: line -- separated from the heading by the same // exotic separator -- is correctly NOT read across that boundary (that is // the attribution guard I-U28 below exists to prove), yielding // 'missing' rather than 'blocked'. Either token is a non-passing, // blocking state, so the gate still BLOCKS -- the outcome origin/next // produced, restored. const LS = String.fromCharCode(0x2028); // never a raw separator in source: a formatter that normalizes line separators would silently turn this fixture into an ordinary-character control that still passes const content = 'Notes.' + LS + '### 2. B' + LS + 'result: blocked'; const items = parseUatResultItems(content); assert.strictEqual(items.length, 1, 'a U+2028-delimited heading must still be found'); assert.strictEqual(items[0].test, 2); assert.strictEqual(items[0].name, 'B'); // IDENTITY, not a proxy: the exact token. `notStrictEqual(..., 'passed')` also passes on // 'pass', which IS in UAT_PASS_RESULTS -- so it could not catch a regression that // attributed a PASSING result to the recovered heading, which is the whole risk here. assert.strictEqual(items[0].result, 'missing', 'the result: line sits across the same exotic separator, so it is correctly NOT attributed -- ' + 'missing is a non-passing, blocking state'); }); test('H-U28 restored: evaluateUatPassed BLOCKS on the U+2028-delimited heading shape', () => { const tmpDir = makeTmpDir(); try { const LS = String.fromCharCode(0x2028); // never a raw separator in source: a formatter that normalizes line separators would silently turn this fixture into an ordinary-character control that still passes const content = 'Notes.' + LS + '### 2. B' + LS + 'result: blocked'; const body = ['---', 'status: passed', '---', '', '# UAT', '', content, ''].join('\n'); writeFile(tmpDir, '01-h28-UAT.md', body); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'gate must block on the U+2028-delimited heading -- origin/next parity'); assert.ok(report.blockers.length > 0, 'a blocker must be recorded'); } finally { rmDir(tmpDir); } }); test('CR-fenced case: gate and audit surface (parseUatItemsWithStats) agree -- neither silently clean', () => { // #3078-CR MEDIUM 1 follow-up evidence: a lone-CR document // (see:CRfenceCR### 2. BCRresult: blockedCRfence) has its CRs normalized // to \n before parsing, which turns a literal fence-marker sequence into // a REAL fence delimiter it was not before normalization -- the row ends // up fenced and stripped on the gate side. Both surfaces must agree this // phase is NOT clean (the gate must not pass while the audit surface // reports a shortfall/gap for the same document). const crBody = ['see:', '```', '### 2. B', 'result: blocked', '```'].join('\r'); const tmpDir = makeTmpDir(); try { writeFile(tmpDir, '01-crfence-UAT.md', crBody); const gateReport = evaluateUatPassed(tmpDir); const auditReport = parseUatItemsWithStats(crBody); assert.strictEqual(gateReport.passed, false, 'gate must not report a clean pass for this document'); assert.ok(auditReport.headingsSeen > 0, 'audit surface must see the heading exists'); assert.ok(auditReport.items.length === 0 && auditReport.shortfallBlocks > 0, 'audit surface must record the fenced row as an unresolved shortfall, not silently drop it'); } finally { rmDir(tmpDir); } }); test('lone-CR frontmatter: gate no longer silently drops a blocking status hidden by an unnormalized read', () => { // A lone-CR-terminated frontmatter fence (`---\rstatus: partial\r---`) must // still be recognised as frontmatter — the raw, unnormalized read this // fix replaces treated the whole fence as one unbroken line, so // `extractFrontmatter` never matched it and the blocking `status: partial` // was silently dropped (fail-OPEN, the false-clean this fix closes). const body = [ '---\rstatus: partial\r---', '', '# UAT', '', '### 1. Alpha\rresult: passed', '### 2. Beta\rresult: pass', '', ].join('\r'); writeFile(tmpDir, '01-beta-UAT.md', body); const gateReport = evaluateUatPassed(tmpDir); assert.equal(gateReport.passed, false, 'gate: the hidden status: partial must now block'); assert.ok( gateReport.blockers.some((b) => b.includes('status=partial')), 'gate: the frontmatter status blocker must be surfaced, not silently dropped', ); }); test('clean control: a normally-passing file agrees as passed on both surfaces', () => { const body = [ '---', 'status: passed', '---', '', '# UAT', '', '### 1. Alpha', 'result: passed', '', '### 2. Beta', 'result: pass', '', ].join('\n'); writeFile(tmpDir, '01-gamma-UAT.md', body); const gateReport = evaluateUatPassed(tmpDir); const auditReport = parseUatItemsWithStats(body); // AGREEMENT: the gate accepts, and the audit surface reports NO outstanding // (non-passing) rows for the same bytes. assert.equal(gateReport.passed, true, 'gate: a clean file must still pass'); assert.equal(auditReport.items.length, 0, 'audit: a clean file must have no outstanding rows'); }); }); // ─── #4546 — deferred follow-up skips are non-blocking ──────────────────────── describe('#4546 — deferred follow-up skips', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); function makeDeferredUatItem(n, name, reason) { const lines = [`### ${n}. ${name}`, `expected: ${name} works`]; if (reason === null) { lines.push('result: skipped'); } else { lines.push('result: skipped', `reason: ${reason}`); } return lines; } test('deferred follow-up skip is non-blocking (#4546)', () => { const content = [ '---', 'status: complete', '---', '', '### 1. Test A', 'expected: A', 'result: passed', '', ...makeDeferredUatItem(2, 'Test B', '"Deferred follow-up: nice to have, next version"'), '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, true, `a deliberately deferred follow-up must not block: ${JSON.stringify(report.blockers)}`); assert.strictEqual(report.blockers.length, 0); const deferred = report.checks.find(c => c.test === 2); assert.ok(deferred, 'deferred check present'); assert.strictEqual(deferred.passing, true, 'deferred skip counts as passing'); assert.strictEqual(deferred.deferred, true, 'deferred skip is flagged deferred in the report'); const passing = report.checks.find(c => c.test === 1); assert.strictEqual(passing.deferred, false, 'a real pass is not flagged deferred'); }); test('plain skipped without a reason still blocks (#4546 negative space)', () => { const content = [ '---', 'status: complete', '---', '', '### 1. Test A', 'expected: A', 'result: passed', '', ...makeDeferredUatItem(2, 'Test B', null), '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'a skipped test with no reason is unresolved and must block'); assert.ok(report.blockers.some(b => /test 2/.test(b)), `expected a blocker for test 2, got: ${JSON.stringify(report.blockers)}`); }); test('skipped with a non-deferral reason still blocks (#4546 negative space)', () => { const content = [ '---', 'status: complete', '---', '', '### 1. Test A', 'expected: A', 'result: passed', '', ...makeDeferredUatItem(2, 'Test B', '"waiting on credentials"'), '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'a skipped test whose reason is not a deferral must block'); }); test('deferred reason match is case-insensitive (#4546 boundary)', () => { const content = [ '---', 'status: complete', '---', '', ...makeDeferredUatItem(1, 'Test A', '"deferred follow-up: later"'), '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, true, 'the matcher anchors on the shipped template text case-insensitively'); }); test('a deferred skip does not mask other blockers (#4546 independence)', () => { const content = [ '---', 'status: complete', '---', '', ...makeDeferredUatItem(1, 'Test A', '"Deferred follow-up: later"'), '', '### 2. Test B', 'expected: B', 'result: pending', '', '### 3. Test C', 'expected: C', 'result: blocked', 'blocked_by: server', '', '### 4. Test D', 'expected: D', 'result: issue', 'reported: "crashes"', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'deferred skip must not mask pending/blocked/issue blockers'); assert.ok(report.blockers.some(b => /test 2/.test(b)), 'pending still blocks'); assert.ok(report.blockers.some(b => /test 3/.test(b)), 'blocked still blocks'); assert.ok(report.blockers.some(b => /test 4/.test(b)), 'issue still blocks'); assert.ok(!report.blockers.some(b => /test 1/.test(b)), `the deferred item must not appear among blockers: ${JSON.stringify(report.blockers)}`); }); test('property: deferred-skip acceptance drives the real gate over arbitrary result/reason pairs (#4546)', () => { // Gate-driven: expectations are derived from the INPUT (the spec sentence // in #4546), then asserted against evaluateUatPassed's report — the // property never restates the implementation's regex. The generated // classes include the no-result-line shape (the parser's 'missing' // branch) and reasonless skips. const deferredRe = /^["']?deferred follow-up\b/i; fc.assert( fc.property( fc.constantFrom('passed', 'pass', 'skipped', 'pending', 'blocked', 'issue', 'missing'), fc.option(fc.stringMatching(/^["']?[a-z ]{0,30}$/), { nil: undefined }), (result, reason) => { const itemLines = [`### 1. Test X`, 'expected: X works']; if (result !== 'missing') { itemLines.push(`result: ${result}`); if (reason !== undefined) itemLines.push(`reason: ${reason}`); } const content = [ '---', 'status: complete', '---', '', ...itemLines, '', ].join('\n'); fs.writeFileSync(path.join(tmpDir, 'phase-UAT.md'), content, 'utf-8'); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.checks.length, 1); const check = report.checks[0]; const isDeferral = result === 'skipped' && typeof reason === 'string' && deferredRe.test(reason); const specPassing = result === 'passed' || result === 'pass' || isDeferral; assert.strictEqual(check.result, result === 'missing' ? 'missing' : result); assert.strictEqual(check.passing, specPassing, `result=${result} reason=${JSON.stringify(reason)}: gate must ${specPassing ? 'pass' : 'block'}`); assert.strictEqual(check.deferred, isDeferral, `result=${result} reason=${JSON.stringify(reason)}: deferred flag`); assert.strictEqual(report.passed, specPassing, 'a single-item file passes exactly when the item passes'); assert.deepStrictEqual(report.blockers, specPassing ? [] : [report.blockers[0]]); } ), { numRuns: 120, seed: 4546 } ); }); });