'use strict'; /** * Unit tests for uat-predicate.cjs * Tests the pure-computation module: stripFalsePositiveContexts, * parseUatResultItems, evaluateUatPassed. * * Issue #247 — phase uat-passed predicate */ const { test, describe, beforeEach, afterEach } = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const os = require('node:os'); const fc = require('fast-check'); const { stripFalsePositiveContexts, parseUatResultItems, analyzeMarkdown, evaluateUatPassed, } = require('../gsd-core/bin/lib/uat-predicate.cjs'); const { cleanup } = require('./helpers.cjs'); // ─── Helpers ────────────────────────────────────────────────────────────────── function makeTmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-uat-pred-test-')); } function rmDir(dir) { cleanup(dir); } function writeFile(dir, name, content) { fs.writeFileSync(path.join(dir, name), content, 'utf-8'); } function makePassingUat(n = 1) { const tests = Array.from({ length: n }, (_, i) => [ `### ${i + 1}. Test ${i + 1}`, `expected: It works`, `result: passed`, '', ].join('\n')).join('\n'); return `---\nstatus: passed\n---\n\n# UAT\n\n${tests}`; } // ─── stripFalsePositiveContexts ──────────────────────────────────────────────── describe('stripFalsePositiveContexts — frontmatter', () => { test('removes leading frontmatter block', () => { const input = '---\nstatus: pending\nresult: pending\n---\n\nReal content here.'; const out = stripFalsePositiveContexts(input); assert.ok(!out.includes('result: pending'), 'frontmatter result: pending should be stripped'); assert.ok(out.includes('Real content here.'), 'body content must be preserved'); }); test('does not strip non-frontmatter --- dividers later in document', () => { const input = '---\nstatus: ok\n---\n\n# Section\n\n---\n\nMore content.'; const out = stripFalsePositiveContexts(input); assert.ok(out.includes('More content.'), 'content after a non-frontmatter divider must survive'); }); test('handles CRLF frontmatter', () => { const input = '---\r\nstatus: partial\r\nresult: pending\r\n---\r\n\r\nBody text.'; const out = stripFalsePositiveContexts(input); assert.ok(!out.includes('result: pending'), 'CRLF frontmatter must be stripped'); assert.ok(out.includes('Body text.'), 'body after CRLF frontmatter must survive'); }); }); describe('stripFalsePositiveContexts — HTML comments', () => { test('removes single-line HTML comment', () => { const input = 'Before\n\nAfter'; const out = stripFalsePositiveContexts(input); assert.ok(!out.includes('result: pending'), 'HTML comment content must be stripped'); assert.ok(out.includes('Before'), 'content before comment must survive'); assert.ok(out.includes('After'), 'content after comment must survive'); }); test('removes multi-line HTML comment', () => { const input = 'A\n\nB'; const out = stripFalsePositiveContexts(input); assert.ok(!out.includes('result: pending'), 'multi-line HTML comment content must be stripped'); assert.ok(out.includes('A'), 'content before comment must survive'); assert.ok(out.includes('B'), 'content after comment must survive'); }); test('unterminated HTML comment swallows to EOF (fail-closed)', () => { const input = 'Before\n', ].join('\n'); const clean = stripFalsePositiveContexts(rawContent); const items = parseUatResultItems(clean); assert.strictEqual(items.length, 0, 'Fake result inside HTML comment must produce no items'); writeFile(tmpDir, 'phase-UAT.md', rawContent); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.strictEqual(report.no_uat_artifacts, true); }); test('#247: result:passed inside frontmatter: parseUatResultItems returns [] + evaluateUatPassed → passed:false + no_uat_artifacts:true', () => { const rawContent = [ '---', 'example_result: passed', '---', '', 'No real test blocks here.', ].join('\n'); const clean = stripFalsePositiveContexts(rawContent); const items = parseUatResultItems(clean); assert.strictEqual(items.length, 0, 'Fake result inside frontmatter must produce no items'); writeFile(tmpDir, 'phase-UAT.md', rawContent); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.strictEqual(report.no_uat_artifacts, true); }); test('#247: result:passed inside fenced block is NOT treated as passing test (with real failing test)', () => { const content = [ '---', 'status: partial', '---', '', '# Example', '', '```', '### 1. Test', 'expected: Example output', 'result: passed', '```', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'result:passed inside a fenced block must not flip passed to true'); assert.ok(report.checks.some(c => c.result === 'pending' && !c.passing), 'Real pending test must be captured'); }); test('#247: result:passed inside blockquote is NOT treated as passing test', () => { const content = [ '---', 'status: partial', '---', '', '> ### 1. Test', '> expected: Example', '> result: passed', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'result:passed inside a blockquote must not flip passed to true'); }); test('#247: result:passed inside HTML comment is NOT treated as passing test', () => { const content = [ '---', 'status: partial', '---', '', '', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'result:passed inside an HTML comment must not flip passed to true'); }); test('#247: result:passed inside frontmatter example is NOT treated as passing test', () => { const content = [ '---', 'status: partial', 'example_result: passed', '---', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'result:passed in frontmatter must not flip passed to true'); }); test('#247: real passing test block outside all contexts → passed:true', () => { const content = [ '---', 'status: passed', '---', '', '# UAT Results', '', '### 1. Login works', 'expected: User logs in', 'result: passed', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, true, 'A real passing test outside false-positive contexts must pass'); }); test('block-scalar expected: followed by blank line + result: pending → parsed as blocker (not dropped)', () => { const content = [ '### 1. Test A', 'expected: |', ' multi', ' line expected output', '', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'Block-scalar expected: with result: pending must be captured as a blocker'); assert.ok(report.checks.some(c => c.result === 'pending' && !c.passing), `Expected pending check, got: ${JSON.stringify(report.checks)}`); }); }); // ─── evaluateUatPassed — output shape contract ─────────────────────────────── describe('evaluateUatPassed — output shape (Hyrum\'s Law contract)', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('returns all required fields in the locked shape including no_uat_artifacts', () => { writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); const report = evaluateUatPassed(tmpDir); // Locked field names assert.ok('passed' in report, 'report.passed must exist'); assert.ok('uat_files' in report, 'report.uat_files must exist'); assert.ok('verification_files' in report, 'report.verification_files must exist'); assert.ok('checks' in report, 'report.checks must exist'); assert.ok('blockers' in report, 'report.blockers must exist'); assert.ok('no_uat_artifacts' in report, 'report.no_uat_artifacts must exist'); assert.ok('policy' in report, 'report.policy must exist'); assert.ok('require_verification' in report.policy, 'report.policy.require_verification must exist'); // checks item shape if (report.checks.length > 0) { const c = report.checks[0]; assert.ok('file' in c, 'check.file must exist'); assert.ok('test' in c, 'check.test must exist'); assert.ok('name' in c, 'check.name must exist'); assert.ok('result' in c, 'check.result must exist'); assert.ok('passing' in c, 'check.passing must exist'); } }); test('no_uat_artifacts is false when real checks exist', () => { writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.no_uat_artifacts, false); }); test('no_uat_artifacts is true when no checks exist', () => { const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.no_uat_artifacts, true); }); test('uat_files contains the filename', () => { writeFile(tmpDir, 'my-UAT.md', makePassingUat(1)); const report = evaluateUatPassed(tmpDir); assert.ok(report.uat_files.includes('my-UAT.md'), `uat_files should include 'my-UAT.md', got: ${JSON.stringify(report.uat_files)}`); }); test('verification_files contains the filename', () => { writeFile(tmpDir, 'phase-UAT.md', makePassingUat(1)); writeFile(tmpDir, 'phase-VERIFICATION.md', '---\nstatus: passed\n---\n'); const report = evaluateUatPassed(tmpDir); assert.ok(report.verification_files.includes('phase-VERIFICATION.md'), `verification_files should include 'phase-VERIFICATION.md', got: ${JSON.stringify(report.verification_files)}`); }); }); // ─── #3511: phase-scoped UAT/VERIFICATION scanning — the transition gate ───── // // evaluateUatPassed is the CRITICAL anchor for #3511: its `passed`/`blockers` // fields directly gate a phase transition. A cross-phase stray file sitting // in a phase directory must never contribute a blocker to a phase it does not // belong to, and the phase's own artifacts must keep behaving exactly as // before. describe('#3511: evaluateUatPassed — cross-phase stray files do not contribute blockers', () => { let baseDir; let phaseDir; beforeEach(() => { baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-3511-uat-pred-')); // Phase-shaped basename ("03-…") so isPhaseArtifact actually scopes — // extractPhaseToken('03-uat-predicate') derives token "03". phaseDir = path.join(baseDir, '03-uat-predicate'); fs.mkdirSync(phaseDir); }); afterEach(() => { rmDir(baseDir); }); test('passed:true with a passing own UAT + own VERIFICATION, despite a blocking cross-phase stray VERIFICATION', () => { writeFile(phaseDir, '03-UAT.md', makePassingUat(1)); writeFile(phaseDir, '03-VERIFICATION.md', '---\nstatus: passed\n---\n\nOK.'); // Cross-phase stray: a "04" VERIFICATION file sitting in phase 03's // directory, with a BLOCKING status. Pre-#3511, this unscoped scan would // have picked it up and blocked phase 03's transition. writeFile(phaseDir, '04-VERIFICATION.md', '---\nstatus: human_needed\n---\n\nNeeds human check.'); const report = evaluateUatPassed(phaseDir); assert.strictEqual(report.passed, true, 'a cross-phase stray VERIFICATION file must not block this phase\'s transition'); assert.ok( !report.blockers.some(b => /04-VERIFICATION\.md/.test(b) || /human_needed/i.test(b)), `blockers must not name the stray file; got: ${JSON.stringify(report.blockers)}`, ); assert.strictEqual(report.verification_files.includes('04-VERIFICATION.md'), false, 'the stray must not even be counted as a verification_files entry for this phase'); }); test('passed:false with own report still failing, unaffected by an unrelated passing cross-phase stray (non-stray case unchanged)', () => { writeFile(phaseDir, '03-UAT.md', makePassingUat(1)); // This phase's own VERIFICATION is blocking. writeFile(phaseDir, '03-VERIFICATION.md', '---\nstatus: gaps_found\n---\n\nHas gaps.'); // A cross-phase stray that is itself passing must not paper over the // phase's own real failure either — over-exclusion is as dangerous as // under-exclusion here. writeFile(phaseDir, '99-VERIFICATION.md', '---\nstatus: passed\n---\n\nOK.'); const report = evaluateUatPassed(phaseDir); assert.strictEqual(report.passed, false, 'the phase\'s own gaps_found VERIFICATION must still block, exactly as before #3511'); assert.ok(report.blockers.some(b => /gaps_found/i.test(b)), `blockers must still name this phase's own gaps_found status; got: ${JSON.stringify(report.blockers)}`); }); test('#3511 follow-up: passed:true from a NON-canonical dir shape "1-unpadded" (over-exclusion / no_uat_artifacts check)', () => { // "1-unpadded" tokenizes to literal "1"; scaffold writes the PADDED // "01-…" form (normalizePhaseName). A literal token compare excluded the // phase's own artifacts here, flipping `no_uat_artifacts: true` and // false-blocking the transition gate this predicate feeds. const unpaddedDir = path.join(baseDir, '1-unpadded'); fs.mkdirSync(unpaddedDir); writeFile(unpaddedDir, '01-UAT.md', makePassingUat(1)); writeFile(unpaddedDir, '01-VERIFICATION.md', '---\nstatus: passed\n---\n\nOK.'); const report = evaluateUatPassed(unpaddedDir); assert.strictEqual(report.no_uat_artifacts, false, `own UAT/VERIFICATION files in an unpadded-dir phase must be found; got: ${JSON.stringify(report)}`); assert.strictEqual(report.passed, true, `the phase's own passing files in a non-canonical dir must pass the gate; got: ${JSON.stringify(report)}`); }); }); // ─── FIX A regression: nested-fence (~~~ inside ```) ───────────────────────── describe('FIX A — nested fence: ~~~ inside ``` does not prematurely close outer fence', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('parseUatResultItems sees [] for fake inside ``` that encloses ~~~', () => { // A backtick fence that contains an inner ~~~ fence with a fake test block. // The ~~~ must NOT close the ``` fence — the whole interior is content and is dropped. const raw = [ '```', '~~~', '### 1. Fake', 'expected: X', 'result: passed', '~~~', '```', ].join('\n'); const clean = stripFalsePositiveContexts(raw); const items = parseUatResultItems(clean); assert.strictEqual(items.length, 0, 'fake inside nested fence must not leak through'); }); test('evaluateUatPassed → passed:false + no_uat_artifacts:true for nested-fence-only file', () => { const raw = [ '```', '~~~', '### 1. Fake', 'expected: X', 'result: passed', '~~~', '```', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', raw); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false, 'nested-fence fake must not flip passed'); assert.strictEqual(report.no_uat_artifacts, true, 'no real items → no_uat_artifacts:true'); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake must not appear in checks'); }); test('balanced nested fence (``` inside ~~~) is NOT flagged as malformed', () => { // ~~~ outer, ``` inner — properly closed — should not trigger unterminatedFence const raw = [ '~~~', '```', 'code', '```', '~~~', '', '### 1. Real Test', 'expected: Works', 'result: passed', ].join('\n'); const { unterminatedFence } = analyzeMarkdown(raw); assert.strictEqual(unterminatedFence, false, 'balanced nested fence must not be flagged'); const clean = stripFalsePositiveContexts(raw); const items = parseUatResultItems(clean); assert.strictEqual(items.length, 1, 'real test outside fence must still be found'); assert.strictEqual(items[0].result, 'passed'); }); test('real result:passed outside ``` that encloses ~~~ → passed:true (no false-block)', () => { const raw = [ '---', 'status: passed', '---', '', '```', '~~~', '### 1. Fake', 'result: passed', '~~~', '```', '', '### 1. Real Test', 'expected: Works', 'result: passed', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', raw); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, true, 'real result outside nested fence must still pass'); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake must not appear in checks'); }); }); // ─── FIX B regression: cross-line result value ──────────────────────────────── describe('FIX B — cross-line result: value must be on the same line', () => { test('result: with value on next line → result:missing (not passed)', () => { const content = [ '### 1. Cross-line Test', 'expected: Y', 'result:', '', 'passed', ].join('\n'); const items = parseUatResultItems(content); assert.strictEqual(items.length, 1); assert.notStrictEqual(items[0].result, 'passed', 'result value on a subsequent line must not be captured as passed'); assert.strictEqual(items[0].result, 'missing', 'cross-line result must yield missing (blocker)'); }); test('evaluateUatPassed → passed:false for cross-line result:passed', () => { const tmpDir = makeTmpDir(); try { const content = [ '### 1. Cross-line Test', 'expected: Y', 'result:', '', 'passed', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.ok(!report.checks.some(c => c.result === 'passed'), 'cross-line result must not produce a passing check'); } finally { rmDir(tmpDir); } }); }); // ─── FIX C regression: masked-comment (earlier closed comment + later unterminated) ── describe('FIX C — dangling comment survives earlier balanced comment', () => { test('analyzeMarkdown detects unterminated comment after a properly closed one', () => { const raw = [ '', 'Some text', '', '', '', 'Some text', '', '', '### 1. Real Test', 'result: passed', ].join('\n'); const { unterminatedComment } = analyzeMarkdown(raw); assert.strictEqual(unterminatedComment, false, 'only balanced comments must not be flagged as unterminated'); }); }); // ─── FIX D regression: balanced mixed fences are NOT flagged ───────────────── describe('FIX D — odd-fence-count heuristic replaced: balanced mixed fences not flagged', () => { test('analyzeMarkdown: ``` followed by ~~~ (both balanced) → unterminatedFence:false', () => { const raw = [ '```', 'code', '```', '', '~~~', 'more code', '~~~', ].join('\n'); const { unterminatedFence } = analyzeMarkdown(raw); assert.strictEqual(unterminatedFence, false, 'two separate balanced fences must not trigger unterminatedFence'); }); test('evaluateUatPassed: multiple balanced fences + real passing test → passed:true, no malformed blocker', () => { const tmpDir = makeTmpDir(); try { const raw = [ '---', 'status: passed', '---', '', '```', 'result: fake', '```', '', '~~~', 'result: also fake', '~~~', '', '### 1. Real Test', 'expected: Works', 'result: passed', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', raw); const report = evaluateUatPassed(tmpDir); assert.ok(!report.blockers.some(b => /malformed/i.test(b)), `Balanced mixed fences must not produce malformed blocker, got: ${JSON.stringify(report.blockers)}`); assert.strictEqual(report.passed, true, 'real test outside balanced fences must still pass'); } finally { rmDir(tmpDir); } }); }); // ─── FIX E extra: fake-not-in-checks assertions for existing mixed tests ────── describe('FIX E — fake items must NOT appear in checks (absence assertions)', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('fake inside fence + real pending: fake NOT in checks', () => { const content = [ '---', 'status: partial', '---', '', '```', '### 10. Fake', 'expected: Fake', 'result: passed', '```', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake test inside fence must not appear in checks'); }); test('fake inside blockquote + real pending: fake NOT in checks', () => { const content = [ '---', 'status: partial', '---', '', '> ### 10. Fake', '> expected: Fake', '> result: passed', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake test inside blockquote must not appear in checks'); }); test('fake inside HTML comment + real pending: fake NOT in checks', () => { const content = [ '---', 'status: partial', '---', '', '', '', '### 1. Real Test', 'expected: Something', 'result: pending', '', ].join('\n'); writeFile(tmpDir, 'phase-UAT.md', content); const report = evaluateUatPassed(tmpDir); assert.strictEqual(report.passed, false); assert.ok(!report.checks.some(c => c.name === 'Fake'), 'fake test inside HTML comment must not appear in checks'); }); }); // ─── Property-based test (fast-check) ───────────────────────────────────────── describe('evaluateUatPassed — property: wrapping in false-positive context never flips to passed', () => { let tmpDir; beforeEach(() => { tmpDir = makeTmpDir(); }); afterEach(() => { rmDir(tmpDir); }); test('fc: inserting result:passed inside wrapper context never flips a failing UAT to passed', () => { // A baseline UAT file that has a pending item — it must always evaluate to passed:false // regardless of how many "result: passed" lines we inject inside fenced/blockquote/comment wrappers. const baseFailingBody = [ '### 1. Real Test', 'expected: It works', 'result: pending', '', ].join('\n'); const wrappers = fc.constantFrom( // backtick fence (inner) => '```\n' + inner + '\n```', // tilde fence (inner) => '~~~\n' + inner + '\n~~~', // HTML comment (inner) => '', // blockquote — prefix each line (inner) => inner.split('\n').map(l => '> ' + l).join('\n'), ); fc.assert( fc.property(wrappers, fc.nat(3), (wrap, extraCount) => { // Build a "fake passing block" that would fool a naive regex const fakePassingLines = Array.from({ length: extraCount + 1 }, (_, i) => `### ${i + 10}. Fake Test ${i + 10}\nexpected: Fake\nresult: passed` ).join('\n'); const fullContent = [ '---', 'status: partial', '---', '', wrap(fakePassingLines), '', baseFailingBody, ].join('\n'); // Write to a unique tmp file to avoid cross-test state const fcDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-fc-uat-')); try { fs.writeFileSync(path.join(fcDir, 'feature-UAT.md'), fullContent, 'utf-8'); const report = evaluateUatPassed(fcDir); // The pending item must always keep passed:false // AND fake items injected via wrappers must never appear in checks const hasFakeInChecks = report.checks.some(c => c.name.startsWith('Fake Test')); return report.passed === false && !hasFakeInChecks; } finally { cleanup(fcDir); } }), { numRuns: 50 } ); }); });