'use strict'; /** * Tests for the UAT-evidence seam (src/uat-evidence.cts) and the stale-reason * split on readVerificationStatus. * * The bug: a phase whose human UAT passed could not reach `passed`. A * gap-closure plan (or a later phase editing a covered file) made the report * `stale`; verify-work only recorded a passed UAT against `human_needed`; the * re-run verifier could not see the UAT file and re-emitted `human_needed`; * and execute-phase's human_needed branch rewrote `*-UAT.md` with every row * `[pending]`. Each acceptance scenario below simulates the verifier re-run by * writing the report a verifier would write, then drives the real verbs. * * All assertions are on structured values (typed results, `--json` CLI output, * parsed UAT rows) — never on rendered prose. */ const { describe, test, beforeEach, afterEach } = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const { createTempProject, cleanup, runMsdTools } = require('./helpers.cjs'); const { readVerificationStatus, computeCoveredDigest, STALE_REASONS, } = require('../msd-core/bin/lib/verification.cjs'); const { CANONICALIZE_REASON, canonicalizeHumanNeeded, readUatEvidence, seedUatFromVerification, scanUatRows, } = require('../msd-core/bin/lib/uat-evidence.cjs'); const PHASE = '03-demo'; const ITEMS = [ { test: 'Login form renders', expected: 'Two fields and a submit button' }, { test: 'Logout clears the session', expected: 'Redirected to /login' }, ]; let root; let phaseDir; function rel(...segments) { return ['.planning', 'phases', PHASE, ...segments].join('/'); } function writePlan(n) { fs.writeFileSync(path.join(phaseDir, `03-${n}-PLAN.md`), `plan ${n}\n`); fs.writeFileSync(path.join(phaseDir, `03-${n}-SUMMARY.md`), `summary ${n}\n`); } function artifacts(...plans) { return plans.flatMap((n) => [rel(`03-${n}-PLAN.md`), rel(`03-${n}-SUMMARY.md`)]); } /** Write the report a verifier run would write over `covered`, digest computed for real. */ function writeReport({ status, covered, items = [] }) { const digest = computeCoveredDigest(root, covered, undefined, { phaseDir }); assert.ok(digest, 'fixture covered files must fingerprint'); const lines = [ '---', `phase: ${PHASE}`, `status: ${status}`, `covered_files: [${covered.join(', ')}]`, `covered_digest: "${digest}"`, ]; if (items.length > 0) { lines.push('human_verification:'); for (const item of items) { lines.push(` - test: "${item.test}"`, ` expected: "${item.expected}"`, ' why_human: "visual"'); if (item.retest_reason) lines.push(` retest_reason: "${item.retest_reason}"`); } } lines.push('---', '', '# Verification', ''); fs.writeFileSync(path.join(phaseDir, '03-VERIFICATION.md'), lines.join('\n')); } function writePassedUat() { fs.writeFileSync( path.join(phaseDir, '03-UAT.md'), [ '---', 'status: complete', `phase: ${PHASE}`, 'source: [03-VERIFICATION.md]', 'started: 2026-10-01T10:00:00.000Z', 'updated: 2026-10-01T11:00:00.000Z', '---', '', '## Current Test', '', '[testing complete]', '', '## Tests', '', '### 1. Login form renders', 'expected: Two fields and a submit button', 'result: pass', 'reported: "both fields there, button works"', '', '### 2. Logout clears the session', 'expected: Redirected to /login', 'result: pass', 'reported: "landed on /login"', '', '## Summary', '', 'total: 2', 'passed: 2', 'issues: 0', 'pending: 0', 'skipped: 0', 'blocked: 0', '', '## Gaps', '', '- truth: "Logout clears the session"', ' status: resolved', ' test: 2', '', ].join('\n'), ); } function uatBytes() { return fs.readFileSync(path.join(phaseDir, '03-UAT.md'), 'utf-8'); } function uatRows() { return scanUatRows(uatBytes().split('\n')); } /** Pin a file's mtime (the fixture is not a git repo, so mtime is the clock). */ function setMtime(file, isoTime) { const t = new Date(isoTime); fs.utimesSync(path.join(root, file), t, t); } beforeEach(() => { root = createTempProject('msd-uat-evidence-'); phaseDir = path.join(root, '.planning', 'phases', PHASE); fs.mkdirSync(phaseDir, { recursive: true }); fs.mkdirSync(path.join(root, 'src'), { recursive: true }); fs.writeFileSync(path.join(root, 'src', 'auth.js'), 'export const login = 1;\n'); fs.writeFileSync(path.join(root, 'src', 'billing.js'), 'export const bill = 1;\n'); writePlan('01'); writePlan('02'); }); afterEach(() => { cleanup(root); }); describe('readVerificationStatus — stale_reason', () => { test('a fresh report carries no stale_reason', () => { writeReport({ status: 'passed', covered: [...artifacts('01', '02'), 'src/auth.js'] }); const result = readVerificationStatus(phaseDir); assert.equal(result.status, 'passed'); assert.equal('stale_reason' in result, false); assert.equal('raw_status' in result, false); }); test('a PLAN/SUMMARY the report never declared reads uncovered_artifacts', () => { writeReport({ status: 'human_needed', covered: [...artifacts('01', '02'), 'src/auth.js'], items: ITEMS }); writePlan('03'); const result = readVerificationStatus(phaseDir); assert.equal(result.status, 'stale'); assert.equal(result.stale_reason, 'uncovered_artifacts'); assert.equal(result.raw_status, 'human_needed'); }); test('an edited covered file reads covered_changed', () => { writeReport({ status: 'passed', covered: [...artifacts('01', '02'), 'src/auth.js'] }); fs.writeFileSync(path.join(root, 'src', 'auth.js'), 'export const login = 2;\n'); const result = readVerificationStatus(phaseDir); assert.equal(result.status, 'stale'); assert.equal(result.stale_reason, 'covered_changed'); assert.equal(result.raw_status, 'passed'); }); test('a malformed fingerprint still fails closed, as fingerprint_malformed', () => { fs.writeFileSync( path.join(phaseDir, '03-VERIFICATION.md'), ['---', 'status: passed', 'covered_files: [src/auth.js]', '---', ''].join('\n'), ); const result = readVerificationStatus(phaseDir); assert.equal(result.status, 'stale'); assert.equal(result.stale_reason, 'fingerprint_malformed'); }); test('every emitted reason belongs to the frozen STALE_REASONS set', () => { assert.deepEqual([...STALE_REASONS].sort(), [ 'covered_changed', 'fingerprint_malformed', 'summary_newer', 'uncovered_artifacts', ]); }); }); describe('verification.seed-uat — merge, never overwrite', () => { test('creates the UAT file with every item pending when none exists', () => { writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS }); const seed = seedUatFromVerification(phaseDir); assert.equal(seed.created, true); assert.equal(seed.pending, 2); assert.deepEqual(seed.appended, [ { test: 1, name: 'Login form renders' }, { test: 2, name: 'Logout clears the session' }, ]); assert.deepEqual(uatRows().map((r) => r.result), ['pending', 'pending']); }); test('no-clobber: passed rows and their reported evidence are left byte-identical', () => { writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS }); writePassedUat(); const before = uatBytes(); const seed = seedUatFromVerification(phaseDir); assert.equal(seed.changed, false); assert.equal(seed.created, false); assert.equal(seed.pending, 0); assert.equal(seed.all_passing, true); assert.equal(uatBytes(), before); }); test('an unseen item is appended as pending; existing results are kept', () => { writePassedUat(); writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: [...ITEMS, { test: 'Password reset email arrives', expected: 'Email within a minute' }], }); const seed = seedUatFromVerification(phaseDir, { now: new Date('2026-10-10T00:00:00.000Z') }); assert.deepEqual(seed.appended, [{ test: 3, name: 'Password reset email arrives' }]); assert.deepEqual(seed.kept.map((r) => r.result), ['pass', 'pass']); assert.equal(seed.pending, 1); assert.deepEqual(uatRows().map((r) => [r.test, r.result]), [[1, 'pass'], [2, 'pass'], [3, 'pending']]); const fm = runMsdTools(['frontmatter', 'get', rel('03-UAT.md')], root); const parsed = JSON.parse(fm.output); assert.equal(parsed.status, 'testing'); assert.equal(parsed.updated, '2026-10-10T00:00:00.000Z'); }); test('item matching ignores case and punctuation', () => { writePassedUat(); writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: [{ test: 'login form renders.', expected: 'x' }], }); const seed = seedUatFromVerification(phaseDir); assert.equal(seed.changed, false); assert.deepEqual(seed.appended, []); }); test('a passing row is reset only with a recorded retest_reason, written into the row', () => { writePassedUat(); writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: [ { ...ITEMS[0], retest_reason: 'src/auth.js changed after the UAT pass' }, ITEMS[1], ], }); const seed = seedUatFromVerification(phaseDir); assert.deepEqual(seed.retested, [ { test: 1, name: 'Login form renders', previous_result: 'pass', reason: 'src/auth.js changed after the UAT pass' }, ]); assert.equal(seed.pending, 1); const rows = uatRows(); assert.equal(rows[0].result, 'pending'); assert.equal(rows[0].retest_reason, 'src/auth.js changed after the UAT pass'); assert.equal(rows[1].result, 'pass'); }); test('a row inside a fenced block is not a row', () => { const rows = scanUatRows( ['## Tests', '', '```', '### 9. Fake', 'result: pass', '```', '', '### 1. Real', 'result: [pending]', ''], ); assert.deepEqual(rows.map((r) => [r.test, r.result]), [[1, 'pending']]); }); test('returns null when the phase has no verification report', () => { assert.equal(seedUatFromVerification(phaseDir), null); }); }); describe('verification.canonicalize-uat — the single flip seam', () => { test('flips human_needed to passed when every UAT row passed', () => { writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS }); writePassedUat(); const result = canonicalizeHumanNeeded(phaseDir); assert.deepEqual(result, { canonicalized: true, status: 'passed', reason: CANONICALIZE_REASON.UAT_PASSED, blockers: [], }); assert.equal(readVerificationStatus(phaseDir).status, 'passed'); }); test('refuses while a UAT row is pending, and names the blocking rows (#4663)', () => { writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS }); seedUatFromVerification(phaseDir); const result = canonicalizeHumanNeeded(phaseDir); assert.equal(result.canonicalized, false); assert.equal(result.reason, CANONICALIZE_REASON.UAT_NOT_PASSED); assert.equal(result.blockers.length, 2); assert.equal(readVerificationStatus(phaseDir).status, 'human_needed'); }); test('refuses when there is no UAT evidence at all (zero issues is not a pass)', () => { writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS }); const result = canonicalizeHumanNeeded(phaseDir); assert.equal(result.canonicalized, false); assert.equal(result.reason, CANONICALIZE_REASON.UAT_NOT_PASSED); }); test('never flips a stale report, even with a fully passed UAT', () => { writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS }); writePassedUat(); writePlan('03'); const result = canonicalizeHumanNeeded(phaseDir); assert.equal(result.canonicalized, false); assert.equal(result.status, 'stale'); assert.equal(result.reason, CANONICALIZE_REASON.NOT_HUMAN_NEEDED); }); test('CLI: verification canonicalize-uat emits the typed result', () => { writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS }); writePassedUat(); const run = runMsdTools(['verification', 'canonicalize-uat', rel()], root); assert.equal(run.success, true); assert.equal(JSON.parse(run.output).reason, 'uat_passed'); }); }); describe('acceptance — a passed UAT is neither discarded nor re-requested', () => { test('gap-closure fixture: gap plan + passed UAT reaches passed without re-testing', () => { writeReport({ status: 'human_needed', covered: [...artifacts('01', '02'), 'src/auth.js'], items: ITEMS }); writePlan('03'); // the gap-closure plan and its summary writePassedUat(); // the human re-ran UAT at the gap plan's checkpoint assert.equal(readVerificationStatus(phaseDir).stale_reason, 'uncovered_artifacts'); // Worst-case verifier re-run: it covers plan 03 but re-lists both human // items without consulting the UAT file (no retest_reason). writeReport({ status: 'human_needed', covered: [...artifacts('01', '02', '03'), 'src/auth.js'], items: ITEMS }); const before = uatBytes(); const seed = seedUatFromVerification(phaseDir); assert.equal(seed.pending, 0); assert.equal(uatBytes(), before); assert.equal(canonicalizeHumanNeeded(phaseDir).canonicalized, true); const status = readVerificationStatus(phaseDir); assert.equal(status.status, 'passed'); assert.equal(status.next_command, ''); assert.deepEqual(uatRows().map((r) => r.result), ['pass', 'pass']); }); test('real change still stales, and the affected row is marked for re-test', () => { const covered = [...artifacts('01', '02'), 'src/auth.js', 'src/billing.js']; writeReport({ status: 'human_needed', covered, items: ITEMS }); writePassedUat(); assert.equal(canonicalizeHumanNeeded(phaseDir).canonicalized, true); setMtime(rel('03-UAT.md'), '2026-10-01T11:00:00.000Z'); setMtime('src/billing.js', '2026-09-30T00:00:00.000Z'); fs.writeFileSync(path.join(root, 'src', 'auth.js'), 'export const login = 2;\n'); setMtime('src/auth.js', '2026-10-05T00:00:00.000Z'); const status = readVerificationStatus(phaseDir); assert.equal(status.status, 'stale'); assert.equal(status.stale_reason, 'covered_changed'); const evidence = readUatEvidence(phaseDir); assert.deepEqual(evidence.changed_since_uat, [{ file: 'src/auth.js', reason: 'modified' }]); assert.equal(evidence.code_unchanged_since_uat, false); assert.deepEqual(evidence.checked_files, ['src/auth.js', 'src/billing.js']); assert.deepEqual(evidence.rows.map((r) => r.passing), [true, true]); // The verifier re-lists only the row the changed file bears on. writeReport({ status: 'human_needed', covered, items: [{ ...ITEMS[0], retest_reason: 'src/auth.js changed after the UAT pass' }], }); const seed = seedUatFromVerification(phaseDir); assert.deepEqual(seed.retested.map((r) => r.test), [1]); assert.deepEqual(uatRows().map((r) => r.result), ['pending', 'pass']); const refused = canonicalizeHumanNeeded(phaseDir); assert.equal(refused.canonicalized, false); assert.equal(refused.reason, CANONICALIZE_REASON.UAT_NOT_PASSED); assert.equal(readVerificationStatus(phaseDir).status, 'human_needed'); }); test('a deleted covered file counts as changed since the UAT', () => { writePassedUat(); const evidence = readUatEvidence(phaseDir, ['src/auth.js', 'src/gone.js', rel('03-01-SUMMARY.md')]); assert.deepEqual(evidence.checked_files, ['src/auth.js', 'src/gone.js']); assert.deepEqual( evidence.changed_since_uat.filter((c) => c.reason === 'missing'), [{ file: 'src/gone.js', reason: 'missing' }], ); }); test('no UAT file is no evidence', () => { const evidence = readUatEvidence(phaseDir, ['src/auth.js']); assert.equal(evidence.uat_file, ''); assert.equal(evidence.code_unchanged_since_uat, false); assert.deepEqual(evidence.rows, []); }); test('loop guard: after UAT passes, progress does not route the phase anywhere', () => { writeReport({ status: 'human_needed', covered: [...artifacts('01', '02'), 'src/auth.js'], items: ITEMS }); writePassedUat(); canonicalizeHumanNeeded(phaseDir); const run = runMsdTools(['init', 'progress'], root); assert.equal(run.success, true); const progress = JSON.parse(run.output); const phase = progress.phases.find((p) => p.number === '03'); assert.equal(phase.verification_status, 'passed'); assert.equal(phase.phase_complete, true); assert.equal(phase.verification_next_command, ''); assert.equal(phase.verification_stale_reason, ''); assert.deepEqual(progress.reverify_phases, []); }); test('cross-phase drift: finished phases surface as ONE re-verify action and keep their UAT', () => { // Phase 03 passed with UAT; a second finished phase shares src/auth.js. const covered03 = [...artifacts('01', '02'), 'src/auth.js', 'src/billing.js']; writeReport({ status: 'human_needed', covered: covered03, items: ITEMS }); writePassedUat(); canonicalizeHumanNeeded(phaseDir); setMtime(rel('03-UAT.md'), '2026-10-01T11:00:00.000Z'); setMtime('src/auth.js', '2026-09-30T00:00:00.000Z'); const otherDir = path.join(root, '.planning', 'phases', '04-other'); fs.mkdirSync(otherDir, { recursive: true }); fs.writeFileSync(path.join(otherDir, '04-01-PLAN.md'), 'plan\n'); fs.writeFileSync(path.join(otherDir, '04-01-SUMMARY.md'), 'summary\n'); const covered04 = ['.planning/phases/04-other/04-01-PLAN.md', '.planning/phases/04-other/04-01-SUMMARY.md', 'src/billing.js']; fs.writeFileSync( path.join(otherDir, '04-VERIFICATION.md'), [ '---', 'status: passed', `covered_files: [${covered04.join(', ')}]`, `covered_digest: "${computeCoveredDigest(root, covered04, undefined, { phaseDir: otherDir })}"`, '---', '', ].join('\n'), ); // A later phase edits the shared framework file. fs.writeFileSync(path.join(root, 'src', 'billing.js'), 'export const bill = 2;\n'); setMtime('src/billing.js', '2026-10-08T00:00:00.000Z'); const progress = JSON.parse(runMsdTools(['init', 'progress'], root).output); assert.deepEqual( progress.reverify_phases.map((p) => [p.number, p.stale_reason]), [['03', 'covered_changed'], ['04', 'covered_changed']], ); // Evidence names only the drifted file; both UAT rows still stand. const evidence = readUatEvidence(phaseDir); assert.deepEqual(evidence.changed_since_uat, [{ file: 'src/billing.js', reason: 'modified' }]); assert.deepEqual(evidence.rows.map((r) => r.passing), [true, true]); // The verifier finds neither row exercises billing: it re-verifies to // `passed` with no human items. The UAT file is untouched and nothing is // left to re-verify for phase 03. const before = uatBytes(); writeReport({ status: 'passed', covered: covered03 }); assert.equal(seedUatFromVerification(phaseDir).changed, false); assert.equal(uatBytes(), before); const after = JSON.parse(runMsdTools(['init', 'progress'], root).output); assert.deepEqual(after.reverify_phases.map((p) => p.number), ['04']); }); test('CLI: verification uat-evidence and seed-uat emit JSON; unknown flags are usage errors', () => { writeReport({ status: 'human_needed', covered: [...artifacts('01', '02'), 'src/auth.js'], items: ITEMS }); const seed = runMsdTools(['verification', 'seed-uat', rel()], root); assert.equal(seed.success, true); assert.equal(JSON.parse(seed.output).created, true); const evidence = runMsdTools(['verification', 'uat-evidence', rel(), '--files', 'src/auth.js,src/billing.js'], root); assert.equal(evidence.success, true); assert.deepEqual(JSON.parse(evidence.output).checked_files, ['src/auth.js', 'src/billing.js']); const bad = runMsdTools(['verification', 'uat-evidence', rel(), '--nope'], root); assert.equal(bad.success, false); }); });