msd-core/tests/uat-evidence.test.cjs

'use strict';

/**
 * Tests for the UAT-evidence seam (src/uat-evidence.cts) and the stale-reason
 * split on readVerificationStatus.
 *
 * The bug: a phase whose human UAT passed could not reach `passed`. A
 * gap-closure plan (or a later phase editing a covered file) made the report
 * `stale`; verify-work only recorded a passed UAT against `human_needed`; the
 * re-run verifier could not see the UAT file and re-emitted `human_needed`;
 * and execute-phase's human_needed branch rewrote `*-UAT.md` with every row
 * `[pending]`. Each acceptance scenario below simulates the verifier re-run by
 * writing the report a verifier would write, then drives the real verbs.
 *
 * All assertions are on structured values (typed results, `--json` CLI output,
 * parsed UAT rows) — never on rendered prose.
 */

const { describe, test, beforeEach, afterEach } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');

const { createTempProject, cleanup, runMsdTools } = require('./helpers.cjs');
const {
  readVerificationStatus,
  computeCoveredDigest,
  STALE_REASONS,
} = require('../msd-core/bin/lib/verification.cjs');
const {
  CANONICALIZE_REASON,
  canonicalizeHumanNeeded,
  readUatEvidence,
  seedUatFromVerification,
  scanUatRows,
} = require('../msd-core/bin/lib/uat-evidence.cjs');

const PHASE = '03-demo';
const ITEMS = [
  { test: 'Login form renders', expected: 'Two fields and a submit button' },
  { test: 'Logout clears the session', expected: 'Redirected to /login' },
];

let root;
let phaseDir;

function rel(...segments) {
  return ['.planning', 'phases', PHASE, ...segments].join('/');
}

function writePlan(n) {
  fs.writeFileSync(path.join(phaseDir, `03-${n}-PLAN.md`), `plan ${n}\n`);
  fs.writeFileSync(path.join(phaseDir, `03-${n}-SUMMARY.md`), `summary ${n}\n`);
}

function artifacts(...plans) {
  return plans.flatMap((n) => [rel(`03-${n}-PLAN.md`), rel(`03-${n}-SUMMARY.md`)]);
}

/** Write the report a verifier run would write over `covered`, digest computed for real. */
function writeReport({ status, covered, items = [] }) {
  const digest = computeCoveredDigest(root, covered, undefined, { phaseDir });
  assert.ok(digest, 'fixture covered files must fingerprint');
  const lines = [
    '---',
    `phase: ${PHASE}`,
    `status: ${status}`,
    `covered_files: [${covered.join(', ')}]`,
    `covered_digest: "${digest}"`,
  ];
  if (items.length > 0) {
    lines.push('human_verification:');
    for (const item of items) {
      lines.push(`  - test: "${item.test}"`, `    expected: "${item.expected}"`, '    why_human: "visual"');
      if (item.retest_reason) lines.push(`    retest_reason: "${item.retest_reason}"`);
    }
  }
  lines.push('---', '', '# Verification', '');
  fs.writeFileSync(path.join(phaseDir, '03-VERIFICATION.md'), lines.join('\n'));
}

function writePassedUat() {
  fs.writeFileSync(
    path.join(phaseDir, '03-UAT.md'),
    [
      '---',
      'status: complete',
      `phase: ${PHASE}`,
      'source: [03-VERIFICATION.md]',
      'started: 2026-10-01T10:00:00.000Z',
      'updated: 2026-10-01T11:00:00.000Z',
      '---',
      '',
      '## Current Test',
      '',
      '[testing complete]',
      '',
      '## Tests',
      '',
      '### 1. Login form renders',
      'expected: Two fields and a submit button',
      'result: pass',
      'reported: "both fields there, button works"',
      '',
      '### 2. Logout clears the session',
      'expected: Redirected to /login',
      'result: pass',
      'reported: "landed on /login"',
      '',
      '## Summary',
      '',
      'total: 2',
      'passed: 2',
      'issues: 0',
      'pending: 0',
      'skipped: 0',
      'blocked: 0',
      '',
      '## Gaps',
      '',
      '- truth: "Logout clears the session"',
      '  status: resolved',
      '  test: 2',
      '',
    ].join('\n'),
  );
}

function uatBytes() {
  return fs.readFileSync(path.join(phaseDir, '03-UAT.md'), 'utf-8');
}

function uatRows() {
  return scanUatRows(uatBytes().split('\n'));
}

/** Pin a file's mtime (the fixture is not a git repo, so mtime is the clock). */
function setMtime(file, isoTime) {
  const t = new Date(isoTime);
  fs.utimesSync(path.join(root, file), t, t);
}

beforeEach(() => {
  root = createTempProject('msd-uat-evidence-');
  phaseDir = path.join(root, '.planning', 'phases', PHASE);
  fs.mkdirSync(phaseDir, { recursive: true });
  fs.mkdirSync(path.join(root, 'src'), { recursive: true });
  fs.writeFileSync(path.join(root, 'src', 'auth.js'), 'export const login = 1;\n');
  fs.writeFileSync(path.join(root, 'src', 'billing.js'), 'export const bill = 1;\n');
  writePlan('01');
  writePlan('02');
});

afterEach(() => {
  cleanup(root);
});

describe('readVerificationStatus — stale_reason', () => {
  test('a fresh report carries no stale_reason', () => {
    writeReport({ status: 'passed', covered: [...artifacts('01', '02'), 'src/auth.js'] });
    const result = readVerificationStatus(phaseDir);
    assert.equal(result.status, 'passed');
    assert.equal('stale_reason' in result, false);
    assert.equal('raw_status' in result, false);
  });

  test('a PLAN/SUMMARY the report never declared reads uncovered_artifacts', () => {
    writeReport({ status: 'human_needed', covered: [...artifacts('01', '02'), 'src/auth.js'], items: ITEMS });
    writePlan('03');
    const result = readVerificationStatus(phaseDir);
    assert.equal(result.status, 'stale');
    assert.equal(result.stale_reason, 'uncovered_artifacts');
    assert.equal(result.raw_status, 'human_needed');
  });

  test('an edited covered file reads covered_changed', () => {
    writeReport({ status: 'passed', covered: [...artifacts('01', '02'), 'src/auth.js'] });
    fs.writeFileSync(path.join(root, 'src', 'auth.js'), 'export const login = 2;\n');
    const result = readVerificationStatus(phaseDir);
    assert.equal(result.status, 'stale');
    assert.equal(result.stale_reason, 'covered_changed');
    assert.equal(result.raw_status, 'passed');
  });

  test('a malformed fingerprint still fails closed, as fingerprint_malformed', () => {
    fs.writeFileSync(
      path.join(phaseDir, '03-VERIFICATION.md'),
      ['---', 'status: passed', 'covered_files: [src/auth.js]', '---', ''].join('\n'),
    );
    const result = readVerificationStatus(phaseDir);
    assert.equal(result.status, 'stale');
    assert.equal(result.stale_reason, 'fingerprint_malformed');
  });

  test('every emitted reason belongs to the frozen STALE_REASONS set', () => {
    assert.deepEqual([...STALE_REASONS].sort(), [
      'covered_changed',
      'fingerprint_malformed',
      'summary_newer',
      'uncovered_artifacts',
    ]);
  });
});

describe('verification.seed-uat — merge, never overwrite', () => {
  test('creates the UAT file with every item pending when none exists', () => {
    writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS });
    const seed = seedUatFromVerification(phaseDir);
    assert.equal(seed.created, true);
    assert.equal(seed.pending, 2);
    assert.deepEqual(seed.appended, [
      { test: 1, name: 'Login form renders' },
      { test: 2, name: 'Logout clears the session' },
    ]);
    assert.deepEqual(uatRows().map((r) => r.result), ['pending', 'pending']);
  });

  test('no-clobber: passed rows and their reported evidence are left byte-identical', () => {
    writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS });
    writePassedUat();
    const before = uatBytes();
    const seed = seedUatFromVerification(phaseDir);
    assert.equal(seed.changed, false);
    assert.equal(seed.created, false);
    assert.equal(seed.pending, 0);
    assert.equal(seed.all_passing, true);
    assert.equal(uatBytes(), before);
  });

  test('an unseen item is appended as pending; existing results are kept', () => {
    writePassedUat();
    writeReport({
      status: 'human_needed',
      covered: artifacts('01', '02'),
      items: [...ITEMS, { test: 'Password reset email arrives', expected: 'Email within a minute' }],
    });
    const seed = seedUatFromVerification(phaseDir, { now: new Date('2026-10-10T00:00:00.000Z') });
    assert.deepEqual(seed.appended, [{ test: 3, name: 'Password reset email arrives' }]);
    assert.deepEqual(seed.kept.map((r) => r.result), ['pass', 'pass']);
    assert.equal(seed.pending, 1);
    assert.deepEqual(uatRows().map((r) => [r.test, r.result]), [[1, 'pass'], [2, 'pass'], [3, 'pending']]);
    const fm = runMsdTools(['frontmatter', 'get', rel('03-UAT.md')], root);
    const parsed = JSON.parse(fm.output);
    assert.equal(parsed.status, 'testing');
    assert.equal(parsed.updated, '2026-10-10T00:00:00.000Z');
  });

  test('item matching ignores case and punctuation', () => {
    writePassedUat();
    writeReport({
      status: 'human_needed',
      covered: artifacts('01', '02'),
      items: [{ test: 'login form renders.', expected: 'x' }],
    });
    const seed = seedUatFromVerification(phaseDir);
    assert.equal(seed.changed, false);
    assert.deepEqual(seed.appended, []);
  });

  test('a passing row is reset only with a recorded retest_reason, written into the row', () => {
    writePassedUat();
    writeReport({
      status: 'human_needed',
      covered: artifacts('01', '02'),
      items: [
        { ...ITEMS[0], retest_reason: 'src/auth.js changed after the UAT pass' },
        ITEMS[1],
      ],
    });
    const seed = seedUatFromVerification(phaseDir);
    assert.deepEqual(seed.retested, [
      { test: 1, name: 'Login form renders', previous_result: 'pass', reason: 'src/auth.js changed after the UAT pass' },
    ]);
    assert.equal(seed.pending, 1);
    const rows = uatRows();
    assert.equal(rows[0].result, 'pending');
    assert.equal(rows[0].retest_reason, 'src/auth.js changed after the UAT pass');
    assert.equal(rows[1].result, 'pass');
  });

  test('a row inside a fenced block is not a row', () => {
    const rows = scanUatRows(
      ['## Tests', '', '```', '### 9. Fake', 'result: pass', '```', '', '### 1. Real', 'result: [pending]', ''],
    );
    assert.deepEqual(rows.map((r) => [r.test, r.result]), [[1, 'pending']]);
  });

  test('returns null when the phase has no verification report', () => {
    assert.equal(seedUatFromVerification(phaseDir), null);
  });
});

describe('verification.canonicalize-uat — the single flip seam', () => {
  test('flips human_needed to passed when every UAT row passed', () => {
    writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS });
    writePassedUat();
    const result = canonicalizeHumanNeeded(phaseDir);
    assert.deepEqual(result, {
      canonicalized: true,
      status: 'passed',
      reason: CANONICALIZE_REASON.UAT_PASSED,
      blockers: [],
    });
    assert.equal(readVerificationStatus(phaseDir).status, 'passed');
  });

  test('refuses while a UAT row is pending, and names the blocking rows (#4663)', () => {
    writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS });
    seedUatFromVerification(phaseDir);
    const result = canonicalizeHumanNeeded(phaseDir);
    assert.equal(result.canonicalized, false);
    assert.equal(result.reason, CANONICALIZE_REASON.UAT_NOT_PASSED);
    assert.equal(result.blockers.length, 2);
    assert.equal(readVerificationStatus(phaseDir).status, 'human_needed');
  });

  test('refuses when there is no UAT evidence at all (zero issues is not a pass)', () => {
    writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS });
    const result = canonicalizeHumanNeeded(phaseDir);
    assert.equal(result.canonicalized, false);
    assert.equal(result.reason, CANONICALIZE_REASON.UAT_NOT_PASSED);
  });

  test('never flips a stale report, even with a fully passed UAT', () => {
    writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS });
    writePassedUat();
    writePlan('03');
    const result = canonicalizeHumanNeeded(phaseDir);
    assert.equal(result.canonicalized, false);
    assert.equal(result.status, 'stale');
    assert.equal(result.reason, CANONICALIZE_REASON.NOT_HUMAN_NEEDED);
  });

  test('CLI: verification canonicalize-uat emits the typed result', () => {
    writeReport({ status: 'human_needed', covered: artifacts('01', '02'), items: ITEMS });
    writePassedUat();
    const run = runMsdTools(['verification', 'canonicalize-uat', rel()], root);
    assert.equal(run.success, true);
    assert.equal(JSON.parse(run.output).reason, 'uat_passed');
  });
});

describe('acceptance — a passed UAT is neither discarded nor re-requested', () => {
  test('gap-closure fixture: gap plan + passed UAT reaches passed without re-testing', () => {
    writeReport({ status: 'human_needed', covered: [...artifacts('01', '02'), 'src/auth.js'], items: ITEMS });
    writePlan('03'); // the gap-closure plan and its summary
    writePassedUat(); // the human re-ran UAT at the gap plan's checkpoint
    assert.equal(readVerificationStatus(phaseDir).stale_reason, 'uncovered_artifacts');

    // Worst-case verifier re-run: it covers plan 03 but re-lists both human
    // items without consulting the UAT file (no retest_reason).
    writeReport({ status: 'human_needed', covered: [...artifacts('01', '02', '03'), 'src/auth.js'], items: ITEMS });
    const before = uatBytes();
    const seed = seedUatFromVerification(phaseDir);
    assert.equal(seed.pending, 0);
    assert.equal(uatBytes(), before);

    assert.equal(canonicalizeHumanNeeded(phaseDir).canonicalized, true);
    const status = readVerificationStatus(phaseDir);
    assert.equal(status.status, 'passed');
    assert.equal(status.next_command, '');
    assert.deepEqual(uatRows().map((r) => r.result), ['pass', 'pass']);
  });

  test('real change still stales, and the affected row is marked for re-test', () => {
    const covered = [...artifacts('01', '02'), 'src/auth.js', 'src/billing.js'];
    writeReport({ status: 'human_needed', covered, items: ITEMS });
    writePassedUat();
    assert.equal(canonicalizeHumanNeeded(phaseDir).canonicalized, true);
    setMtime(rel('03-UAT.md'), '2026-10-01T11:00:00.000Z');
    setMtime('src/billing.js', '2026-09-30T00:00:00.000Z');

    fs.writeFileSync(path.join(root, 'src', 'auth.js'), 'export const login = 2;\n');
    setMtime('src/auth.js', '2026-10-05T00:00:00.000Z');

    const status = readVerificationStatus(phaseDir);
    assert.equal(status.status, 'stale');
    assert.equal(status.stale_reason, 'covered_changed');

    const evidence = readUatEvidence(phaseDir);
    assert.deepEqual(evidence.changed_since_uat, [{ file: 'src/auth.js', reason: 'modified' }]);
    assert.equal(evidence.code_unchanged_since_uat, false);
    assert.deepEqual(evidence.checked_files, ['src/auth.js', 'src/billing.js']);
    assert.deepEqual(evidence.rows.map((r) => r.passing), [true, true]);

    // The verifier re-lists only the row the changed file bears on.
    writeReport({
      status: 'human_needed',
      covered,
      items: [{ ...ITEMS[0], retest_reason: 'src/auth.js changed after the UAT pass' }],
    });
    const seed = seedUatFromVerification(phaseDir);
    assert.deepEqual(seed.retested.map((r) => r.test), [1]);
    assert.deepEqual(uatRows().map((r) => r.result), ['pending', 'pass']);

    const refused = canonicalizeHumanNeeded(phaseDir);
    assert.equal(refused.canonicalized, false);
    assert.equal(refused.reason, CANONICALIZE_REASON.UAT_NOT_PASSED);
    assert.equal(readVerificationStatus(phaseDir).status, 'human_needed');
  });

  test('a deleted covered file counts as changed since the UAT', () => {
    writePassedUat();
    const evidence = readUatEvidence(phaseDir, ['src/auth.js', 'src/gone.js', rel('03-01-SUMMARY.md')]);
    assert.deepEqual(evidence.checked_files, ['src/auth.js', 'src/gone.js']);
    assert.deepEqual(
      evidence.changed_since_uat.filter((c) => c.reason === 'missing'),
      [{ file: 'src/gone.js', reason: 'missing' }],
    );
  });

  test('no UAT file is no evidence', () => {
    const evidence = readUatEvidence(phaseDir, ['src/auth.js']);
    assert.equal(evidence.uat_file, '');
    assert.equal(evidence.code_unchanged_since_uat, false);
    assert.deepEqual(evidence.rows, []);
  });

  test('loop guard: after UAT passes, progress does not route the phase anywhere', () => {
    writeReport({ status: 'human_needed', covered: [...artifacts('01', '02'), 'src/auth.js'], items: ITEMS });
    writePassedUat();
    canonicalizeHumanNeeded(phaseDir);

    const run = runMsdTools(['init', 'progress'], root);
    assert.equal(run.success, true);
    const progress = JSON.parse(run.output);
    const phase = progress.phases.find((p) => p.number === '03');
    assert.equal(phase.verification_status, 'passed');
    assert.equal(phase.phase_complete, true);
    assert.equal(phase.verification_next_command, '');
    assert.equal(phase.verification_stale_reason, '');
    assert.deepEqual(progress.reverify_phases, []);
  });

  test('cross-phase drift: finished phases surface as ONE re-verify action and keep their UAT', () => {
    // Phase 03 passed with UAT; a second finished phase shares src/auth.js.
    const covered03 = [...artifacts('01', '02'), 'src/auth.js', 'src/billing.js'];
    writeReport({ status: 'human_needed', covered: covered03, items: ITEMS });
    writePassedUat();
    canonicalizeHumanNeeded(phaseDir);
    setMtime(rel('03-UAT.md'), '2026-10-01T11:00:00.000Z');
    setMtime('src/auth.js', '2026-09-30T00:00:00.000Z');

    const otherDir = path.join(root, '.planning', 'phases', '04-other');
    fs.mkdirSync(otherDir, { recursive: true });
    fs.writeFileSync(path.join(otherDir, '04-01-PLAN.md'), 'plan\n');
    fs.writeFileSync(path.join(otherDir, '04-01-SUMMARY.md'), 'summary\n');
    const covered04 = ['.planning/phases/04-other/04-01-PLAN.md', '.planning/phases/04-other/04-01-SUMMARY.md', 'src/billing.js'];
    fs.writeFileSync(
      path.join(otherDir, '04-VERIFICATION.md'),
      [
        '---',
        'status: passed',
        `covered_files: [${covered04.join(', ')}]`,
        `covered_digest: "${computeCoveredDigest(root, covered04, undefined, { phaseDir: otherDir })}"`,
        '---',
        '',
      ].join('\n'),
    );

    // A later phase edits the shared framework file.
    fs.writeFileSync(path.join(root, 'src', 'billing.js'), 'export const bill = 2;\n');
    setMtime('src/billing.js', '2026-10-08T00:00:00.000Z');

    const progress = JSON.parse(runMsdTools(['init', 'progress'], root).output);
    assert.deepEqual(
      progress.reverify_phases.map((p) => [p.number, p.stale_reason]),
      [['03', 'covered_changed'], ['04', 'covered_changed']],
    );

    // Evidence names only the drifted file; both UAT rows still stand.
    const evidence = readUatEvidence(phaseDir);
    assert.deepEqual(evidence.changed_since_uat, [{ file: 'src/billing.js', reason: 'modified' }]);
    assert.deepEqual(evidence.rows.map((r) => r.passing), [true, true]);

    // The verifier finds neither row exercises billing: it re-verifies to
    // `passed` with no human items. The UAT file is untouched and nothing is
    // left to re-verify for phase 03.
    const before = uatBytes();
    writeReport({ status: 'passed', covered: covered03 });
    assert.equal(seedUatFromVerification(phaseDir).changed, false);
    assert.equal(uatBytes(), before);
    const after = JSON.parse(runMsdTools(['init', 'progress'], root).output);
    assert.deepEqual(after.reverify_phases.map((p) => p.number), ['04']);
  });

  test('CLI: verification uat-evidence and seed-uat emit JSON; unknown flags are usage errors', () => {
    writeReport({ status: 'human_needed', covered: [...artifacts('01', '02'), 'src/auth.js'], items: ITEMS });
    const seed = runMsdTools(['verification', 'seed-uat', rel()], root);
    assert.equal(seed.success, true);
    assert.equal(JSON.parse(seed.output).created, true);

    const evidence = runMsdTools(['verification', 'uat-evidence', rel(), '--files', 'src/auth.js,src/billing.js'], root);
    assert.equal(evidence.success, true);
    assert.deepEqual(JSON.parse(evidence.output).checked_files, ['src/auth.js', 'src/billing.js']);

    const bad = runMsdTools(['verification', 'uat-evidence', rel(), '--nope'], root);
    assert.equal(bad.success, false);
  });
});