parseUatItems only scanned '### N.' expected/result blocks and parseVerificationItems only recognized table/bullet/numbered shapes, so audit-uat returned a false-clean total_items:0 when a file recorded open findings in a '## Gaps' section, declared items in a frontmatter human_verification: array, or used the '### N. <label>'+bold-paragraph verification shape. parseUatItems now also scans '## Gaps' (via collectSection + iterateBullets) and surfaces any entry whose status != resolved. parseVerificationItems now treats the frontmatter human_verification: array (via extractFrontmatter) as the primary source when present, and adds a tokenizeHeadings fallback for the '### N.'+bold-paragraph shape, preserving the existing table/bullet/numbered recognition (no double-count, no regression). Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -423,6 +423,399 @@ All checks passed.
|
||||
`status: passed file should produce 0 outstanding items, got ${output.summary.total_items}`);
|
||||
assert.strictEqual(output.summary.total_files, 0);
|
||||
});
|
||||
|
||||
// Regression: #2286 — parseUatItems never scanned a `## Gaps` section, so a
|
||||
// *-UAT.md file recording its only outstanding findings there returned
|
||||
// total_items: 0 (false-clean). Boundary: 0 / 1 / 2+ unresolved entries.
|
||||
describe('Gaps section scanning (#2286)', () => {
|
||||
test('a Gaps-only UAT file with 0 unresolved entries (all resolved) yields no items', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "SC1: Widget renders with data"',
|
||||
' status: resolved',
|
||||
' reason: "Fixed in follow-up commit"',
|
||||
'',
|
||||
'- truth: "SC2: Second finding also fixed"',
|
||||
' status: resolved',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 0,
|
||||
'resolved Gaps entries must not be counted as outstanding items');
|
||||
assert.strictEqual(output.summary.total_files, 0);
|
||||
});
|
||||
|
||||
test('a Gaps-only UAT file with exactly 1 unresolved entry and zero ### N. test blocks yields 1 item', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "SC1: Widget renders with data"',
|
||||
' status: open',
|
||||
' reason: "Missing data binding"',
|
||||
' severity: major',
|
||||
' test: 2',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1, 'total_items must be > 0, not the false-clean 0');
|
||||
assert.strictEqual(output.results[0].type, 'uat');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'SC1: Widget renders with data');
|
||||
assert.strictEqual(output.results[0].items[0].result, 'open');
|
||||
assert.strictEqual(output.results[0].items[0].reason, 'Missing data binding');
|
||||
assert.strictEqual(output.results[0].items[0].test, 2);
|
||||
});
|
||||
|
||||
test('a Gaps section with 2+ unresolved entries surfaces all of them and skips the resolved one', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '02-api');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '02-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 02-api',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "SC1: First outstanding gap"',
|
||||
' status: failed',
|
||||
' reason: "Endpoint returns 500"',
|
||||
'',
|
||||
'- truth: "SC2: Second outstanding gap"',
|
||||
' status: open',
|
||||
'',
|
||||
'- truth: "SC3: Already fixed gap"',
|
||||
' status: resolved',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 2,
|
||||
'exactly the 2 unresolved gaps should be counted, resolved gap excluded');
|
||||
const names = output.results[0].items.map((item) => item.name).sort();
|
||||
assert.deepStrictEqual(names, ['SC1: First outstanding gap', 'SC2: Second outstanding gap']);
|
||||
});
|
||||
|
||||
// Regression: #2286 review HIGH finding — a naive whole-string `key:`
|
||||
// scan over a Gaps entry's flattened text matches the FIRST `key:`-shaped
|
||||
// substring anywhere, including one embedded inside an EARLIER field's
|
||||
// own quoted free-text value. A `truth`/`reason` value that itself
|
||||
// contains the literal text "status: resolved" (or "reason:"/"test:")
|
||||
// must never hijack the real, later `status:`/`reason:`/`test:` field —
|
||||
// the fix parses each field anchored to the START of its own line.
|
||||
test('a truth value containing the literal substring "status: resolved" does not suppress the real open status', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "The status: resolved workflow should trigger a banner"',
|
||||
' status: failed',
|
||||
' reason: "Contains a reason: field embedded phrase, and test: 9 too"',
|
||||
' test: 3',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1,
|
||||
'the genuinely open gap must be surfaced, not dropped because its truth text contains "status: resolved"');
|
||||
const item = output.results[0].items[0];
|
||||
assert.strictEqual(item.name, 'The status: resolved workflow should trigger a banner');
|
||||
assert.strictEqual(item.result, 'failed', 'the REAL status: field must win, not the embedded phrase inside truth');
|
||||
assert.strictEqual(item.reason, 'Contains a reason: field embedded phrase, and test: 9 too',
|
||||
'the reason value is taken verbatim, including its own embedded colon-bearing phrases');
|
||||
assert.strictEqual(item.test, 3, 'the REAL test: field (3) must win, not the "test: 9" phrase embedded in reason');
|
||||
});
|
||||
|
||||
// Regression: #2286 review LOW finding — a nested `artifacts:` sub-list
|
||||
// (per templates/UAT.md's `## Gaps` schema) must be folded into its
|
||||
// parent entry, not mis-split into spurious standalone items.
|
||||
test('a Gaps entry with a nested artifacts sub-list parses as exactly one item', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "SC1: Some behavior"',
|
||||
' status: failed',
|
||||
' reason: "reason text"',
|
||||
' severity: major',
|
||||
' test: 1',
|
||||
' root_cause: ""',
|
||||
' artifacts:',
|
||||
' - src/foo.ts',
|
||||
' - src/bar.ts',
|
||||
' missing: []',
|
||||
' debug_session: ""',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1,
|
||||
'the nested artifacts sub-list items must not spawn spurious extra Gaps items');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'SC1: Some behavior');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'unknown',
|
||||
'a Gaps item with no dedicated category mapping falls back to unknown');
|
||||
});
|
||||
|
||||
// Regression: #2286 review item 5 (fail-safe direction) — #2286 is a
|
||||
// false-NEGATIVE bug, so a Gaps entry with no parseable `status:` field
|
||||
// is surfaced (as result: 'unknown') rather than silently dropped.
|
||||
test('a Gaps entry with no status field is surfaced as an unknown-status item (fail-safe)', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'- truth: "SC1: Missing status field entirely"',
|
||||
' reason: "why it is open"',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1,
|
||||
'a garbled/missing status must SURFACE the entry, not silently drop it');
|
||||
assert.strictEqual(output.results[0].items[0].result, 'unknown');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'SC1: Missing status field entirely');
|
||||
});
|
||||
|
||||
test('an empty Gaps section (heading present, no bullets) yields 0 items without throwing', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 0);
|
||||
assert.strictEqual(output.summary.total_files, 0);
|
||||
});
|
||||
});
|
||||
|
||||
// Regression: #2286 — parseVerificationItems never read the frontmatter's
|
||||
// structured `human_verification:` YAML array, and never recognized the
|
||||
// `### N. <label>` + bold-paragraph body shape shipped by
|
||||
// templates/verification-report.md. Boundary: array length 0 / 1 / 2+.
|
||||
describe('human_verification frontmatter array + heading shape (#2286)', () => {
|
||||
test('an empty human_verification array (length 0) falls back to the body scan', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 04-auth',
|
||||
'human_verification: []',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification',
|
||||
'',
|
||||
'1. Test SSO login with Google account',
|
||||
'2. Test password reset flow end-to-end',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 2,
|
||||
'an empty structured array must fall back to the existing body scan, not report 0');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'Test SSO login with Google account');
|
||||
});
|
||||
|
||||
test('a populated human_verification array of length 1 is sourced from frontmatter as primary', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 04-auth',
|
||||
'human_verification:',
|
||||
' - test: "Confirm the widget renders correctly"',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification',
|
||||
'',
|
||||
'None — see frontmatter human_verification array.',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1,
|
||||
'total_items must reflect the frontmatter array, not the unstructured body prose');
|
||||
// #2286 review LOW finding: extractFrontmatter's generic array-item
|
||||
// parser has no notion of nested key/value objects — a `- test: "..."`
|
||||
// entry is ALWAYS flattened to the raw post-"- " text, verbatim (only
|
||||
// its own wrapping quote is stripped, and only at the string's outer
|
||||
// edges). normalizeHumanVerificationEntry deliberately does NOT strip
|
||||
// a leading "key:"-shaped prefix (see its doc comment) because doing
|
||||
// so is indistinguishable from truncating a legitimate plain string
|
||||
// that starts with a word and a colon — so this documented, slightly
|
||||
// ugly artifact is the CORRECT (non-data-lossy) output for this shape.
|
||||
assert.strictEqual(output.results[0].items[0].name, 'test: "Confirm the widget renders correctly');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'human_uat');
|
||||
});
|
||||
|
||||
// Regression: #2286 review LOW finding — a plain-string human_verification
|
||||
// entry that itself starts with "Word: " must be preserved verbatim, not
|
||||
// truncated by a (removed) leading-key-prefix strip.
|
||||
test('a plain-string human_verification entry beginning with "Word: " is preserved verbatim', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 04-auth',
|
||||
'human_verification:',
|
||||
' - "Confirm: the button responds"',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification',
|
||||
'',
|
||||
'None.',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1);
|
||||
assert.strictEqual(output.results[0].items[0].name, 'Confirm: the button responds',
|
||||
'a plain string beginning with a word and a colon must not be truncated');
|
||||
});
|
||||
|
||||
test('a populated human_verification array of length 2+ takes priority over a differently-shaped body', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 04-auth',
|
||||
'human_verification:',
|
||||
' - "Confirm SSO login works end to end"',
|
||||
' - "Confirm MFA enrollment banner appears"',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification',
|
||||
'',
|
||||
'1. A body-scan item that must NOT be double-counted',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 2,
|
||||
'the structured array is the PRIMARY source and must not union with the body scan');
|
||||
const names = output.results[0].items.map((item) => item.name).sort();
|
||||
assert.deepStrictEqual(names, ['Confirm MFA enrollment banner appears', 'Confirm SSO login works end to end']);
|
||||
});
|
||||
|
||||
test('recognizes the ### N. <label> + bold-paragraph Human Verification body shape', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '05-widgets');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '05-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 05-widgets',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification Required',
|
||||
'',
|
||||
'### 1. Widget render check',
|
||||
'**Test:** Confirm the widget appears as expected on the dashboard.',
|
||||
'**Expected:** Widget renders with live data within 2 seconds.',
|
||||
'**Why human:** Visual rendering cannot be verified by static analysis.',
|
||||
'',
|
||||
'### 2. Notification banner check',
|
||||
'**Test:** Trigger a new notification and confirm the banner appears.',
|
||||
'**Expected:** Banner appears within 1 second and auto-dismisses after 5 seconds.',
|
||||
'**Why human:** Timing-based UI behavior requires visual confirmation.',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 2,
|
||||
'the ### N. + bold-paragraph shape must be recognized instead of returning 0 items');
|
||||
assert.strictEqual(output.results[0].items[0].test, 1);
|
||||
assert.strictEqual(output.results[0].items[0].name, 'Widget render check');
|
||||
assert.strictEqual(output.results[0].items[1].test, 2);
|
||||
assert.strictEqual(output.results[0].items[1].name, 'Notification banner check');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'human_uat');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('uat render-checkpoint', () => {
|
||||
|
||||
Reference in New Issue
Block a user