parseUatItems only scanned '### N.' expected/result blocks and parseVerificationItems only recognized table/bullet/numbered shapes, so audit-uat returned a false-clean total_items:0 when a file recorded open findings in a '## Gaps' section, declared items in a frontmatter human_verification: array, or used the '### N. <label>'+bold-paragraph verification shape. parseUatItems now also scans '## Gaps' (via collectSection + iterateBullets) and surfaces any entry whose status != resolved. parseVerificationItems now treats the frontmatter human_verification: array (via extractFrontmatter) as the primary source when present, and adds a tokenizeHeadings fallback for the '### N.'+bold-paragraph shape, preserving the existing table/bullet/numbered recognition (no double-count, no regression). Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
5
.changeset/quick-elks-climb.md
Normal file
5
.changeset/quick-elks-climb.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
type: Fixed
|
||||
pr: 2317
|
||||
---
|
||||
**`audit-uat` no longer reports a false-clean `total_items: 0` when real items exist** — the parsers ignored two artifact shapes: a `## Gaps` section recording open findings, and verification items declared in frontmatter (`human_verification:` array) or as `### N.`+bold-paragraph blocks. audit-uat now surfaces unresolved `## Gaps` entries and reads the frontmatter array / heading shape, so a phase with outstanding UAT/verification work is no longer waved through as clean. (#2286)
|
||||
252
src/uat.cts
252
src/uat.cts
@@ -365,14 +365,203 @@ function parseUatItems(content: string): UatItem[] {
|
||||
items.push(item);
|
||||
}
|
||||
}
|
||||
items.push(...parseGapsItems(content));
|
||||
return items;
|
||||
}
|
||||
|
||||
// ─── parseGapsItems ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Extract unresolved entries from a UAT file's `## Gaps` section (#2286).
|
||||
*
|
||||
* `## Gaps` records open findings as a YAML-lite bullet list (see
|
||||
* `templates/UAT.md`'s `## Gaps` block: `- truth: "..."` followed by indented
|
||||
* continuation lines `status:` / `reason:` / `severity:` / `test:` / etc.,
|
||||
* and — for `artifacts:` / `missing:` — a further-nested `- ` sub-list).
|
||||
* `parseUatItems`'s `### N.` test-block regex never looks at this section at
|
||||
* all, so a UAT file whose only outstanding findings live in `## Gaps` was
|
||||
* silently invisible — the false-negative this fix addresses.
|
||||
*
|
||||
* Reuses the existing `collectSection` seam (already used elsewhere in this
|
||||
* file for `## Current Test` / `## Tests`) to locate the section. Field
|
||||
* extraction is deliberately NOT done via `iterateBullets`: that seam folds
|
||||
* every continuation line onto ONE space-joined `text` string per bullet,
|
||||
* which erases line boundaries — a `key:` scan against that flattened text
|
||||
* matches the FIRST `key:`-shaped substring anywhere, including one that
|
||||
* happens to appear inside an EARLIER field's own quoted free-text value
|
||||
* (e.g. `truth: "The status: resolved workflow should trigger"` — a real
|
||||
* `status: failed` on the next line would never be reached, silently
|
||||
* DROPPING a genuinely open gap — the exact false-negative class #2286
|
||||
* exists to fix, so the fix must not reintroduce it). `splitGapsEntries` /
|
||||
* `extractGapEntryFields` below instead walk the section PER LINE and only
|
||||
* recognise a field at the START of its own (trimmed) line, so a `key:`
|
||||
* embedded inside another field's quoted value can never be mistaken for a
|
||||
* field declaration.
|
||||
*
|
||||
* Every entry whose `status` is present and NOT `resolved` (case-insensitive)
|
||||
* is surfaced — mirroring the "ignore passing/resolved" convention already
|
||||
* used for `### N.` test blocks (`result: pass` is never surfaced) and the
|
||||
* VERIFICATION table-row PASS/resolved skip (`hasPassResult`, below). An
|
||||
* entry with NO parseable `status:` field is surfaced too, as `result:
|
||||
* 'unknown'` — #2286 is a false-NEGATIVE bug, and a `## Gaps` entry only
|
||||
* exists to record an outstanding finding (a template-conformant RESOLVED
|
||||
* entry always carries an explicit `status: resolved`); a garbled or
|
||||
* non-conformant entry is far more likely to be an unresolved finding whose
|
||||
* `status:` line failed to parse than a genuinely resolved one, so the
|
||||
* fail-safe direction is to surface it rather than silently drop it.
|
||||
*/
|
||||
function parseGapsItems(content: string): UatItem[] {
|
||||
const gapsSection = collectSection(
|
||||
content,
|
||||
(h) => /^gaps$/i.test(h.text) && h.level === 2,
|
||||
{ levelBounded: true },
|
||||
);
|
||||
if (!gapsSection) return [];
|
||||
|
||||
const items: UatItem[] = [];
|
||||
|
||||
for (const entryLines of splitGapsEntries(gapsSection.body)) {
|
||||
const fields = extractGapEntryFields(entryLines);
|
||||
const rawStatus = fields.status;
|
||||
if (rawStatus && rawStatus.toLowerCase() === 'resolved') continue;
|
||||
// Fail-safe: missing/garbled status surfaces as 'unknown' rather than
|
||||
// being dropped (see doc comment above).
|
||||
const status = rawStatus || 'unknown';
|
||||
|
||||
const truth = fields.truth;
|
||||
const reason = fields.reason;
|
||||
const testNum = fields.test;
|
||||
|
||||
const item: UatItem = {
|
||||
name: truth || rawGapEntryText(entryLines),
|
||||
result: status,
|
||||
category: categorizeItem(status, reason, undefined),
|
||||
};
|
||||
if (testNum && /^\d+$/.test(testNum)) item.test = parseInt(testNum, 10);
|
||||
if (reason) item.reason = reason;
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a `## Gaps` section body into per-entry line groups on TOP-LEVEL
|
||||
* `- ` bullet openers.
|
||||
*
|
||||
* The indentation of the FIRST bullet line encountered establishes the
|
||||
* "top-level" indent for the whole section; any subsequent `- `-opening line
|
||||
* at that same indent (or shallower) starts a NEW entry, while everything
|
||||
* more deeply indented — field continuation lines (` status: ...`) AND
|
||||
* nested sub-lists (` - src/foo.ts` under ` artifacts:`) — is folded into
|
||||
* the CURRENT entry. This keeps a `artifacts:`/`missing:` sub-list's `- `
|
||||
* items from being mis-split into spurious standalone entries (#2286 review
|
||||
* LOW finding).
|
||||
*
|
||||
* Lines before the first bullet (e.g. the `<!-- YAML format ... -->` comment
|
||||
* the template emits) are discarded. An empty/whitespace-only section body
|
||||
* (heading present, no bullets) returns `[]`.
|
||||
*/
|
||||
function splitGapsEntries(sectionBody: string): string[][] {
|
||||
const lines = sectionBody.split('\n');
|
||||
const entries: string[][] = [];
|
||||
let current: string[] | null = null;
|
||||
let baseIndent: number | null = null;
|
||||
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.replace(/\r$/, '');
|
||||
const bulletMatch = line.match(/^(\s*)-\s/);
|
||||
if (bulletMatch) {
|
||||
const indent = bulletMatch[1].length;
|
||||
if (baseIndent === null) baseIndent = indent;
|
||||
if (indent <= baseIndent) {
|
||||
if (current) entries.push(current);
|
||||
current = [line];
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (current) current.push(line);
|
||||
// else: pre-first-bullet content (e.g. the template's HTML comment) — discarded.
|
||||
}
|
||||
if (current) entries.push(current);
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract `key: value` fields from one Gaps entry's lines, anchored to the
|
||||
* START of each (bullet-marker-stripped, trimmed) line — never scanning the
|
||||
* REST of a line, so a colon-bearing phrase inside a quoted `truth`/`reason`
|
||||
* value is never misread as a field declaration (see `parseGapsItems`'s doc
|
||||
* comment for the false-negative this specifically guards against).
|
||||
*
|
||||
* Recognises a double-quoted value (`truth: "..."`, stripped of its wrapping
|
||||
* quotes — the value may itself contain any character, including `:`) or a
|
||||
* bare value (`status: open`, `test: 2`, `artifacts: []`) taken verbatim.
|
||||
* The FIRST occurrence of a given key wins (top-level fields always precede
|
||||
* any nested sub-list content in the template's field ordering); later
|
||||
* `key:`-shaped nested-list content is captured, if it parses as one, but
|
||||
* never overrides an already-seen top-level field.
|
||||
*/
|
||||
function extractGapEntryFields(entryLines: string[]): Record<string, string> {
|
||||
const fields: Record<string, string> = {};
|
||||
const fieldLineRe = /^([A-Za-z_][A-Za-z0-9_-]*):\s*(.*)$/;
|
||||
|
||||
entryLines.forEach((rawLine, idx) => {
|
||||
const line = rawLine.replace(/\r$/, '');
|
||||
// Strip ONLY the entry-opening bullet marker (idx 0); a bullet marker on
|
||||
// a later line belongs to a nested sub-list and is handled by
|
||||
// `splitGapsEntries` already folding it in — it is not itself a field
|
||||
// line unless it independently matches `key: value` after stripping.
|
||||
const bulletStripped = line.match(/^(\s*)-\s+(.*)$/);
|
||||
const content = idx === 0 && bulletStripped ? bulletStripped[2] : line.trim();
|
||||
|
||||
const m = fieldLineRe.exec(content);
|
||||
if (!m) return;
|
||||
const key = m[1];
|
||||
let value = m[2].trim();
|
||||
if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
if (!(key in fields)) fields[key] = value;
|
||||
});
|
||||
|
||||
return fields;
|
||||
}
|
||||
|
||||
/** Fallback display text for a Gaps entry with no parseable `truth:` field. */
|
||||
function rawGapEntryText(entryLines: string[]): string {
|
||||
return entryLines
|
||||
.map((l, i) => (i === 0 ? l.replace(/^(\s*)-\s+/, '') : l.trim()))
|
||||
.join(' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ─── parseVerificationItems ───────────────────────────────────────────────────
|
||||
|
||||
function parseVerificationItems(content: string, status: string): UatItem[] {
|
||||
const items: UatItem[] = [];
|
||||
if (status === 'human_needed') {
|
||||
// #2286: the frontmatter's structured `human_verification:` YAML array
|
||||
// (extractFrontmatter) is the PRIMARY source of truth when present and
|
||||
// non-empty — it fully bypasses the body-shape scan below, so a file
|
||||
// whose frontmatter declares the array doesn't require any particular
|
||||
// `## Human Verification` body shape at all. An absent or empty array
|
||||
// (length 0) falls back to the body scan unchanged.
|
||||
const frontmatter = extractFrontmatter(content);
|
||||
const humanVerification = frontmatter.human_verification;
|
||||
if (Array.isArray(humanVerification) && humanVerification.length > 0) {
|
||||
humanVerification.forEach((entry, idx) => {
|
||||
items.push({
|
||||
test: idx + 1,
|
||||
name: normalizeHumanVerificationEntry(entry),
|
||||
result: 'human_needed',
|
||||
category: 'human_uat',
|
||||
});
|
||||
});
|
||||
return items;
|
||||
}
|
||||
|
||||
// Use the seam to locate the ## Human Verification section (ADR-1372 T5).
|
||||
const hvSection = collectSection(
|
||||
content,
|
||||
@@ -459,12 +648,75 @@ function parseVerificationItems(content: string, status: string): UatItem[] {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// #2286: fall back to the `### N. <label>` heading + bold-led paragraph
|
||||
// shape (the canonical form emitted by `templates/verification-report.md`
|
||||
// — `### 1. {Test Name}` followed by `**Test:** ... **Expected:** ...
|
||||
// **Why human:** ...`), which the table/bullet/numbered per-line scan
|
||||
// above never recognises (a `###`-prefixed line matches none of those
|
||||
// three patterns). Uses the same `tokenizeHeadings` seam
|
||||
// `parseFirstPendingTest` already uses for `### N.` sub-headings,
|
||||
// applied here to the Human Verification section body. Runs in
|
||||
// addition to (a union with) the scan above — the two shapes don't
|
||||
// collide, so this only adds items a `###` heading page would have
|
||||
// silently produced zero for.
|
||||
const hvSubHeadings = tokenizeHeadings(hvSection.body).filter(
|
||||
(h) => h.level === 3 && /^\d+\.\s+/.test(h.text),
|
||||
);
|
||||
for (let i = 0; i < hvSubHeadings.length; i += 1) {
|
||||
const current = hvSubHeadings[i];
|
||||
const next = hvSubHeadings[i + 1];
|
||||
const block = next
|
||||
? hvSection.body.slice(current.offset, next.offset)
|
||||
: hvSection.body.slice(current.offset);
|
||||
const bodyAfterHeading = block.slice(block.indexOf('\n') + 1);
|
||||
// Require a bold-led paragraph body (`**Test:** ...`) to distinguish
|
||||
// a genuine verification item from an unrelated numbered heading.
|
||||
if (!/^\s*\*\*/.test(bodyAfterHeading)) continue;
|
||||
|
||||
const headingParts = current.text.match(/^(\d+)\.\s+(.+)$/);
|
||||
if (!headingParts) continue;
|
||||
items.push({
|
||||
test: parseInt(headingParts[1], 10),
|
||||
name: headingParts[2].trim(),
|
||||
result: 'human_needed',
|
||||
category: 'human_uat',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// gaps_found items are already handled by plan-phase --gaps pipeline
|
||||
return items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a single `human_verification:` frontmatter array entry (#2286)
|
||||
* into a display-ready name.
|
||||
*
|
||||
* #2286 review (LOW finding): `extractFrontmatter`'s generic array-item
|
||||
* parser (`src/frontmatter.cts`, the `line.trim().startsWith('- ')` branch)
|
||||
* has NO notion of nested key/value objects — regardless of whether the
|
||||
* source YAML was authored as `- test: "..."` (an implied-but-unsupported
|
||||
* shorthand) or `- "plain string"`, it ALWAYS pushes the raw post-`- ` text
|
||||
* (with only a single layer of wrapping quotes stripped) as a plain string.
|
||||
* There is therefore no reliable signal here to distinguish a genuine
|
||||
* `key: value`-shaped pseudo-field from a legitimate plain string that
|
||||
* itself happens to start with a word and a colon (e.g. `"Confirm: the
|
||||
* button responds"`). A prior version of this function stripped a leading
|
||||
* `word:` prefix on the assumption it was always a flattened nested-object
|
||||
* key — that assumption is false, and it silently truncated real plain-string
|
||||
* content. No such stripping is applied: any residual wrapping-quote noise
|
||||
* left by `extractFrontmatter`'s own (anchor-only) quote handling is cleaned
|
||||
* up, and everything else is preserved verbatim.
|
||||
*/
|
||||
function normalizeHumanVerificationEntry(raw: unknown): string {
|
||||
if (typeof raw !== 'string') {
|
||||
return raw === null || raw === undefined ? '' : JSON.stringify(raw);
|
||||
}
|
||||
const s = raw.trim().replace(/^["']+|["']+$/g, '').trim();
|
||||
return s || raw.trim();
|
||||
}
|
||||
|
||||
// ─── categorizeItem ───────────────────────────────────────────────────────────
|
||||
|
||||
function categorizeItem(result: string, reason?: string, blockedBy?: string): UatCategory {
|
||||
|
||||
@@ -423,6 +423,399 @@ All checks passed.
|
||||
`status: passed file should produce 0 outstanding items, got ${output.summary.total_items}`);
|
||||
assert.strictEqual(output.summary.total_files, 0);
|
||||
});
|
||||
|
||||
// Regression: #2286 — parseUatItems never scanned a `## Gaps` section, so a
|
||||
// *-UAT.md file recording its only outstanding findings there returned
|
||||
// total_items: 0 (false-clean). Boundary: 0 / 1 / 2+ unresolved entries.
|
||||
describe('Gaps section scanning (#2286)', () => {
|
||||
test('a Gaps-only UAT file with 0 unresolved entries (all resolved) yields no items', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "SC1: Widget renders with data"',
|
||||
' status: resolved',
|
||||
' reason: "Fixed in follow-up commit"',
|
||||
'',
|
||||
'- truth: "SC2: Second finding also fixed"',
|
||||
' status: resolved',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 0,
|
||||
'resolved Gaps entries must not be counted as outstanding items');
|
||||
assert.strictEqual(output.summary.total_files, 0);
|
||||
});
|
||||
|
||||
test('a Gaps-only UAT file with exactly 1 unresolved entry and zero ### N. test blocks yields 1 item', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "SC1: Widget renders with data"',
|
||||
' status: open',
|
||||
' reason: "Missing data binding"',
|
||||
' severity: major',
|
||||
' test: 2',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1, 'total_items must be > 0, not the false-clean 0');
|
||||
assert.strictEqual(output.results[0].type, 'uat');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'SC1: Widget renders with data');
|
||||
assert.strictEqual(output.results[0].items[0].result, 'open');
|
||||
assert.strictEqual(output.results[0].items[0].reason, 'Missing data binding');
|
||||
assert.strictEqual(output.results[0].items[0].test, 2);
|
||||
});
|
||||
|
||||
test('a Gaps section with 2+ unresolved entries surfaces all of them and skips the resolved one', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '02-api');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '02-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 02-api',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "SC1: First outstanding gap"',
|
||||
' status: failed',
|
||||
' reason: "Endpoint returns 500"',
|
||||
'',
|
||||
'- truth: "SC2: Second outstanding gap"',
|
||||
' status: open',
|
||||
'',
|
||||
'- truth: "SC3: Already fixed gap"',
|
||||
' status: resolved',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 2,
|
||||
'exactly the 2 unresolved gaps should be counted, resolved gap excluded');
|
||||
const names = output.results[0].items.map((item) => item.name).sort();
|
||||
assert.deepStrictEqual(names, ['SC1: First outstanding gap', 'SC2: Second outstanding gap']);
|
||||
});
|
||||
|
||||
// Regression: #2286 review HIGH finding — a naive whole-string `key:`
|
||||
// scan over a Gaps entry's flattened text matches the FIRST `key:`-shaped
|
||||
// substring anywhere, including one embedded inside an EARLIER field's
|
||||
// own quoted free-text value. A `truth`/`reason` value that itself
|
||||
// contains the literal text "status: resolved" (or "reason:"/"test:")
|
||||
// must never hijack the real, later `status:`/`reason:`/`test:` field —
|
||||
// the fix parses each field anchored to the START of its own line.
|
||||
test('a truth value containing the literal substring "status: resolved" does not suppress the real open status', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "The status: resolved workflow should trigger a banner"',
|
||||
' status: failed',
|
||||
' reason: "Contains a reason: field embedded phrase, and test: 9 too"',
|
||||
' test: 3',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1,
|
||||
'the genuinely open gap must be surfaced, not dropped because its truth text contains "status: resolved"');
|
||||
const item = output.results[0].items[0];
|
||||
assert.strictEqual(item.name, 'The status: resolved workflow should trigger a banner');
|
||||
assert.strictEqual(item.result, 'failed', 'the REAL status: field must win, not the embedded phrase inside truth');
|
||||
assert.strictEqual(item.reason, 'Contains a reason: field embedded phrase, and test: 9 too',
|
||||
'the reason value is taken verbatim, including its own embedded colon-bearing phrases');
|
||||
assert.strictEqual(item.test, 3, 'the REAL test: field (3) must win, not the "test: 9" phrase embedded in reason');
|
||||
});
|
||||
|
||||
// Regression: #2286 review LOW finding — a nested `artifacts:` sub-list
|
||||
// (per templates/UAT.md's `## Gaps` schema) must be folded into its
|
||||
// parent entry, not mis-split into spurious standalone items.
|
||||
test('a Gaps entry with a nested artifacts sub-list parses as exactly one item', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'<!-- YAML format for plan-phase --gaps consumption -->',
|
||||
'- truth: "SC1: Some behavior"',
|
||||
' status: failed',
|
||||
' reason: "reason text"',
|
||||
' severity: major',
|
||||
' test: 1',
|
||||
' root_cause: ""',
|
||||
' artifacts:',
|
||||
' - src/foo.ts',
|
||||
' - src/bar.ts',
|
||||
' missing: []',
|
||||
' debug_session: ""',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1,
|
||||
'the nested artifacts sub-list items must not spawn spurious extra Gaps items');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'SC1: Some behavior');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'unknown',
|
||||
'a Gaps item with no dedicated category mapping falls back to unknown');
|
||||
});
|
||||
|
||||
// Regression: #2286 review item 5 (fail-safe direction) — #2286 is a
|
||||
// false-NEGATIVE bug, so a Gaps entry with no parseable `status:` field
|
||||
// is surfaced (as result: 'unknown') rather than silently dropped.
|
||||
test('a Gaps entry with no status field is surfaced as an unknown-status item (fail-safe)', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
'- truth: "SC1: Missing status field entirely"',
|
||||
' reason: "why it is open"',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1,
|
||||
'a garbled/missing status must SURFACE the entry, not silently drop it');
|
||||
assert.strictEqual(output.results[0].items[0].result, 'unknown');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'SC1: Missing status field entirely');
|
||||
});
|
||||
|
||||
test('an empty Gaps section (heading present, no bullets) yields 0 items without throwing', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), [
|
||||
'---',
|
||||
'status: partial',
|
||||
'phase: 01-foundation',
|
||||
'---',
|
||||
'',
|
||||
'## Gaps',
|
||||
'',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 0);
|
||||
assert.strictEqual(output.summary.total_files, 0);
|
||||
});
|
||||
});
|
||||
|
||||
// Regression: #2286 — parseVerificationItems never read the frontmatter's
|
||||
// structured `human_verification:` YAML array, and never recognized the
|
||||
// `### N. <label>` + bold-paragraph body shape shipped by
|
||||
// templates/verification-report.md. Boundary: array length 0 / 1 / 2+.
|
||||
describe('human_verification frontmatter array + heading shape (#2286)', () => {
|
||||
test('an empty human_verification array (length 0) falls back to the body scan', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 04-auth',
|
||||
'human_verification: []',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification',
|
||||
'',
|
||||
'1. Test SSO login with Google account',
|
||||
'2. Test password reset flow end-to-end',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 2,
|
||||
'an empty structured array must fall back to the existing body scan, not report 0');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'Test SSO login with Google account');
|
||||
});
|
||||
|
||||
test('a populated human_verification array of length 1 is sourced from frontmatter as primary', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 04-auth',
|
||||
'human_verification:',
|
||||
' - test: "Confirm the widget renders correctly"',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification',
|
||||
'',
|
||||
'None — see frontmatter human_verification array.',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1,
|
||||
'total_items must reflect the frontmatter array, not the unstructured body prose');
|
||||
// #2286 review LOW finding: extractFrontmatter's generic array-item
|
||||
// parser has no notion of nested key/value objects — a `- test: "..."`
|
||||
// entry is ALWAYS flattened to the raw post-"- " text, verbatim (only
|
||||
// its own wrapping quote is stripped, and only at the string's outer
|
||||
// edges). normalizeHumanVerificationEntry deliberately does NOT strip
|
||||
// a leading "key:"-shaped prefix (see its doc comment) because doing
|
||||
// so is indistinguishable from truncating a legitimate plain string
|
||||
// that starts with a word and a colon — so this documented, slightly
|
||||
// ugly artifact is the CORRECT (non-data-lossy) output for this shape.
|
||||
assert.strictEqual(output.results[0].items[0].name, 'test: "Confirm the widget renders correctly');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'human_uat');
|
||||
});
|
||||
|
||||
// Regression: #2286 review LOW finding — a plain-string human_verification
|
||||
// entry that itself starts with "Word: " must be preserved verbatim, not
|
||||
// truncated by a (removed) leading-key-prefix strip.
|
||||
test('a plain-string human_verification entry beginning with "Word: " is preserved verbatim', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 04-auth',
|
||||
'human_verification:',
|
||||
' - "Confirm: the button responds"',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification',
|
||||
'',
|
||||
'None.',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1);
|
||||
assert.strictEqual(output.results[0].items[0].name, 'Confirm: the button responds',
|
||||
'a plain string beginning with a word and a colon must not be truncated');
|
||||
});
|
||||
|
||||
test('a populated human_verification array of length 2+ takes priority over a differently-shaped body', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 04-auth',
|
||||
'human_verification:',
|
||||
' - "Confirm SSO login works end to end"',
|
||||
' - "Confirm MFA enrollment banner appears"',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification',
|
||||
'',
|
||||
'1. A body-scan item that must NOT be double-counted',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 2,
|
||||
'the structured array is the PRIMARY source and must not union with the body scan');
|
||||
const names = output.results[0].items.map((item) => item.name).sort();
|
||||
assert.deepStrictEqual(names, ['Confirm MFA enrollment banner appears', 'Confirm SSO login works end to end']);
|
||||
});
|
||||
|
||||
test('recognizes the ### N. <label> + bold-paragraph Human Verification body shape', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '05-widgets');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '05-VERIFICATION.md'), [
|
||||
'---',
|
||||
'status: human_needed',
|
||||
'phase: 05-widgets',
|
||||
'---',
|
||||
'',
|
||||
'## Human Verification Required',
|
||||
'',
|
||||
'### 1. Widget render check',
|
||||
'**Test:** Confirm the widget appears as expected on the dashboard.',
|
||||
'**Expected:** Widget renders with live data within 2 seconds.',
|
||||
'**Why human:** Visual rendering cannot be verified by static analysis.',
|
||||
'',
|
||||
'### 2. Notification banner check',
|
||||
'**Test:** Trigger a new notification and confirm the banner appears.',
|
||||
'**Expected:** Banner appears within 1 second and auto-dismisses after 5 seconds.',
|
||||
'**Why human:** Timing-based UI behavior requires visual confirmation.',
|
||||
].join('\n'));
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 2,
|
||||
'the ### N. + bold-paragraph shape must be recognized instead of returning 0 items');
|
||||
assert.strictEqual(output.results[0].items[0].test, 1);
|
||||
assert.strictEqual(output.results[0].items[0].name, 'Widget render check');
|
||||
assert.strictEqual(output.results[0].items[1].test, 2);
|
||||
assert.strictEqual(output.results[0].items[1].name, 'Notification banner check');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'human_uat');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('uat render-checkpoint', () => {
|
||||
|
||||
Reference in New Issue
Block a user