diff --git a/.changeset/quick-elks-climb.md b/.changeset/quick-elks-climb.md new file mode 100644 index 000000000..d1c83556f --- /dev/null +++ b/.changeset/quick-elks-climb.md @@ -0,0 +1,5 @@ +--- +type: Fixed +pr: 2317 +--- +**`audit-uat` no longer reports a false-clean `total_items: 0` when real items exist** — the parsers ignored two artifact shapes: a `## Gaps` section recording open findings, and verification items declared in frontmatter (`human_verification:` array) or as `### N.`+bold-paragraph blocks. audit-uat now surfaces unresolved `## Gaps` entries and reads the frontmatter array / heading shape, so a phase with outstanding UAT/verification work is no longer waved through as clean. (#2286) diff --git a/src/uat.cts b/src/uat.cts index 9e271c2be..5d3b00ec8 100644 --- a/src/uat.cts +++ b/src/uat.cts @@ -365,14 +365,203 @@ function parseUatItems(content: string): UatItem[] { items.push(item); } } + items.push(...parseGapsItems(content)); return items; } +// ─── parseGapsItems ─────────────────────────────────────────────────────────── + +/** + * Extract unresolved entries from a UAT file's `## Gaps` section (#2286). + * + * `## Gaps` records open findings as a YAML-lite bullet list (see + * `templates/UAT.md`'s `## Gaps` block: `- truth: "..."` followed by indented + * continuation lines `status:` / `reason:` / `severity:` / `test:` / etc., + * and — for `artifacts:` / `missing:` — a further-nested `- ` sub-list). + * `parseUatItems`'s `### N.` test-block regex never looks at this section at + * all, so a UAT file whose only outstanding findings live in `## Gaps` was + * silently invisible — the false-negative this fix addresses. + * + * Reuses the existing `collectSection` seam (already used elsewhere in this + * file for `## Current Test` / `## Tests`) to locate the section. Field + * extraction is deliberately NOT done via `iterateBullets`: that seam folds + * every continuation line onto ONE space-joined `text` string per bullet, + * which erases line boundaries — a `key:` scan against that flattened text + * matches the FIRST `key:`-shaped substring anywhere, including one that + * happens to appear inside an EARLIER field's own quoted free-text value + * (e.g. `truth: "The status: resolved workflow should trigger"` — a real + * `status: failed` on the next line would never be reached, silently + * DROPPING a genuinely open gap — the exact false-negative class #2286 + * exists to fix, so the fix must not reintroduce it). `splitGapsEntries` / + * `extractGapEntryFields` below instead walk the section PER LINE and only + * recognise a field at the START of its own (trimmed) line, so a `key:` + * embedded inside another field's quoted value can never be mistaken for a + * field declaration. + * + * Every entry whose `status` is present and NOT `resolved` (case-insensitive) + * is surfaced — mirroring the "ignore passing/resolved" convention already + * used for `### N.` test blocks (`result: pass` is never surfaced) and the + * VERIFICATION table-row PASS/resolved skip (`hasPassResult`, below). An + * entry with NO parseable `status:` field is surfaced too, as `result: + * 'unknown'` — #2286 is a false-NEGATIVE bug, and a `## Gaps` entry only + * exists to record an outstanding finding (a template-conformant RESOLVED + * entry always carries an explicit `status: resolved`); a garbled or + * non-conformant entry is far more likely to be an unresolved finding whose + * `status:` line failed to parse than a genuinely resolved one, so the + * fail-safe direction is to surface it rather than silently drop it. + */ +function parseGapsItems(content: string): UatItem[] { + const gapsSection = collectSection( + content, + (h) => /^gaps$/i.test(h.text) && h.level === 2, + { levelBounded: true }, + ); + if (!gapsSection) return []; + + const items: UatItem[] = []; + + for (const entryLines of splitGapsEntries(gapsSection.body)) { + const fields = extractGapEntryFields(entryLines); + const rawStatus = fields.status; + if (rawStatus && rawStatus.toLowerCase() === 'resolved') continue; + // Fail-safe: missing/garbled status surfaces as 'unknown' rather than + // being dropped (see doc comment above). + const status = rawStatus || 'unknown'; + + const truth = fields.truth; + const reason = fields.reason; + const testNum = fields.test; + + const item: UatItem = { + name: truth || rawGapEntryText(entryLines), + result: status, + category: categorizeItem(status, reason, undefined), + }; + if (testNum && /^\d+$/.test(testNum)) item.test = parseInt(testNum, 10); + if (reason) item.reason = reason; + items.push(item); + } + + return items; +} + +/** + * Split a `## Gaps` section body into per-entry line groups on TOP-LEVEL + * `- ` bullet openers. + * + * The indentation of the FIRST bullet line encountered establishes the + * "top-level" indent for the whole section; any subsequent `- `-opening line + * at that same indent (or shallower) starts a NEW entry, while everything + * more deeply indented — field continuation lines (` status: ...`) AND + * nested sub-lists (` - src/foo.ts` under ` artifacts:`) — is folded into + * the CURRENT entry. This keeps a `artifacts:`/`missing:` sub-list's `- ` + * items from being mis-split into spurious standalone entries (#2286 review + * LOW finding). + * + * Lines before the first bullet (e.g. the `` comment + * the template emits) are discarded. An empty/whitespace-only section body + * (heading present, no bullets) returns `[]`. + */ +function splitGapsEntries(sectionBody: string): string[][] { + const lines = sectionBody.split('\n'); + const entries: string[][] = []; + let current: string[] | null = null; + let baseIndent: number | null = null; + + for (const rawLine of lines) { + const line = rawLine.replace(/\r$/, ''); + const bulletMatch = line.match(/^(\s*)-\s/); + if (bulletMatch) { + const indent = bulletMatch[1].length; + if (baseIndent === null) baseIndent = indent; + if (indent <= baseIndent) { + if (current) entries.push(current); + current = [line]; + continue; + } + } + if (current) current.push(line); + // else: pre-first-bullet content (e.g. the template's HTML comment) — discarded. + } + if (current) entries.push(current); + + return entries; +} + +/** + * Extract `key: value` fields from one Gaps entry's lines, anchored to the + * START of each (bullet-marker-stripped, trimmed) line — never scanning the + * REST of a line, so a colon-bearing phrase inside a quoted `truth`/`reason` + * value is never misread as a field declaration (see `parseGapsItems`'s doc + * comment for the false-negative this specifically guards against). + * + * Recognises a double-quoted value (`truth: "..."`, stripped of its wrapping + * quotes — the value may itself contain any character, including `:`) or a + * bare value (`status: open`, `test: 2`, `artifacts: []`) taken verbatim. + * The FIRST occurrence of a given key wins (top-level fields always precede + * any nested sub-list content in the template's field ordering); later + * `key:`-shaped nested-list content is captured, if it parses as one, but + * never overrides an already-seen top-level field. + */ +function extractGapEntryFields(entryLines: string[]): Record { + const fields: Record = {}; + const fieldLineRe = /^([A-Za-z_][A-Za-z0-9_-]*):\s*(.*)$/; + + entryLines.forEach((rawLine, idx) => { + const line = rawLine.replace(/\r$/, ''); + // Strip ONLY the entry-opening bullet marker (idx 0); a bullet marker on + // a later line belongs to a nested sub-list and is handled by + // `splitGapsEntries` already folding it in — it is not itself a field + // line unless it independently matches `key: value` after stripping. + const bulletStripped = line.match(/^(\s*)-\s+(.*)$/); + const content = idx === 0 && bulletStripped ? bulletStripped[2] : line.trim(); + + const m = fieldLineRe.exec(content); + if (!m) return; + const key = m[1]; + let value = m[2].trim(); + if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) { + value = value.slice(1, -1); + } + if (!(key in fields)) fields[key] = value; + }); + + return fields; +} + +/** Fallback display text for a Gaps entry with no parseable `truth:` field. */ +function rawGapEntryText(entryLines: string[]): string { + return entryLines + .map((l, i) => (i === 0 ? l.replace(/^(\s*)-\s+/, '') : l.trim())) + .join(' ') + .trim(); +} + // ─── parseVerificationItems ─────────────────────────────────────────────────── function parseVerificationItems(content: string, status: string): UatItem[] { const items: UatItem[] = []; if (status === 'human_needed') { + // #2286: the frontmatter's structured `human_verification:` YAML array + // (extractFrontmatter) is the PRIMARY source of truth when present and + // non-empty — it fully bypasses the body-shape scan below, so a file + // whose frontmatter declares the array doesn't require any particular + // `## Human Verification` body shape at all. An absent or empty array + // (length 0) falls back to the body scan unchanged. + const frontmatter = extractFrontmatter(content); + const humanVerification = frontmatter.human_verification; + if (Array.isArray(humanVerification) && humanVerification.length > 0) { + humanVerification.forEach((entry, idx) => { + items.push({ + test: idx + 1, + name: normalizeHumanVerificationEntry(entry), + result: 'human_needed', + category: 'human_uat', + }); + }); + return items; + } + // Use the seam to locate the ## Human Verification section (ADR-1372 T5). const hvSection = collectSection( content, @@ -459,12 +648,75 @@ function parseVerificationItems(content: string, status: string): UatItem[] { }); } } + + // #2286: fall back to the `### N.