Merge pull request #1212 from jecanore/feat/audit-uat

feat: add verification debt tracking and /gsd:audit-uat command

get-shit-done/bin/gsd-tools.cjs

@@ -64,6 +64,9 @@
  * Todos:
  *   todo complete <filename>           Move todo from pending to completed
  *
+ * UAT Audit:
+ *   audit-uat                           Scan all phases for unresolved UAT/verification items
+ *
  * Scaffolding:
  *   scaffold context --phase <N>       Create CONTEXT.md template
  *   scaffold uat --phase <N>           Create UAT.md template
@@ -536,6 +539,12 @@ async function main() {
       break;
     }
 
+    case 'audit-uat': {
+      const uat = require('./lib/uat.cjs');
+      uat.cmdAuditUat(cwd, raw);
+      break;
+    }
+
     case 'stats': {
       const subcommand = args[1] || 'json';
       commands.cmdStats(cwd, subcommand, raw);

get-shit-done/bin/lib/phase.cjs

@@ -735,6 +735,27 @@ function cmdPhaseComplete(cwd, phaseNum, raw) {
   const summaryCount = phaseInfo.summaries.length;
   let requirementsUpdated = false;
 
+  // Check for unresolved verification debt (non-blocking warnings)
+  const warnings = [];
+  try {
+    const phaseFullDir = path.join(cwd, phaseInfo.directory);
+    const phaseFiles = fs.readdirSync(phaseFullDir);
+
+    for (const file of phaseFiles.filter(f => f.includes('-UAT') && f.endsWith('.md'))) {
+      const content = fs.readFileSync(path.join(phaseFullDir, file), 'utf-8');
+      if (/result: pending/.test(content)) warnings.push(`${file}: has pending tests`);
+      if (/result: blocked/.test(content)) warnings.push(`${file}: has blocked tests`);
+      if (/status: partial/.test(content)) warnings.push(`${file}: testing incomplete (partial)`);
+      if (/status: diagnosed/.test(content)) warnings.push(`${file}: has diagnosed gaps`);
+    }
+
+    for (const file of phaseFiles.filter(f => f.includes('-VERIFICATION') && f.endsWith('.md'))) {
+      const content = fs.readFileSync(path.join(phaseFullDir, file), 'utf-8');
+      if (/status: human_needed/.test(content)) warnings.push(`${file}: needs human verification`);
+      if (/status: gaps_found/.test(content)) warnings.push(`${file}: has unresolved gaps`);
+    }
+  } catch {}
+
   // Update ROADMAP.md: mark phase complete
   if (fs.existsSync(roadmapPath)) {
     let roadmapContent = fs.readFileSync(roadmapPath, 'utf-8');
@@ -936,6 +957,8 @@ function cmdPhaseComplete(cwd, phaseNum, raw) {
     roadmap_updated: fs.existsSync(roadmapPath),
     state_updated: fs.existsSync(statePath),
     requirements_updated: requirementsUpdated,
+    warnings,
+    has_warnings: warnings.length > 0,
   };
 
   output(result, raw);

get-shit-done/bin/lib/uat.cjs

@@ -0,0 +1,189 @@
+/**
+ * UAT Audit — Cross-phase UAT/VERIFICATION scanner
+ *
+ * Reads all *-UAT.md and *-VERIFICATION.md files across all phases.
+ * Extracts non-passing items. Returns structured JSON for workflow consumption.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { output, error, getMilestonePhaseFilter } = require('./core.cjs');
+const { extractFrontmatter } = require('./frontmatter.cjs');
+
+function cmdAuditUat(cwd, raw) {
+  const phasesDir = path.join(cwd, '.planning', 'phases');
+  if (!fs.existsSync(phasesDir)) {
+    error('No .planning/phases directory found');
+  }
+
+  const isDirInMilestone = getMilestonePhaseFilter(cwd);
+  const results = [];
+
+  // Scan all phase directories
+  const dirs = fs.readdirSync(phasesDir, { withFileTypes: true })
+    .filter(e => e.isDirectory())
+    .map(e => e.name)
+    .filter(isDirInMilestone)
+    .sort();
+
+  for (const dir of dirs) {
+    const phaseMatch = dir.match(/^(\d+[A-Z]?(?:\.\d+)*)/i);
+    const phaseNum = phaseMatch ? phaseMatch[1] : dir;
+    const phaseDir = path.join(phasesDir, dir);
+    const files = fs.readdirSync(phaseDir);
+
+    // Process UAT files
+    for (const file of files.filter(f => f.includes('-UAT') && f.endsWith('.md'))) {
+      const content = fs.readFileSync(path.join(phaseDir, file), 'utf-8');
+      const items = parseUatItems(content);
+      if (items.length > 0) {
+        results.push({
+          phase: phaseNum,
+          phase_dir: dir,
+          file,
+          file_path: `.planning/phases/${dir}/${file}`,
+          type: 'uat',
+          status: (extractFrontmatter(content).status || 'unknown'),
+          items,
+        });
+      }
+    }
+
+    // Process VERIFICATION files
+    for (const file of files.filter(f => f.includes('-VERIFICATION') && f.endsWith('.md'))) {
+      const content = fs.readFileSync(path.join(phaseDir, file), 'utf-8');
+      const status = extractFrontmatter(content).status || 'unknown';
+      if (status === 'human_needed' || status === 'gaps_found') {
+        const items = parseVerificationItems(content, status);
+        if (items.length > 0) {
+          results.push({
+            phase: phaseNum,
+            phase_dir: dir,
+            file,
+            file_path: `.planning/phases/${dir}/${file}`,
+            type: 'verification',
+            status,
+            items,
+          });
+        }
+      }
+    }
+  }
+
+  // Compute summary
+  const summary = {
+    total_files: results.length,
+    total_items: results.reduce((sum, r) => sum + r.items.length, 0),
+    by_category: {},
+    by_phase: {},
+  };
+
+  for (const r of results) {
+    if (!summary.by_phase[r.phase]) summary.by_phase[r.phase] = 0;
+    for (const item of r.items) {
+      summary.by_phase[r.phase]++;
+      const cat = item.category || 'unknown';
+      summary.by_category[cat] = (summary.by_category[cat] || 0) + 1;
+    }
+  }
+
+  output({ results, summary }, raw);
+}
+
+function parseUatItems(content) {
+  const items = [];
+  // Match test blocks: ### N. Name\nexpected: ...\nresult: ...\n
+  const testPattern = /###\s*(\d+)\.\s*([^\n]+)\nexpected:\s*([^\n]+)\nresult:\s*(\w+)(?:\n(?:reported|reason|blocked_by):\s*[^\n]*)?/g;
+  let match;
+  while ((match = testPattern.exec(content)) !== null) {
+    const [, num, name, expected, result] = match;
+    if (result === 'pending' || result === 'skipped' || result === 'blocked') {
+      // Extract optional fields — limit to current test block (up to next ### or EOF)
+      const afterMatch = content.slice(match.index);
+      const nextHeading = afterMatch.indexOf('\n###', 1);
+      const blockText = nextHeading > 0 ? afterMatch.slice(0, nextHeading) : afterMatch;
+      const reasonMatch = blockText.match(/reason:\s*(.+)/);
+      const blockedByMatch = blockText.match(/blocked_by:\s*(.+)/);
+
+      const item = {
+        test: parseInt(num, 10),
+        name: name.trim(),
+        expected: expected.trim(),
+        result,
+        category: categorizeItem(result, reasonMatch?.[1], blockedByMatch?.[1]),
+      };
+      if (reasonMatch) item.reason = reasonMatch[1].trim();
+      if (blockedByMatch) item.blocked_by = blockedByMatch[1].trim();
+      items.push(item);
+    }
+  }
+  return items;
+}
+
+function parseVerificationItems(content, status) {
+  const items = [];
+  if (status === 'human_needed') {
+    // Extract from human_verification section — look for numbered items or table rows
+    const hvSection = content.match(/##\s*Human Verification.*?\n([\s\S]*?)(?=\n##\s|\n---\s|$)/i);
+    if (hvSection) {
+      const lines = hvSection[1].split('\n');
+      for (const line of lines) {
+        // Match table rows: | N | description | ... |
+        const tableMatch = line.match(/\|\s*(\d+)\s*\|\s*([^|]+)/);
+        // Match bullet items: - description
+        const bulletMatch = line.match(/^[-*]\s+(.+)/);
+        // Match numbered items: 1. description
+        const numberedMatch = line.match(/^(\d+)\.\s+(.+)/);
+
+        if (tableMatch) {
+          items.push({
+            test: parseInt(tableMatch[1], 10),
+            name: tableMatch[2].trim(),
+            result: 'human_needed',
+            category: 'human_uat',
+          });
+        } else if (numberedMatch) {
+          items.push({
+            test: parseInt(numberedMatch[1], 10),
+            name: numberedMatch[2].trim(),
+            result: 'human_needed',
+            category: 'human_uat',
+          });
+        } else if (bulletMatch && bulletMatch[1].length > 10) {
+          items.push({
+            name: bulletMatch[1].trim(),
+            result: 'human_needed',
+            category: 'human_uat',
+          });
+        }
+      }
+    }
+  }
+  // gaps_found items are already handled by plan-phase --gaps pipeline
+  return items;
+}
+
+function categorizeItem(result, reason, blockedBy) {
+  if (result === 'blocked' || blockedBy) {
+    if (blockedBy) {
+      if (/server/i.test(blockedBy)) return 'server_blocked';
+      if (/device|physical/i.test(blockedBy)) return 'device_needed';
+      if (/build|release|preview/i.test(blockedBy)) return 'build_needed';
+      if (/third.party|twilio|stripe/i.test(blockedBy)) return 'third_party';
+    }
+    return 'blocked';
+  }
+  if (result === 'skipped') {
+    if (reason) {
+      if (/server|not running|not available/i.test(reason)) return 'server_blocked';
+      if (/simulator|physical|device/i.test(reason)) return 'device_needed';
+      if (/build|release|preview/i.test(reason)) return 'build_needed';
+    }
+    return 'skipped_unresolved';
+  }
+  if (result === 'pending') return 'pending';
+  if (result === 'human_needed') return 'human_uat';
+  return 'unknown';
+}
+
+module.exports = { cmdAuditUat };