From 60a76ae06e00e30926fdc897e3b85aad13c071c9 Mon Sep 17 00:00:00 2001 From: jecanore Date: Wed, 18 Mar 2026 23:30:09 -0500 Subject: [PATCH] feat: add verification debt tracking and /gsd:audit-uat command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prevent silent loss of UAT/verification items when projects advance. Surfaces outstanding items across all prior phases so nothing is forgotten. New command: - /gsd:audit-uat — cross-phase audit with categorized report and test plan New capabilities: - Cross-phase health check in /gsd:progress (Step 1.6) - status: partial for incomplete UAT sessions - result: blocked with blocked_by tag for dependency-gated tests - human_needed items persisted as trackable HUMAN-UAT.md files - Phase completion and transition warnings for verification debt Files: 4 new, 14 modified (9 feature + 5 docs) Co-Authored-By: Claude Opus 4.6 (1M context) --- CHANGELOG.md | 7 + commands/gsd/audit-uat.md | 24 ++ docs/ARCHITECTURE.md | 6 +- docs/CLI-TOOLS.md | 8 +- docs/COMMANDS.md | 13 + docs/FEATURES.md | 34 +++ get-shit-done/bin/gsd-tools.cjs | 9 + get-shit-done/bin/lib/phase.cjs | 23 ++ get-shit-done/bin/lib/uat.cjs | 189 +++++++++++++ get-shit-done/templates/UAT.md | 24 +- get-shit-done/workflows/audit-uat.md | 109 ++++++++ get-shit-done/workflows/execute-phase.md | 63 ++++- get-shit-done/workflows/help.md | 11 + get-shit-done/workflows/progress.md | 60 ++++- get-shit-done/workflows/transition.md | 24 ++ get-shit-done/workflows/verify-work.md | 41 ++- tests/copilot-install.test.cjs | 4 +- tests/uat.test.cjs | 326 +++++++++++++++++++++++ 18 files changed, 962 insertions(+), 13 deletions(-) create mode 100644 commands/gsd/audit-uat.md create mode 100644 get-shit-done/bin/lib/uat.cjs create mode 100644 get-shit-done/workflows/audit-uat.md create mode 100644 tests/uat.test.cjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 43618437d..adebc2e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,13 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - **Model alias-to-full-ID resolution** — Task API compatibility for model alias strings (#991) - **Execution hardening** — Pre-wave dependency checks, cross-plan data contracts, and export-level spot checks (#1082) - **Markdown normalization** — Generated markdown conforms to markdownlint standards (#1112) +- **`/gsd:audit-uat` command** — Cross-phase audit of all outstanding UAT and verification items. Scans every phase for pending, skipped, blocked, and human_needed items. Cross-references against codebase to detect stale documentation. Produces prioritized human test plan grouped by testability +- **Verification debt tracking** — Five structural improvements to prevent silent loss of UAT/verification items when projects advance: + - Cross-phase health check in `/gsd:progress` (Step 1.6) surfaces outstanding items from ALL prior phases + - `status: partial` in UAT files distinguishes incomplete testing from completed sessions + - `result: blocked` with `blocked_by` tag for tests blocked by external dependencies (server, device, build, third-party) + - `human_needed` verification items now persist as HUMAN-UAT.md files (trackable across sessions) + - Phase completion and transition warnings surface verification debt non-blockingly ### Changed - Test suite consolidated: runtime converters deduplicated, helpers standardized (#1169) diff --git a/commands/gsd/audit-uat.md b/commands/gsd/audit-uat.md new file mode 100644 index 000000000..604e1be12 --- /dev/null +++ b/commands/gsd/audit-uat.md @@ -0,0 +1,24 @@ +--- +name: gsd:audit-uat +description: Cross-phase audit of all outstanding UAT and verification items +allowed-tools: + - Read + - Glob + - Grep + - Bash +--- + +Scan all phases for pending, skipped, blocked, and human_needed UAT items. Cross-reference against codebase to detect stale documentation. Produce prioritized human test plan. + + + +@~/.claude/get-shit-done/workflows/audit-uat.md + + + +Core planning files are loaded in-workflow via CLI. + +**Scope:** +Glob: .planning/phases/*/*-UAT.md +Glob: .planning/phases/*/*-VERIFICATION.md + diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3d8b4108a..f54ce8816 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -171,7 +171,7 @@ Runtime hooks that integrate with the host AI agent: ### CLI Tools (`get-shit-done/bin/`) -Node.js CLI utility (`gsd-tools.cjs`) with 11 domain modules: +Node.js CLI utility (`gsd-tools.cjs`) with 15 domain modules: | Module | Responsibility | |--------|---------------| @@ -342,8 +342,8 @@ UI-SPEC.md (per phase) ─────────────────── ├── commands/gsd/*.md # 37 slash commands ├── get-shit-done/ │ ├── bin/gsd-tools.cjs # CLI utility -│ ├── bin/lib/*.cjs # 11 domain modules -│ ├── workflows/*.md # 41 workflow definitions +│ ├── bin/lib/*.cjs # 15 domain modules +│ ├── workflows/*.md # 42 workflow definitions │ ├── references/*.md # 13 shared reference docs │ └── templates/ # Planning artifact templates ├── agents/*.md # 15 agent definitions diff --git a/docs/CLI-TOOLS.md b/docs/CLI-TOOLS.md index 2621e7769..2e1dcfb0f 100644 --- a/docs/CLI-TOOLS.md +++ b/docs/CLI-TOOLS.md @@ -9,7 +9,7 @@ `gsd-tools.cjs` is a Node.js CLI utility that replaces repetitive inline bash patterns across GSD's ~50 command, workflow, and agent files. It centralizes: config parsing, model resolution, phase lookup, git commits, summary verification, state management, and template operations. **Location:** `get-shit-done/bin/gsd-tools.cjs` -**Modules:** 11 domain modules in `get-shit-done/bin/lib/` +**Modules:** 15 domain modules in `get-shit-done/bin/lib/` **Usage:** ```bash @@ -330,6 +330,9 @@ node gsd-tools.cjs progress [json|table|bar] # Complete a todo node gsd-tools.cjs todo complete +# UAT audit — scan all phases for unresolved items +node gsd-tools.cjs audit-uat + # Git commit with config checks node gsd-tools.cjs commit [--files f1 f2] [--amend] [--no-verify] ``` @@ -358,3 +361,6 @@ node gsd-tools.cjs websearch [--limit N] [--freshness day|week|month] | Milestone | `lib/milestone.cjs` | Milestone archival, requirements marking | | Commands | `lib/commands.cjs` | Misc: slug, timestamp, todos, scaffold, stats, websearch | | Model Profiles | `lib/model-profiles.cjs` | Profile resolution table | +| UAT | `lib/uat.cjs` | Cross-phase UAT/verification audit | +| Profile Output | `lib/profile-output.cjs` | Developer profile formatting | +| Profile Pipeline | `lib/profile-pipeline.cjs` | Session analysis pipeline | diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 1e2870342..1e5ab4dd3 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -215,6 +215,19 @@ Retroactive 6-pillar visual audit of implemented frontend. --- +### `/gsd:audit-uat` + +Cross-phase audit of all outstanding UAT and verification items. + +**Prerequisites:** At least one phase has been executed with UAT or verification +**Produces:** Categorized audit report with human test plan + +```bash +/gsd:audit-uat +``` + +--- + ### `/gsd:audit-milestone` Verify milestone met its definition of done. diff --git a/docs/FEATURES.md b/docs/FEATURES.md index f62553412..cce53eb26 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -52,6 +52,7 @@ - [Hook System](#37-hook-system) - [Developer Profiling](#38-developer-profiling) - [Execution Hardening](#39-execution-hardening) + - [Verification Debt Tracking](#40-verification-debt-tracking) --- @@ -939,3 +940,36 @@ After Level 3 wiring verification passes, spot-check individual exports for actu - REQ-HARD-01: Pre-wave check MUST verify key-links from all prior wave artifacts before spawning next wave - REQ-HARD-02: Cross-plan contract check MUST detect incompatible data transformations between plans - REQ-HARD-03: Export spot-check MUST identify dead stores in wired files + +--- + +### 40. Verification Debt Tracking + +**Command:** `/gsd:audit-uat` + +**Purpose:** Prevent silent loss of UAT/verification items when projects advance past phases with outstanding tests. Surfaces verification debt across all prior phases so items are never forgotten. + +**Components:** + +**1. Cross-Phase Health Check** (progress.md Step 1.6) +Every `/gsd:progress` call scans ALL phases in the current milestone for outstanding items (pending, skipped, blocked, human_needed). Displays a non-blocking warning section with actionable links. + +**2. `status: partial`** (verify-work.md, UAT.md) +New UAT status that distinguishes between "session ended" and "all tests resolved". Prevents `status: complete` when tests are still pending, blocked, or skipped without reason. + +**3. `result: blocked` with `blocked_by` tag** (verify-work.md, UAT.md) +New test result type for tests blocked by external dependencies (server, physical device, release build, third-party services). Categorized separately from skipped tests. + +**4. HUMAN-UAT.md Persistence** (execute-phase.md) +When verification returns `human_needed`, items are persisted as a trackable HUMAN-UAT.md file with `status: partial`. Feeds into the cross-phase health check and audit systems. + +**5. Phase Completion Warnings** (phase.cjs, transition.md) +`phase complete` CLI returns verification debt warnings in its JSON output. Transition workflow surfaces outstanding items before confirmation. + +**Requirements:** +- REQ-DEBT-01: System MUST surface outstanding UAT/verification items from ALL prior phases in `/gsd:progress` +- REQ-DEBT-02: System MUST distinguish incomplete testing (partial) from completed testing (complete) +- REQ-DEBT-03: System MUST categorize blocked tests with `blocked_by` tags +- REQ-DEBT-04: System MUST persist human_needed verification items as trackable UAT files +- REQ-DEBT-05: System MUST warn (non-blocking) during phase completion and transition when verification debt exists +- REQ-DEBT-06: `/gsd:audit-uat` MUST scan all phases, categorize items by testability, and produce a human test plan diff --git a/get-shit-done/bin/gsd-tools.cjs b/get-shit-done/bin/gsd-tools.cjs index 633d921ff..c8df61a06 100755 --- a/get-shit-done/bin/gsd-tools.cjs +++ b/get-shit-done/bin/gsd-tools.cjs @@ -64,6 +64,9 @@ * Todos: * todo complete Move todo from pending to completed * + * UAT Audit: + * audit-uat Scan all phases for unresolved UAT/verification items + * * Scaffolding: * scaffold context --phase Create CONTEXT.md template * scaffold uat --phase Create UAT.md template @@ -525,6 +528,12 @@ async function main() { break; } + case 'audit-uat': { + const uat = require('./lib/uat.cjs'); + uat.cmdAuditUat(cwd, raw); + break; + } + case 'stats': { const subcommand = args[1] || 'json'; commands.cmdStats(cwd, subcommand, raw); diff --git a/get-shit-done/bin/lib/phase.cjs b/get-shit-done/bin/lib/phase.cjs index dc2c52f99..f01a59c55 100644 --- a/get-shit-done/bin/lib/phase.cjs +++ b/get-shit-done/bin/lib/phase.cjs @@ -721,6 +721,27 @@ function cmdPhaseComplete(cwd, phaseNum, raw) { const summaryCount = phaseInfo.summaries.length; let requirementsUpdated = false; + // Check for unresolved verification debt (non-blocking warnings) + const warnings = []; + try { + const phaseFullDir = path.join(cwd, phaseInfo.directory); + const phaseFiles = fs.readdirSync(phaseFullDir); + + for (const file of phaseFiles.filter(f => f.includes('-UAT') && f.endsWith('.md'))) { + const content = fs.readFileSync(path.join(phaseFullDir, file), 'utf-8'); + if (/result: pending/.test(content)) warnings.push(`${file}: has pending tests`); + if (/result: blocked/.test(content)) warnings.push(`${file}: has blocked tests`); + if (/status: partial/.test(content)) warnings.push(`${file}: testing incomplete (partial)`); + if (/status: diagnosed/.test(content)) warnings.push(`${file}: has diagnosed gaps`); + } + + for (const file of phaseFiles.filter(f => f.includes('-VERIFICATION') && f.endsWith('.md'))) { + const content = fs.readFileSync(path.join(phaseFullDir, file), 'utf-8'); + if (/status: human_needed/.test(content)) warnings.push(`${file}: needs human verification`); + if (/status: gaps_found/.test(content)) warnings.push(`${file}: has unresolved gaps`); + } + } catch {} + // Update ROADMAP.md: mark phase complete if (fs.existsSync(roadmapPath)) { let roadmapContent = fs.readFileSync(roadmapPath, 'utf-8'); @@ -922,6 +943,8 @@ function cmdPhaseComplete(cwd, phaseNum, raw) { roadmap_updated: fs.existsSync(roadmapPath), state_updated: fs.existsSync(statePath), requirements_updated: requirementsUpdated, + warnings, + has_warnings: warnings.length > 0, }; output(result, raw); diff --git a/get-shit-done/bin/lib/uat.cjs b/get-shit-done/bin/lib/uat.cjs new file mode 100644 index 000000000..652fae16e --- /dev/null +++ b/get-shit-done/bin/lib/uat.cjs @@ -0,0 +1,189 @@ +/** + * UAT Audit — Cross-phase UAT/VERIFICATION scanner + * + * Reads all *-UAT.md and *-VERIFICATION.md files across all phases. + * Extracts non-passing items. Returns structured JSON for workflow consumption. + */ + +const fs = require('fs'); +const path = require('path'); +const { output, error, getMilestonePhaseFilter } = require('./core.cjs'); +const { extractFrontmatter } = require('./frontmatter.cjs'); + +function cmdAuditUat(cwd, raw) { + const phasesDir = path.join(cwd, '.planning', 'phases'); + if (!fs.existsSync(phasesDir)) { + error('No .planning/phases directory found'); + } + + const isDirInMilestone = getMilestonePhaseFilter(cwd); + const results = []; + + // Scan all phase directories + const dirs = fs.readdirSync(phasesDir, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name) + .filter(isDirInMilestone) + .sort(); + + for (const dir of dirs) { + const phaseMatch = dir.match(/^(\d+[A-Z]?(?:\.\d+)*)/i); + const phaseNum = phaseMatch ? phaseMatch[1] : dir; + const phaseDir = path.join(phasesDir, dir); + const files = fs.readdirSync(phaseDir); + + // Process UAT files + for (const file of files.filter(f => f.includes('-UAT') && f.endsWith('.md'))) { + const content = fs.readFileSync(path.join(phaseDir, file), 'utf-8'); + const items = parseUatItems(content); + if (items.length > 0) { + results.push({ + phase: phaseNum, + phase_dir: dir, + file, + file_path: `.planning/phases/${dir}/${file}`, + type: 'uat', + status: (extractFrontmatter(content).status || 'unknown'), + items, + }); + } + } + + // Process VERIFICATION files + for (const file of files.filter(f => f.includes('-VERIFICATION') && f.endsWith('.md'))) { + const content = fs.readFileSync(path.join(phaseDir, file), 'utf-8'); + const status = extractFrontmatter(content).status || 'unknown'; + if (status === 'human_needed' || status === 'gaps_found') { + const items = parseVerificationItems(content, status); + if (items.length > 0) { + results.push({ + phase: phaseNum, + phase_dir: dir, + file, + file_path: `.planning/phases/${dir}/${file}`, + type: 'verification', + status, + items, + }); + } + } + } + } + + // Compute summary + const summary = { + total_files: results.length, + total_items: results.reduce((sum, r) => sum + r.items.length, 0), + by_category: {}, + by_phase: {}, + }; + + for (const r of results) { + if (!summary.by_phase[r.phase]) summary.by_phase[r.phase] = 0; + for (const item of r.items) { + summary.by_phase[r.phase]++; + const cat = item.category || 'unknown'; + summary.by_category[cat] = (summary.by_category[cat] || 0) + 1; + } + } + + output({ results, summary }, raw); +} + +function parseUatItems(content) { + const items = []; + // Match test blocks: ### N. Name\nexpected: ...\nresult: ...\n + const testPattern = /###\s*(\d+)\.\s*([^\n]+)\nexpected:\s*([^\n]+)\nresult:\s*(\w+)(?:\n(?:reported|reason|blocked_by):\s*[^\n]*)?/g; + let match; + while ((match = testPattern.exec(content)) !== null) { + const [, num, name, expected, result] = match; + if (result === 'pending' || result === 'skipped' || result === 'blocked') { + // Extract optional fields — limit to current test block (up to next ### or EOF) + const afterMatch = content.slice(match.index); + const nextHeading = afterMatch.indexOf('\n###', 1); + const blockText = nextHeading > 0 ? afterMatch.slice(0, nextHeading) : afterMatch; + const reasonMatch = blockText.match(/reason:\s*(.+)/); + const blockedByMatch = blockText.match(/blocked_by:\s*(.+)/); + + const item = { + test: parseInt(num, 10), + name: name.trim(), + expected: expected.trim(), + result, + category: categorizeItem(result, reasonMatch?.[1], blockedByMatch?.[1]), + }; + if (reasonMatch) item.reason = reasonMatch[1].trim(); + if (blockedByMatch) item.blocked_by = blockedByMatch[1].trim(); + items.push(item); + } + } + return items; +} + +function parseVerificationItems(content, status) { + const items = []; + if (status === 'human_needed') { + // Extract from human_verification section — look for numbered items or table rows + const hvSection = content.match(/##\s*Human Verification.*?\n([\s\S]*?)(?=\n##\s|\n---\s|$)/i); + if (hvSection) { + const lines = hvSection[1].split('\n'); + for (const line of lines) { + // Match table rows: | N | description | ... | + const tableMatch = line.match(/\|\s*(\d+)\s*\|\s*([^|]+)/); + // Match bullet items: - description + const bulletMatch = line.match(/^[-*]\s+(.+)/); + // Match numbered items: 1. description + const numberedMatch = line.match(/^(\d+)\.\s+(.+)/); + + if (tableMatch) { + items.push({ + test: parseInt(tableMatch[1], 10), + name: tableMatch[2].trim(), + result: 'human_needed', + category: 'human_uat', + }); + } else if (numberedMatch) { + items.push({ + test: parseInt(numberedMatch[1], 10), + name: numberedMatch[2].trim(), + result: 'human_needed', + category: 'human_uat', + }); + } else if (bulletMatch && bulletMatch[1].length > 10) { + items.push({ + name: bulletMatch[1].trim(), + result: 'human_needed', + category: 'human_uat', + }); + } + } + } + } + // gaps_found items are already handled by plan-phase --gaps pipeline + return items; +} + +function categorizeItem(result, reason, blockedBy) { + if (result === 'blocked' || blockedBy) { + if (blockedBy) { + if (/server/i.test(blockedBy)) return 'server_blocked'; + if (/device|physical/i.test(blockedBy)) return 'device_needed'; + if (/build|release|preview/i.test(blockedBy)) return 'build_needed'; + if (/third.party|twilio|stripe/i.test(blockedBy)) return 'third_party'; + } + return 'blocked'; + } + if (result === 'skipped') { + if (reason) { + if (/server|not running|not available/i.test(reason)) return 'server_blocked'; + if (/simulator|physical|device/i.test(reason)) return 'device_needed'; + if (/build|release|preview/i.test(reason)) return 'build_needed'; + } + return 'skipped_unresolved'; + } + if (result === 'pending') return 'pending'; + if (result === 'human_needed') return 'human_uat'; + return 'unknown'; +} + +module.exports = { cmdAuditUat }; diff --git a/get-shit-done/templates/UAT.md b/get-shit-done/templates/UAT.md index 5e2118ed2..fd2345a98 100644 --- a/get-shit-done/templates/UAT.md +++ b/get-shit-done/templates/UAT.md @@ -8,7 +8,7 @@ Template for `.planning/phases/XX-name/{phase_num}-UAT.md` — persistent UAT se ```markdown --- -status: testing | complete | diagnosed +status: testing | partial | complete | diagnosed phase: XX-name source: [list of SUMMARY.md files tested] started: [ISO timestamp] @@ -45,6 +45,12 @@ expected: [observable behavior] result: skipped reason: [why skipped] +### 5. [Test Name] +expected: [observable behavior] +result: blocked +blocked_by: server | physical-device | release-build | third-party | prior-phase +reason: [why blocked] + ... ## Summary @@ -54,6 +60,7 @@ passed: [N] issues: [N] pending: [N] skipped: [N] +blocked: [N] ## Gaps @@ -74,7 +81,7 @@ skipped: [N] **Frontmatter:** -- `status`: OVERWRITE - "testing" or "complete" +- `status`: OVERWRITE - "testing", "partial", or "complete" - `phase`: IMMUTABLE - set on creation - `source`: IMMUTABLE - SUMMARY files being tested - `started`: IMMUTABLE - set on creation @@ -87,9 +94,10 @@ skipped: [N] **Tests:** - Each test: OVERWRITE result field when user responds -- `result` values: [pending], pass, issue, skipped +- `result` values: [pending], pass, issue, skipped, blocked - If issue: add `reported` (verbatim) and `severity` (inferred) - If skipped: add `reason` if provided +- If blocked: add `blocked_by` (tag) and `reason` (if provided) **Summary:** - OVERWRITE counts after each response @@ -156,6 +164,16 @@ skipped: [N] - Commit file - Present summary with next steps +**Partial completion:** +- status → "partial" (if pending, blocked, or unresolved skipped tests remain) +- Current Test → "[testing paused — {N} items outstanding]" +- Commit file +- Present summary with outstanding items highlighted + +**Resuming partial session:** +- `/gsd:verify-work {phase}` picks up from first pending/blocked test +- When all items resolved, status advances to "complete" + **Resume after /clear:** 1. Read frontmatter → know phase and status 2. Read Current Test → know where we are diff --git a/get-shit-done/workflows/audit-uat.md b/get-shit-done/workflows/audit-uat.md new file mode 100644 index 000000000..13e95ce79 --- /dev/null +++ b/get-shit-done/workflows/audit-uat.md @@ -0,0 +1,109 @@ + +Cross-phase audit of all UAT and verification files. Finds every outstanding item (pending, skipped, blocked, human_needed), optionally verifies against the codebase to detect stale docs, and produces a prioritized human test plan. + + + + + +Run the CLI audit: + +```bash +AUDIT=$(node "$HOME/.claude/get-shit-done/bin/gsd-tools.cjs" audit-uat --raw) +``` + +Parse JSON for `results` array and `summary` object. + +If `summary.total_items` is 0: +``` +## All Clear + +No outstanding UAT or verification items found across all phases. +All tests are passing, resolved, or diagnosed with fix plans. +``` +Stop here. + + + +Group items by what's actionable NOW vs. what needs prerequisites: + +**Testable Now** (no external dependencies): +- `pending` — tests never run +- `human_uat` — human verification items +- `skipped_unresolved` — skipped without clear blocking reason + +**Needs Prerequisites:** +- `server_blocked` — needs external server running +- `device_needed` — needs physical device (not simulator) +- `build_needed` — needs release/preview build +- `third_party` — needs external service configuration + +For each item in "Testable Now", use Grep/Read to check if the underlying feature still exists in the codebase: +- If the test references a component/function that no longer exists → mark as `stale` +- If the test references code that has been significantly rewritten → mark as `needs_update` +- Otherwise → mark as `active` + + + +Present the audit report: + +``` +## UAT Audit Report + +**{total_items} outstanding items across {total_files} files in {phase_count} phases** + +### Testable Now ({count}) + +| # | Phase | Test | Description | Status | +|---|-------|------|-------------|--------| +| 1 | {phase} | {test_name} | {expected} | {active/stale/needs_update} | +... + +### Needs Prerequisites ({count}) + +| # | Phase | Test | Blocked By | Description | +|---|-------|------|------------|-------------| +| 1 | {phase} | {test_name} | {category} | {expected} | +... + +### Stale (can be closed) ({count}) + +| # | Phase | Test | Why Stale | +|---|-------|------|-----------| +| 1 | {phase} | {test_name} | {reason} | +... + +--- + +## Recommended Actions + +1. **Close stale items:** `/gsd:verify-work {phase}` — mark stale tests as resolved +2. **Run active tests:** Human UAT test plan below +3. **When prerequisites met:** Retest blocked items with `/gsd:verify-work {phase}` +``` + + + +Generate a human UAT test plan for "Testable Now" + "active" items only: + +Group by what can be tested together (same screen, same feature, same prerequisite): + +``` +## Human UAT Test Plan + +### Group 1: {category — e.g., "Billing Flow"} +Prerequisites: {what needs to be running/configured} + +1. **{Test name}** (Phase {N}) + - Navigate to: {where} + - Do: {action} + - Expected: {expected behavior} + +2. **{Test name}** (Phase {N}) + ... + +### Group 2: {category} +... +``` + + + diff --git a/get-shit-done/workflows/execute-phase.md b/get-shit-done/workflows/execute-phase.md index 552afb318..0d6c42704 100644 --- a/get-shit-done/workflows/execute-phase.md +++ b/get-shit-done/workflows/execute-phase.md @@ -525,6 +525,51 @@ grep "^status:" "$PHASE_DIR"/*-VERIFICATION.md | cut -d: -f2 | tr -d ' ' | `gaps_found` | Present gap summary, offer `/gsd:plan-phase {phase} --gaps` | **If human_needed:** + +**Step A: Persist human verification items as UAT file.** + +Create `{phase_dir}/{phase_num}-HUMAN-UAT.md` using UAT template format: + +```markdown +--- +status: partial +phase: {phase_num}-{phase_name} +source: [{phase_num}-VERIFICATION.md] +started: [now ISO] +updated: [now ISO] +--- + +## Current Test + +[awaiting human testing] + +## Tests + +{For each human_verification item from VERIFICATION.md:} + +### {N}. {item description} +expected: {expected behavior from VERIFICATION.md} +result: [pending] + +## Summary + +total: {count} +passed: 0 +issues: 0 +pending: {count} +skipped: 0 +blocked: 0 + +## Gaps +``` + +Commit the file: +```bash +node "$HOME/.claude/get-shit-done/bin/gsd-tools.cjs" commit "test({phase_num}): persist human verification items as UAT" --files "{phase_dir}/{phase_num}-HUMAN-UAT.md" +``` + +**Step B: Present to user:** + ``` ## ✓ Phase {X}: {Name} — Human Verification Required @@ -532,9 +577,15 @@ All automated checks passed. {N} items need human testing: {From VERIFICATION.md human_verification section} +Items saved to `{phase_num}-HUMAN-UAT.md` — they will appear in `/gsd:progress` and `/gsd:audit-uat`. + "approved" → continue | Report issues → gap closure ``` +**If user says "approved":** Proceed to `update_roadmap`. The HUMAN-UAT.md file persists with `status: partial` and will surface in future progress checks until the user runs `/gsd:verify-work` on it. + +**If user reports issues:** Proceed to gap closure as currently implemented. + **If gaps_found:** ``` ## ⚠ Phase {X}: {Name} — Gaps Found @@ -572,8 +623,18 @@ The CLI handles: - Updating plan count to final - Advancing STATE.md to next phase - Updating REQUIREMENTS.md traceability +- Scanning for verification debt (returns `warnings` array) -Extract from result: `next_phase`, `next_phase_name`, `is_last_phase`. +Extract from result: `next_phase`, `next_phase_name`, `is_last_phase`, `warnings`, `has_warnings`. + +**If has_warnings is true:** +``` +## Phase {X} marked complete with {N} warnings: + +{list each warning} + +These items are tracked and will appear in `/gsd:progress` and `/gsd:audit-uat`. +``` ```bash node "$HOME/.claude/get-shit-done/bin/gsd-tools.cjs" commit "docs(phase-{X}): complete phase execution" --files .planning/ROADMAP.md .planning/STATE.md .planning/REQUIREMENTS.md {phase_dir}/*-VERIFICATION.md diff --git a/get-shit-done/workflows/help.md b/get-shit-done/workflows/help.md index 76106e909..d002edf42 100644 --- a/get-shit-done/workflows/help.md +++ b/get-shit-done/workflows/help.md @@ -371,6 +371,17 @@ Capture a forward-looking idea with trigger conditions for automatic surfacing. Usage: `/gsd:plant-seed "add real-time notifications when we build the events system"` +--- + +**`/gsd:audit-uat`** +Cross-phase audit of all outstanding UAT and verification items. +- Scans every phase for pending, skipped, blocked, and human_needed items +- Cross-references against codebase to detect stale documentation +- Produces prioritized human test plan grouped by testability +- Use before starting a new milestone to clear verification debt + +Usage: `/gsd:audit-uat` + ### Milestone Auditing **`/gsd:audit-milestone [version]`** diff --git a/get-shit-done/workflows/progress.md b/get-shit-done/workflows/progress.md index 83bf5825b..c0ba54a65 100644 --- a/get-shit-done/workflows/progress.md +++ b/get-shit-done/workflows/progress.md @@ -150,17 +150,47 @@ State: "This phase has {X} plans, {Y} summaries." Check for UAT.md files with status "diagnosed" (has gaps needing fixes). ```bash -# Check for diagnosed UAT with gaps -grep -l "status: diagnosed" .planning/phases/[current-phase-dir]/*-UAT.md 2>/dev/null +# Check for diagnosed UAT with gaps or partial (incomplete) testing +grep -l "status: diagnosed\|status: partial" .planning/phases/[current-phase-dir]/*-UAT.md 2>/dev/null ``` Track: - `uat_with_gaps`: UAT.md files with status "diagnosed" (gaps need fixing) +- `uat_partial`: UAT.md files with status "partial" (incomplete testing) + +**Step 1.6: Cross-phase health check** + +Scan ALL phases in the current milestone for outstanding verification debt using the CLI (which respects milestone boundaries via `getMilestonePhaseFilter`): + +```bash +DEBT=$(node "$HOME/.claude/get-shit-done/bin/gsd-tools.cjs" audit-uat --raw 2>/dev/null) +``` + +Parse JSON for `summary.total_items` and `summary.total_files`. + +Track: `outstanding_debt` — `summary.total_items` from the audit. + +**If outstanding_debt > 0:** Add a warning section to the progress report output (in the `report` step), placed between "## What's Next" and the route suggestion: + +```markdown +## Verification Debt ({N} files across prior phases) + +| Phase | File | Issue | +|-------|------|-------| +| {phase} | {filename} | {pending_count} pending, {skipped_count} skipped, {blocked_count} blocked | +| {phase} | {filename} | human_needed — {count} items | + +Review: `/gsd:audit-uat` — full cross-phase audit +Resume testing: `/gsd:verify-work {phase}` — retest specific phase +``` + +This is a WARNING, not a blocker — routing proceeds normally. The debt is visible so the user can make an informed choice. **Step 2: Route based on counts** | Condition | Meaning | Action | |-----------|---------|--------| +| uat_partial > 0 | UAT testing incomplete | Go to **Route E.2** | | uat_with_gaps > 0 | UAT gaps need fix plans | Go to **Route E** | | summaries < plans | Unexecuted plans exist | Go to **Route A** | | summaries = plans AND plans > 0 | Phase complete | Go to Step 3 | @@ -260,6 +290,32 @@ UAT.md exists with gaps (diagnosed issues). User needs to plan fixes. --- +**Route E.2: UAT testing incomplete (partial)** + +UAT.md exists with `status: partial` — testing session ended before all items resolved. + +``` +--- + +## Incomplete UAT Testing + +**{phase_num}-UAT.md** has {N} unresolved tests (pending, blocked, or skipped). + +`/gsd:verify-work {phase}` — resume testing from where you left off + +`/clear` first → fresh context window + +--- + +**Also available:** +- `/gsd:audit-uat` — full cross-phase UAT audit +- `/gsd:execute-phase {phase}` — execute phase plans + +--- +``` + +--- + **Step 3: Check milestone status (only when phase complete)** Read ROADMAP.md and identify: diff --git a/get-shit-done/workflows/transition.md b/get-shit-done/workflows/transition.md index d3073cb6a..dec8af17a 100644 --- a/get-shit-done/workflows/transition.md +++ b/get-shit-done/workflows/transition.md @@ -74,6 +74,30 @@ cat .planning/config.json 2>/dev/null +**Check for verification debt in this phase:** + +```bash +# Count outstanding items in current phase +OUTSTANDING="" +for f in .planning/phases/XX-current/*-UAT.md .planning/phases/XX-current/*-VERIFICATION.md; do + [ -f "$f" ] || continue + grep -q "result: pending\|result: blocked\|status: partial\|status: human_needed\|status: diagnosed" "$f" && OUTSTANDING="$OUTSTANDING\n$(basename $f)" +done +``` + +**If OUTSTANDING is not empty:** + +Append to the completion confirmation message (regardless of mode): + +``` +Outstanding verification items in this phase: +{list filenames} + +These will carry forward as debt. Review: `/gsd:audit-uat` +``` + +This does NOT block transition — it ensures the user sees the debt before confirming. + **If all plans complete:** diff --git a/get-shit-done/workflows/verify-work.md b/get-shit-done/workflows/verify-work.md index deed81d19..7cade7f32 100644 --- a/get-shit-done/workflows/verify-work.md +++ b/get-shit-done/workflows/verify-work.md @@ -231,6 +231,29 @@ result: skipped reason: [user's reason if provided] ``` +**If response indicates blocked:** +- "blocked", "can't test - server not running", "need physical device", "need release build" +- Or any response containing: "server", "blocked", "not running", "physical device", "release build" + +Infer blocked_by tag from response: +- Contains: server, not running, gateway, API → `server` +- Contains: physical, device, hardware, real phone → `physical-device` +- Contains: release, preview, build, EAS → `release-build` +- Contains: stripe, twilio, third-party, configure → `third-party` +- Contains: depends on, prior phase, prerequisite → `prior-phase` +- Default: `other` + +Update Tests section: +``` +### {N}. {name} +expected: {expected} +result: blocked +blocked_by: {inferred tag} +reason: "{verbatim user response}" +``` + +Note: Blocked tests do NOT go into the Gaps section (they aren't code issues — they're prerequisite gates). + **If response is anything else:** - Treat as issue description @@ -293,8 +316,24 @@ Proceed to `present_test`. **Complete testing and commit:** +**Determine final status:** + +Count results: +- `pending_count`: tests with `result: [pending]` +- `blocked_count`: tests with `result: blocked` +- `skipped_no_reason`: tests with `result: skipped` and no `reason` field + +``` +if pending_count > 0 OR blocked_count > 0 OR skipped_no_reason > 0: + status: partial + # Session ended but not all tests resolved +else: + status: complete + # All tests have a definitive result (pass, issue, or skipped-with-reason) +``` + Update frontmatter: -- status: complete +- status: {computed status} - updated: [now] Clear Current Test section: diff --git a/tests/copilot-install.test.cjs b/tests/copilot-install.test.cjs index 249d8a89a..f82ecdac7 100644 --- a/tests/copilot-install.test.cjs +++ b/tests/copilot-install.test.cjs @@ -625,7 +625,7 @@ describe('copyCommandsAsCopilotSkills', () => { // Count gsd-* directories — should be 31 const dirs = fs.readdirSync(tempDir, { withFileTypes: true }) .filter(e => e.isDirectory() && e.name.startsWith('gsd-')); - assert.strictEqual(dirs.length, 46, `expected 46 skill folders, got ${dirs.length}`); + assert.strictEqual(dirs.length, 47, `expected 47 skill folders, got ${dirs.length}`); } finally { fs.rmSync(tempDir, { recursive: true }); } @@ -1119,7 +1119,7 @@ const { execFileSync } = require('child_process'); const crypto = require('crypto'); const INSTALL_PATH = path.join(__dirname, '..', 'bin', 'install.js'); -const EXPECTED_SKILLS = 46; +const EXPECTED_SKILLS = 47; const EXPECTED_AGENTS = 16; function runCopilotInstall(cwd) { diff --git a/tests/uat.test.cjs b/tests/uat.test.cjs new file mode 100644 index 000000000..3c3fbc424 --- /dev/null +++ b/tests/uat.test.cjs @@ -0,0 +1,326 @@ +/** + * GSD Tools Tests - UAT Audit + */ + +const { test, describe, beforeEach, afterEach } = require('node:test'); +const assert = require('node:assert'); +const fs = require('fs'); +const path = require('path'); +const { runGsdTools, createTempProject, cleanup } = require('./helpers.cjs'); + +describe('audit-uat command', () => { + let tmpDir; + + beforeEach(() => { + tmpDir = createTempProject(); + }); + + afterEach(() => { + cleanup(tmpDir); + }); + + test('returns empty results when no UAT files exist', () => { + // Create a phase directory with no UAT files + fs.mkdirSync(path.join(tmpDir, '.planning', 'phases', '01-foundation'), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, '.planning', 'phases', '01-foundation', '.gitkeep'), ''); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + assert.deepStrictEqual(output.results, []); + assert.strictEqual(output.summary.total_items, 0); + assert.strictEqual(output.summary.total_files, 0); + }); + + test('detects UAT with pending items', () => { + const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation'); + fs.mkdirSync(phaseDir, { recursive: true }); + + fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), `--- +status: testing +phase: 01-foundation +started: 2025-01-01T00:00:00Z +updated: 2025-01-01T00:00:00Z +--- + +## Tests + +### 1. Login Form +expected: Form displays with email and password fields +result: pass + +### 2. Submit Button +expected: Submitting shows loading state +result: pending +`); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + assert.strictEqual(output.summary.total_items, 1); + assert.strictEqual(output.results[0].phase, '01'); + assert.strictEqual(output.results[0].items[0].result, 'pending'); + assert.strictEqual(output.results[0].items[0].category, 'pending'); + assert.strictEqual(output.results[0].items[0].name, 'Submit Button'); + }); + + test('detects UAT with blocked items and categorizes blocked_by', () => { + const phaseDir = path.join(tmpDir, '.planning', 'phases', '02-api'); + fs.mkdirSync(phaseDir, { recursive: true }); + + fs.writeFileSync(path.join(phaseDir, '02-UAT.md'), `--- +status: partial +phase: 02-api +started: 2025-01-01T00:00:00Z +updated: 2025-01-01T00:00:00Z +--- + +## Tests + +### 1. API Health Check +expected: Returns 200 OK +result: blocked +blocked_by: server +reason: Server not running locally +`); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + assert.strictEqual(output.summary.total_items, 1); + assert.strictEqual(output.results[0].items[0].result, 'blocked'); + assert.strictEqual(output.results[0].items[0].category, 'server_blocked'); + assert.strictEqual(output.results[0].items[0].blocked_by, 'server'); + }); + + test('detects false completion (complete status with pending items)', () => { + const phaseDir = path.join(tmpDir, '.planning', 'phases', '03-ui'); + fs.mkdirSync(phaseDir, { recursive: true }); + + fs.writeFileSync(path.join(phaseDir, '03-UAT.md'), `--- +status: complete +phase: 03-ui +started: 2025-01-01T00:00:00Z +updated: 2025-01-01T00:00:00Z +--- + +## Tests + +### 1. Dashboard Layout +expected: Cards render in grid +result: pass + +### 2. Mobile Responsive +expected: Grid collapses to single column on mobile +result: pending +`); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + assert.strictEqual(output.summary.total_items, 1); + assert.strictEqual(output.results[0].status, 'complete'); + assert.strictEqual(output.results[0].items[0].result, 'pending'); + }); + + test('extracts human_needed items from VERIFICATION files', () => { + const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth'); + fs.mkdirSync(phaseDir, { recursive: true }); + + fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), `--- +status: human_needed +phase: 04-auth +--- + +## Automated Checks + +All passed. + +## Human Verification + +1. Test SSO login with Google account +2. Test password reset flow end-to-end +3. Verify MFA enrollment on new device +`); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + assert.strictEqual(output.summary.total_items, 3); + assert.strictEqual(output.results[0].type, 'verification'); + assert.strictEqual(output.results[0].status, 'human_needed'); + assert.strictEqual(output.results[0].items[0].category, 'human_uat'); + assert.strictEqual(output.results[0].items[0].name, 'Test SSO login with Google account'); + }); + + test('scans and aggregates across multiple phases', () => { + // Phase 1 with pending + const phase1 = path.join(tmpDir, '.planning', 'phases', '01-foundation'); + fs.mkdirSync(phase1, { recursive: true }); + fs.writeFileSync(path.join(phase1, '01-UAT.md'), `--- +status: partial +phase: 01-foundation +started: 2025-01-01T00:00:00Z +updated: 2025-01-01T00:00:00Z +--- + +## Tests + +### 1. Test A +expected: Works +result: pending +`); + + // Phase 2 with blocked + const phase2 = path.join(tmpDir, '.planning', 'phases', '02-api'); + fs.mkdirSync(phase2, { recursive: true }); + fs.writeFileSync(path.join(phase2, '02-UAT.md'), `--- +status: partial +phase: 02-api +started: 2025-01-01T00:00:00Z +updated: 2025-01-01T00:00:00Z +--- + +## Tests + +### 1. Test B +expected: Responds +result: blocked +blocked_by: server + +### 2. Test C +expected: Returns data +result: skipped +reason: device not available +`); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + assert.strictEqual(output.summary.total_files, 2); + assert.strictEqual(output.summary.total_items, 3); + assert.strictEqual(output.summary.by_phase['01'], 1); + assert.strictEqual(output.summary.by_phase['02'], 2); + }); + + test('milestone scoping filters phases to current milestone', () => { + // Create a ROADMAP.md that only references Phase 2 + fs.writeFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), `# Roadmap + +### Phase 2: API Layer +**Goal:** Build API +`); + + // Phase 1 (not in current milestone) with pending + const phase1 = path.join(tmpDir, '.planning', 'phases', '01-foundation'); + fs.mkdirSync(phase1, { recursive: true }); + fs.writeFileSync(path.join(phase1, '01-UAT.md'), `--- +status: partial +phase: 01-foundation +started: 2025-01-01T00:00:00Z +updated: 2025-01-01T00:00:00Z +--- + +## Tests + +### 1. Old Test +expected: Old behavior +result: pending +`); + + // Phase 2 (in current milestone) with pending + const phase2 = path.join(tmpDir, '.planning', 'phases', '02-api'); + fs.mkdirSync(phase2, { recursive: true }); + fs.writeFileSync(path.join(phase2, '02-UAT.md'), `--- +status: partial +phase: 02-api +started: 2025-01-01T00:00:00Z +updated: 2025-01-01T00:00:00Z +--- + +## Tests + +### 1. New Test +expected: New behavior +result: pending +`); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + // Only Phase 2 should be included (Phase 1 not in ROADMAP) + assert.strictEqual(output.summary.total_files, 1); + assert.strictEqual(output.results[0].phase, '02'); + }); + + test('summary by_category counts are correct', () => { + const phaseDir = path.join(tmpDir, '.planning', 'phases', '05-billing'); + fs.mkdirSync(phaseDir, { recursive: true }); + + fs.writeFileSync(path.join(phaseDir, '05-UAT.md'), `--- +status: partial +phase: 05-billing +started: 2025-01-01T00:00:00Z +updated: 2025-01-01T00:00:00Z +--- + +## Tests + +### 1. Payment Form +expected: Stripe elements load +result: pending + +### 2. Webhook Handler +expected: Processes payment events +result: blocked +blocked_by: third-party Stripe + +### 3. Invoice PDF +expected: Generates downloadable PDF +result: skipped +reason: needs release build + +### 4. Refund Flow +expected: Processes refund +result: pending +`); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + assert.strictEqual(output.summary.total_items, 4); + assert.strictEqual(output.summary.by_category.pending, 2); + assert.strictEqual(output.summary.by_category.third_party, 1); + assert.strictEqual(output.summary.by_category.build_needed, 1); + }); + + test('ignores VERIFICATION files without human_needed or gaps_found status', () => { + const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation'); + fs.mkdirSync(phaseDir, { recursive: true }); + + fs.writeFileSync(path.join(phaseDir, '01-VERIFICATION.md'), `--- +status: passed +phase: 01-foundation +--- + +## Results + +All checks passed. +`); + + const result = runGsdTools('audit-uat --raw', tmpDir); + assert.ok(result.success, `Command failed: ${result.error}`); + + const output = JSON.parse(result.output); + assert.strictEqual(output.summary.total_items, 0); + assert.strictEqual(output.summary.total_files, 0); + }); +});