Merge pull request #1212 from jecanore/feat/audit-uat
feat: add verification debt tracking and /gsd:audit-uat command
This commit is contained in:
Tom Boucher
@@ -625,7 +625,7 @@ describe('copyCommandsAsCopilotSkills', () => {
|
||||
// Count gsd-* directories — should be 31
|
||||
const dirs = fs.readdirSync(tempDir, { withFileTypes: true })
|
||||
.filter(e => e.isDirectory() && e.name.startsWith('gsd-'));
|
||||
assert.strictEqual(dirs.length, 46, `expected 46 skill folders, got ${dirs.length}`);
|
||||
assert.strictEqual(dirs.length, 47, `expected 47 skill folders, got ${dirs.length}`);
|
||||
} finally {
|
||||
fs.rmSync(tempDir, { recursive: true });
|
||||
}
|
||||
@@ -1119,7 +1119,7 @@ const { execFileSync } = require('child_process');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const INSTALL_PATH = path.join(__dirname, '..', 'bin', 'install.js');
|
||||
const EXPECTED_SKILLS = 46;
|
||||
const EXPECTED_SKILLS = 47;
|
||||
const EXPECTED_AGENTS = 16;
|
||||
|
||||
function runCopilotInstall(cwd) {
|
||||
|
||||
326
tests/uat.test.cjs
Normal file
326
tests/uat.test.cjs
Normal file
@@ -0,0 +1,326 @@
|
||||
/**
|
||||
* GSD Tools Tests - UAT Audit
|
||||
*/
|
||||
|
||||
const { test, describe, beforeEach, afterEach } = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { runGsdTools, createTempProject, cleanup } = require('./helpers.cjs');
|
||||
|
||||
describe('audit-uat command', () => {
|
||||
let tmpDir;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = createTempProject();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup(tmpDir);
|
||||
});
|
||||
|
||||
test('returns empty results when no UAT files exist', () => {
|
||||
// Create a phase directory with no UAT files
|
||||
fs.mkdirSync(path.join(tmpDir, '.planning', 'phases', '01-foundation'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpDir, '.planning', 'phases', '01-foundation', '.gitkeep'), '');
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.deepStrictEqual(output.results, []);
|
||||
assert.strictEqual(output.summary.total_items, 0);
|
||||
assert.strictEqual(output.summary.total_files, 0);
|
||||
});
|
||||
|
||||
test('detects UAT with pending items', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-UAT.md'), `---
|
||||
status: testing
|
||||
phase: 01-foundation
|
||||
started: 2025-01-01T00:00:00Z
|
||||
updated: 2025-01-01T00:00:00Z
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
### 1. Login Form
|
||||
expected: Form displays with email and password fields
|
||||
result: pass
|
||||
|
||||
### 2. Submit Button
|
||||
expected: Submitting shows loading state
|
||||
result: pending
|
||||
`);
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1);
|
||||
assert.strictEqual(output.results[0].phase, '01');
|
||||
assert.strictEqual(output.results[0].items[0].result, 'pending');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'pending');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'Submit Button');
|
||||
});
|
||||
|
||||
test('detects UAT with blocked items and categorizes blocked_by', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '02-api');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '02-UAT.md'), `---
|
||||
status: partial
|
||||
phase: 02-api
|
||||
started: 2025-01-01T00:00:00Z
|
||||
updated: 2025-01-01T00:00:00Z
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
### 1. API Health Check
|
||||
expected: Returns 200 OK
|
||||
result: blocked
|
||||
blocked_by: server
|
||||
reason: Server not running locally
|
||||
`);
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1);
|
||||
assert.strictEqual(output.results[0].items[0].result, 'blocked');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'server_blocked');
|
||||
assert.strictEqual(output.results[0].items[0].blocked_by, 'server');
|
||||
});
|
||||
|
||||
test('detects false completion (complete status with pending items)', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '03-ui');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '03-UAT.md'), `---
|
||||
status: complete
|
||||
phase: 03-ui
|
||||
started: 2025-01-01T00:00:00Z
|
||||
updated: 2025-01-01T00:00:00Z
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
### 1. Dashboard Layout
|
||||
expected: Cards render in grid
|
||||
result: pass
|
||||
|
||||
### 2. Mobile Responsive
|
||||
expected: Grid collapses to single column on mobile
|
||||
result: pending
|
||||
`);
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 1);
|
||||
assert.strictEqual(output.results[0].status, 'complete');
|
||||
assert.strictEqual(output.results[0].items[0].result, 'pending');
|
||||
});
|
||||
|
||||
test('extracts human_needed items from VERIFICATION files', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '04-auth');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '04-VERIFICATION.md'), `---
|
||||
status: human_needed
|
||||
phase: 04-auth
|
||||
---
|
||||
|
||||
## Automated Checks
|
||||
|
||||
All passed.
|
||||
|
||||
## Human Verification
|
||||
|
||||
1. Test SSO login with Google account
|
||||
2. Test password reset flow end-to-end
|
||||
3. Verify MFA enrollment on new device
|
||||
`);
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 3);
|
||||
assert.strictEqual(output.results[0].type, 'verification');
|
||||
assert.strictEqual(output.results[0].status, 'human_needed');
|
||||
assert.strictEqual(output.results[0].items[0].category, 'human_uat');
|
||||
assert.strictEqual(output.results[0].items[0].name, 'Test SSO login with Google account');
|
||||
});
|
||||
|
||||
test('scans and aggregates across multiple phases', () => {
|
||||
// Phase 1 with pending
|
||||
const phase1 = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phase1, { recursive: true });
|
||||
fs.writeFileSync(path.join(phase1, '01-UAT.md'), `---
|
||||
status: partial
|
||||
phase: 01-foundation
|
||||
started: 2025-01-01T00:00:00Z
|
||||
updated: 2025-01-01T00:00:00Z
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
### 1. Test A
|
||||
expected: Works
|
||||
result: pending
|
||||
`);
|
||||
|
||||
// Phase 2 with blocked
|
||||
const phase2 = path.join(tmpDir, '.planning', 'phases', '02-api');
|
||||
fs.mkdirSync(phase2, { recursive: true });
|
||||
fs.writeFileSync(path.join(phase2, '02-UAT.md'), `---
|
||||
status: partial
|
||||
phase: 02-api
|
||||
started: 2025-01-01T00:00:00Z
|
||||
updated: 2025-01-01T00:00:00Z
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
### 1. Test B
|
||||
expected: Responds
|
||||
result: blocked
|
||||
blocked_by: server
|
||||
|
||||
### 2. Test C
|
||||
expected: Returns data
|
||||
result: skipped
|
||||
reason: device not available
|
||||
`);
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_files, 2);
|
||||
assert.strictEqual(output.summary.total_items, 3);
|
||||
assert.strictEqual(output.summary.by_phase['01'], 1);
|
||||
assert.strictEqual(output.summary.by_phase['02'], 2);
|
||||
});
|
||||
|
||||
test('milestone scoping filters phases to current milestone', () => {
|
||||
// Create a ROADMAP.md that only references Phase 2
|
||||
fs.writeFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), `# Roadmap
|
||||
|
||||
### Phase 2: API Layer
|
||||
**Goal:** Build API
|
||||
`);
|
||||
|
||||
// Phase 1 (not in current milestone) with pending
|
||||
const phase1 = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phase1, { recursive: true });
|
||||
fs.writeFileSync(path.join(phase1, '01-UAT.md'), `---
|
||||
status: partial
|
||||
phase: 01-foundation
|
||||
started: 2025-01-01T00:00:00Z
|
||||
updated: 2025-01-01T00:00:00Z
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
### 1. Old Test
|
||||
expected: Old behavior
|
||||
result: pending
|
||||
`);
|
||||
|
||||
// Phase 2 (in current milestone) with pending
|
||||
const phase2 = path.join(tmpDir, '.planning', 'phases', '02-api');
|
||||
fs.mkdirSync(phase2, { recursive: true });
|
||||
fs.writeFileSync(path.join(phase2, '02-UAT.md'), `---
|
||||
status: partial
|
||||
phase: 02-api
|
||||
started: 2025-01-01T00:00:00Z
|
||||
updated: 2025-01-01T00:00:00Z
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
### 1. New Test
|
||||
expected: New behavior
|
||||
result: pending
|
||||
`);
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
// Only Phase 2 should be included (Phase 1 not in ROADMAP)
|
||||
assert.strictEqual(output.summary.total_files, 1);
|
||||
assert.strictEqual(output.results[0].phase, '02');
|
||||
});
|
||||
|
||||
test('summary by_category counts are correct', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '05-billing');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '05-UAT.md'), `---
|
||||
status: partial
|
||||
phase: 05-billing
|
||||
started: 2025-01-01T00:00:00Z
|
||||
updated: 2025-01-01T00:00:00Z
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
### 1. Payment Form
|
||||
expected: Stripe elements load
|
||||
result: pending
|
||||
|
||||
### 2. Webhook Handler
|
||||
expected: Processes payment events
|
||||
result: blocked
|
||||
blocked_by: third-party Stripe
|
||||
|
||||
### 3. Invoice PDF
|
||||
expected: Generates downloadable PDF
|
||||
result: skipped
|
||||
reason: needs release build
|
||||
|
||||
### 4. Refund Flow
|
||||
expected: Processes refund
|
||||
result: pending
|
||||
`);
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 4);
|
||||
assert.strictEqual(output.summary.by_category.pending, 2);
|
||||
assert.strictEqual(output.summary.by_category.third_party, 1);
|
||||
assert.strictEqual(output.summary.by_category.build_needed, 1);
|
||||
});
|
||||
|
||||
test('ignores VERIFICATION files without human_needed or gaps_found status', () => {
|
||||
const phaseDir = path.join(tmpDir, '.planning', 'phases', '01-foundation');
|
||||
fs.mkdirSync(phaseDir, { recursive: true });
|
||||
|
||||
fs.writeFileSync(path.join(phaseDir, '01-VERIFICATION.md'), `---
|
||||
status: passed
|
||||
phase: 01-foundation
|
||||
---
|
||||
|
||||
## Results
|
||||
|
||||
All checks passed.
|
||||
`);
|
||||
|
||||
const result = runGsdTools('audit-uat --raw', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.summary.total_items, 0);
|
||||
assert.strictEqual(output.summary.total_files, 0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user