'use strict'; // allow-test-rule: pending-migration-to-typed-ir [#2974] // Tracked in #2974 for migration to typed-IR assertions per CONTRIBUTING.md // "Prohibited: Raw Text Matching on Test Outputs". Per-file review may // reclassify some entries as source-text-is-the-product during migration. /** * Planner Language Regression Tests (#2091, #2092) * * Prevents time-based reasoning and complexity-as-scope-justification * from leaking back into planning artifacts via future PRs. * * These tests scan agent definitions, workflow files, and references * for prohibited patterns that import human-world constraints into * an AI execution context where those constraints do not exist. */ const { test, describe } = require('node:test'); const assert = require('node:assert/strict'); const fs = require('fs'); const path = require('path'); const ROOT = path.join(__dirname, '..'); const AGENTS_DIR = path.join(ROOT, 'agents'); const WORKFLOWS_DIR = path.join(ROOT, 'get-shit-done', 'workflows'); const REFERENCES_DIR = path.join(ROOT, 'get-shit-done', 'references'); const TEMPLATES_DIR = path.join(ROOT, 'get-shit-done', 'templates'); /** * Collect all .md files from a directory (non-recursive). */ function mdFiles(dir) { if (!fs.existsSync(dir)) return []; return fs.readdirSync(dir) .filter(f => f.endsWith('.md')) .map(f => ({ name: f, path: path.join(dir, f) })); } /** * Collect all .md files recursively. */ function mdFilesRecursive(dir) { if (!fs.existsSync(dir)) return []; const results = []; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const full = path.join(dir, entry.name); if (entry.isDirectory()) { results.push(...mdFilesRecursive(full)); } else if (entry.name.endsWith('.md')) { results.push({ name: entry.name, path: full }); } } return results; } /** * Files that define planning behavior — agents, workflows, references. * These are the files where time-based and complexity-based scope * reasoning must never appear. */ const PLANNING_FILES = [ ...mdFiles(AGENTS_DIR), ...mdFiles(WORKFLOWS_DIR), ...mdFiles(REFERENCES_DIR), ...mdFilesRecursive(TEMPLATES_DIR), ]; // -- Prohibited patterns -- /** * Time-based task sizing patterns. * Matches "15-60 minutes", "X minutes Claude execution time", etc. * Does NOT match operational timeouts ("timeout: 5 minutes"), * API docs examples ("100 requests per 15 minutes"), * or human-readable timeout descriptions in workflow execution steps. */ const TIME_SIZING_PATTERNS = [ // "N-M minutes" in task sizing context (not timeout context) /each task[:\s]*\*?\*?\d+[-–]\d+\s*min/i, // "minutes Claude execution time" or "minutes execution time" /minutes?\s+(claude\s+)?execution\s+time/i, // Duration-based sizing table rows: "< 15 min", "15-60 min", "> 60 min" /[<>]\s*\d+\s*min\s*\|/i, ]; /** * Complexity-as-scope-justification patterns. * Matches "too complex to implement", "challenging feature", etc. * Does NOT match legitimate uses like: * - "complex domains" in research/discovery context (describing what to research) * - "non-trivial" in verification context (confirming substantive code exists) * - "challenging" in user-profiling context (quoting user reactions) */ const COMPLEXITY_SCOPE_PATTERNS = [ // "too complex to" — always a scope-reduction justification /too\s+complex\s+to/i, // "too difficult" — always a scope-reduction justification /too\s+difficult/i, // "is too complex for" — scope justification (e.g. "Phase X is too complex for") /is\s+too\s+complex\s+for/i, ]; /** * Files allowed to contain certain patterns because they document * the prohibition itself, or use the terms in non-scope-reduction context. */ const ALLOWLIST = { // Plan-checker scans FOR these patterns — it's a detection list, not usage 'gsd-plan-checker.md': ['complexity_scope', 'time_sizing'], // Planner defines the prohibition and the authority limits — uses terms to explain what NOT to do 'gsd-planner.md': ['complexity_scope'], // Debugger uses "30+ minutes" as anti-pattern detection, not task sizing 'gsd-debugger.md': ['time_sizing'], // Doc-writer uses "15 minutes" in API rate limit example, "2 minutes" for doc quality 'gsd-doc-writer.md': ['time_sizing'], // Discovery-phase uses time for level descriptions (operational, not scope) 'discovery-phase.md': ['time_sizing'], // Explore uses "~30 seconds" as operational estimate 'explore.md': ['time_sizing'], // Review uses "up to 5 minutes" for CodeRabbit timeout 'review.md': ['time_sizing'], // Fast uses "under 2 minutes wall time" as operational constraint 'fast.md': ['time_sizing'], // Execute-phase uses "timeout: 5 minutes" for test runner 'execute-phase.md': ['time_sizing'], // Verify-phase uses "timeout: 5 minutes" for test runner 'verify-phase.md': ['time_sizing'], // Map-codebase documents subagent_timeout 'map-codebase.md': ['time_sizing'], // Help documents CodeRabbit timing 'help.md': ['time_sizing'], }; function isAllowlisted(fileName, category) { const entry = ALLOWLIST[fileName]; return entry && entry.includes(category); } // -- Tests -- describe('Planner language regression — time-based task sizing (#2092)', () => { for (const file of PLANNING_FILES) { test(`${file.name} must not use time-based task sizing`, () => { if (isAllowlisted(file.name, 'time_sizing')) return; const content = fs.readFileSync(file.path, 'utf-8'); for (const pattern of TIME_SIZING_PATTERNS) { const match = content.match(pattern); assert.ok( !match, [ `${file.name} contains time-based task sizing: "${match?.[0]}"`, 'Task sizing must use context-window percentage, not time units.', 'See issue #2092 for rationale.', ].join('\n') ); } }); } }); describe('Planner language regression — complexity-as-scope-justification (#2092)', () => { for (const file of PLANNING_FILES) { test(`${file.name} must not use complexity to justify scope reduction`, () => { if (isAllowlisted(file.name, 'complexity_scope')) return; const content = fs.readFileSync(file.path, 'utf-8'); for (const pattern of COMPLEXITY_SCOPE_PATTERNS) { const match = content.match(pattern); assert.ok( !match, [ `${file.name} contains complexity-as-scope-justification: "${match?.[0]}"`, 'Scope decisions must be based on context cost, missing information,', 'or dependency conflicts — not perceived difficulty.', 'See issue #2092 for rationale.', ].join('\n') ); } }); } }); describe('gsd-planner.md — required structural sections (#2091, #2092)', () => { let plannerContent; test('planner file exists and is readable', () => { const plannerPath = path.join(AGENTS_DIR, 'gsd-planner.md'); assert.ok(fs.existsSync(plannerPath), 'agents/gsd-planner.md must exist'); plannerContent = fs.readFileSync(plannerPath, 'utf-8'); }); test('contains section', () => { assert.ok( plannerContent.includes(''), 'gsd-planner.md must contain a section defining what the planner cannot decide' ); }); test('authority limits prohibit difficulty-based scope decisions', () => { assert.ok( plannerContent.includes('The planner has no authority to'), 'planner_authority_limits must explicitly state what the planner cannot decide' ); }); test('authority limits list three legitimate split reasons: context cost, missing info, dependency', () => { assert.ok( plannerContent.includes('Context cost') || plannerContent.includes('context cost'), 'authority limits must list context cost as a legitimate split reason' ); assert.ok( plannerContent.includes('Missing information') || plannerContent.includes('missing information'), 'authority limits must list missing information as a legitimate split reason' ); assert.ok( plannerContent.includes('Dependency conflict') || plannerContent.includes('dependency conflict'), 'authority limits must list dependency conflict as a legitimate split reason' ); }); test('task sizing uses context percentage, not time units', () => { assert.ok( plannerContent.includes('context consumption') || plannerContent.includes('context cost'), 'task sizing must reference context consumption, not time' ); assert.ok( !(/each task[:\s]*\*?\*?\d+[-–]\d+\s*min/i.test(plannerContent)), 'task sizing must not use minutes as sizing unit' ); }); test('contains multi-source coverage audit (not just D-XX decisions)', () => { assert.ok( plannerContent.includes('Multi-Source Coverage Audit') || plannerContent.includes('multi-source coverage audit'), 'gsd-planner.md must contain a multi-source coverage audit, not just D-XX decision matrix' ); }); test('coverage audit includes all four source types: GOAL, REQ, RESEARCH, CONTEXT', () => { // The planner file or its referenced planner-source-audit.md must define all four types. // The inline compact version uses **GOAL**, **REQ**, **RESEARCH**, **CONTEXT**. const refPath = path.join(ROOT, 'get-shit-done', 'references', 'planner-source-audit.md'); const combined = plannerContent + (fs.existsSync(refPath) ? fs.readFileSync(refPath, 'utf-8') : ''); const hasGoal = combined.includes('**GOAL**'); const hasReq = combined.includes('**REQ**'); const hasResearch = combined.includes('**RESEARCH**'); const hasContext = combined.includes('**CONTEXT**'); assert.ok(hasGoal, 'coverage audit must include GOAL source type (ROADMAP.md phase goal)'); assert.ok(hasReq, 'coverage audit must include REQ source type (REQUIREMENTS.md)'); assert.ok(hasResearch, 'coverage audit must include RESEARCH source type (RESEARCH.md)'); assert.ok(hasContext, 'coverage audit must include CONTEXT source type (CONTEXT.md decisions)'); }); test('coverage audit defines MISSING item handling with developer escalation', () => { assert.ok( plannerContent.includes('Source Audit: Unplanned Items Found') || plannerContent.includes('MISSING'), 'coverage audit must define handling for MISSING items' ); assert.ok( plannerContent.includes('Awaiting developer decision') || plannerContent.includes('developer confirmation'), 'MISSING items must escalate to developer, not be silently dropped' ); }); }); describe('plan-phase.md — source audit orchestration (#2091)', () => { let workflowContent; test('plan-phase workflow exists and is readable', () => { const workflowPath = path.join(WORKFLOWS_DIR, 'plan-phase.md'); assert.ok(fs.existsSync(workflowPath), 'workflows/plan-phase.md must exist'); workflowContent = fs.readFileSync(workflowPath, 'utf-8'); }); test('step 9 handles Source Audit return from planner', () => { assert.ok( workflowContent.includes('Source Audit: Unplanned Items Found'), 'plan-phase.md step 9 must handle the Source Audit return from the planner' ); }); test('step 9c exists for source audit gap handling', () => { assert.ok( workflowContent.includes('9c') && workflowContent.includes('Source Audit'), 'plan-phase.md must have a step 9c for handling source audit gaps' ); }); test('step 9b does not use "too complex" language', () => { // Extract just step 9b content (between "## 9b" and "## 9c" or "## 10") const step9bMatch = workflowContent.match(/## 9b\.([\s\S]*?)(?=## 9c|## 10)/); if (step9bMatch) { const step9b = step9bMatch[1]; assert.ok( !step9b.includes('too complex'), 'step 9b must not use "too complex" — use context budget language instead' ); } }); test('phase split recommendation uses context budget framing', () => { assert.ok( workflowContent.includes('context budget') || workflowContent.includes('context cost'), 'phase split recommendation must be framed in terms of context budget, not complexity' ); }); }); describe('gsd-plan-checker.md — scope reduction detection includes time/complexity (#2092)', () => { let checkerContent; test('plan-checker exists and is readable', () => { const checkerPath = path.join(AGENTS_DIR, 'gsd-plan-checker.md'); assert.ok(fs.existsSync(checkerPath), 'agents/gsd-plan-checker.md must exist'); checkerContent = fs.readFileSync(checkerPath, 'utf-8'); }); test('scope reduction scan includes complexity-based justification patterns', () => { assert.ok( checkerContent.includes('too complex') || checkerContent.includes('too difficult'), 'plan-checker scope reduction scan must detect complexity-based justification language' ); }); test('scope reduction scan includes time-based justification patterns', () => { assert.ok( checkerContent.includes('would take') || checkerContent.includes('hours') || checkerContent.includes('minutes'), 'plan-checker scope reduction scan must detect time-based justification language' ); }); });