diff --git a/.changeset/humble-pandas-purr.md b/.changeset/humble-pandas-purr.md new file mode 100644 index 000000000..59f0baa02 --- /dev/null +++ b/.changeset/humble-pandas-purr.md @@ -0,0 +1,5 @@ +--- +type: Fixed +pr: 1079 +--- +`validate agents` (and `validate health`) now cross-reference the install manifest to detect manifest-backed Codex agent pair drift: when a generated `agents/gsd-*.md` / `agents/gsd-*.toml` pair has one side missing on disk, the agent is reported as incomplete and `agents_found` is `false` (previously a false-healthy `agents_found: true, missing: []`). `validate health` names the incomplete agents and recommends re-running the installer. The check no-ops when no manifest is present. (#1058) diff --git a/src/core.cts b/src/core.cts index 1e9f23288..952d29889 100644 --- a/src/core.cts +++ b/src/core.cts @@ -211,6 +211,7 @@ interface AgentsInstalledResult { agents_installed: boolean; missing_agents: string[]; installed_agents: string[]; + incomplete_agents: string[]; agents_dir: string; agent_runtime: string; } @@ -232,6 +233,7 @@ function checkAgentsInstalled(runtime?: string): AgentsInstalledResult { agents_installed: false, missing_agents: expectedAgents, installed_agents: [], + incomplete_agents: [], agents_dir: agentsDir, agent_runtime: resolvedRuntime, }; @@ -259,10 +261,61 @@ function checkAgentsInstalled(runtime?: string): AgentsInstalledResult { } } + // ── Manifest-backed completeness check ────────────────────────────────────── + // If a gsd-file-manifest.json exists alongside the agents dir (parent dir), + // verify that every manifest-tracked file for each expected agent is present + // on disk. Missing manifest-tracked files indicate an incomplete install even + // when the plain presence check above passed (e.g. .md present, .toml absent). + // If no manifest is found the check is a no-op (graceful for claude/bundled). + const incomplete: string[] = []; + const manifestPath = path.join(path.dirname(agentsDir), 'gsd-file-manifest.json'); + let manifestFiles: Record = {}; + try { + const raw = fs.readFileSync(manifestPath, 'utf8'); + const parsed: unknown = JSON.parse(raw); + if ( + parsed !== null && + typeof parsed === 'object' && + 'files' in parsed && + typeof (parsed as Record)['files'] === 'object' && + (parsed as Record)['files'] !== null + ) { + manifestFiles = (parsed as Record>)['files']; + } + } catch { + // No manifest or unreadable — completeness check is skipped + } + + if (Object.keys(manifestFiles).length > 0) { + for (const agent of expectedAgents) { + // Find all manifest keys that belong to this agent: + // key must be "agents/." with no further path segments. + const agentPrefix = `agents/${agent}.`; + const agentManifestKeys = Object.keys(manifestFiles).filter(key => { + if (!key.startsWith(agentPrefix)) return false; + const rest = key.slice(agentPrefix.length); + // rest must be a bare extension (no slashes, non-empty) + return rest.length > 0 && !rest.includes('/'); + }); + if (agentManifestKeys.length === 0) { + // Agent not tracked in manifest — skip completeness check for this agent + continue; + } + const allPresent = agentManifestKeys.every(key => { + const basename = key.slice('agents/'.length); + return fs.existsSync(path.join(agentsDir, basename)); + }); + if (!allPresent) { + incomplete.push(agent); + } + } + } + return { - agents_installed: installed.length > 0 && missing.length === 0, + agents_installed: installed.length > 0 && missing.length === 0 && incomplete.length === 0, missing_agents: missing, installed_agents: installed, + incomplete_agents: incomplete, agents_dir: agentsDir, agent_runtime: resolvedRuntime, }; diff --git a/src/verify.cts b/src/verify.cts index 94587dfed..3346ff2bb 100644 --- a/src/verify.cts +++ b/src/verify.cts @@ -1107,6 +1107,20 @@ function cmdValidateHealth( `No GSD agents found in ${agentStatus.agents_dir} — Task(subagent_type="gsd-*") will fall back to general-purpose`, `Run the GSD installer: npx ${PACKAGE_NAME}@latest`, ); + } else if ((agentStatus.incomplete_agents).length > 0 && (agentStatus.missing_agents).length === 0) { + addIssue( + 'warning', + 'W010', + `Incomplete agent installs (missing generated file): ${(agentStatus.incomplete_agents).join(', ')} — affected workflows may fall back to general-purpose`, + `Re-run the GSD installer to complete the install: npx ${PACKAGE_NAME}@latest`, + ); + } else if ((agentStatus.incomplete_agents).length > 0) { + addIssue( + 'warning', + 'W010', + `Missing ${(agentStatus.missing_agents).length} GSD agents: ${(agentStatus.missing_agents).join(', ')}; incomplete agent installs (missing generated file): ${(agentStatus.incomplete_agents).join(', ')} — affected workflows will fall back to general-purpose`, + `Run the GSD installer: npx ${PACKAGE_NAME}@latest`, + ); } else { addIssue( 'warning', @@ -1624,6 +1638,7 @@ function cmdValidateAgents(cwd: string, raw: boolean): void { agents_found: agentStatus.agents_installed, installed: agentStatus.installed_agents, missing: agentStatus.missing_agents, + incomplete: agentStatus.incomplete_agents, expected, }, raw, diff --git a/tests/agent-install-validation.test.cjs b/tests/agent-install-validation.test.cjs index 4dd8bce6f..27b71b9f0 100644 --- a/tests/agent-install-validation.test.cjs +++ b/tests/agent-install-validation.test.cjs @@ -384,3 +384,209 @@ describe('validate agents subcommand (#1371)', () => { assert.ok(output.expected.length > 0, 'Must have expected agents'); }); }); + +// ─── Bug #1058: validate agents detects manifest-backed .md/.toml pair drift ── + +describe('bug #1058: validate agents detects manifest-backed .md/.toml pair drift', () => { + // All real agents/gsd-*.md files in the repo (used to populate fixtures) + const REPO_AGENTS_DIR_1058 = path.resolve(__dirname, '..', 'agents'); + + /** + * Copy all gsd-*.md files from the repo agents dir into destDir. + */ + function copyAgentMdFiles(destDir) { + const files = fs.readdirSync(REPO_AGENTS_DIR_1058).filter(f => /^gsd-.*\.md$/.test(f)); + for (const file of files) { + const src = path.join(REPO_AGENTS_DIR_1058, file); + const dst = path.join(destDir, file); + fs.copyFileSync(src, dst); + } + } + + /** + * Build a gsd-file-manifest.json whose files map includes both .md and .toml + * entries for every EXPECTED_AGENTS entry. + */ + function buildManifestBothPairs(agents) { + const files = {}; + for (const name of agents) { + files[`agents/${name}.md`] = 'deadbeef'; + files[`agents/${name}.toml`] = 'deadbeef'; + } + return { version: '1.0.0', files }; + } + + let tmpDir; + + afterEach(() => { + if (tmpDir) { + cleanup(tmpDir); + tmpDir = null; + } + }); + + test('agents_found=false and incomplete non-empty when .toml files are absent but manifest expects them', () => { + // Arrange: all .md files present, manifest says both .md and .toml are expected, + // but NO .toml files are created on disk. + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-missing-toml-')); + const agentsDir = path.join(tmpDir, 'agents'); + fs.mkdirSync(agentsDir, { recursive: true }); + + // Copy real .md files + copyAgentMdFiles(agentsDir); + + // Write manifest with both .md and .toml entries but no .toml files on disk + fs.writeFileSync( + path.join(tmpDir, 'gsd-file-manifest.json'), + JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2), + ); + + // Act + const result = runGsdTools('validate agents --raw', tmpDir, { + GSD_RUNTIME: 'codex', + GSD_AGENTS_DIR: agentsDir, + }); + assert.ok(result.success, `Command failed: ${result.error}`); + const output = JSON.parse(result.output); + + // Assert: the completeness check must catch the missing .toml files + assert.strictEqual( + output.agents_found, + false, + `Expected agents_found=false when manifest-tracked .toml files are absent, got agents_found=${output.agents_found}`, + ); + assert.ok( + Array.isArray(output.incomplete), + 'Expected output.incomplete to be an array', + ); + // Every agent whose .toml is absent must appear in incomplete (sorted deep-equal) + assert.deepStrictEqual( + [...output.incomplete].sort(), + [...EXPECTED_AGENTS].sort(), + `Expected incomplete to equal full EXPECTED_AGENTS set; got ${JSON.stringify(output.incomplete)}`, + ); + // missing (entirely-absent .md) must be empty — these agents are not missing, only incomplete + assert.deepStrictEqual( + output.missing, + [], + `Expected missing=[] when all .md files are present; got ${JSON.stringify(output.missing)}`, + ); + }); + + test('agents_found=false and incomplete non-empty when .md files are absent but manifest expects them', () => { + // Arrange: only .toml files present on disk, manifest says both .md and .toml are expected, + // but NO .md files exist. This is the opposite-side pair-drift case. + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-missing-md-')); + const agentsDir = path.join(tmpDir, 'agents'); + fs.mkdirSync(agentsDir, { recursive: true }); + + // Create ONLY .toml files — deliberately skip .md files + for (const name of EXPECTED_AGENTS) { + fs.writeFileSync(path.join(agentsDir, `${name}.toml`), `name = "${name}" +`); + } + + // Write manifest with both .md and .toml entries + fs.writeFileSync( + path.join(tmpDir, 'gsd-file-manifest.json'), + JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2), + ); + + // Act + const result = runGsdTools('validate agents --raw', tmpDir, { + GSD_RUNTIME: 'codex', + GSD_AGENTS_DIR: agentsDir, + }); + assert.ok(result.success, `Command failed: ${result.error}`); + const output = JSON.parse(result.output); + + // Assert: missing .md files must surface as incomplete (pair-drift detected symmetrically) + assert.strictEqual( + output.agents_found, + false, + `Expected agents_found=false when manifest-tracked .md files are absent, got agents_found=${output.agents_found}`, + ); + assert.ok( + Array.isArray(output.incomplete), + 'Expected output.incomplete to be an array', + ); + assert.ok( + output.incomplete.length > 0, + `Expected incomplete to be non-empty when .md files are absent; got ${JSON.stringify(output.incomplete)}`, + ); + }); + + test('agents_found=true and incomplete=[] when both .md and .toml files are present', () => { + // Arrange: all .md AND .toml present, manifest says both expected + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-complete-pair-')); + const agentsDir = path.join(tmpDir, 'agents'); + fs.mkdirSync(agentsDir, { recursive: true }); + + // Copy real .md files + copyAgentMdFiles(agentsDir); + + // Create .toml files for each expected agent + for (const name of EXPECTED_AGENTS) { + fs.writeFileSync(path.join(agentsDir, `${name}.toml`), `name = "${name}"\n`); + } + + // Write manifest with both .md and .toml entries + fs.writeFileSync( + path.join(tmpDir, 'gsd-file-manifest.json'), + JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2), + ); + + // Act + const result = runGsdTools('validate agents --raw', tmpDir, { + GSD_RUNTIME: 'codex', + GSD_AGENTS_DIR: agentsDir, + }); + assert.ok(result.success, `Command failed: ${result.error}`); + const output = JSON.parse(result.output); + + // Assert: complete pair => no false positive + assert.strictEqual( + output.agents_found, + true, + `Expected agents_found=true when both .md and .toml files are present; got ${output.agents_found}`, + ); + assert.deepStrictEqual( + output.incomplete, + [], + `Expected incomplete=[] when all manifest-tracked files are present; got ${JSON.stringify(output.incomplete)}`, + ); + }); + + test('no spurious incomplete flags when no manifest is present (protects claude/bundled installs)', () => { + // Arrange: all .md files present, NO manifest file at all + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-no-manifest-')); + const agentsDir = path.join(tmpDir, 'agents'); + fs.mkdirSync(agentsDir, { recursive: true }); + + // Copy real .md files + copyAgentMdFiles(agentsDir); + + // Explicitly do NOT write gsd-file-manifest.json + + // Act + const result = runGsdTools('validate agents --raw', tmpDir, { + GSD_RUNTIME: 'codex', + GSD_AGENTS_DIR: agentsDir, + }); + assert.ok(result.success, `Command failed: ${result.error}`); + const output = JSON.parse(result.output); + + // Assert: without a manifest, completeness check must no-op (incomplete empty) + assert.deepStrictEqual( + output.incomplete, + [], + `Expected incomplete=[] when no manifest present; got ${JSON.stringify(output.incomplete)}`, + ); + // agents_found should reflect plain file presence (all .md files copied => true) + assert.strictEqual( + output.agents_found, + true, + `Expected agents_found=true when all .md files are present and no manifest; got ${output.agents_found}`, + ); + }); +});