fix(#1058): cross-reference install manifest in validate agents to catch .md/.toml pair drift (#1079)
* fix(#1058): cross-reference install manifest in validate agents to catch pair drift `validate agents` considered an agent installed if ANY supported file format was present on disk. The Codex installer generates a per-agent PAIR (agents/gsd-*.md AND agents/gsd-*.toml) and records both in gsd-file-manifest.json, so a partial generated install — one side of the pair missing — was reported as healthy (agents_found: true, missing: []), masking an incomplete Codex agent install. checkAgentsInstalled now cross-references the install manifest beside the agents dir (path.dirname(agentsDir)/gsd-file-manifest.json): for each expected agent, if the manifest tracks files for it and any tracked file is absent on disk, the agent is reported in a new `incomplete` list and agents_found becomes false. The check no-ops when no manifest is present (preserves bundled/claude behavior) and is scoped to expected agents so retired/stale manifest entries cannot false-flag. Regression cases added to tests/agent-install-validation.test.cjs cover the drift case, the complete-pair (no false positive), and the no-manifest no-op. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * docs(#1058): add changeset for validate-agents manifest pair-drift fix Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
5
.changeset/humble-pandas-purr.md
Normal file
5
.changeset/humble-pandas-purr.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
type: Fixed
|
||||
pr: 1079
|
||||
---
|
||||
`validate agents` (and `validate health`) now cross-reference the install manifest to detect manifest-backed Codex agent pair drift: when a generated `agents/gsd-*.md` / `agents/gsd-*.toml` pair has one side missing on disk, the agent is reported as incomplete and `agents_found` is `false` (previously a false-healthy `agents_found: true, missing: []`). `validate health` names the incomplete agents and recommends re-running the installer. The check no-ops when no manifest is present. (#1058)
|
||||
55
src/core.cts
55
src/core.cts
@@ -211,6 +211,7 @@ interface AgentsInstalledResult {
|
||||
agents_installed: boolean;
|
||||
missing_agents: string[];
|
||||
installed_agents: string[];
|
||||
incomplete_agents: string[];
|
||||
agents_dir: string;
|
||||
agent_runtime: string;
|
||||
}
|
||||
@@ -232,6 +233,7 @@ function checkAgentsInstalled(runtime?: string): AgentsInstalledResult {
|
||||
agents_installed: false,
|
||||
missing_agents: expectedAgents,
|
||||
installed_agents: [],
|
||||
incomplete_agents: [],
|
||||
agents_dir: agentsDir,
|
||||
agent_runtime: resolvedRuntime,
|
||||
};
|
||||
@@ -259,10 +261,61 @@ function checkAgentsInstalled(runtime?: string): AgentsInstalledResult {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Manifest-backed completeness check ──────────────────────────────────────
|
||||
// If a gsd-file-manifest.json exists alongside the agents dir (parent dir),
|
||||
// verify that every manifest-tracked file for each expected agent is present
|
||||
// on disk. Missing manifest-tracked files indicate an incomplete install even
|
||||
// when the plain presence check above passed (e.g. .md present, .toml absent).
|
||||
// If no manifest is found the check is a no-op (graceful for claude/bundled).
|
||||
const incomplete: string[] = [];
|
||||
const manifestPath = path.join(path.dirname(agentsDir), 'gsd-file-manifest.json');
|
||||
let manifestFiles: Record<string, unknown> = {};
|
||||
try {
|
||||
const raw = fs.readFileSync(manifestPath, 'utf8');
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (
|
||||
parsed !== null &&
|
||||
typeof parsed === 'object' &&
|
||||
'files' in parsed &&
|
||||
typeof (parsed as Record<string, unknown>)['files'] === 'object' &&
|
||||
(parsed as Record<string, unknown>)['files'] !== null
|
||||
) {
|
||||
manifestFiles = (parsed as Record<string, Record<string, unknown>>)['files'];
|
||||
}
|
||||
} catch {
|
||||
// No manifest or unreadable — completeness check is skipped
|
||||
}
|
||||
|
||||
if (Object.keys(manifestFiles).length > 0) {
|
||||
for (const agent of expectedAgents) {
|
||||
// Find all manifest keys that belong to this agent:
|
||||
// key must be "agents/<agentName>.<ext>" with no further path segments.
|
||||
const agentPrefix = `agents/${agent}.`;
|
||||
const agentManifestKeys = Object.keys(manifestFiles).filter(key => {
|
||||
if (!key.startsWith(agentPrefix)) return false;
|
||||
const rest = key.slice(agentPrefix.length);
|
||||
// rest must be a bare extension (no slashes, non-empty)
|
||||
return rest.length > 0 && !rest.includes('/');
|
||||
});
|
||||
if (agentManifestKeys.length === 0) {
|
||||
// Agent not tracked in manifest — skip completeness check for this agent
|
||||
continue;
|
||||
}
|
||||
const allPresent = agentManifestKeys.every(key => {
|
||||
const basename = key.slice('agents/'.length);
|
||||
return fs.existsSync(path.join(agentsDir, basename));
|
||||
});
|
||||
if (!allPresent) {
|
||||
incomplete.push(agent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
agents_installed: installed.length > 0 && missing.length === 0,
|
||||
agents_installed: installed.length > 0 && missing.length === 0 && incomplete.length === 0,
|
||||
missing_agents: missing,
|
||||
installed_agents: installed,
|
||||
incomplete_agents: incomplete,
|
||||
agents_dir: agentsDir,
|
||||
agent_runtime: resolvedRuntime,
|
||||
};
|
||||
|
||||
@@ -1107,6 +1107,20 @@ function cmdValidateHealth(
|
||||
`No GSD agents found in ${agentStatus.agents_dir} — Task(subagent_type="gsd-*") will fall back to general-purpose`,
|
||||
`Run the GSD installer: npx ${PACKAGE_NAME}@latest`,
|
||||
);
|
||||
} else if ((agentStatus.incomplete_agents).length > 0 && (agentStatus.missing_agents).length === 0) {
|
||||
addIssue(
|
||||
'warning',
|
||||
'W010',
|
||||
`Incomplete agent installs (missing generated file): ${(agentStatus.incomplete_agents).join(', ')} — affected workflows may fall back to general-purpose`,
|
||||
`Re-run the GSD installer to complete the install: npx ${PACKAGE_NAME}@latest`,
|
||||
);
|
||||
} else if ((agentStatus.incomplete_agents).length > 0) {
|
||||
addIssue(
|
||||
'warning',
|
||||
'W010',
|
||||
`Missing ${(agentStatus.missing_agents).length} GSD agents: ${(agentStatus.missing_agents).join(', ')}; incomplete agent installs (missing generated file): ${(agentStatus.incomplete_agents).join(', ')} — affected workflows will fall back to general-purpose`,
|
||||
`Run the GSD installer: npx ${PACKAGE_NAME}@latest`,
|
||||
);
|
||||
} else {
|
||||
addIssue(
|
||||
'warning',
|
||||
@@ -1624,6 +1638,7 @@ function cmdValidateAgents(cwd: string, raw: boolean): void {
|
||||
agents_found: agentStatus.agents_installed,
|
||||
installed: agentStatus.installed_agents,
|
||||
missing: agentStatus.missing_agents,
|
||||
incomplete: agentStatus.incomplete_agents,
|
||||
expected,
|
||||
},
|
||||
raw,
|
||||
|
||||
@@ -384,3 +384,209 @@ describe('validate agents subcommand (#1371)', () => {
|
||||
assert.ok(output.expected.length > 0, 'Must have expected agents');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Bug #1058: validate agents detects manifest-backed .md/.toml pair drift ──
|
||||
|
||||
describe('bug #1058: validate agents detects manifest-backed .md/.toml pair drift', () => {
|
||||
// All real agents/gsd-*.md files in the repo (used to populate fixtures)
|
||||
const REPO_AGENTS_DIR_1058 = path.resolve(__dirname, '..', 'agents');
|
||||
|
||||
/**
|
||||
* Copy all gsd-*.md files from the repo agents dir into destDir.
|
||||
*/
|
||||
function copyAgentMdFiles(destDir) {
|
||||
const files = fs.readdirSync(REPO_AGENTS_DIR_1058).filter(f => /^gsd-.*\.md$/.test(f));
|
||||
for (const file of files) {
|
||||
const src = path.join(REPO_AGENTS_DIR_1058, file);
|
||||
const dst = path.join(destDir, file);
|
||||
fs.copyFileSync(src, dst);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a gsd-file-manifest.json whose files map includes both .md and .toml
|
||||
* entries for every EXPECTED_AGENTS entry.
|
||||
*/
|
||||
function buildManifestBothPairs(agents) {
|
||||
const files = {};
|
||||
for (const name of agents) {
|
||||
files[`agents/${name}.md`] = 'deadbeef';
|
||||
files[`agents/${name}.toml`] = 'deadbeef';
|
||||
}
|
||||
return { version: '1.0.0', files };
|
||||
}
|
||||
|
||||
let tmpDir;
|
||||
|
||||
afterEach(() => {
|
||||
if (tmpDir) {
|
||||
cleanup(tmpDir);
|
||||
tmpDir = null;
|
||||
}
|
||||
});
|
||||
|
||||
test('agents_found=false and incomplete non-empty when .toml files are absent but manifest expects them', () => {
|
||||
// Arrange: all .md files present, manifest says both .md and .toml are expected,
|
||||
// but NO .toml files are created on disk.
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-missing-toml-'));
|
||||
const agentsDir = path.join(tmpDir, 'agents');
|
||||
fs.mkdirSync(agentsDir, { recursive: true });
|
||||
|
||||
// Copy real .md files
|
||||
copyAgentMdFiles(agentsDir);
|
||||
|
||||
// Write manifest with both .md and .toml entries but no .toml files on disk
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, 'gsd-file-manifest.json'),
|
||||
JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2),
|
||||
);
|
||||
|
||||
// Act
|
||||
const result = runGsdTools('validate agents --raw', tmpDir, {
|
||||
GSD_RUNTIME: 'codex',
|
||||
GSD_AGENTS_DIR: agentsDir,
|
||||
});
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
|
||||
// Assert: the completeness check must catch the missing .toml files
|
||||
assert.strictEqual(
|
||||
output.agents_found,
|
||||
false,
|
||||
`Expected agents_found=false when manifest-tracked .toml files are absent, got agents_found=${output.agents_found}`,
|
||||
);
|
||||
assert.ok(
|
||||
Array.isArray(output.incomplete),
|
||||
'Expected output.incomplete to be an array',
|
||||
);
|
||||
// Every agent whose .toml is absent must appear in incomplete (sorted deep-equal)
|
||||
assert.deepStrictEqual(
|
||||
[...output.incomplete].sort(),
|
||||
[...EXPECTED_AGENTS].sort(),
|
||||
`Expected incomplete to equal full EXPECTED_AGENTS set; got ${JSON.stringify(output.incomplete)}`,
|
||||
);
|
||||
// missing (entirely-absent .md) must be empty — these agents are not missing, only incomplete
|
||||
assert.deepStrictEqual(
|
||||
output.missing,
|
||||
[],
|
||||
`Expected missing=[] when all .md files are present; got ${JSON.stringify(output.missing)}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('agents_found=false and incomplete non-empty when .md files are absent but manifest expects them', () => {
|
||||
// Arrange: only .toml files present on disk, manifest says both .md and .toml are expected,
|
||||
// but NO .md files exist. This is the opposite-side pair-drift case.
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-missing-md-'));
|
||||
const agentsDir = path.join(tmpDir, 'agents');
|
||||
fs.mkdirSync(agentsDir, { recursive: true });
|
||||
|
||||
// Create ONLY .toml files — deliberately skip .md files
|
||||
for (const name of EXPECTED_AGENTS) {
|
||||
fs.writeFileSync(path.join(agentsDir, `${name}.toml`), `name = "${name}"
|
||||
`);
|
||||
}
|
||||
|
||||
// Write manifest with both .md and .toml entries
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, 'gsd-file-manifest.json'),
|
||||
JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2),
|
||||
);
|
||||
|
||||
// Act
|
||||
const result = runGsdTools('validate agents --raw', tmpDir, {
|
||||
GSD_RUNTIME: 'codex',
|
||||
GSD_AGENTS_DIR: agentsDir,
|
||||
});
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
|
||||
// Assert: missing .md files must surface as incomplete (pair-drift detected symmetrically)
|
||||
assert.strictEqual(
|
||||
output.agents_found,
|
||||
false,
|
||||
`Expected agents_found=false when manifest-tracked .md files are absent, got agents_found=${output.agents_found}`,
|
||||
);
|
||||
assert.ok(
|
||||
Array.isArray(output.incomplete),
|
||||
'Expected output.incomplete to be an array',
|
||||
);
|
||||
assert.ok(
|
||||
output.incomplete.length > 0,
|
||||
`Expected incomplete to be non-empty when .md files are absent; got ${JSON.stringify(output.incomplete)}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('agents_found=true and incomplete=[] when both .md and .toml files are present', () => {
|
||||
// Arrange: all .md AND .toml present, manifest says both expected
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-complete-pair-'));
|
||||
const agentsDir = path.join(tmpDir, 'agents');
|
||||
fs.mkdirSync(agentsDir, { recursive: true });
|
||||
|
||||
// Copy real .md files
|
||||
copyAgentMdFiles(agentsDir);
|
||||
|
||||
// Create .toml files for each expected agent
|
||||
for (const name of EXPECTED_AGENTS) {
|
||||
fs.writeFileSync(path.join(agentsDir, `${name}.toml`), `name = "${name}"\n`);
|
||||
}
|
||||
|
||||
// Write manifest with both .md and .toml entries
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, 'gsd-file-manifest.json'),
|
||||
JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2),
|
||||
);
|
||||
|
||||
// Act
|
||||
const result = runGsdTools('validate agents --raw', tmpDir, {
|
||||
GSD_RUNTIME: 'codex',
|
||||
GSD_AGENTS_DIR: agentsDir,
|
||||
});
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
|
||||
// Assert: complete pair => no false positive
|
||||
assert.strictEqual(
|
||||
output.agents_found,
|
||||
true,
|
||||
`Expected agents_found=true when both .md and .toml files are present; got ${output.agents_found}`,
|
||||
);
|
||||
assert.deepStrictEqual(
|
||||
output.incomplete,
|
||||
[],
|
||||
`Expected incomplete=[] when all manifest-tracked files are present; got ${JSON.stringify(output.incomplete)}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('no spurious incomplete flags when no manifest is present (protects claude/bundled installs)', () => {
|
||||
// Arrange: all .md files present, NO manifest file at all
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-no-manifest-'));
|
||||
const agentsDir = path.join(tmpDir, 'agents');
|
||||
fs.mkdirSync(agentsDir, { recursive: true });
|
||||
|
||||
// Copy real .md files
|
||||
copyAgentMdFiles(agentsDir);
|
||||
|
||||
// Explicitly do NOT write gsd-file-manifest.json
|
||||
|
||||
// Act
|
||||
const result = runGsdTools('validate agents --raw', tmpDir, {
|
||||
GSD_RUNTIME: 'codex',
|
||||
GSD_AGENTS_DIR: agentsDir,
|
||||
});
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
|
||||
// Assert: without a manifest, completeness check must no-op (incomplete empty)
|
||||
assert.deepStrictEqual(
|
||||
output.incomplete,
|
||||
[],
|
||||
`Expected incomplete=[] when no manifest present; got ${JSON.stringify(output.incomplete)}`,
|
||||
);
|
||||
// agents_found should reflect plain file presence (all .md files copied => true)
|
||||
assert.strictEqual(
|
||||
output.agents_found,
|
||||
true,
|
||||
`Expected agents_found=true when all .md files are present and no manifest; got ${output.agents_found}`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user