fix(#1058): cross-reference install manifest in validate agents to catch .md/.toml pair drift (#1079)
* fix(#1058): cross-reference install manifest in validate agents to catch pair drift `validate agents` considered an agent installed if ANY supported file format was present on disk. The Codex installer generates a per-agent PAIR (agents/gsd-*.md AND agents/gsd-*.toml) and records both in gsd-file-manifest.json, so a partial generated install — one side of the pair missing — was reported as healthy (agents_found: true, missing: []), masking an incomplete Codex agent install. checkAgentsInstalled now cross-references the install manifest beside the agents dir (path.dirname(agentsDir)/gsd-file-manifest.json): for each expected agent, if the manifest tracks files for it and any tracked file is absent on disk, the agent is reported in a new `incomplete` list and agents_found becomes false. The check no-ops when no manifest is present (preserves bundled/claude behavior) and is scoped to expected agents so retired/stale manifest entries cannot false-flag. Regression cases added to tests/agent-install-validation.test.cjs cover the drift case, the complete-pair (no false positive), and the no-manifest no-op. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * docs(#1058): add changeset for validate-agents manifest pair-drift fix Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
@@ -384,3 +384,209 @@ describe('validate agents subcommand (#1371)', () => {
|
||||
assert.ok(output.expected.length > 0, 'Must have expected agents');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Bug #1058: validate agents detects manifest-backed .md/.toml pair drift ──
|
||||
|
||||
describe('bug #1058: validate agents detects manifest-backed .md/.toml pair drift', () => {
|
||||
// All real agents/gsd-*.md files in the repo (used to populate fixtures)
|
||||
const REPO_AGENTS_DIR_1058 = path.resolve(__dirname, '..', 'agents');
|
||||
|
||||
/**
|
||||
* Copy all gsd-*.md files from the repo agents dir into destDir.
|
||||
*/
|
||||
function copyAgentMdFiles(destDir) {
|
||||
const files = fs.readdirSync(REPO_AGENTS_DIR_1058).filter(f => /^gsd-.*\.md$/.test(f));
|
||||
for (const file of files) {
|
||||
const src = path.join(REPO_AGENTS_DIR_1058, file);
|
||||
const dst = path.join(destDir, file);
|
||||
fs.copyFileSync(src, dst);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a gsd-file-manifest.json whose files map includes both .md and .toml
|
||||
* entries for every EXPECTED_AGENTS entry.
|
||||
*/
|
||||
function buildManifestBothPairs(agents) {
|
||||
const files = {};
|
||||
for (const name of agents) {
|
||||
files[`agents/${name}.md`] = 'deadbeef';
|
||||
files[`agents/${name}.toml`] = 'deadbeef';
|
||||
}
|
||||
return { version: '1.0.0', files };
|
||||
}
|
||||
|
||||
let tmpDir;
|
||||
|
||||
afterEach(() => {
|
||||
if (tmpDir) {
|
||||
cleanup(tmpDir);
|
||||
tmpDir = null;
|
||||
}
|
||||
});
|
||||
|
||||
test('agents_found=false and incomplete non-empty when .toml files are absent but manifest expects them', () => {
|
||||
// Arrange: all .md files present, manifest says both .md and .toml are expected,
|
||||
// but NO .toml files are created on disk.
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-missing-toml-'));
|
||||
const agentsDir = path.join(tmpDir, 'agents');
|
||||
fs.mkdirSync(agentsDir, { recursive: true });
|
||||
|
||||
// Copy real .md files
|
||||
copyAgentMdFiles(agentsDir);
|
||||
|
||||
// Write manifest with both .md and .toml entries but no .toml files on disk
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, 'gsd-file-manifest.json'),
|
||||
JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2),
|
||||
);
|
||||
|
||||
// Act
|
||||
const result = runGsdTools('validate agents --raw', tmpDir, {
|
||||
GSD_RUNTIME: 'codex',
|
||||
GSD_AGENTS_DIR: agentsDir,
|
||||
});
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
|
||||
// Assert: the completeness check must catch the missing .toml files
|
||||
assert.strictEqual(
|
||||
output.agents_found,
|
||||
false,
|
||||
`Expected agents_found=false when manifest-tracked .toml files are absent, got agents_found=${output.agents_found}`,
|
||||
);
|
||||
assert.ok(
|
||||
Array.isArray(output.incomplete),
|
||||
'Expected output.incomplete to be an array',
|
||||
);
|
||||
// Every agent whose .toml is absent must appear in incomplete (sorted deep-equal)
|
||||
assert.deepStrictEqual(
|
||||
[...output.incomplete].sort(),
|
||||
[...EXPECTED_AGENTS].sort(),
|
||||
`Expected incomplete to equal full EXPECTED_AGENTS set; got ${JSON.stringify(output.incomplete)}`,
|
||||
);
|
||||
// missing (entirely-absent .md) must be empty — these agents are not missing, only incomplete
|
||||
assert.deepStrictEqual(
|
||||
output.missing,
|
||||
[],
|
||||
`Expected missing=[] when all .md files are present; got ${JSON.stringify(output.missing)}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('agents_found=false and incomplete non-empty when .md files are absent but manifest expects them', () => {
|
||||
// Arrange: only .toml files present on disk, manifest says both .md and .toml are expected,
|
||||
// but NO .md files exist. This is the opposite-side pair-drift case.
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-missing-md-'));
|
||||
const agentsDir = path.join(tmpDir, 'agents');
|
||||
fs.mkdirSync(agentsDir, { recursive: true });
|
||||
|
||||
// Create ONLY .toml files — deliberately skip .md files
|
||||
for (const name of EXPECTED_AGENTS) {
|
||||
fs.writeFileSync(path.join(agentsDir, `${name}.toml`), `name = "${name}"
|
||||
`);
|
||||
}
|
||||
|
||||
// Write manifest with both .md and .toml entries
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, 'gsd-file-manifest.json'),
|
||||
JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2),
|
||||
);
|
||||
|
||||
// Act
|
||||
const result = runGsdTools('validate agents --raw', tmpDir, {
|
||||
GSD_RUNTIME: 'codex',
|
||||
GSD_AGENTS_DIR: agentsDir,
|
||||
});
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
|
||||
// Assert: missing .md files must surface as incomplete (pair-drift detected symmetrically)
|
||||
assert.strictEqual(
|
||||
output.agents_found,
|
||||
false,
|
||||
`Expected agents_found=false when manifest-tracked .md files are absent, got agents_found=${output.agents_found}`,
|
||||
);
|
||||
assert.ok(
|
||||
Array.isArray(output.incomplete),
|
||||
'Expected output.incomplete to be an array',
|
||||
);
|
||||
assert.ok(
|
||||
output.incomplete.length > 0,
|
||||
`Expected incomplete to be non-empty when .md files are absent; got ${JSON.stringify(output.incomplete)}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('agents_found=true and incomplete=[] when both .md and .toml files are present', () => {
|
||||
// Arrange: all .md AND .toml present, manifest says both expected
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-complete-pair-'));
|
||||
const agentsDir = path.join(tmpDir, 'agents');
|
||||
fs.mkdirSync(agentsDir, { recursive: true });
|
||||
|
||||
// Copy real .md files
|
||||
copyAgentMdFiles(agentsDir);
|
||||
|
||||
// Create .toml files for each expected agent
|
||||
for (const name of EXPECTED_AGENTS) {
|
||||
fs.writeFileSync(path.join(agentsDir, `${name}.toml`), `name = "${name}"\n`);
|
||||
}
|
||||
|
||||
// Write manifest with both .md and .toml entries
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, 'gsd-file-manifest.json'),
|
||||
JSON.stringify(buildManifestBothPairs(EXPECTED_AGENTS), null, 2),
|
||||
);
|
||||
|
||||
// Act
|
||||
const result = runGsdTools('validate agents --raw', tmpDir, {
|
||||
GSD_RUNTIME: 'codex',
|
||||
GSD_AGENTS_DIR: agentsDir,
|
||||
});
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
|
||||
// Assert: complete pair => no false positive
|
||||
assert.strictEqual(
|
||||
output.agents_found,
|
||||
true,
|
||||
`Expected agents_found=true when both .md and .toml files are present; got ${output.agents_found}`,
|
||||
);
|
||||
assert.deepStrictEqual(
|
||||
output.incomplete,
|
||||
[],
|
||||
`Expected incomplete=[] when all manifest-tracked files are present; got ${JSON.stringify(output.incomplete)}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('no spurious incomplete flags when no manifest is present (protects claude/bundled installs)', () => {
|
||||
// Arrange: all .md files present, NO manifest file at all
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-bug-1058-no-manifest-'));
|
||||
const agentsDir = path.join(tmpDir, 'agents');
|
||||
fs.mkdirSync(agentsDir, { recursive: true });
|
||||
|
||||
// Copy real .md files
|
||||
copyAgentMdFiles(agentsDir);
|
||||
|
||||
// Explicitly do NOT write gsd-file-manifest.json
|
||||
|
||||
// Act
|
||||
const result = runGsdTools('validate agents --raw', tmpDir, {
|
||||
GSD_RUNTIME: 'codex',
|
||||
GSD_AGENTS_DIR: agentsDir,
|
||||
});
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
|
||||
// Assert: without a manifest, completeness check must no-op (incomplete empty)
|
||||
assert.deepStrictEqual(
|
||||
output.incomplete,
|
||||
[],
|
||||
`Expected incomplete=[] when no manifest present; got ${JSON.stringify(output.incomplete)}`,
|
||||
);
|
||||
// agents_found should reflect plain file presence (all .md files copied => true)
|
||||
assert.strictEqual(
|
||||
output.agents_found,
|
||||
true,
|
||||
`Expected agents_found=true when all .md files are present and no manifest; got ${output.agents_found}`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user