Makes docs/AGENTS.md section structure the single source of truth for the
primary-vs-advanced agent classification and fails when docs/INVENTORY.md's
"Primary doc" column or the AGENTS.md prose counts drift from it.
The classification ("primary" = full role card, "advanced stub" = concise
stub) was hand-duplicated across three doc surfaces with no enforcement.
This drift guard derives the expected class from AGENTS.md section placement
and cross-checks the INVENTORY.md column, the prose counts, the parenthetical
advanced-agent list, and full roster completeness (21 primary / 12 advanced
/ 0 inventory-only today).
No agent frontmatter field is added (avoids the capability-registry `tier`
collision and the research-profiles.cjs ripple); the classification stays
documentary, enforced from where it is defined.
Closes #1171
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
378
tests/agent-classification-parity.test.cjs
Normal file
378
tests/agent-classification-parity.test.cjs
Normal file
@@ -0,0 +1,378 @@
|
||||
// allow-test-rule: runtime-contract-is-the-product — docs/AGENTS.md section layout + docs/INVENTORY.md table ARE the classification surface being validated
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Agent classification parity test (#1171)
|
||||
*
|
||||
* Makes docs/AGENTS.md section structure the single source of truth for
|
||||
* primary-vs-advanced agent classification, and fails when docs/INVENTORY.md
|
||||
* or the AGENTS.md prose counts drift from it.
|
||||
*
|
||||
* Classification rules (derived from AGENTS.md section placement):
|
||||
* - ### gsd-<name> headings BEFORE "## Advanced and Specialized Agents" → "primary"
|
||||
* - ### gsd-<name> headings INSIDE/AFTER that section → "advanced stub"
|
||||
* - agents/gsd-*.md files with NO heading in AGENTS.md → "inventory only"
|
||||
*/
|
||||
|
||||
const { describe, test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const AGENTS_MD = path.join(ROOT, 'docs', 'AGENTS.md');
|
||||
const INVENTORY_MD = path.join(ROOT, 'docs', 'INVENTORY.md');
|
||||
const AGENTS_DIR = path.join(ROOT, 'agents');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parsing helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Parse AGENTS.md and return two arrays:
|
||||
* primaryHeadings — agent names (without "gsd-" prefix) before the advanced section
|
||||
* advancedHeadings — agent names inside "## Advanced and Specialized Agents"
|
||||
*
|
||||
* We work with the full "gsd-<name>" slug so the names are unambiguous.
|
||||
*
|
||||
* State machine (three-state):
|
||||
* 'before' — before "## Advanced and Specialized Agents"
|
||||
* 'advanced' — inside that section
|
||||
* 'past' — after a subsequent ## heading that follows the advanced section
|
||||
*
|
||||
* A "## " heading (h2, NOT h3) that appears BEFORE the advanced section does NOT
|
||||
* trigger 'past'. Only a "## " heading encountered WHILE in 'advanced' does.
|
||||
* ### gsd-* headings are counted: primary if 'before', advanced if 'advanced',
|
||||
* ignored if 'past'.
|
||||
*/
|
||||
function parseAgentsMd(raw) {
|
||||
const lines = raw.split('\n');
|
||||
const ADVANCED_SECTION = /^##\s+Advanced and Specialized Agents\s*$/;
|
||||
// Matches an h2 heading (exactly two hashes, not three or more)
|
||||
const H2 = /^##(?!#)\s+\S/;
|
||||
const GSD_H3 = /^###\s+(gsd-[\w-]+)\s*$/;
|
||||
|
||||
const primaryHeadings = [];
|
||||
const advancedHeadings = [];
|
||||
// 'before' | 'advanced' | 'past'
|
||||
let state = 'before';
|
||||
|
||||
for (const line of lines) {
|
||||
if (ADVANCED_SECTION.test(line)) {
|
||||
state = 'advanced';
|
||||
continue;
|
||||
}
|
||||
// Any other h2 heading while in 'advanced' terminates the advanced region
|
||||
if (state === 'advanced' && H2.test(line)) {
|
||||
state = 'past';
|
||||
continue;
|
||||
}
|
||||
const m = GSD_H3.exec(line);
|
||||
if (m) {
|
||||
if (state === 'before') {
|
||||
primaryHeadings.push(m[1]);
|
||||
} else if (state === 'advanced') {
|
||||
advancedHeadings.push(m[1]);
|
||||
}
|
||||
// state === 'past': silently ignored
|
||||
}
|
||||
}
|
||||
|
||||
return { primaryHeadings, advancedHeadings };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse INVENTORY.md agent table.
|
||||
* Returns a Map<agentSlug, primaryDocValue> e.g. "primary" | "advanced stub" | "inventory only"
|
||||
*
|
||||
* Scoping: only rows that fall between the "## Agents" heading and the NEXT
|
||||
* "## " heading are considered. This prevents a future non-agent table that
|
||||
* happens to contain a "| gsd-..." row from polluting the result.
|
||||
*
|
||||
* Column resolution: the header row "| Agent | ... | Primary doc |" is parsed
|
||||
* to find the 0-based index of the "Primary doc" column. Rows are split on "|"
|
||||
* and only the leading/trailing empty edge cells are dropped (slice(1,-1)) so
|
||||
* that empty middle cells do NOT shift column positions.
|
||||
*/
|
||||
function parseInventoryMd(raw) {
|
||||
const lines = raw.split('\n');
|
||||
// Matches any h2 heading (exactly two hashes, not three or more)
|
||||
const H2 = /^##(?!#)\s+\S/;
|
||||
// Matches the Agents section heading (e.g. "## Agents (33 shipped)")
|
||||
const AGENTS_SECTION = /^##\s+Agents\b/;
|
||||
|
||||
const result = new Map();
|
||||
let inAgentsSection = false;
|
||||
let primaryDocColIndex = -1; // column index within the trimmed, edge-stripped cell array
|
||||
|
||||
for (const line of lines) {
|
||||
if (AGENTS_SECTION.test(line)) {
|
||||
inAgentsSection = true;
|
||||
primaryDocColIndex = -1; // reset in case file is re-parsed
|
||||
continue;
|
||||
}
|
||||
// Any subsequent h2 heading ends the agents section
|
||||
if (inAgentsSection && H2.test(line)) {
|
||||
break;
|
||||
}
|
||||
if (!inAgentsSection) continue;
|
||||
|
||||
// Every table row starts and ends with "|"
|
||||
if (!line.startsWith('|')) continue;
|
||||
|
||||
// Split on "|", drop the leading and trailing empty strings that result
|
||||
// from the leading/trailing "|", but preserve empty middle cells so column
|
||||
// indices stay stable.
|
||||
const rawCells = line.split('|');
|
||||
// rawCells[0] is '' (before the first |), rawCells[last] is '' (after last |)
|
||||
const cells = rawCells.slice(1, -1).map((c) => c.trim());
|
||||
|
||||
// Detect the header row by looking for an "Agent" cell followed by a "Primary doc" cell
|
||||
if (primaryDocColIndex === -1) {
|
||||
const pdIdx = cells.findIndex((c) => c === 'Primary doc');
|
||||
if (pdIdx !== -1 && cells[0] === 'Agent') {
|
||||
primaryDocColIndex = pdIdx;
|
||||
}
|
||||
continue; // header row (or rows before the header is found) — not a data row
|
||||
}
|
||||
|
||||
// Skip separator rows (---|---|...)
|
||||
if (cells.every((c) => /^[-: ]+$/.test(c))) continue;
|
||||
|
||||
// Data rows: first cell must be a gsd-* slug
|
||||
if (!cells[0].startsWith('gsd-')) continue;
|
||||
if (cells.length <= primaryDocColIndex) continue;
|
||||
|
||||
const agentSlug = cells[0];
|
||||
const primaryDoc = cells[primaryDocColIndex];
|
||||
result.set(agentSlug, primaryDoc);
|
||||
}
|
||||
|
||||
assert.ok(
|
||||
primaryDocColIndex !== -1,
|
||||
'INVENTORY.md: "Primary doc" column header not found in the Agents table — check the ## Agents section heading and table header row',
|
||||
);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all agents/gsd-*.md basenames (without .md extension).
|
||||
*/
|
||||
function listAgentFiles() {
|
||||
return fs
|
||||
.readdirSync(AGENTS_DIR)
|
||||
.filter((f) => /^gsd-.*\.md$/.test(f))
|
||||
.map((f) => f.replace(/\.md$/, ''))
|
||||
.sort();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Load and parse
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const rawAgentsMd = fs.readFileSync(AGENTS_MD, 'utf8');
|
||||
const rawInventoryMd = fs.readFileSync(INVENTORY_MD, 'utf8');
|
||||
|
||||
const { primaryHeadings, advancedHeadings } = parseAgentsMd(rawAgentsMd);
|
||||
const inventoryMap = parseInventoryMd(rawInventoryMd);
|
||||
const agentFiles = listAgentFiles();
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Robustness guards — must pass before any assertion block runs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
assert.ok(
|
||||
primaryHeadings.length > 0,
|
||||
'AGENTS.md: no ### gsd-* headings found before Advanced section — raw head:\n' + rawAgentsMd.slice(0, 300),
|
||||
);
|
||||
|
||||
assert.ok(
|
||||
advancedHeadings.length > 0,
|
||||
'AGENTS.md: no ### gsd-* headings found inside Advanced section — raw head:\n' + rawAgentsMd.slice(0, 300),
|
||||
);
|
||||
|
||||
assert.ok(
|
||||
inventoryMap.size > 0,
|
||||
'INVENTORY.md: no | gsd-* | rows parsed — raw head:\n' + rawInventoryMd.slice(0, 300),
|
||||
);
|
||||
|
||||
assert.ok(
|
||||
agentFiles.length > 0,
|
||||
'agents/: no gsd-*.md files found — check AGENTS_DIR path: ' + AGENTS_DIR,
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Derived sets
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const primarySet = new Set(primaryHeadings);
|
||||
const advancedSet = new Set(advancedHeadings);
|
||||
const agentFileSet = new Set(agentFiles);
|
||||
|
||||
// Agents that exist on disk but have no heading in AGENTS.md
|
||||
const inventoryOnly = agentFiles.filter(
|
||||
(a) => !primarySet.has(a) && !advancedSet.has(a),
|
||||
);
|
||||
const inventoryOnlySet = new Set(inventoryOnly);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('agent-classification-parity: AGENTS.md section structure is the single source of truth', () => {
|
||||
|
||||
/**
|
||||
* Test 1 — INVENTORY.md "Primary doc" column matches AGENTS.md-derived class.
|
||||
* For every agent, the column value must equal:
|
||||
* "primary" if the agent's ### heading is before ## Advanced and Specialized Agents
|
||||
* "advanced stub" if the agent's ### heading is inside/after that section
|
||||
* "inventory only" if the agent has no ### heading in AGENTS.md
|
||||
*/
|
||||
test('INVENTORY.md "Primary doc" values match AGENTS.md section placement', () => {
|
||||
const mismatches = [];
|
||||
|
||||
for (const [slug, inventoryValue] of inventoryMap) {
|
||||
let expected;
|
||||
if (primarySet.has(slug)) {
|
||||
expected = 'primary';
|
||||
} else if (advancedSet.has(slug)) {
|
||||
expected = 'advanced stub';
|
||||
} else if (inventoryOnlySet.has(slug)) {
|
||||
expected = 'inventory only';
|
||||
} else {
|
||||
// Row in INVENTORY.md for an agent with no file — handled in test 3
|
||||
continue;
|
||||
}
|
||||
|
||||
if (inventoryValue !== expected) {
|
||||
mismatches.push(` ${slug}: INVENTORY.md="${inventoryValue}" but AGENTS.md section says "${expected}"`);
|
||||
}
|
||||
}
|
||||
|
||||
assert.strictEqual(
|
||||
mismatches.length,
|
||||
0,
|
||||
'INVENTORY.md "Primary doc" column disagrees with AGENTS.md section placement:\n' + mismatches.join('\n'),
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* Test 2 — AGENTS.md prose counts and parenthetical list are accurate.
|
||||
*
|
||||
* Checks three sub-facts from line ~13:
|
||||
* a) "21 primary agents" → count of primary headings
|
||||
* b) "Twelve additional" → count of advanced headings
|
||||
* c) The parenthetical slug list → equals the set of advanced heading names
|
||||
* (slugs without the "gsd-" prefix, as the prose uses)
|
||||
*/
|
||||
test('AGENTS.md prose counts and advanced-agent parenthetical list are accurate', () => {
|
||||
|
||||
// --- (a) prose primary count ---
|
||||
const primaryCountMatch = rawAgentsMd.match(/\*\*(\d+)\s+primary\s+agents?\*\*/);
|
||||
assert.ok(
|
||||
primaryCountMatch,
|
||||
'AGENTS.md: could not find "**N primary agents**" in prose — raw head:\n' + rawAgentsMd.slice(0, 500),
|
||||
);
|
||||
const prosePrimaryCount = parseInt(primaryCountMatch[1], 10);
|
||||
assert.strictEqual(
|
||||
prosePrimaryCount,
|
||||
primaryHeadings.length,
|
||||
`AGENTS.md prose says "${prosePrimaryCount} primary agents" but there are ${primaryHeadings.length} ### gsd-* headings before the Advanced section`,
|
||||
);
|
||||
|
||||
// --- (b) prose advanced count (cardinal word or digit) ---
|
||||
// Look for "Twelve additional" or "12 additional" (case-insensitive cardinal)
|
||||
const CARDINALS = {
|
||||
one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7,
|
||||
eight: 8, nine: 9, ten: 10, eleven: 11, twelve: 12, thirteen: 13,
|
||||
};
|
||||
const advancedCountMatch = rawAgentsMd.match(
|
||||
/\b((?:\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen))\s+additional\s+shipped\s+agents?\b/i,
|
||||
);
|
||||
assert.ok(
|
||||
advancedCountMatch,
|
||||
'AGENTS.md: could not find "N additional shipped agents" in prose — raw head:\n' + rawAgentsMd.slice(0, 500),
|
||||
);
|
||||
const advancedCountRaw = advancedCountMatch[1].toLowerCase();
|
||||
const proseAdvancedCount = CARDINALS[advancedCountRaw] !== undefined
|
||||
? CARDINALS[advancedCountRaw]
|
||||
: parseInt(advancedCountRaw, 10);
|
||||
assert.strictEqual(
|
||||
proseAdvancedCount,
|
||||
advancedHeadings.length,
|
||||
`AGENTS.md prose says "${advancedCountRaw} additional shipped agents" but there are ${advancedHeadings.length} ### gsd-* headings in the Advanced section`,
|
||||
);
|
||||
|
||||
// --- (c) parenthetical slug list ---
|
||||
// The prose lists short slugs without "gsd-" prefix, e.g.:
|
||||
// (pattern-mapper, debug-session-manager, ...)
|
||||
const parenMatch = rawAgentsMd.match(/\(([^)]+)\)\s+have concise stubs/);
|
||||
assert.ok(
|
||||
parenMatch,
|
||||
'AGENTS.md: could not find parenthetical advanced-agent list "(slug, slug, ...) have concise stubs" — raw head:\n' + rawAgentsMd.slice(0, 500),
|
||||
);
|
||||
const proseSlugs = parenMatch[1].split(',').map((s) => s.trim().toLowerCase());
|
||||
const proseSlugsSet = new Set(proseSlugs);
|
||||
|
||||
// Derive expected slugs from AGENTS.md headings (strip "gsd-" prefix)
|
||||
const expectedSlugs = new Set(advancedHeadings.map((h) => h.replace(/^gsd-/, '')));
|
||||
|
||||
const missingFromProse = [...expectedSlugs].filter((s) => !proseSlugsSet.has(s));
|
||||
const extraInProse = [...proseSlugsSet].filter((s) => !expectedSlugs.has(s));
|
||||
|
||||
assert.deepStrictEqual(
|
||||
{ missingFromProse, extraInProse },
|
||||
{ missingFromProse: [], extraInProse: [] },
|
||||
'AGENTS.md parenthetical slug list disagrees with ### headings in the Advanced section.\n' +
|
||||
` Missing from prose: ${JSON.stringify(missingFromProse)}\n` +
|
||||
` Extra in prose: ${JSON.stringify(extraInProse)}`,
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* Test 3 — Roster completeness.
|
||||
*
|
||||
* Sub-checks:
|
||||
* a) Every agents/gsd-*.md appears exactly once in (primary ∪ advanced ∪ inventoryOnly)
|
||||
* — i.e. no agent is double-classified.
|
||||
* b) INVENTORY.md has a row for every agent file.
|
||||
* c) INVENTORY.md has no row for a non-existent agent file.
|
||||
* d) primary.length + advanced.length + inventoryOnly.length === total agent file count
|
||||
*/
|
||||
test('Roster completeness: every agent file is classified exactly once', () => {
|
||||
|
||||
// (a) no agent appears in more than one classification bucket
|
||||
const overlap = primaryHeadings.filter((a) => advancedSet.has(a));
|
||||
assert.deepStrictEqual(
|
||||
overlap,
|
||||
[],
|
||||
'Agents appear in BOTH primary and advanced heading sections: ' + JSON.stringify(overlap),
|
||||
);
|
||||
|
||||
// (b) INVENTORY.md has a row for every agent file
|
||||
const missingFromInventory = agentFiles.filter((a) => !inventoryMap.has(a));
|
||||
assert.deepStrictEqual(
|
||||
missingFromInventory,
|
||||
[],
|
||||
'agents/gsd-*.md files missing from INVENTORY.md: ' + JSON.stringify(missingFromInventory),
|
||||
);
|
||||
|
||||
// (c) INVENTORY.md has no row for a non-existent agent file
|
||||
const phantomRows = [...inventoryMap.keys()].filter((slug) => !agentFileSet.has(slug));
|
||||
assert.deepStrictEqual(
|
||||
phantomRows,
|
||||
[],
|
||||
'INVENTORY.md rows reference agents that have no agents/gsd-*.md file: ' + JSON.stringify(phantomRows),
|
||||
);
|
||||
|
||||
// (d) counts add up
|
||||
const total = primaryHeadings.length + advancedHeadings.length + inventoryOnly.length;
|
||||
assert.strictEqual(
|
||||
total,
|
||||
agentFiles.length,
|
||||
`primary(${primaryHeadings.length}) + advanced(${advancedHeadings.length}) + inventoryOnly(${inventoryOnly.length}) = ${total} ≠ ${agentFiles.length} agent files`,
|
||||
);
|
||||
});
|
||||
|
||||
});
|
||||
Reference in New Issue
Block a user