* docs(adr): add ADR-0003 model catalog module * fix(#3229): add shared model catalog as source of truth for agent profiles and runtime tier defaults Research / design (ADR-0003): - Existing drift came from 4 independent model truths: 1. CJS model-profiles.cjs 2. SDK config-query.ts stale copy (18 agents) 3. settings-advanced.md runtime tier table 4. session-runner Claude-only profile map - New design: one machine-readable Model Catalog Module in sdk/shared/ that both packages ship and consume. Implementation: - sdk/shared/model-catalog.json — canonical source of truth for: - full 33-agent registry - per-agent golden (quality) alias + balanced/budget aliases - adaptive derivation from routingTier - agent→phaseType map - agent→dynamic-routing default tier map - runtime tier defaults for all supported runtimes - get-shit-done/bin/lib/model-catalog.cjs — CJS adapter over the catalog - sdk/src/model-catalog.ts — SDK adapter over the same catalog - CJS model-profiles.cjs now re-exports derived data from model-catalog.cjs - SDK config-query.ts now re-exports MODEL_PROFILES/VALID_PROFILES from model-catalog.ts instead of maintaining its own list - sdk/src/query/helpers.ts runtime list now comes from the catalog (fixes hermes drift) - sdk/src/session-runner.ts Claude profile→model-id mapping now resolves via catalog - docs/CONFIGURATION.md + settings-advanced.md runtime tables updated to match catalog Behavior changes: - resolve-model now covers every shipped agent file on disk (33 agents) - unknown-agent fallback is profile-semantic, not hardcoded sonnet: quality→opus, budget→haiku, balanced/adaptive→sonnet, inherit→inherit - Group B runtimes remain known runtimes but do not get built-in tier defaults Tests (RED→GREEN): - root tests: shipped agent files must equal MODEL_PROFILES keys - sdk tests: shipped agent files must equal MODEL_PROFILES keys - direct fix assertion: gsd-code-reviewer resolves to opus under quality with no unknown_agent - runtime defaults parity test: settings-advanced.md + CONFIGURATION.md tables must match catalog - helper tests: hermes included in SUPPORTED_RUNTIMES and getRuntimeConfigDir() Closes #3229 * chore(changeset): update #3229 changeset pr field to 3230 * fix(ci): update inherit fallback expectations and inventory parity for model catalog
This commit is contained in:
@@ -94,9 +94,9 @@ describe('bug #1829: model_profile "inherit" — resolveModelInternal', () => {
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-verifier'), 'inherit');
|
||||
});
|
||||
|
||||
test('returns "sonnet" (default) for unknown agent even with inherit profile', () => {
|
||||
test('returns "inherit" for unknown agent with inherit profile', () => {
|
||||
writeConfig(tmpDir, { model_profile: 'inherit' });
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-nonexistent'), 'sonnet');
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-nonexistent'), 'inherit');
|
||||
});
|
||||
|
||||
test('per-agent override takes precedence over inherit profile', () => {
|
||||
|
||||
@@ -1071,6 +1071,16 @@ describe('resolve-model command', () => {
|
||||
assert.strictEqual(output.unknown_agent, undefined, 'should not have unknown_agent for known agent');
|
||||
});
|
||||
|
||||
test('shipped-but-previously-missing agent resolves under quality profile (#3229)', () => {
|
||||
fs.writeFileSync(path.join(tmpDir, '.planning', 'config.json'), JSON.stringify({ model_profile: 'quality' }));
|
||||
const result = runGsdTools('resolve-model gsd-code-reviewer', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.model, 'opus');
|
||||
assert.strictEqual(output.profile, 'quality');
|
||||
assert.strictEqual(output.unknown_agent, undefined);
|
||||
});
|
||||
|
||||
test('unknown agent returns unknown_agent=true', () => {
|
||||
const result = runGsdTools('resolve-model fake-nonexistent-agent', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
@@ -1079,6 +1089,24 @@ describe('resolve-model command', () => {
|
||||
assert.strictEqual(output.unknown_agent, true, 'should flag unknown agent');
|
||||
});
|
||||
|
||||
test('unknown agent uses quality-semantic fallback (opus)', () => {
|
||||
fs.writeFileSync(path.join(tmpDir, '.planning', 'config.json'), JSON.stringify({ model_profile: 'quality' }));
|
||||
const result = runGsdTools('resolve-model fake-nonexistent-agent', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.model, 'opus');
|
||||
assert.strictEqual(output.unknown_agent, true);
|
||||
});
|
||||
|
||||
test('unknown agent uses budget-semantic fallback (haiku)', () => {
|
||||
fs.writeFileSync(path.join(tmpDir, '.planning', 'config.json'), JSON.stringify({ model_profile: 'budget' }));
|
||||
const result = runGsdTools('resolve-model fake-nonexistent-agent', tmpDir);
|
||||
assert.ok(result.success, `Command failed: ${result.error}`);
|
||||
const output = JSON.parse(result.output);
|
||||
assert.strictEqual(output.model, 'haiku');
|
||||
assert.strictEqual(output.unknown_agent, true);
|
||||
});
|
||||
|
||||
test('default profile fallback when no config exists', () => {
|
||||
// tmpDir has no config.json, so defaults to balanced profile
|
||||
const result = runGsdTools('resolve-model gsd-executor', tmpDir);
|
||||
|
||||
@@ -434,9 +434,9 @@ describe('resolveModelInternal', () => {
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-nonexistent'), 'sonnet');
|
||||
});
|
||||
|
||||
test('returns sonnet for unknown agent type even with inherit profile', () => {
|
||||
test('returns inherit for unknown agent type with inherit profile', () => {
|
||||
writeConfig({ model_profile: 'inherit' });
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-nonexistent'), 'sonnet');
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-nonexistent'), 'inherit');
|
||||
});
|
||||
|
||||
test('defaults to balanced profile when model_profile missing', () => {
|
||||
|
||||
63
tests/model-catalog-runtime-defaults.test.cjs
Normal file
63
tests/model-catalog-runtime-defaults.test.cjs
Normal file
@@ -0,0 +1,63 @@
|
||||
// allow-test-rule: source-text-is-the-product
|
||||
// These docs tables are the shipped operator surface for runtime model tiers.
|
||||
|
||||
'use strict';
|
||||
|
||||
const { describe, test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const { catalog, KNOWN_RUNTIMES } = require('../get-shit-done/bin/lib/model-catalog.cjs');
|
||||
|
||||
const ROOT = path.join(__dirname, '..');
|
||||
const SETTINGS_ADVANCED = fs.readFileSync(path.join(ROOT, 'get-shit-done', 'workflows', 'settings-advanced.md'), 'utf8');
|
||||
const CONFIG_DOC = fs.readFileSync(path.join(ROOT, 'docs', 'CONFIGURATION.md'), 'utf8');
|
||||
|
||||
describe('model catalog runtime defaults parity (#3229)', () => {
|
||||
test('known runtimes include hermes and match catalog keys', () => {
|
||||
assert.ok(KNOWN_RUNTIMES.has('hermes'));
|
||||
assert.deepStrictEqual([...KNOWN_RUNTIMES].sort(), Object.keys(catalog.runtimeTierDefaults).sort());
|
||||
});
|
||||
|
||||
test('settings-advanced runtime defaults table matches catalog for concrete runtimes', () => {
|
||||
for (const [runtime, tiers] of Object.entries(catalog.runtimeTierDefaults)) {
|
||||
if (!tiers.opus) continue; // Group B runtimes intentionally have no built-ins
|
||||
assert.ok(SETTINGS_ADVANCED.includes(`| \`${runtime}\``), `settings-advanced.md missing ${runtime} row`);
|
||||
for (const alias of ['opus', 'sonnet', 'haiku']) {
|
||||
const entry = tiers[alias];
|
||||
assert.ok(entry?.model, `${runtime}.${alias} missing model in catalog`);
|
||||
assert.ok(
|
||||
SETTINGS_ADVANCED.includes(`\`${entry.model}\``),
|
||||
`settings-advanced.md missing ${runtime}.${alias} model ${entry.model}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('CONFIGURATION runtime defaults table matches catalog for concrete runtimes', () => {
|
||||
for (const [runtime, tiers] of Object.entries(catalog.runtimeTierDefaults)) {
|
||||
if (!tiers.opus) continue;
|
||||
assert.ok(CONFIG_DOC.includes(`| \`${runtime}\``), `CONFIGURATION.md missing ${runtime} row`);
|
||||
for (const alias of ['opus', 'sonnet', 'haiku']) {
|
||||
const entry = tiers[alias];
|
||||
assert.ok(
|
||||
CONFIG_DOC.includes(`\`${entry.model}\``),
|
||||
`CONFIGURATION.md missing ${runtime}.${alias} model ${entry.model}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('Group B runtimes remain documented as having no built-in defaults', () => {
|
||||
const groupB = ['kilo', 'cline', 'cursor', 'windsurf', 'augment', 'trae', 'codebuddy', 'antigravity'];
|
||||
for (const runtime of groupB) {
|
||||
const tiers = catalog.runtimeTierDefaults[runtime];
|
||||
assert.equal(tiers.opus, null);
|
||||
assert.equal(tiers.sonnet, null);
|
||||
assert.equal(tiers.haiku, null);
|
||||
}
|
||||
assert.ok(SETTINGS_ADVANCED.includes('Group B'));
|
||||
assert.ok(CONFIG_DOC.includes('Group B'));
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,9 @@
|
||||
const { test, describe } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const {
|
||||
MODEL_PROFILES,
|
||||
VALID_PROFILES,
|
||||
@@ -15,20 +18,20 @@ const {
|
||||
getAgentToModelMapForProfile,
|
||||
} = require('../get-shit-done/bin/lib/model-profiles.cjs');
|
||||
|
||||
function agentFilesOnDisk() {
|
||||
return fs.readdirSync(path.join(__dirname, '..', 'agents'))
|
||||
.filter((f) => /^gsd-.*\.md$/.test(f))
|
||||
.map((f) => f.replace(/\.md$/, ''))
|
||||
.sort();
|
||||
}
|
||||
|
||||
// ─── MODEL_PROFILES data integrity ────────────────────────────────────────────
|
||||
|
||||
describe('MODEL_PROFILES', () => {
|
||||
test('contains all expected GSD agents', () => {
|
||||
const expectedAgents = [
|
||||
'gsd-planner', 'gsd-roadmapper', 'gsd-executor',
|
||||
'gsd-phase-researcher', 'gsd-project-researcher', 'gsd-research-synthesizer',
|
||||
'gsd-debugger', 'gsd-codebase-mapper', 'gsd-verifier',
|
||||
'gsd-plan-checker', 'gsd-integration-checker', 'gsd-nyquist-auditor',
|
||||
'gsd-ui-researcher', 'gsd-ui-checker', 'gsd-ui-auditor',
|
||||
];
|
||||
for (const agent of expectedAgents) {
|
||||
assert.ok(MODEL_PROFILES[agent], `Missing agent: ${agent}`);
|
||||
}
|
||||
test('contains every shipped gsd agent file on disk (#3229)', () => {
|
||||
const expectedAgents = agentFilesOnDisk();
|
||||
const actualAgents = Object.keys(MODEL_PROFILES).sort();
|
||||
assert.deepStrictEqual(actualAgents, expectedAgents);
|
||||
});
|
||||
|
||||
test('every agent has quality, balanced, budget, and adaptive profiles', () => {
|
||||
|
||||
Reference in New Issue
Block a user