Verify-first audit: gemini opus gemini-3-pro→gemini-3.1-pro-preview (undefined in gemini-cli source), codex sonnet gpt-5.3-codex→gpt-5.4 (deprecated per OpenAI); qwen3-coder-next verified valid, unchanged. Adds a regression guard + sourcing note. Closes #779.
This commit is contained in:
Tom Boucher
5
.changeset/audit-779-model-catalog-ids.md
Normal file
5
.changeset/audit-779-model-catalog-ids.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
type: Changed
|
||||
pr: 1047
|
||||
---
|
||||
audit(#779): correct stale model-catalog IDs verified against live provider sources. The gemini opus default `gemini-3-pro` → `gemini-3.1-pro-preview` (the bare `gemini-3-pro` ID is undefined in gemini-cli source — only `gemini-3-pro-preview`/`gemini-3.1-pro-preview` exist) and the codex sonnet default `gpt-5.3-codex` → `gpt-5.4` (deprecated per OpenAI's Codex models page); the same two IDs are also updated in the `google`/`openai` provider-preset entries. `qwen3-coder-next` was verified valid (callable on Alibaba Model Studio) and left unchanged. Adds a regression guard against the retired IDs and a sourcing/verification note in CONFIGURATION.md. Catalog IDs are internal defaults; users who pinned the old IDs must update their config.
|
||||
@@ -5748,7 +5748,7 @@ function mergeCodexConfig(configPath, gsdBlock) {
|
||||
|
||||
/**
|
||||
* Repair config.toml files corrupted by pre-#1346 GSD installs.
|
||||
* Non-boolean keys (e.g. model = "gpt-5.3-codex") that ended up under [features]
|
||||
* Non-boolean keys (e.g. model = "gpt-5.4") that ended up under [features]
|
||||
* are relocated before the [features] header so Codex can parse them correctly.
|
||||
* Returns the content unchanged if no trapped keys are found.
|
||||
*/
|
||||
|
||||
@@ -1203,14 +1203,16 @@ When `runtime` is set, profile tiers (`opus`/`sonnet`/`haiku`) resolve to runtim
|
||||
| Runtime | `opus` | `sonnet` | `haiku` | reasoning_effort |
|
||||
|---------|--------|----------|---------|------------------|
|
||||
| `claude` | `claude-opus-4-8` | `claude-sonnet-4-6` | `claude-haiku-4-5` | (not used) |
|
||||
| `codex` | `gpt-5.5` | `gpt-5.3-codex` | `gpt-5.4-mini` | `xhigh` / `medium` / `medium` |
|
||||
| `gemini` | `gemini-3-pro` | `gemini-3-flash` | `gemini-2.5-flash-lite` | (not used) |
|
||||
| `codex` | `gpt-5.5` | `gpt-5.4` | `gpt-5.4-mini` | `xhigh` / `medium` / `medium` |
|
||||
| `gemini` | `gemini-3.1-pro-preview` | `gemini-3-flash` | `gemini-2.5-flash-lite` | (not used) |
|
||||
| `qwen` | `qwen3-max-2026-01-23` | `qwen3-coder-plus` | `qwen3-coder-next` | (not used) |
|
||||
| `opencode` | `anthropic/claude-opus-4-8` | `anthropic/claude-sonnet-4-6` | `anthropic/claude-haiku-4-5` | (not used) |
|
||||
| `copilot` | `claude-opus-4-8` | `claude-sonnet-4-6` | `claude-haiku-4-5` | (not used) |
|
||||
| `hermes` | `anthropic/claude-opus-4-8` | `anthropic/claude-sonnet-4-6` | `anthropic/claude-haiku-4-5` | (not used) |
|
||||
| Group B (`kilo`, `cline`, `cursor`, `windsurf`, `augment`, `trae`, `codebuddy`, `antigravity`) | (no built-in default — your runtime handles model selection) | | | |
|
||||
|
||||
> **How these model IDs are sourced.** The catalog (`bin/shared/model-catalog.json`) pins each runtime's tier defaults to that provider's current frontier IDs, and may intentionally carry forward-dated IDs ahead of a provider's public docs. To verify an ID is live before changing it, check the provider's own source/API — e.g. Gemini: gemini-cli `packages/core/src/config/models.ts` or `gemini --model <id> --prompt ping`; Codex: `codex debug models` or the OpenAI Codex models page; Qwen: Alibaba Model Studio model list. Only change an ID that the provider actually rejects — absence from documentation alone is not proof of invalidity.
|
||||
|
||||
**Codex example** — one config, tiered models, no large `model_overrides` block:
|
||||
|
||||
```json
|
||||
@@ -1220,7 +1222,7 @@ When `runtime` is set, profile tiers (`opus`/`sonnet`/`haiku`) resolve to runtim
|
||||
}
|
||||
```
|
||||
|
||||
This resolves `gsd-planner` → `gpt-5.5` (xhigh), `gsd-executor` → `gpt-5.3-codex` (medium), `gsd-codebase-mapper` → `gpt-5.4-mini` (medium). The Codex installer embeds `model = "..."` and `model_reasoning_effort = "..."` in each generated agent TOML.
|
||||
This resolves `gsd-planner` → `gpt-5.5` (xhigh), `gsd-executor` → `gpt-5.4` (medium), `gsd-codebase-mapper` → `gpt-5.4-mini` (medium). The Codex installer embeds `model = "..."` and `model_reasoning_effort = "..."` in each generated agent TOML.
|
||||
|
||||
**Claude example** — explicit opt-in resolves to full Claude IDs (no `resolve_model_ids: true` needed):
|
||||
|
||||
@@ -1286,7 +1288,7 @@ Choose a provider and budget level via the settings workflow; GSD writes the can
|
||||
"provider": "openai",
|
||||
"budget": "medium",
|
||||
"high": "gpt-5.5",
|
||||
"medium": "gpt-5.3-codex",
|
||||
"medium": "gpt-5.4",
|
||||
"low": "gpt-5.4-mini"
|
||||
}
|
||||
}
|
||||
@@ -1304,7 +1306,7 @@ For advanced per-runtime control, `runtime_tiers` accepts explicit entries using
|
||||
"runtime_tiers": {
|
||||
"codex": {
|
||||
"opus": { "model": "gpt-5.5", "reasoning_effort": "high" },
|
||||
"sonnet": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" },
|
||||
"sonnet": { "model": "gpt-5.4", "reasoning_effort": "medium" },
|
||||
"haiku": { "model": "gpt-5.4-mini", "reasoning_effort": "low" }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,11 +14,11 @@
|
||||
},
|
||||
"codex": {
|
||||
"opus": { "model": "gpt-5.5", "reasoning_effort": "xhigh" },
|
||||
"sonnet": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" },
|
||||
"sonnet": { "model": "gpt-5.4", "reasoning_effort": "medium" },
|
||||
"haiku": { "model": "gpt-5.4-mini", "reasoning_effort": "medium" }
|
||||
},
|
||||
"gemini": {
|
||||
"opus": { "model": "gemini-3-pro" },
|
||||
"opus": { "model": "gemini-3.1-pro-preview" },
|
||||
"sonnet": { "model": "gemini-3-flash" },
|
||||
"haiku": { "model": "gemini-2.5-flash-lite" }
|
||||
},
|
||||
@@ -100,12 +100,12 @@
|
||||
"haiku": { "low": { "model": "claude-haiku-4-5" }, "medium": { "model": "claude-haiku-4-5" }, "high": { "model": "claude-sonnet-4-6" } }
|
||||
},
|
||||
"openai": {
|
||||
"opus": { "low": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" }, "medium": { "model": "gpt-5.5", "reasoning_effort": "high" }, "high": { "model": "gpt-5.5", "reasoning_effort": "xhigh" } },
|
||||
"sonnet": { "low": { "model": "gpt-5.4-mini", "reasoning_effort": "low" }, "medium": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" }, "high": { "model": "gpt-5.5", "reasoning_effort": "medium" } },
|
||||
"haiku": { "low": { "model": "gpt-5.4-mini", "reasoning_effort": "minimal" }, "medium": { "model": "gpt-5.4-mini", "reasoning_effort": "medium" }, "high": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" } }
|
||||
"opus": { "low": { "model": "gpt-5.4", "reasoning_effort": "medium" }, "medium": { "model": "gpt-5.5", "reasoning_effort": "high" }, "high": { "model": "gpt-5.5", "reasoning_effort": "xhigh" } },
|
||||
"sonnet": { "low": { "model": "gpt-5.4-mini", "reasoning_effort": "low" }, "medium": { "model": "gpt-5.4", "reasoning_effort": "medium" }, "high": { "model": "gpt-5.5", "reasoning_effort": "medium" } },
|
||||
"haiku": { "low": { "model": "gpt-5.4-mini", "reasoning_effort": "minimal" }, "medium": { "model": "gpt-5.4-mini", "reasoning_effort": "medium" }, "high": { "model": "gpt-5.4", "reasoning_effort": "medium" } }
|
||||
},
|
||||
"google": {
|
||||
"opus": { "low": { "model": "gemini-2.5-flash-lite" }, "medium": { "model": "gemini-3-flash" }, "high": { "model": "gemini-3-pro" } },
|
||||
"opus": { "low": { "model": "gemini-2.5-flash-lite" }, "medium": { "model": "gemini-3-flash" }, "high": { "model": "gemini-3.1-pro-preview" } },
|
||||
"sonnet": { "low": { "model": "gemini-2.5-flash-lite" }, "medium": { "model": "gemini-3-flash" }, "high": { "model": "gemini-3-flash" } },
|
||||
"haiku": { "low": { "model": "gemini-2.5-flash-lite" }, "medium": { "model": "gemini-2.5-flash-lite" }, "high": { "model": "gemini-3-flash" } }
|
||||
},
|
||||
|
||||
@@ -354,8 +354,8 @@ Built-in tier defaults by runtime:
|
||||
| Runtime | `opus` | `sonnet` | `haiku` |
|
||||
|------------|-------------------------------|---------------------------------|-------------------------------|
|
||||
| `claude` | `claude-opus-4-8` | `claude-sonnet-4-6` | `claude-haiku-4-5` |
|
||||
| `codex` | `gpt-5.5` | `gpt-5.3-codex` | `gpt-5.4-mini` |
|
||||
| `gemini` | `gemini-3-pro` | `gemini-3-flash` | `gemini-2.5-flash-lite` |
|
||||
| `codex` | `gpt-5.5` | `gpt-5.4` | `gpt-5.4-mini` |
|
||||
| `gemini` | `gemini-3.1-pro-preview` | `gemini-3-flash` | `gemini-2.5-flash-lite` |
|
||||
| `qwen` | `qwen3-max-2026-01-23` | `qwen3-coder-plus` | `qwen3-coder-next` |
|
||||
| `opencode` | `anthropic/claude-opus-4-8` | `anthropic/claude-sonnet-4-6` | `anthropic/claude-haiku-4-5` |
|
||||
| `copilot` | `claude-opus-4-8` | `claude-sonnet-4-6` | `claude-haiku-4-5` |
|
||||
@@ -625,7 +625,7 @@ AskUserQuestion([
|
||||
options: [
|
||||
{ label: "anthropic", description: "claude-opus-4-8 / claude-sonnet-4-6 / claude-haiku-4-5 (Anthropic / Claude)" },
|
||||
{ label: "anthropic-fable", description: "claude-fable-5 / claude-sonnet-4-6 / claude-haiku-4-5 (Anthropic / Claude Fable opt-in)" },
|
||||
{ label: "openai", description: "gpt-5.5 / gpt-5.3-codex / gpt-5.4-mini (OpenAI / Codex)" },
|
||||
{ label: "openai", description: "gpt-5.5 / gpt-5.4 / gpt-5.4-mini (OpenAI / Codex)" },
|
||||
{ label: "Other known provider", description: "Type google or qwen; both still use the canonical tier mapping." }
|
||||
]
|
||||
}
|
||||
@@ -661,10 +661,10 @@ Canonical tier mappings by provider and budget:
|
||||
| anthropic-fable | medium | claude-opus-4-8 | claude-sonnet-4-6 | claude-haiku-4-5 |
|
||||
| anthropic-fable | low | claude-haiku-4-5 | claude-haiku-4-5 | claude-haiku-4-5 |
|
||||
| openai | high | gpt-5.5 | gpt-5.5 | gpt-5.5 |
|
||||
| openai | medium | gpt-5.5 | gpt-5.3-codex | gpt-5.4-mini |
|
||||
| openai | medium | gpt-5.5 | gpt-5.4 | gpt-5.4-mini |
|
||||
| openai | low | gpt-5.4-mini | gpt-5.4-mini | gpt-5.4-mini |
|
||||
| google | high | gemini-3-pro | gemini-3-pro | gemini-3-pro |
|
||||
| google | medium | gemini-3-pro | gemini-3-flash | gemini-2.5-flash-lite |
|
||||
| google | high | gemini-3.1-pro-preview | gemini-3.1-pro-preview | gemini-3.1-pro-preview |
|
||||
| google | medium | gemini-3.1-pro-preview | gemini-3-flash | gemini-2.5-flash-lite |
|
||||
| google | low | gemini-2.5-flash-lite | gemini-2.5-flash-lite | gemini-2.5-flash-lite |
|
||||
| qwen | high | qwen3-max-2026-01-23 | qwen3-max-2026-01-23 | qwen3-max-2026-01-23 |
|
||||
| qwen | medium | qwen3-max-2026-01-23 | qwen3-coder-plus | qwen3-coder-next |
|
||||
|
||||
@@ -73,7 +73,7 @@ Parse current values (default to `true` if not present):
|
||||
```
|
||||
Note: Quality, Balanced, Budget, and Adaptive profiles assign semantic tiers
|
||||
(Opus/Sonnet/Haiku) to each agent. When `runtime` is set in .planning/config.json,
|
||||
tiers resolve to runtime-native model IDs — on Codex that's gpt-5.4 / gpt-5.3-codex /
|
||||
tiers resolve to runtime-native model IDs — on Codex that's gpt-5.5 / gpt-5.4 /
|
||||
gpt-5.4-mini with appropriate reasoning effort. See "Runtime-Aware Profiles" in
|
||||
docs/CONFIGURATION.md.
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
* When `runtime` is set to a non-Claude value, profile tiers resolve to runtime-
|
||||
* native model IDs.
|
||||
*
|
||||
* Codex: opus -> gpt-5.4 (xhigh), sonnet -> gpt-5.3-codex (medium), haiku -> gpt-5.4-mini (medium)
|
||||
* Codex: opus -> gpt-5.5 (xhigh), sonnet -> gpt-5.4 (medium), haiku -> gpt-5.4-mini (medium)
|
||||
*
|
||||
* `runtime: "claude"` is the implicit default and is treated as a no-op for
|
||||
* resolution — it does not override `resolve_model_ids: "omit"` or any other
|
||||
@@ -176,9 +176,9 @@ describe('issue #2517: runtime "codex" — Codex tier resolution', () => {
|
||||
assert.strictEqual(rendered.value, 'xhigh');
|
||||
});
|
||||
|
||||
test('sonnet tier -> gpt-5.3-codex model; heavy-tier agent -> xhigh effort on codex', () => {
|
||||
test('sonnet tier -> gpt-5.4 model; heavy-tier agent -> xhigh effort on codex', () => {
|
||||
writeConfig(tmpDir, { runtime: 'codex', model_profile: 'balanced' });
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-roadmapper'), 'gpt-5.3-codex');
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-roadmapper'), 'gpt-5.4');
|
||||
// gsd-roadmapper is heavy routing tier → effort 'xhigh' (not catalog medium)
|
||||
const eff = resolveEffortInternal(tmpDir, 'gsd-roadmapper');
|
||||
const rendered = renderEffortForRuntime('codex', eff);
|
||||
@@ -251,8 +251,8 @@ describe('issue #2517: precedence chain', () => {
|
||||
// gsd-planner quality -> opus -> overridden to gpt-5-pro
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'gpt-5-pro');
|
||||
// haiku not overridden — fall back to spec defaults
|
||||
// gsd-codebase-mapper quality -> sonnet -> gpt-5.3-codex
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-codebase-mapper'), 'gpt-5.3-codex');
|
||||
// gsd-codebase-mapper quality -> sonnet -> gpt-5.4
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-codebase-mapper'), 'gpt-5.4');
|
||||
});
|
||||
|
||||
test('partial profile_overrides — only opus overridden, sonnet uses default', () => {
|
||||
@@ -266,7 +266,7 @@ describe('issue #2517: precedence chain', () => {
|
||||
// gsd-planner balanced -> opus -> overridden to gpt-5-pro
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'gpt-5-pro');
|
||||
// gsd-roadmapper balanced -> sonnet -> spec default
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-roadmapper'), 'gpt-5.3-codex');
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-roadmapper'), 'gpt-5.4');
|
||||
});
|
||||
|
||||
test('per-agent override beats profile override beats default', () => {
|
||||
@@ -643,9 +643,9 @@ describe('issue #2612: runtime "gemini" — Gemini tier resolution', () => {
|
||||
beforeEach(() => { isolateHome(); tmpDir = createTempProject(); _resetRuntimeWarningCacheForTests(); });
|
||||
afterEach(() => { cleanup(tmpDir); restoreHome(); });
|
||||
|
||||
test('opus tier -> gemini-3-pro', () => {
|
||||
test('opus tier -> gemini-3.1-pro-preview', () => {
|
||||
writeConfig(tmpDir, { runtime: 'gemini', model_profile: 'quality' });
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'gemini-3-pro');
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'gemini-3.1-pro-preview');
|
||||
});
|
||||
|
||||
test('sonnet tier -> gemini-3-flash', () => {
|
||||
|
||||
@@ -14,6 +14,8 @@ const { allRuntimes } = require('../bin/install.js');
|
||||
const ROOT = path.join(__dirname, '..');
|
||||
const SETTINGS_ADVANCED = fs.readFileSync(path.join(ROOT, 'gsd-core', 'workflows', 'settings-advanced.md'), 'utf8');
|
||||
const CONFIG_DOC = fs.readFileSync(path.join(ROOT, 'docs', 'CONFIGURATION.md'), 'utf8');
|
||||
const catalogPath = path.join(ROOT, 'gsd-core', 'bin', 'shared', 'model-catalog.json');
|
||||
const CATALOG_RAW = fs.readFileSync(catalogPath, 'utf8');
|
||||
|
||||
describe('model catalog runtime defaults parity (#3229)', () => {
|
||||
test('known runtimes include hermes and match catalog keys', () => {
|
||||
@@ -68,4 +70,12 @@ describe('model catalog runtime defaults parity (#3229)', () => {
|
||||
assert.ok(SETTINGS_ADVANCED.includes('Group B'));
|
||||
assert.ok(CONFIG_DOC.includes('Group B'));
|
||||
});
|
||||
|
||||
test('catalog contains no retired/invalid model IDs', () => {
|
||||
// Retired per issue #779 verify-first audit (gemini-cli source + OpenAI Codex models page).
|
||||
const RETIRED = ['"gemini-3-pro"', '"gpt-5.3-codex"'];
|
||||
for (const id of RETIRED) {
|
||||
assert.ok(!CATALOG_RAW.includes(id), `retired model ID ${id} must not appear in model-catalog.json (see #779)`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user