From 9e3b056b153e61f6454e36cdbd963c0c670755ff Mon Sep 17 00:00:00 2001 From: Tom Boucher Date: Thu, 11 Jun 2026 13:36:23 -0400 Subject: [PATCH] fix(#779): correct stale model-catalog model IDs verified against live providers (#1047) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verify-first audit: gemini opus gemini-3-pro→gemini-3.1-pro-preview (undefined in gemini-cli source), codex sonnet gpt-5.3-codex→gpt-5.4 (deprecated per OpenAI); qwen3-coder-next verified valid, unchanged. Adds a regression guard + sourcing note. Closes #779. --- .changeset/audit-779-model-catalog-ids.md | 5 +++++ bin/install.js | 2 +- docs/CONFIGURATION.md | 12 +++++++----- gsd-core/bin/shared/model-catalog.json | 12 ++++++------ gsd-core/workflows/settings-advanced.md | 12 ++++++------ gsd-core/workflows/settings.md | 2 +- tests/issue-2517-runtime-aware-profiles.test.cjs | 16 ++++++++-------- tests/model-catalog-runtime-defaults.test.cjs | 10 ++++++++++ 8 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 .changeset/audit-779-model-catalog-ids.md diff --git a/.changeset/audit-779-model-catalog-ids.md b/.changeset/audit-779-model-catalog-ids.md new file mode 100644 index 000000000..54bb0bdf3 --- /dev/null +++ b/.changeset/audit-779-model-catalog-ids.md @@ -0,0 +1,5 @@ +--- +type: Changed +pr: 1047 +--- +audit(#779): correct stale model-catalog IDs verified against live provider sources. The gemini opus default `gemini-3-pro` → `gemini-3.1-pro-preview` (the bare `gemini-3-pro` ID is undefined in gemini-cli source — only `gemini-3-pro-preview`/`gemini-3.1-pro-preview` exist) and the codex sonnet default `gpt-5.3-codex` → `gpt-5.4` (deprecated per OpenAI's Codex models page); the same two IDs are also updated in the `google`/`openai` provider-preset entries. `qwen3-coder-next` was verified valid (callable on Alibaba Model Studio) and left unchanged. Adds a regression guard against the retired IDs and a sourcing/verification note in CONFIGURATION.md. Catalog IDs are internal defaults; users who pinned the old IDs must update their config. diff --git a/bin/install.js b/bin/install.js index f25ae7af4..5fcd0298c 100755 --- a/bin/install.js +++ b/bin/install.js @@ -5748,7 +5748,7 @@ function mergeCodexConfig(configPath, gsdBlock) { /** * Repair config.toml files corrupted by pre-#1346 GSD installs. - * Non-boolean keys (e.g. model = "gpt-5.3-codex") that ended up under [features] + * Non-boolean keys (e.g. model = "gpt-5.4") that ended up under [features] * are relocated before the [features] header so Codex can parse them correctly. * Returns the content unchanged if no trapped keys are found. */ diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 2e1dc86b0..64396a17f 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -1203,14 +1203,16 @@ When `runtime` is set, profile tiers (`opus`/`sonnet`/`haiku`) resolve to runtim | Runtime | `opus` | `sonnet` | `haiku` | reasoning_effort | |---------|--------|----------|---------|------------------| | `claude` | `claude-opus-4-8` | `claude-sonnet-4-6` | `claude-haiku-4-5` | (not used) | -| `codex` | `gpt-5.5` | `gpt-5.3-codex` | `gpt-5.4-mini` | `xhigh` / `medium` / `medium` | -| `gemini` | `gemini-3-pro` | `gemini-3-flash` | `gemini-2.5-flash-lite` | (not used) | +| `codex` | `gpt-5.5` | `gpt-5.4` | `gpt-5.4-mini` | `xhigh` / `medium` / `medium` | +| `gemini` | `gemini-3.1-pro-preview` | `gemini-3-flash` | `gemini-2.5-flash-lite` | (not used) | | `qwen` | `qwen3-max-2026-01-23` | `qwen3-coder-plus` | `qwen3-coder-next` | (not used) | | `opencode` | `anthropic/claude-opus-4-8` | `anthropic/claude-sonnet-4-6` | `anthropic/claude-haiku-4-5` | (not used) | | `copilot` | `claude-opus-4-8` | `claude-sonnet-4-6` | `claude-haiku-4-5` | (not used) | | `hermes` | `anthropic/claude-opus-4-8` | `anthropic/claude-sonnet-4-6` | `anthropic/claude-haiku-4-5` | (not used) | | Group B (`kilo`, `cline`, `cursor`, `windsurf`, `augment`, `trae`, `codebuddy`, `antigravity`) | (no built-in default — your runtime handles model selection) | | | | +> **How these model IDs are sourced.** The catalog (`bin/shared/model-catalog.json`) pins each runtime's tier defaults to that provider's current frontier IDs, and may intentionally carry forward-dated IDs ahead of a provider's public docs. To verify an ID is live before changing it, check the provider's own source/API — e.g. Gemini: gemini-cli `packages/core/src/config/models.ts` or `gemini --model --prompt ping`; Codex: `codex debug models` or the OpenAI Codex models page; Qwen: Alibaba Model Studio model list. Only change an ID that the provider actually rejects — absence from documentation alone is not proof of invalidity. + **Codex example** — one config, tiered models, no large `model_overrides` block: ```json @@ -1220,7 +1222,7 @@ When `runtime` is set, profile tiers (`opus`/`sonnet`/`haiku`) resolve to runtim } ``` -This resolves `gsd-planner` → `gpt-5.5` (xhigh), `gsd-executor` → `gpt-5.3-codex` (medium), `gsd-codebase-mapper` → `gpt-5.4-mini` (medium). The Codex installer embeds `model = "..."` and `model_reasoning_effort = "..."` in each generated agent TOML. +This resolves `gsd-planner` → `gpt-5.5` (xhigh), `gsd-executor` → `gpt-5.4` (medium), `gsd-codebase-mapper` → `gpt-5.4-mini` (medium). The Codex installer embeds `model = "..."` and `model_reasoning_effort = "..."` in each generated agent TOML. **Claude example** — explicit opt-in resolves to full Claude IDs (no `resolve_model_ids: true` needed): @@ -1286,7 +1288,7 @@ Choose a provider and budget level via the settings workflow; GSD writes the can "provider": "openai", "budget": "medium", "high": "gpt-5.5", - "medium": "gpt-5.3-codex", + "medium": "gpt-5.4", "low": "gpt-5.4-mini" } } @@ -1304,7 +1306,7 @@ For advanced per-runtime control, `runtime_tiers` accepts explicit entries using "runtime_tiers": { "codex": { "opus": { "model": "gpt-5.5", "reasoning_effort": "high" }, - "sonnet": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" }, + "sonnet": { "model": "gpt-5.4", "reasoning_effort": "medium" }, "haiku": { "model": "gpt-5.4-mini", "reasoning_effort": "low" } } } diff --git a/gsd-core/bin/shared/model-catalog.json b/gsd-core/bin/shared/model-catalog.json index e7b546de3..8fad7e216 100644 --- a/gsd-core/bin/shared/model-catalog.json +++ b/gsd-core/bin/shared/model-catalog.json @@ -14,11 +14,11 @@ }, "codex": { "opus": { "model": "gpt-5.5", "reasoning_effort": "xhigh" }, - "sonnet": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" }, + "sonnet": { "model": "gpt-5.4", "reasoning_effort": "medium" }, "haiku": { "model": "gpt-5.4-mini", "reasoning_effort": "medium" } }, "gemini": { - "opus": { "model": "gemini-3-pro" }, + "opus": { "model": "gemini-3.1-pro-preview" }, "sonnet": { "model": "gemini-3-flash" }, "haiku": { "model": "gemini-2.5-flash-lite" } }, @@ -100,12 +100,12 @@ "haiku": { "low": { "model": "claude-haiku-4-5" }, "medium": { "model": "claude-haiku-4-5" }, "high": { "model": "claude-sonnet-4-6" } } }, "openai": { - "opus": { "low": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" }, "medium": { "model": "gpt-5.5", "reasoning_effort": "high" }, "high": { "model": "gpt-5.5", "reasoning_effort": "xhigh" } }, - "sonnet": { "low": { "model": "gpt-5.4-mini", "reasoning_effort": "low" }, "medium": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" }, "high": { "model": "gpt-5.5", "reasoning_effort": "medium" } }, - "haiku": { "low": { "model": "gpt-5.4-mini", "reasoning_effort": "minimal" }, "medium": { "model": "gpt-5.4-mini", "reasoning_effort": "medium" }, "high": { "model": "gpt-5.3-codex", "reasoning_effort": "medium" } } + "opus": { "low": { "model": "gpt-5.4", "reasoning_effort": "medium" }, "medium": { "model": "gpt-5.5", "reasoning_effort": "high" }, "high": { "model": "gpt-5.5", "reasoning_effort": "xhigh" } }, + "sonnet": { "low": { "model": "gpt-5.4-mini", "reasoning_effort": "low" }, "medium": { "model": "gpt-5.4", "reasoning_effort": "medium" }, "high": { "model": "gpt-5.5", "reasoning_effort": "medium" } }, + "haiku": { "low": { "model": "gpt-5.4-mini", "reasoning_effort": "minimal" }, "medium": { "model": "gpt-5.4-mini", "reasoning_effort": "medium" }, "high": { "model": "gpt-5.4", "reasoning_effort": "medium" } } }, "google": { - "opus": { "low": { "model": "gemini-2.5-flash-lite" }, "medium": { "model": "gemini-3-flash" }, "high": { "model": "gemini-3-pro" } }, + "opus": { "low": { "model": "gemini-2.5-flash-lite" }, "medium": { "model": "gemini-3-flash" }, "high": { "model": "gemini-3.1-pro-preview" } }, "sonnet": { "low": { "model": "gemini-2.5-flash-lite" }, "medium": { "model": "gemini-3-flash" }, "high": { "model": "gemini-3-flash" } }, "haiku": { "low": { "model": "gemini-2.5-flash-lite" }, "medium": { "model": "gemini-2.5-flash-lite" }, "high": { "model": "gemini-3-flash" } } }, diff --git a/gsd-core/workflows/settings-advanced.md b/gsd-core/workflows/settings-advanced.md index 4b020d798..3ef0d06fb 100644 --- a/gsd-core/workflows/settings-advanced.md +++ b/gsd-core/workflows/settings-advanced.md @@ -354,8 +354,8 @@ Built-in tier defaults by runtime: | Runtime | `opus` | `sonnet` | `haiku` | |------------|-------------------------------|---------------------------------|-------------------------------| | `claude` | `claude-opus-4-8` | `claude-sonnet-4-6` | `claude-haiku-4-5` | -| `codex` | `gpt-5.5` | `gpt-5.3-codex` | `gpt-5.4-mini` | -| `gemini` | `gemini-3-pro` | `gemini-3-flash` | `gemini-2.5-flash-lite` | +| `codex` | `gpt-5.5` | `gpt-5.4` | `gpt-5.4-mini` | +| `gemini` | `gemini-3.1-pro-preview` | `gemini-3-flash` | `gemini-2.5-flash-lite` | | `qwen` | `qwen3-max-2026-01-23` | `qwen3-coder-plus` | `qwen3-coder-next` | | `opencode` | `anthropic/claude-opus-4-8` | `anthropic/claude-sonnet-4-6` | `anthropic/claude-haiku-4-5` | | `copilot` | `claude-opus-4-8` | `claude-sonnet-4-6` | `claude-haiku-4-5` | @@ -625,7 +625,7 @@ AskUserQuestion([ options: [ { label: "anthropic", description: "claude-opus-4-8 / claude-sonnet-4-6 / claude-haiku-4-5 (Anthropic / Claude)" }, { label: "anthropic-fable", description: "claude-fable-5 / claude-sonnet-4-6 / claude-haiku-4-5 (Anthropic / Claude Fable opt-in)" }, - { label: "openai", description: "gpt-5.5 / gpt-5.3-codex / gpt-5.4-mini (OpenAI / Codex)" }, + { label: "openai", description: "gpt-5.5 / gpt-5.4 / gpt-5.4-mini (OpenAI / Codex)" }, { label: "Other known provider", description: "Type google or qwen; both still use the canonical tier mapping." } ] } @@ -661,10 +661,10 @@ Canonical tier mappings by provider and budget: | anthropic-fable | medium | claude-opus-4-8 | claude-sonnet-4-6 | claude-haiku-4-5 | | anthropic-fable | low | claude-haiku-4-5 | claude-haiku-4-5 | claude-haiku-4-5 | | openai | high | gpt-5.5 | gpt-5.5 | gpt-5.5 | -| openai | medium | gpt-5.5 | gpt-5.3-codex | gpt-5.4-mini | +| openai | medium | gpt-5.5 | gpt-5.4 | gpt-5.4-mini | | openai | low | gpt-5.4-mini | gpt-5.4-mini | gpt-5.4-mini | -| google | high | gemini-3-pro | gemini-3-pro | gemini-3-pro | -| google | medium | gemini-3-pro | gemini-3-flash | gemini-2.5-flash-lite | +| google | high | gemini-3.1-pro-preview | gemini-3.1-pro-preview | gemini-3.1-pro-preview | +| google | medium | gemini-3.1-pro-preview | gemini-3-flash | gemini-2.5-flash-lite | | google | low | gemini-2.5-flash-lite | gemini-2.5-flash-lite | gemini-2.5-flash-lite | | qwen | high | qwen3-max-2026-01-23 | qwen3-max-2026-01-23 | qwen3-max-2026-01-23 | | qwen | medium | qwen3-max-2026-01-23 | qwen3-coder-plus | qwen3-coder-next | diff --git a/gsd-core/workflows/settings.md b/gsd-core/workflows/settings.md index 2ead4abe5..60471556f 100644 --- a/gsd-core/workflows/settings.md +++ b/gsd-core/workflows/settings.md @@ -73,7 +73,7 @@ Parse current values (default to `true` if not present): ``` Note: Quality, Balanced, Budget, and Adaptive profiles assign semantic tiers (Opus/Sonnet/Haiku) to each agent. When `runtime` is set in .planning/config.json, -tiers resolve to runtime-native model IDs — on Codex that's gpt-5.4 / gpt-5.3-codex / +tiers resolve to runtime-native model IDs — on Codex that's gpt-5.5 / gpt-5.4 / gpt-5.4-mini with appropriate reasoning effort. See "Runtime-Aware Profiles" in docs/CONFIGURATION.md. diff --git a/tests/issue-2517-runtime-aware-profiles.test.cjs b/tests/issue-2517-runtime-aware-profiles.test.cjs index 5d9f09e81..c9f8a7006 100644 --- a/tests/issue-2517-runtime-aware-profiles.test.cjs +++ b/tests/issue-2517-runtime-aware-profiles.test.cjs @@ -8,7 +8,7 @@ * When `runtime` is set to a non-Claude value, profile tiers resolve to runtime- * native model IDs. * - * Codex: opus -> gpt-5.4 (xhigh), sonnet -> gpt-5.3-codex (medium), haiku -> gpt-5.4-mini (medium) + * Codex: opus -> gpt-5.5 (xhigh), sonnet -> gpt-5.4 (medium), haiku -> gpt-5.4-mini (medium) * * `runtime: "claude"` is the implicit default and is treated as a no-op for * resolution — it does not override `resolve_model_ids: "omit"` or any other @@ -176,9 +176,9 @@ describe('issue #2517: runtime "codex" — Codex tier resolution', () => { assert.strictEqual(rendered.value, 'xhigh'); }); - test('sonnet tier -> gpt-5.3-codex model; heavy-tier agent -> xhigh effort on codex', () => { + test('sonnet tier -> gpt-5.4 model; heavy-tier agent -> xhigh effort on codex', () => { writeConfig(tmpDir, { runtime: 'codex', model_profile: 'balanced' }); - assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-roadmapper'), 'gpt-5.3-codex'); + assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-roadmapper'), 'gpt-5.4'); // gsd-roadmapper is heavy routing tier → effort 'xhigh' (not catalog medium) const eff = resolveEffortInternal(tmpDir, 'gsd-roadmapper'); const rendered = renderEffortForRuntime('codex', eff); @@ -251,8 +251,8 @@ describe('issue #2517: precedence chain', () => { // gsd-planner quality -> opus -> overridden to gpt-5-pro assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'gpt-5-pro'); // haiku not overridden — fall back to spec defaults - // gsd-codebase-mapper quality -> sonnet -> gpt-5.3-codex - assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-codebase-mapper'), 'gpt-5.3-codex'); + // gsd-codebase-mapper quality -> sonnet -> gpt-5.4 + assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-codebase-mapper'), 'gpt-5.4'); }); test('partial profile_overrides — only opus overridden, sonnet uses default', () => { @@ -266,7 +266,7 @@ describe('issue #2517: precedence chain', () => { // gsd-planner balanced -> opus -> overridden to gpt-5-pro assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'gpt-5-pro'); // gsd-roadmapper balanced -> sonnet -> spec default - assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-roadmapper'), 'gpt-5.3-codex'); + assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-roadmapper'), 'gpt-5.4'); }); test('per-agent override beats profile override beats default', () => { @@ -643,9 +643,9 @@ describe('issue #2612: runtime "gemini" — Gemini tier resolution', () => { beforeEach(() => { isolateHome(); tmpDir = createTempProject(); _resetRuntimeWarningCacheForTests(); }); afterEach(() => { cleanup(tmpDir); restoreHome(); }); - test('opus tier -> gemini-3-pro', () => { + test('opus tier -> gemini-3.1-pro-preview', () => { writeConfig(tmpDir, { runtime: 'gemini', model_profile: 'quality' }); - assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'gemini-3-pro'); + assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'gemini-3.1-pro-preview'); }); test('sonnet tier -> gemini-3-flash', () => { diff --git a/tests/model-catalog-runtime-defaults.test.cjs b/tests/model-catalog-runtime-defaults.test.cjs index a619bb98b..95a4c02e7 100644 --- a/tests/model-catalog-runtime-defaults.test.cjs +++ b/tests/model-catalog-runtime-defaults.test.cjs @@ -14,6 +14,8 @@ const { allRuntimes } = require('../bin/install.js'); const ROOT = path.join(__dirname, '..'); const SETTINGS_ADVANCED = fs.readFileSync(path.join(ROOT, 'gsd-core', 'workflows', 'settings-advanced.md'), 'utf8'); const CONFIG_DOC = fs.readFileSync(path.join(ROOT, 'docs', 'CONFIGURATION.md'), 'utf8'); +const catalogPath = path.join(ROOT, 'gsd-core', 'bin', 'shared', 'model-catalog.json'); +const CATALOG_RAW = fs.readFileSync(catalogPath, 'utf8'); describe('model catalog runtime defaults parity (#3229)', () => { test('known runtimes include hermes and match catalog keys', () => { @@ -68,4 +70,12 @@ describe('model catalog runtime defaults parity (#3229)', () => { assert.ok(SETTINGS_ADVANCED.includes('Group B')); assert.ok(CONFIG_DOC.includes('Group B')); }); + + test('catalog contains no retired/invalid model IDs', () => { + // Retired per issue #779 verify-first audit (gemini-cli source + OpenAI Codex models page). + const RETIRED = ['"gemini-3-pro"', '"gpt-5.3-codex"']; + for (const id of RETIRED) { + assert.ok(!CATALOG_RAW.includes(id), `retired model ID ${id} must not appear in model-catalog.json (see #779)`); + } + }); });