enhance(#2229): add three-way claim disposition (admit/refute/abstain) to /gsd-explore research pass (#2543)
Closes #2229. Each claim surfaced by /gsd-explore's research pass is dispositioned admit, refute, or abstain, with abstentions routed to a visible ledger instead of being smoothed into confident prose. Refute and abstain are separated by whether the disagreeing source is authoritative for that claim; a strong prior is never authoritative alone. Two guards ride with it: conflict-abstention, and a tier floor that presents a would-be admit as an abstain when the researcher's resolved tier is the budget tier or cannot be determined. To make that floor enforceable, resolve-model now emits the effective tier (--pick tier). It was already computed above the resolve_model_ids omit gate but was unreachable from a workflow, which left the floor inert on every non-Claude install - the model id is blank under omit and runtime-substituted where a tier map exists, and the profile defaults to balanced. The tier signal mirrors every resolution step that can change which tier runs, including the model_policy preset, and reports unknown rather than guessing. Output is additive; model, profile and effort are unchanged. Two residuals are disclosed in the workflow rather than papered over: a raw-model-id model_overrides pin reports unknown and is floored (fails closed), and a model_profile_overrides entry repointing a tier at another tier's model can under-report (fails open, and predates this change). Admin merge used only to satisfy the missing secondary reviewer on a single-maintainer PR. No CI failure and no conflict were bypassed: 38 checks green, remote runner 32255/32255 on both Node lanes.
This commit is contained in:
Rezolv
@@ -44,6 +44,8 @@ const {
|
||||
resolveEffortInternal,
|
||||
resolveFastModeInternal,
|
||||
resolveEffortForTier,
|
||||
resolveTierFromConfig,
|
||||
resolveTierInternal,
|
||||
} = modelResolver;
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
@@ -4060,5 +4062,313 @@ describe('#49 resolveModelForTier: model_policy beats dynamic_routing', () => {
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-planner'), 'fable');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── #2229: computeProfileTier / resolveTierFromConfig / resolveTierInternal ──
|
||||
//
|
||||
// `tier` is additive on top of resolveModelInternal's model/profile/effort keys
|
||||
// (cmdResolveModel, src/commands.cts) so a caller can learn the effective tier
|
||||
// even under resolve_model_ids:"omit", where `model` is deliberately blank.
|
||||
|
||||
describe('#2229 resolveTierInternal / resolveTierFromConfig — config rows', () => {
|
||||
let tmpDir;
|
||||
beforeEach(() => { tmpDir = makeTempProject(); });
|
||||
afterEach(() => { if (tmpDir) cleanup(tmpDir); tmpDir = null; });
|
||||
|
||||
test('no config -> balanced profile -> "sonnet"', () => {
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'sonnet');
|
||||
});
|
||||
|
||||
test('model_profile=budget -> "haiku"', () => {
|
||||
writeConfig(tmpDir, { model_profile: 'budget' });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'haiku');
|
||||
});
|
||||
|
||||
test('model_overrides="haiku" alias -> "haiku"', () => {
|
||||
writeConfig(tmpDir, { model_overrides: { 'gsd-phase-researcher': 'haiku' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'haiku');
|
||||
});
|
||||
|
||||
test('model_overrides full Claude id "claude-haiku-4-5" maps back to its alias -> "haiku"', () => {
|
||||
writeConfig(tmpDir, { model_overrides: { 'gsd-phase-researcher': 'claude-haiku-4-5' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'haiku');
|
||||
});
|
||||
|
||||
test('model_overrides pinning a non-Claude id (gemini-2.5-flash-lite) is never guessed -> "unknown"', () => {
|
||||
writeConfig(tmpDir, { model_overrides: { 'gsd-phase-researcher': 'gemini-2.5-flash-lite' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'unknown');
|
||||
});
|
||||
|
||||
// Regression: CLAUDE_POLICY_ID_TO_ALIAS is a plain object literal indexed
|
||||
// with this config-supplied override value with no own-property guard. A
|
||||
// prototype-chain override ("toString", "constructor", "__proto__",
|
||||
// "valueOf", "hasOwnProperty") returned the inherited Function/Object
|
||||
// member (typeof "function"/"object") instead of falling through to
|
||||
// "unknown" — and a function-valued tier is silently dropped by
|
||||
// JSON.stringify in the CLI output, so `tier` vanished from `query
|
||||
// resolve-model` entirely.
|
||||
for (const proto of ['toString', 'constructor', '__proto__', 'valueOf', 'hasOwnProperty']) {
|
||||
test(`REGRESSION: model_overrides="${proto}" (prototype-chain key) -> string "unknown", not an inherited member`, () => {
|
||||
writeConfig(tmpDir, { model_overrides: { 'gsd-phase-researcher': proto } });
|
||||
const result = resolveTierInternal(tmpDir, 'gsd-phase-researcher');
|
||||
assert.strictEqual(typeof result, 'string', `expected a string, got ${typeof result}: ${String(result)}`);
|
||||
assert.strictEqual(result, 'unknown');
|
||||
});
|
||||
}
|
||||
|
||||
test('model_profile=inherit -> "inherit"', () => {
|
||||
writeConfig(tmpDir, { model_profile: 'inherit' });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'inherit');
|
||||
});
|
||||
|
||||
test('ADVERSARIAL: models=0 (non-object) does not throw and falls back to "sonnet"', () => {
|
||||
writeConfig(tmpDir, { models: 0 });
|
||||
assert.doesNotThrow(() => resolveTierInternal(tmpDir, 'gsd-phase-researcher'));
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'sonnet');
|
||||
});
|
||||
|
||||
for (const hostileModels of ['nope', [], null]) {
|
||||
test(`ADVERSARIAL: models=${JSON.stringify(hostileModels)} does not throw and falls back to "sonnet"`, () => {
|
||||
writeConfig(tmpDir, { models: hostileModels });
|
||||
assert.doesNotThrow(() => resolveTierInternal(tmpDir, 'gsd-phase-researcher'));
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'sonnet');
|
||||
});
|
||||
}
|
||||
|
||||
test('ADVERSARIAL: empty config object {} does not throw -> "sonnet"', () => {
|
||||
writeConfig(tmpDir, {});
|
||||
assert.doesNotThrow(() => resolveTierInternal(tmpDir, 'gsd-phase-researcher'));
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'sonnet');
|
||||
});
|
||||
|
||||
test('ADVERSARIAL: zero-byte config.json does not throw -> "sonnet"', () => {
|
||||
fs.writeFileSync(path.join(tmpDir, '.planning', 'config.json'), '', 'utf-8');
|
||||
assert.doesNotThrow(() => resolveTierInternal(tmpDir, 'gsd-phase-researcher'));
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'sonnet');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2229 resolveTierInternal — computeProfileTier is blind to model-id and profile-only checks', () => {
|
||||
let tmpDir;
|
||||
beforeEach(() => { tmpDir = makeTempProject(); });
|
||||
afterEach(() => { if (tmpDir) cleanup(tmpDir); tmpDir = null; });
|
||||
|
||||
test('models.research="haiku" wins over model_profile=balanced (a profile-only check would miss this)', () => {
|
||||
writeConfig(tmpDir, { model_profile: 'balanced', models: { research: 'haiku' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'haiku');
|
||||
});
|
||||
});
|
||||
|
||||
// Regression (#3282): resolveTierFromConfig used to report only the PROFILE
|
||||
// tier and ignore the model_policy preset step that resolveModelInternal
|
||||
// applies afterward (its "2.5" step) — so a haiku-tier run under
|
||||
// model_policy.budget:"low" reported tier "sonnet", silently defeating any
|
||||
// tier-floor keyed on this value. These pin the fixed behavior: the reported
|
||||
// tier and the resolved model must agree on which tier actually ran.
|
||||
describe('#3282 resolveTierFromConfig mirrors resolveModelInternal step 2.5 (model_policy)', () => {
|
||||
let tmpDir;
|
||||
beforeEach(() => { tmpDir = makeTempProject(); });
|
||||
afterEach(() => { if (tmpDir) cleanup(tmpDir); tmpDir = null; });
|
||||
|
||||
test('model_policy budget:low -> tier "haiku", agreeing with the resolved model', () => {
|
||||
writeConfig(tmpDir, { model_policy: { provider: 'anthropic', budget: 'low' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'haiku');
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-phase-researcher'), 'haiku');
|
||||
});
|
||||
|
||||
test('model_policy budget:high -> tier "opus", agreeing with the resolved model', () => {
|
||||
writeConfig(tmpDir, { model_policy: { provider: 'anthropic', budget: 'high' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'opus');
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-phase-researcher'), 'opus');
|
||||
});
|
||||
|
||||
test('model_policy budget:medium -> tier "sonnet"', () => {
|
||||
writeConfig(tmpDir, { model_policy: { provider: 'anthropic', budget: 'medium' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'sonnet');
|
||||
});
|
||||
|
||||
// Measured (2026-08-09): model_profile:"budget" gives gsd-phase-researcher
|
||||
// a "haiku" profile tier, but model_policy.budget:"high" resolves the
|
||||
// anthropic "haiku" preset's high slot to "claude-sonnet-5" — the POLICY
|
||||
// outranks the PROFILE, so the actually-dispatched tier is "sonnet", not
|
||||
// the profile's "haiku". A profile-only reporter would under-report this.
|
||||
test('model_policy outranks model_profile: budget profile + policy budget:high -> tier "sonnet"', () => {
|
||||
writeConfig(tmpDir, { model_profile: 'budget', model_policy: { provider: 'anthropic', budget: 'high' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'sonnet');
|
||||
});
|
||||
|
||||
// Measured (2026-08-09): on a non-Claude runtime, resolveModelInternal
|
||||
// returns the policy-resolved model id VERBATIM (no Claude-alias mapping
|
||||
// is attempted) — "qwen3-coder-plus" carries no tier we can name. Must
|
||||
// report "unknown", never fall back to the profile tier ("balanced" ->
|
||||
// "sonnet" here), which would silently reintroduce the under-report.
|
||||
test('non-Claude runtime + model_policy resolving a verbatim model id -> tier "unknown"', () => {
|
||||
writeConfig(tmpDir, { runtime: 'qwen', model_policy: { provider: 'qwen', budget: 'medium' } });
|
||||
assert.strictEqual(resolveModelInternal(tmpDir, 'gsd-phase-researcher'), 'qwen3-coder-plus');
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'unknown');
|
||||
});
|
||||
|
||||
// "fable" is a real, reachable tier value (Claude Code's Agent tool accepts
|
||||
// it, and CLAUDE_POLICY_ID_TO_ALIAS maps "claude-fable-5" to it) — it is
|
||||
// NOT the budget tier and must NOT be floored by a budget-tier check.
|
||||
test('model_overrides pinning "fable" -> tier "fable", a valid non-floored tier', () => {
|
||||
writeConfig(tmpDir, { model_overrides: { 'gsd-phase-researcher': 'fable' } });
|
||||
assert.strictEqual(resolveTierInternal(tmpDir, 'gsd-phase-researcher'), 'fable');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2229 resolveTierFromConfig — config-object form matches the cwd/CLI form', () => {
|
||||
// computeProfileTier itself is an internal (unexported) helper — per its own
|
||||
// doc comment, resolveTierFromConfig "calls straight back into it", so this
|
||||
// exercises the same code path through the public API.
|
||||
test('config-object form of the phase-type-override case matches the tmpDir form', () => {
|
||||
const cfg = { model_profile: 'balanced', models: { research: 'haiku' } };
|
||||
assert.strictEqual(resolveTierFromConfig(cfg, 'gsd-phase-researcher'), 'haiku');
|
||||
});
|
||||
|
||||
test('agent with no catalog entry and a non-inherit profile -> "unknown"', () => {
|
||||
assert.strictEqual(resolveTierFromConfig({ model_profile: 'balanced' }, 'not-a-real-agent'), 'unknown');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2229 cmdResolveModel CLI — tier key, additive-output guard, unknown_agent', () => {
|
||||
let tmpDir;
|
||||
// Local require, matching the folded-block idiom elsewhere in this file:
|
||||
// the top-level imports only pull in `cleanup` from helpers.cjs.
|
||||
const { runGsdTools } = require('./helpers.cjs');
|
||||
|
||||
beforeEach(() => { tmpDir = makeTempProject(); });
|
||||
afterEach(() => { if (tmpDir) cleanup(tmpDir); tmpDir = null; });
|
||||
|
||||
test('model_profile=balanced + models.research=haiku -> tier "haiku", profile still "balanced"', () => {
|
||||
writeConfig(tmpDir, { model_profile: 'balanced', models: { research: 'haiku' } });
|
||||
const result = runGsdTools('resolve-model gsd-phase-researcher', tmpDir);
|
||||
assert.ok(result.success, `resolve-model failed: ${result.error}`);
|
||||
const parsed = JSON.parse(result.output);
|
||||
assert.strictEqual(parsed.tier, 'haiku');
|
||||
assert.strictEqual(parsed.profile, 'balanced');
|
||||
});
|
||||
|
||||
test('resolve_model_ids=omit -> tier "sonnet" even though model is blank', () => {
|
||||
writeConfig(tmpDir, { resolve_model_ids: 'omit' });
|
||||
const result = runGsdTools('resolve-model gsd-phase-researcher', tmpDir);
|
||||
assert.ok(result.success, `resolve-model failed: ${result.error}`);
|
||||
const parsed = JSON.parse(result.output);
|
||||
assert.strictEqual(parsed.tier, 'sonnet');
|
||||
assert.strictEqual(parsed.model, '');
|
||||
});
|
||||
|
||||
test('CORE CASE: resolve_model_ids=omit + models.research=haiku -> tier "haiku" though model is blank ' +
|
||||
'and profile alone would also miss it', () => {
|
||||
writeConfig(tmpDir, { resolve_model_ids: 'omit', models: { research: 'haiku' } });
|
||||
const result = runGsdTools('resolve-model gsd-phase-researcher', tmpDir);
|
||||
assert.ok(result.success, `resolve-model failed: ${result.error}`);
|
||||
const parsed = JSON.parse(result.output);
|
||||
assert.strictEqual(parsed.tier, 'haiku');
|
||||
assert.strictEqual(parsed.model, '');
|
||||
});
|
||||
|
||||
test('runtime=codex + model_profile=budget -> tier "haiku" even though the resolved model id ' +
|
||||
'("gpt-5.6-luna") contains no "haiku" substring', () => {
|
||||
writeConfig(tmpDir, { runtime: 'codex', model_profile: 'budget' });
|
||||
const result = runGsdTools('resolve-model gsd-phase-researcher', tmpDir);
|
||||
assert.ok(result.success, `resolve-model failed: ${result.error}`);
|
||||
const parsed = JSON.parse(result.output);
|
||||
assert.strictEqual(parsed.tier, 'haiku');
|
||||
assert.strictEqual(parsed.model, 'gpt-5.6-luna');
|
||||
});
|
||||
|
||||
test('unknown agent -> tier "unknown", unknown_agent:true still present', () => {
|
||||
writeConfig(tmpDir, {});
|
||||
const result = runGsdTools('resolve-model not-a-real-agent', tmpDir);
|
||||
assert.ok(result.success, `resolve-model failed: ${result.error}`);
|
||||
const parsed = JSON.parse(result.output);
|
||||
assert.strictEqual(parsed.tier, 'unknown');
|
||||
assert.strictEqual(parsed.unknown_agent, true);
|
||||
});
|
||||
|
||||
// Regression: without the Object.hasOwn guard, model_overrides:"toString"
|
||||
// resolved a function-valued tier, and JSON.stringify silently DROPS a
|
||||
// function-valued object key — so `tier` disappeared from the parsed
|
||||
// output entirely rather than merely holding a wrong value. Asserting the
|
||||
// key's presence (not just its value) is what would have caught that.
|
||||
test('REGRESSION: model_overrides="toString" (prototype-chain key) -> parsed JSON still HAS a "tier" key, equal to "unknown"', () => {
|
||||
writeConfig(tmpDir, { model_overrides: { 'gsd-phase-researcher': 'toString' } });
|
||||
const result = runGsdTools('resolve-model gsd-phase-researcher', tmpDir);
|
||||
assert.ok(result.success, `resolve-model failed: ${result.error}`);
|
||||
const parsed = JSON.parse(result.output);
|
||||
assert.ok(Object.hasOwn(parsed, 'tier'), `expected a "tier" key in ${result.output}`);
|
||||
assert.strictEqual(parsed.tier, 'unknown');
|
||||
});
|
||||
|
||||
test('--pick tier prints the bare string "sonnet" (no JSON, no quotes) for {}', () => {
|
||||
writeConfig(tmpDir, {});
|
||||
const result = runGsdTools(['resolve-model', 'gsd-phase-researcher', '--pick', 'tier'], tmpDir);
|
||||
assert.ok(result.success, `resolve-model failed: ${result.error}`);
|
||||
assert.strictEqual(result.output, 'sonnet');
|
||||
});
|
||||
|
||||
// Compatibility guard, not a tier test: pins that adding `tier` to
|
||||
// cmdResolveModel's output did not disturb the pre-existing
|
||||
// model/profile/effort keys that other consumers already parse.
|
||||
test('COMPATIBILITY GUARD: adding tier did not disturb model/profile/effort for {}', () => {
|
||||
writeConfig(tmpDir, {});
|
||||
const result = runGsdTools('resolve-model gsd-phase-researcher', tmpDir);
|
||||
assert.ok(result.success, `resolve-model failed: ${result.error}`);
|
||||
const parsed = JSON.parse(result.output);
|
||||
assert.strictEqual(parsed.model, 'sonnet');
|
||||
assert.strictEqual(parsed.profile, 'balanced');
|
||||
assert.strictEqual(parsed.effort, 'high');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2229 PARITY GUARD: catalog budget tier for gsd-phase-researcher backs explore.md\'s tier-floor text', () => {
|
||||
test('MODEL_PROFILES["gsd-phase-researcher"].budget === "haiku"', () => {
|
||||
// gsd-core/workflows/explore.md's tier-floor text names "haiku" as this
|
||||
// agent's budget tier; if the catalog ever moves that tier, this test
|
||||
// fails instead of the floor silently ceasing to fire.
|
||||
const { MODEL_PROFILES } = require('../gsd-core/bin/lib/model-profiles.cjs');
|
||||
assert.strictEqual(MODEL_PROFILES['gsd-phase-researcher'].budget, 'haiku');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2229 PROPERTY: resolveTierFromConfig never throws and always returns a known tier', () => {
|
||||
test('for arbitrary plain-object configs', () => {
|
||||
const fc = require('./helpers/fast-check-setup.cjs');
|
||||
const KNOWN_TIERS = new Set(['opus', 'sonnet', 'haiku', 'fable', 'inherit', 'unknown']);
|
||||
fc.assert(
|
||||
fc.property(fc.object(), (cfg) => {
|
||||
let result;
|
||||
assert.doesNotThrow(() => { result = resolveTierFromConfig(cfg, 'gsd-phase-researcher'); });
|
||||
assert.ok(typeof result === 'string', `expected a string, got: ${JSON.stringify(result)}`);
|
||||
assert.ok(KNOWN_TIERS.has(result), `unexpected tier value: ${JSON.stringify(result)}`);
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
// Regression: the generic fc.object() generator above almost never produces
|
||||
// a prototype-chain string ("toString", "constructor", "__proto__",
|
||||
// "valueOf", "hasOwnProperty") as a model_overrides value, so it never
|
||||
// exercised the CLAUDE_POLICY_ID_TO_ALIAS own-property guard. This variant
|
||||
// pins model_overrides['gsd-phase-researcher'] to a mix of those names and
|
||||
// arbitrary strings so the "always returns a known tier" invariant is
|
||||
// actually checked against the class of input that broke it.
|
||||
test('for configs whose model_overrides value may be a prototype-chain key', () => {
|
||||
const fc = require('./helpers/fast-check-setup.cjs');
|
||||
const KNOWN_TIERS = new Set(['opus', 'sonnet', 'haiku', 'fable', 'inherit', 'unknown']);
|
||||
const overrideArb = fc.oneof(
|
||||
fc.constantFrom('toString', 'constructor', '__proto__', 'valueOf', 'hasOwnProperty'),
|
||||
fc.string(),
|
||||
);
|
||||
fc.assert(
|
||||
fc.property(fc.object(), overrideArb, (baseCfg, overrideValue) => {
|
||||
const cfg = { ...baseCfg, model_overrides: { 'gsd-phase-researcher': overrideValue } };
|
||||
let result;
|
||||
assert.doesNotThrow(() => { result = resolveTierFromConfig(cfg, 'gsd-phase-researcher'); });
|
||||
assert.strictEqual(typeof result, 'string', `expected a string, got: ${typeof result} (${JSON.stringify(result)})`);
|
||||
assert.ok(KNOWN_TIERS.has(result), `unexpected tier value: ${JSON.stringify(result)}`);
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user