fix(#33): add regression test for model_profile adaptive option (#149)

* fix(#33): add regression test asserting adaptive is reachable in settings.md UI

The schema (sdk/shared/model-catalog.json) defines 5 model_profile values
(quality, balanced, budget, adaptive, inherit). The settings.md AskUserQuestion
previously omitted `adaptive`; the fix (two-question split, #3784) is already
applied. This test locks the schema/UI contract so the gap cannot regress.

Closes #33

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* chore(#33): add changeset fragment

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* fix(#33): address review feedback — presence not count, drop structural caps

Replace exact-count assertion (profiles.length === 5) with presence
assertion (profiles.includes('adaptive')). Remove the options-per-question
≤4 cap guard — that is a structural implementation detail, not a
behavioural regression guard for issue #33.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
2026-05-24 13:30:00 -04:00
committed by GitHub
parent 21e3ce53c0
commit 7bf06f7a68
2 changed files with 156 additions and 0 deletions

View File

@@ -0,0 +1,5 @@
---
type: Fixed
pr: 149
---
Added regression test for the `model_profile: adaptive` settings option so future changes cannot silently break adaptive-mode selection.

View File

@@ -0,0 +1,151 @@
'use strict';
// allow-test-rule: source-text-is-the-product
// The deployed settings.md IS the product — testing its text content tests the deployed contract.
/**
* Regression test for issue #33
*
* model_profile UI shows 4 options, schema has 5 — `adaptive` missing from
* `settings.md` AskUserQuestion.
*
* The schema (sdk/shared/model-catalog.json `profiles` array) defines 5 valid
* model_profile values: quality, balanced, budget, adaptive, inherit. The
* settings.md AskUserQuestion block for model_profile originally listed only 4
* options (Quality, Balanced, Budget, Inherit) — `adaptive` was missing.
*
* Fix: the model_profile selection uses a two-question split. Q1 routes between
* Adaptive / Standard-tier / Inherit (3 options). Q2 (only when Q1 = Standard)
* asks Quality / Balanced / Budget. This keeps every individual options array
* within the AskUserQuestion 4-option cap while making all 5 profiles reachable.
*
* Fixes: #33
*/
const { describe, test, before } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const REPO_ROOT = path.join(__dirname, '..');
const SETTINGS_PATH = path.join(REPO_ROOT, 'get-shit-done', 'workflows', 'settings.md');
const CATALOG_PATH = path.join(REPO_ROOT, 'sdk', 'shared', 'model-catalog.json');
/**
* Collect every label: "..." value within a text block, lowercased.
*/
function extractOptionLabels(block) {
const re = /label:\s*"([^"]+)"/g;
const labels = [];
let m;
while ((m = re.exec(block)) !== null) {
labels.push(m[1].toLowerCase());
}
return labels;
}
describe('issue #33: model_profile schema and settings.md UI are in sync', () => {
let catalog;
let settingsContent;
let presentBlock;
before(() => {
catalog = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf-8'));
settingsContent = fs.readFileSync(SETTINGS_PATH, 'utf-8');
const presentMatch = settingsContent.match(/<step name="present_settings">[\s\S]*?<\/step>/);
assert.ok(presentMatch, 'settings.md must contain a present_settings step');
presentBlock = presentMatch[0];
});
// -- (a) Schema contract ---------------------------------------------------
test('schema includes the adaptive model_profile value', () => {
assert.ok(
Array.isArray(catalog.profiles),
'sdk/shared/model-catalog.json must have a "profiles" array'
);
assert.ok(
catalog.profiles.includes('adaptive'),
'model-catalog should include the adaptive profile. Got: [' + catalog.profiles.join(', ') + ']'
);
});
test('schema includes adaptive as a model_profile value', () => {
assert.ok(
catalog.profiles.includes('adaptive'),
'"adaptive" must be in sdk/shared/model-catalog.json profiles. Got: [' + catalog.profiles.join(', ') + ']'
);
});
test('schema includes all expected model_profile values', () => {
const expected = ['quality', 'balanced', 'budget', 'adaptive', 'inherit'];
for (const profile of expected) {
assert.ok(
catalog.profiles.includes(profile),
'Schema must include "' + profile + '" in profiles. Got: [' + catalog.profiles.join(', ') + ']'
);
}
});
// -- (b) UI contract — all 5 profiles reachable via present_settings -------
test('present_settings includes Adaptive as a selectable option (#33)', () => {
const labels = extractOptionLabels(presentBlock);
assert.ok(
labels.some(l => l === 'adaptive' || l.startsWith('adaptive')),
'Issue #33: present_settings must include an "Adaptive" label in its model_profile AskUserQuestion options so users can select it interactively. Got labels: [' + labels.join(', ') + ']'
);
});
test('present_settings includes Quality as a selectable option', () => {
const labels = extractOptionLabels(presentBlock);
assert.ok(
labels.some(l => l === 'quality' || l.startsWith('quality')),
'present_settings must include a "Quality" option. Got: [' + labels.join(', ') + ']'
);
});
test('present_settings includes Balanced as a selectable option', () => {
const labels = extractOptionLabels(presentBlock);
assert.ok(
labels.some(l => l === 'balanced' || l.startsWith('balanced')),
'present_settings must include a "Balanced" option. Got: [' + labels.join(', ') + ']'
);
});
test('present_settings includes Budget as a selectable option', () => {
const labels = extractOptionLabels(presentBlock);
assert.ok(
labels.some(l => l === 'budget' || l.startsWith('budget')),
'present_settings must include a "Budget" option. Got: [' + labels.join(', ') + ']'
);
});
test('present_settings includes Inherit as a selectable option', () => {
const labels = extractOptionLabels(presentBlock);
assert.ok(
labels.some(l => l === 'inherit' || l.startsWith('inherit')),
'present_settings must include an "Inherit" option. Got: [' + labels.join(', ') + ']'
);
});
// -- update_config and confirm steps reference adaptive --------------------
test('update_config step lists adaptive as a valid model_profile value', () => {
const m = settingsContent.match(/<step name="update_config">[\s\S]*?<\/step>/);
assert.ok(m, 'settings.md must have an update_config step');
assert.ok(
m[0].includes('adaptive'),
'update_config step must list "adaptive" as a valid model_profile value'
);
});
test('confirm step table shows adaptive as a possible model profile value', () => {
const m = settingsContent.match(/<step name="confirm">[\s\S]*?<\/step>/);
assert.ok(m, 'settings.md must have a confirm step');
assert.ok(
m[0].includes('adaptive'),
'confirm step must include "adaptive" in the Model Profile row'
);
});
});