a9a7a328e6
Mechanical rename produced by scripts/msd-rename.cjs: gsd/Gsd/GSD -> msd/Msd/MSD across contents and paths, upstream package/repo coordinates -> @golem15/msd-core and golem15com/msd-core. Deep links into upstream history, sibling upstream packages, the GSD-2 import feature, CHANGELOG.md and .changeset/ are kept as-is. Hand edits on top: MSD block-letter banner and logos, LICENSE copyright line, package/plugin identity, regenerated lockfile, install-tree fixtures, derived registries and benchmark baseline; migration checksum baseline re-locked (MSD keeps its own install state, so no install had applied the old sums); sort-order and regex-escaped expectations in tests adjusted.
646 lines
25 KiB
JavaScript
646 lines
25 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* gen-prompt-budget-parity-corpus.cjs
|
|
*
|
|
* Captures a characterization corpus of `prompt-budget.applyBudget`'s observable
|
|
* output, for issue #2929 (epic #1671 Phase 2 — the `context-composer` seam).
|
|
*
|
|
* WHY THIS EXISTS
|
|
* ---------------
|
|
* Phase 2 generalizes `applyBudget`'s trim ladder into a shared composer seam.
|
|
* Its success condition is that `prompt-budget`'s output does not change. The only
|
|
* authority on "did not change" is the behavior that shipped BEFORE the refactor,
|
|
* so this corpus is generated against the pre-refactor implementation and committed
|
|
* as a frozen fixture. `tests/prompt-budget-parity.test.cjs` then replays every case
|
|
* and asserts byte-identical `prompt` plus field-identical `metadata`.
|
|
*
|
|
* FIXTURE PROVENANCE (CONTRIBUTING.md "Fixture provenance (#2371)")
|
|
* -----------------------------------------------------------------
|
|
* That rule forbids fixtures derived from the gate's own writer, grammar, or the
|
|
* author's mental model, because such a fixture can only confirm what the author
|
|
* already believed. This corpus is expressly NOT hand-authored: every `expected`
|
|
* value is computed by executing the shipped implementation, which predates the
|
|
* composer and knows nothing about it. What IS author-chosen is the set of INPUTS —
|
|
* so the inputs are derived from the design's parity-critical behavior table rather
|
|
* than from intuition, and `tests/prompt-budget-parity.test.cjs` additionally proves
|
|
* the corpus is non-vacuous by mutating the ladder and requiring the corpus to fail.
|
|
* Disclosed here rather than assumed.
|
|
*
|
|
* USAGE
|
|
* node scripts/gen-prompt-budget-parity-corpus.cjs --write # regenerate
|
|
* node scripts/gen-prompt-budget-parity-corpus.cjs --check # drift-guard (exit 1 on drift)
|
|
*
|
|
* `--check` is the intentional-change gate: after Phase 2 lands, any diff here means
|
|
* review-prompt output moved, which is a user-visible change requiring justification.
|
|
*/
|
|
|
|
'use strict';
|
|
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
|
|
const REPO_ROOT = path.resolve(__dirname, '..');
|
|
const OUT_PATH = path.join(REPO_ROOT, 'tests', 'fixtures', 'prompt-budget-parity', 'corpus.json');
|
|
const IMPL_PATH = path.join(REPO_ROOT, 'msd-core', 'bin', 'lib', 'prompt-budget.cjs');
|
|
|
|
/** Corpus schema version. Bump only on a deliberate shape change. */
|
|
const CORPUS_VERSION = 1;
|
|
|
|
// ─── Deterministic content builders ──────────────────────────────────────────
|
|
// estimateTokens() is Math.ceil(len / 4), so a string of length 4*N measures
|
|
// exactly N tokens. Every size below is therefore exact, not approximate — which
|
|
// is what makes the cap-1 / cap / cap+1 rows land ON the boundary rather than near it.
|
|
|
|
/** A filler string measuring exactly `tokens` tokens under chars/4. */
|
|
function tokensOf(tokens, fill = 'x') {
|
|
return fill.repeat(tokens * 4);
|
|
}
|
|
|
|
/** `lines` newline-separated lines, each `width` chars wide. */
|
|
function linesOf(lines, width = 19) {
|
|
return Array.from({ length: lines }, (_, i) => String(i).padEnd(width, '-')).join('\n');
|
|
}
|
|
|
|
// ─── Case definitions ────────────────────────────────────────────────────────
|
|
// Each case is { name, why, sections, budget, options }.
|
|
// `why` names the design/test-matrix row the case exists to pin.
|
|
|
|
function buildCases() {
|
|
const cases = [];
|
|
const add = (c) => cases.push(c);
|
|
|
|
// Base shape reused by the boundary family. With safetyMarginPct 0 the effective
|
|
// budget equals the budget, so totals can be placed exactly on the cap.
|
|
// staticBase = est(instructions) + est('## Roadmap\n\n') + est(roadmap)
|
|
// + est('## Plans\n\n') + est('### a.md\n\n')
|
|
const boundaryBase = (planTokens) => ({
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(planTokens, 'P') }],
|
|
});
|
|
|
|
// ── A. Budget boundary (RULESET.TESTS.boundary-coverage.fixtures a/b/c) ────
|
|
add({
|
|
name: 'A1-exact-cap',
|
|
why: 'A1/row 3 — total measures exactly effectiveBudget; must NOT be treated as pressure',
|
|
sections: boundaryBase(71),
|
|
budget: 100,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'A2-one-under-cap',
|
|
why: 'A2/row 5 — one token under; no pressure, no reserve deducted',
|
|
sections: boundaryBase(70),
|
|
budget: 100,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'A3-one-over-cap',
|
|
why: 'A3/row 4 — one token over; pressure engages the ladder',
|
|
sections: boundaryBase(72),
|
|
budget: 100,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
|
|
// A1 pins exact-cap as "no pressure", but with only a floored plan group present that
|
|
// assertion is not load-bearing: the 1024-char floor absorbs any trim, so relaxing the
|
|
// pressure test from `>` to `>=` produces byte-identical output and the corpus cannot
|
|
// see the difference. This case gives the ladder something DROPPABLE at exactly the cap,
|
|
// so `>` vs `>=` becomes observable as context surviving vs being omitted.
|
|
// staticBase 29 + plan 20 = 49, context wrapper 3, so est(context) 148 puts the
|
|
// baseline at exactly 200 = effectiveBudget.
|
|
add({
|
|
name: 'A3b-exact-cap-with-droppable-present',
|
|
why: 'exact-cap discriminator — baseline lands exactly ON effectiveBudget with a droppable available. Pins that `baseline > effectiveBudget` is STRICT: at equality nothing is dropped. Mutating the comparison to >= flips context from kept to omitted, which A1 alone cannot detect.',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(20, 'P') }],
|
|
context: tokensOf(148, 'C'),
|
|
},
|
|
budget: 200,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'A3c-one-under-cap-with-droppable-present',
|
|
why: 'companion to A3b — one token under the cap with the same droppable available. Together they bracket the strict-inequality boundary from below.',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(20, 'P') }],
|
|
context: tokensOf(147, 'C'),
|
|
},
|
|
budget: 200,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
|
|
// Row (d) of RULESET.TESTS.boundary-coverage.fixtures: baseline within
|
|
// reserve-distance of the limit. NOTE_RESERVE_TOKENS is 80, and this is the exact
|
|
// band (effectiveBudget - 80, effectiveBudget] that PR #3708 shipped two
|
|
// regressions in (LEARNING.prompt-budget.boundary-gap). Swept token by token.
|
|
// Budget 200 (not 100) is required here, not incidental: the band is
|
|
// NOTE_RESERVE_TOKENS (80) wide, so the baseline needs >=81 tokens of headroom
|
|
// beneath the cap to be swept. staticBase is 29 tokens, so planTokens = 171 - delta
|
|
// places the baseline exactly `delta` under effectiveBudget and stays positive
|
|
// across the whole sweep. delta 80 and 81 sit just OUTSIDE the band deliberately —
|
|
// they are the negative controls that prove the band's edge is where it is claimed.
|
|
for (const delta of [0, 1, 2, 39, 40, 79, 80, 81]) {
|
|
add({
|
|
name: `A4-reserve-band-minus-${delta}`,
|
|
why: `boundary-coverage.fixtures row (d) — baseline at effectiveBudget-${delta}. Pins the NEGATIVE control: within NOTE_RESERVE_TOKENS distance of the cap, NOTHING may be trimmed. Early-pressure firing here is the exact PR #3708 regression.`,
|
|
sections: boundaryBase(171 - delta),
|
|
budget: 200,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
}
|
|
|
|
// Mirror of A4 on the OTHER side of the cap. A4 proves the reserve is NOT deducted
|
|
// at or below the cap; this proves it IS deducted above it, and that the ladder
|
|
// relieves pressure by dropping. Without both sides the family pins only half the
|
|
// boundary, and an "always reserve" regression would still pass A4.
|
|
// staticBase 29 + planContent 20 = 49, and the context wrapper costs 3, so
|
|
// est(context) = 148 + delta places the baseline exactly `delta` OVER effectiveBudget.
|
|
for (const delta of [1, 2, 40, 79, 80, 81]) {
|
|
add({
|
|
name: `A10-pressure-band-plus-${delta}`,
|
|
why: `boundary-coverage.fixtures row (d), above-cap mirror — baseline at effectiveBudget+${delta}; pressure fires, NOTE_RESERVE_TOKENS is deducted, and context is dropped`,
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(20, 'P') }],
|
|
context: tokensOf(148 + delta, 'C'),
|
|
},
|
|
budget: 200,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
}
|
|
|
|
add({
|
|
name: 'A5-safety-margin-default',
|
|
why: 'A4 — default 10% margin; effectiveBudget = floor(budget * 0.9)',
|
|
sections: boundaryBase(60),
|
|
budget: 100,
|
|
options: {},
|
|
});
|
|
|
|
// The rounding mode of effectiveBudget was unpinned: every other case in this file
|
|
// uses a (budget, safetyMarginPct) pair whose product is a whole number, so
|
|
// Math.floor and Math.round agree and mutating one to the other changed nothing.
|
|
// These three straddle .5 in both directions so the mode is observable.
|
|
// 95 * 0.90 = 85.5 -> floor 85, round 86 (disagree)
|
|
// 97 * 0.90 = 87.3 -> floor 87, round 87 (agree; guards against a ceil mutation)
|
|
// 93 * 0.85 = 79.05 -> floor 79, round 79 (agree; second ceil guard, different margin)
|
|
add({
|
|
name: 'A11-fractional-effective-budget-half',
|
|
why: 'pins the ROUNDING MODE of effectiveBudget. 95 * (1 - 10/100) = 85.5, where Math.floor (85) and Math.round (86) disagree. Without a fractional case the mode is unpinned and floor->round is a byte-invisible mutation, which an isolated review confirmed against the 47-case corpus.',
|
|
sections: boundaryBase(56),
|
|
budget: 95,
|
|
options: { safetyMarginPct: 10 },
|
|
});
|
|
add({
|
|
name: 'A12-fractional-effective-budget-below-half',
|
|
why: 'companion to A11 below the .5 point: 97 * 0.90 = 87.3, where floor and round agree but Math.ceil (88) would not. Guards the mutation A11 does not cover.',
|
|
sections: boundaryBase(58),
|
|
budget: 97,
|
|
options: { safetyMarginPct: 10 },
|
|
});
|
|
add({
|
|
name: 'A13-fractional-effective-budget-odd-margin',
|
|
why: 'third rounding guard at a non-multiple-of-10 margin: 93 * (1 - 15/100) = 79.05. Exercises the margin arithmetic itself, not just the budget, since safetyMarginPct is caller-supplied.',
|
|
sections: boundaryBase(50),
|
|
budget: 93,
|
|
options: { safetyMarginPct: 15 },
|
|
});
|
|
add({
|
|
name: 'A6-safety-margin-zero',
|
|
why: 'A5 — 0% margin means the full budget is usable',
|
|
sections: boundaryBase(60),
|
|
budget: 100,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'A7-safety-margin-hundred',
|
|
why: 'A5 — 100% margin drives effectiveBudget to 0, forcing the min-set hard fail',
|
|
sections: boundaryBase(10),
|
|
budget: 100,
|
|
options: { safetyMarginPct: 100 },
|
|
});
|
|
add({
|
|
name: 'A8-budget-zero',
|
|
why: 'A6 — non-positive budget must hard-fail cleanly, never negative-length slice',
|
|
sections: boundaryBase(10),
|
|
budget: 0,
|
|
options: {},
|
|
});
|
|
add({
|
|
name: 'A9-budget-negative',
|
|
why: 'A6 — negative budget must not crash',
|
|
sections: boundaryBase(10),
|
|
budget: -50,
|
|
options: {},
|
|
});
|
|
|
|
// ── B. Shrink strategies ──────────────────────────────────────────────────
|
|
add({
|
|
name: 'B3-head-shrink-engaged',
|
|
why: 'B3/row 7 — projectMd longer than projectMdHeadLines under pressure; shrunk flag true',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(40, 'P') }],
|
|
projectMd: linesOf(120),
|
|
},
|
|
budget: 200,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B4-head-shrink-noop',
|
|
why: 'B4/row 7+9 — fewer lines than the cap; text unchanged and shrunk flag MUST stay false',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(40, 'P') }],
|
|
projectMd: linesOf(5),
|
|
},
|
|
budget: 60,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B5-head-shrink-zero-lines',
|
|
why: 'B5/row 8 — projectMdHeadLines <= 0 yields the empty string',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(40, 'P') }],
|
|
projectMd: linesOf(120),
|
|
},
|
|
budget: 60,
|
|
options: { safetyMarginPct: 0, projectMdHeadLines: 0 },
|
|
});
|
|
add({
|
|
name: 'B6-plan-truncate-share-above-floor',
|
|
why: 'B6/row 13 — proportional share ABOVE the floor, so the share (not the floor) decides. Budget 700: min-set is 5+5+256+256=522 which fits, so this reaches the truncate step instead of hard-failing the way the original budget-400 version did. groupBudget 598 of 1000 => charsBudget 2392, share 1196 per plan > floor 1024.',
|
|
sections: {
|
|
instructions: tokensOf(5, 'I'),
|
|
roadmap: tokensOf(5, 'R'),
|
|
plans: [
|
|
{ file: 'a.md', content: tokensOf(500, 'A') },
|
|
{ file: 'b.md', content: tokensOf(500, 'B') },
|
|
],
|
|
},
|
|
budget: 700,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B7-plan-truncate-floor-binding',
|
|
why: 'B7/row 10 — share falls BELOW the 1024-char floor so the floor wins, and the resulting total deliberately EXCEEDS the group budget. Budget 1100: min-set 5+5+4*256=1034 fits; groupBudget 992 => charsBudget 3968 => share 992 < 1024, so every plan is held at the floor. This is the case that proves a floor is a per-fragment guarantee, not a budget cap.',
|
|
sections: {
|
|
instructions: tokensOf(5, 'I'),
|
|
roadmap: tokensOf(5, 'R'),
|
|
plans: [
|
|
{ file: 'a.md', content: tokensOf(500, 'A') },
|
|
{ file: 'b.md', content: tokensOf(500, 'B') },
|
|
{ file: 'c.md', content: tokensOf(500, 'C') },
|
|
{ file: 'd.md', content: tokensOf(500, 'D') },
|
|
],
|
|
},
|
|
budget: 1100,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B8-zero-sized-plans',
|
|
why: 'B8/row 11 — total original chars 0; no divide-by-zero, pct stays 0',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [
|
|
{ file: 'a.md', content: '' },
|
|
{ file: 'b.md', content: '' },
|
|
],
|
|
context: tokensOf(200, 'C'),
|
|
},
|
|
budget: 60,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B9-nonpositive-plan-budget',
|
|
why: 'B9/row 12 — computed plan budget <= 0 skips the truncate step ENTIRELY rather than clamping',
|
|
sections: boundaryBase(72),
|
|
budget: 100,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B11-drop-order-all-three',
|
|
why: 'B11/row 14 — droppables leave in declared order: context, research, requirements',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(30, 'P') }],
|
|
context: tokensOf(300, 'C'),
|
|
research: tokensOf(300, 'S'),
|
|
requirements: tokensOf(300, 'Q'),
|
|
},
|
|
budget: 200,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B11b-drop-context-only',
|
|
why: 'B11 — pressure relieved after the first drop; research and requirements survive',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(30, 'P') }],
|
|
context: tokensOf(300, 'C'),
|
|
research: tokensOf(10, 'S'),
|
|
requirements: tokensOf(10, 'Q'),
|
|
},
|
|
budget: 300,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B12-droppables-absent',
|
|
why: 'B12/row 15 — null droppables are not dropped and never appear in omitted. Budget 400 (not 120) so this actually RENDERS: at 120 the min-set pre-check hard-fails and the case would compare two empty strings.',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(300, 'P') }],
|
|
context: null,
|
|
research: null,
|
|
requirements: null,
|
|
},
|
|
budget: 400,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B13-droppables-empty-string',
|
|
why: 'B13/row 23 — empty-string droppables. Paired with B12 to pin whether "" and null are distinguished. They are NOT: applyBudget uses truthy checks throughout, so "" behaves as absent. This case records that fact rather than the design doc\'s claim that they differ.',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(300, 'P') }],
|
|
context: '',
|
|
research: '',
|
|
requirements: '',
|
|
},
|
|
budget: 400,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'B13b-empty-string-droppable-under-pressure',
|
|
why: 'B13 companion — an empty-string droppable while the ladder is ACTIVELY running. Pins that "" is never recorded in omitted even under real pressure, which B12/B13 (no pressure) cannot show. research is real and IS dropped, so the ladder is demonstrably engaged.',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(20, 'P') }],
|
|
context: '',
|
|
research: tokensOf(200, 'S'),
|
|
requirements: '',
|
|
},
|
|
budget: 200,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
|
|
// ── C. Note reserve + hard-fail paths ─────────────────────────────────────
|
|
add({
|
|
name: 'C1-no-trim-no-note',
|
|
why: 'C1/row 1 — everything fits; no note, noteInjected false',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(10, 'P') }],
|
|
projectMd: linesOf(3),
|
|
context: tokensOf(5, 'C'),
|
|
research: tokensOf(5, 'S'),
|
|
requirements: tokensOf(5, 'Q'),
|
|
},
|
|
budget: 1000,
|
|
options: {},
|
|
});
|
|
add({
|
|
name: 'C4-min-set-hard-fail',
|
|
why: 'C4/row 16 — min-set pre-check exceeds budget; empty output and estimatedTokens 0',
|
|
sections: {
|
|
instructions: tokensOf(500, 'I'),
|
|
roadmap: tokensOf(500, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(500, 'P') }],
|
|
},
|
|
budget: 100,
|
|
options: {},
|
|
});
|
|
add({
|
|
name: 'C5-post-assembly-hard-fail',
|
|
why: 'C5/row 17 — survives the min-set check but the assembled prompt still overflows; real measured size reported',
|
|
sections: {
|
|
instructions: tokensOf(80, 'I'),
|
|
roadmap: tokensOf(80, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(2000, 'P') }],
|
|
},
|
|
budget: 200,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'C7-custom-note-template',
|
|
why: 'note template placeholders {budget} {omittedList} {planTruncationPct} all substitute',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(30, 'P') }],
|
|
context: tokensOf(300, 'C'),
|
|
},
|
|
budget: 200,
|
|
options: {
|
|
safetyMarginPct: 0,
|
|
noteTemplate: 'TRIMMED b={budget} o={omittedList} p={planTruncationPct}',
|
|
},
|
|
});
|
|
add({
|
|
name: 'C8-omitted-list-none',
|
|
why: 'renderNote emits the literal "none" when a trim occurred but nothing was dropped',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(30, 'P') }],
|
|
projectMd: linesOf(120),
|
|
},
|
|
budget: 120,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
|
|
// ── D. Shape + encoding ───────────────────────────────────────────────────
|
|
add({
|
|
name: 'D1-all-sections-present',
|
|
why: 'full assembly order: instructions, note, roadmap, project, plans, context, research, requirements',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [
|
|
{ file: 'one.md', content: tokensOf(10, '1') },
|
|
{ file: 'two.md', content: tokensOf(10, '2') },
|
|
],
|
|
projectMd: linesOf(4),
|
|
context: tokensOf(10, 'C'),
|
|
research: tokensOf(10, 'S'),
|
|
requirements: tokensOf(10, 'Q'),
|
|
},
|
|
budget: 5000,
|
|
options: {},
|
|
});
|
|
add({
|
|
name: 'D2-no-optional-sections',
|
|
why: 'minimum viable shape — only the two never-droppables plus one plan',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a.md', content: tokensOf(10, 'P') }],
|
|
},
|
|
budget: 5000,
|
|
options: {},
|
|
});
|
|
add({
|
|
name: 'D3-zero-plans',
|
|
why: 'empty plans array — headers still assembled, reduce over [] is 0',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [],
|
|
},
|
|
budget: 5000,
|
|
options: {},
|
|
});
|
|
add({
|
|
name: 'D4-crlf-content',
|
|
why: 'row 25 — CRLF measured and sliced identically to LF under the token unit',
|
|
sections: {
|
|
instructions: 'alpha\r\nbeta\r\ngamma',
|
|
roadmap: 'one\r\ntwo\r\nthree',
|
|
plans: [{ file: 'a.md', content: Array.from({ length: 60 }, (_, i) => `line ${i}`).join('\r\n') }],
|
|
projectMd: Array.from({ length: 80 }, (_, i) => `p${i}`).join('\r\n'),
|
|
},
|
|
budget: 120,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'D5-unicode-content',
|
|
why: 'row 26 — multibyte and astral content; chars/4 counts UTF-16 units, and slice may split a surrogate pair (pinned as current behavior)',
|
|
sections: {
|
|
instructions: 'héllo wörld ünicode',
|
|
roadmap: '日本語のロードマップ',
|
|
plans: [{ file: 'ünï.md', content: '🎉'.repeat(400) }],
|
|
context: 'Ω'.repeat(200),
|
|
},
|
|
budget: 150,
|
|
options: { safetyMarginPct: 0 },
|
|
});
|
|
add({
|
|
name: 'D6-plan-filename-affects-budget',
|
|
why: 'S5 — per-plan wrapper cost "### <file>\\n\\n" counts toward staticBaseTokens',
|
|
sections: {
|
|
instructions: tokensOf(10, 'I'),
|
|
roadmap: tokensOf(10, 'R'),
|
|
plans: [{ file: 'a-very-long-plan-filename-that-costs-tokens.md', content: tokensOf(10, 'P') }],
|
|
},
|
|
budget: 5000,
|
|
options: {},
|
|
});
|
|
|
|
return cases;
|
|
}
|
|
|
|
// ─── Capture ─────────────────────────────────────────────────────────────────
|
|
|
|
function capture() {
|
|
if (!fs.existsSync(IMPL_PATH)) {
|
|
throw new Error(
|
|
`prompt-budget.cjs not found at ${IMPL_PATH}. Run "npm run build:lib" first ` +
|
|
'(the .cjs is a gitignored tsc output of src/prompt-budget.cts).'
|
|
);
|
|
}
|
|
const { applyBudget } = require(IMPL_PATH);
|
|
|
|
const cases = buildCases();
|
|
|
|
const seen = new Set();
|
|
for (const c of cases) {
|
|
if (seen.has(c.name)) throw new Error(`duplicate corpus case name: ${c.name}`);
|
|
seen.add(c.name);
|
|
}
|
|
|
|
return {
|
|
version: CORPUS_VERSION,
|
|
description:
|
|
'Characterization corpus of prompt-budget.applyBudget captured from the pre-refactor ' +
|
|
'implementation. Oracle for issue #2929 (epic #1671 Phase 2). Regenerate with ' +
|
|
'node scripts/gen-prompt-budget-parity-corpus.cjs --write',
|
|
cases: cases.map((c) => {
|
|
const result = applyBudget({
|
|
sections: c.sections,
|
|
budget: c.budget,
|
|
options: c.options,
|
|
});
|
|
return {
|
|
name: c.name,
|
|
why: c.why,
|
|
input: { sections: c.sections, budget: c.budget, options: c.options },
|
|
expected: { prompt: result.prompt, metadata: result.metadata },
|
|
};
|
|
}),
|
|
};
|
|
}
|
|
|
|
function serialize(corpus) {
|
|
return `${JSON.stringify(corpus, null, 2)}\n`;
|
|
}
|
|
|
|
// ─── CLI ─────────────────────────────────────────────────────────────────────
|
|
|
|
function main(argv) {
|
|
const write = argv.includes('--write');
|
|
const check = argv.includes('--check');
|
|
|
|
if (write === check) {
|
|
process.stderr.write('usage: gen-prompt-budget-parity-corpus.cjs (--write | --check)\n');
|
|
return 2;
|
|
}
|
|
|
|
const serialized = serialize(capture());
|
|
|
|
if (write) {
|
|
fs.mkdirSync(path.dirname(OUT_PATH), { recursive: true });
|
|
fs.writeFileSync(OUT_PATH, serialized);
|
|
const n = JSON.parse(serialized).cases.length;
|
|
process.stdout.write(`wrote ${path.relative(REPO_ROOT, OUT_PATH)} (${n} cases)\n`);
|
|
return 0;
|
|
}
|
|
|
|
if (!fs.existsSync(OUT_PATH)) {
|
|
process.stderr.write(
|
|
`MISSING ${path.relative(REPO_ROOT, OUT_PATH)}\n` +
|
|
'Run: node scripts/gen-prompt-budget-parity-corpus.cjs --write\n'
|
|
);
|
|
return 1;
|
|
}
|
|
|
|
const committed = fs.readFileSync(OUT_PATH, 'utf8');
|
|
if (committed !== serialized) {
|
|
process.stderr.write(
|
|
`DRIFT in ${path.relative(REPO_ROOT, OUT_PATH)}\n\n` +
|
|
"prompt-budget's observable output no longer matches the committed corpus.\n" +
|
|
'This corpus is the parity oracle for #2929: a diff here means review-prompt\n' +
|
|
'output MOVED. That is user-visible.\n\n' +
|
|
'If the change was NOT intended, fix the code — do not regenerate.\n' +
|
|
'If it WAS intended, regenerate and justify the diff in the PR:\n' +
|
|
' node scripts/gen-prompt-budget-parity-corpus.cjs --write\n'
|
|
);
|
|
return 1;
|
|
}
|
|
|
|
process.stdout.write(`ok ${path.relative(REPO_ROOT, OUT_PATH)}\n`);
|
|
return 0;
|
|
}
|
|
|
|
if (require.main === module) {
|
|
process.exitCode = main(process.argv.slice(2));
|
|
}
|
|
|
|
module.exports = { buildCases, capture, serialize, OUT_PATH, CORPUS_VERSION };
|