/** * Codex Agent TOML — typed IR for `~/.codex/agents/.toml` (#3243, ADR-2313). * * A genuine leaf: node builtins only. This is a **document model**, not a policy — * it knows how to parse/render/strip two known keys (`model`, * `model_reasoning_effort`) from a Codex agent `.toml`. It does NOT know which * `model` values are illegal for Codex (that predicate — Anthropic-flavored * detection — stays in `model-catalog.cts`; callers decide what to strip). * * Moved here (not copied) from `agent-install-check.cts` (#3242, Phase 2), which * wrote the hard half: block-range detection, BOM stripping, TOML value * unquoting, and the lenient header scan. That module's behavior is UNCHANGED — * it imports `stripBOM`/`scanTomlLines` from here and its regression suite * (`tests/agent-install-check.test.cjs`) is the proof. * * ── The reconciliation (40-design.md) ────────────────────────────────────── * * Phase 2's reader and this phase's writer disagree on how to handle an * unterminated `developer_instructions` block, deliberately: * * - The READER (`scanTomlLines`, used directly by `checkCodexModelPosture`) * stays LENIENT: an unterminated block still excludes "the rest of the * file" from the header scan (findDeveloperInstructionsBlockRange's * existing fallback), because misreading prompt prose as a pin is only a * false positive — it wastes a user's time, nothing more. * - The WRITER (`parseCodexAgentToml`, used by the Codex sync) is STRICT: an * unterminated block makes the whole document `{ok:false}`, because a * writer that proceeds on a malformed document risks rewriting it. * * One block-range detector, two call sites, two policies — never two detectors * that could silently drift from each other. */ /** Frozen reason enum for a failed {@link parseCodexAgentToml}. */ export const PARSE_REASON = Object.freeze({ UNTERMINATED_BLOCK: 'unterminated_block', }); export type ParseReason = (typeof PARSE_REASON)[keyof typeof PARSE_REASON]; /** * The typed IR. Carries the original `lines` (never re-tokenized once parsed) * plus the detected BOM/trailing-newline metadata so {@link renderCodexAgentToml} * can reproduce the source **byte-identically** when nothing was stripped (the * load-bearing round-trip property — see 50-test-matrix.md row A14), even when * the source mixes line-ending styles (`\r\n`, `\n`, a lone `\r`) within one * file. `stripModel`/`stripReasoningEffort` remove a targeted line AND its own * terminator from `lines`/`terminators`; every other line's content is * untouched, and so is its own terminator — EXCEPT when the removed line was * the file's last line AND the removed line had no terminator (the source had * no trailing newline), in which case the new last line's terminator is * cleared to `''` too (see {@link removeLine}) so the file's trailing-newline * status is preserved rather than silently changed. When the source DID end * with a newline, the new last line keeps its OWN terminator unchanged — * copying the removed line's terminator there would corrupt a mixed-EOL * source by silently changing the new last line's own ending style. */ export interface CodexAgentDoc { /** Content lines (BOM-stripped, terminator-free). Paired 1:1 by index with `terminators`. */ lines: string[]; /** * Each line's OWN terminator (`'\r\n'`, `'\n'`, `'\r'`, or `''` for a line * with none — the last line of a source with no trailing newline). Never * collapsed to one whole-file style: {@link renderCodexAgentToml} rejoins * `lines[i] + terminators[i]` so a mixed-EOL source round-trips exactly. */ terminators: string[]; /** * The line-ending style that appears in the source, informational only — * `renderCodexAgentToml` does NOT use this field (it rejoins each line with * its own `terminators[i]` instead). Recorded as `'\r\n'` if any `\r\n` * appears anywhere in the source, else `'\n'`, purely for callers that want * a human-readable summary (e.g. logging); never a rendering input. */ eol: '\n' | '\r\n'; /** Whether the source began with a UTF-8 BOM (U+FEFF). */ hadBOM: boolean; /** Whether the source ended with a line terminator (any of `\r\n`/`\n`/`\r`). */ trailingNewline: boolean; /** The `developer_instructions = '''...'''` block's line range, or `{start:-1,end:-1}` if absent. */ blockRange: { start: number; end: number }; /** The resolved `model` value (last occurrence outside the block), or null. */ model: string | null; /** Line index of the `model` key, or null if absent. */ modelLineIndex: number | null; /** The resolved `model_reasoning_effort` value (last occurrence outside the block), or null. */ reasoningEffort: string | null; /** Line index of the `model_reasoning_effort` key, or null if absent. */ reasoningEffortLineIndex: number | null; } export type ParseCodexAgentTomlResult = | { ok: true; doc: CodexAgentDoc } | { ok: false; reason: ParseReason }; // The UTF-8 BOM codepoint, spelled as an escape rather than the literal // character so the source file never carries an invisible codepoint. const BOM_CHAR = String.fromCharCode(0xfeff); // Strips a leading UTF-8 BOM (U+FEFF), which fs.readFileSync(..., 'utf8') does not // strip on its own, and unwraps a TOML basic/literal string value's surrounding // quotes so `model = "sonnet"` yields `sonnet`, not `"sonnet"`. export function stripBOM(content: string): string { return content.charCodeAt(0) === 0xfeff ? content.slice(1) : content; } export function unquoteTomlValue(rawValue: string): string { const trimmed = rawValue.trim(); const quoted = trimmed.match(/^"([^"]*)"/) ?? trimmed.match(/^'([^']*)'/); return quoted ? quoted[1] : trimmed; } // The `developer_instructions` block is a TOML multi-line literal string // (`developer_instructions = '''...'''`) that `generateCodexAgentToml` always // emits after the header fields. Prompt prose inside that block discusses models // constantly, so a `model = ...`-shaped line inside it must never be read as a // live pin — but the block can legally appear anywhere in the file (a // hand-reordered agent can move `model` after it), and another key's *value* can // legally contain the literal text `developer_instructions = '''` (e.g. a // `description` field quoting it) without that being the real block opener. So // instead of truncating the file at the first textual occurrence of the marker // anywhere in the content, this locates the block by its anchored line-start // opener (`^\s*developer_instructions\s*=\s*'''`, never a mid-line/mid-value // match) and its closing `'''` line, and excludes only the lines between them — // every other line in the file, before AND after the block, is scanned. // // If no opener is found, nothing is excluded (the whole file is scanned) and // `terminated` is trivially true. If the block IS opened but never closed before // EOF (malformed file), `terminated` is false: the lenient reader (scanTomlLines) // still treats the rest of the file as inside the block (the safe direction for // a reader — see module header comment); the strict writer (parseCodexAgentToml) // reads `terminated` and refuses instead. The emitter always uses `'''` (a TOML // literal string), never a `"""` basic multi-line string, so only `'''` is // treated as the block delimiter here. export function findDeveloperInstructionsBlockRange( lines: string[], ): { start: number; end: number; terminated: boolean } { const openIndex = lines.findIndex((line) => /^\s*developer_instructions\s*=\s*'''/.test(line)); if (openIndex === -1) { return { start: -1, end: -1, terminated: true }; } const afterOpenMarker = lines[openIndex].replace(/^\s*developer_instructions\s*=\s*'''/, ''); if (afterOpenMarker.includes("'''")) { // Same-line block: developer_instructions = '''one line''' return { start: openIndex, end: openIndex, terminated: true }; } for (let i = openIndex + 1; i < lines.length; i++) { if (lines[i].includes("'''")) { return { start: openIndex, end: i, terminated: true }; } } return { start: openIndex, end: lines.length - 1, terminated: false }; } /** Header-scan result shared by the lenient reader ({@link scanTomlLines}). */ export interface HeaderScanResult { model: string | null; hasReasoningEffort: boolean; } interface HeaderLineInfo { model: string | null; modelLineIndex: number | null; reasoningEffort: string | null; reasoningEffortLineIndex: number | null; } // Line-oriented scan of every line OUTSIDE the `developer_instructions` block // (see findDeveloperInstructionsBlockRange). Full-key-name anchoring — // `^([A-Za-z_][\w]*)\s*=` for a bare key, or `^"([^"]*)"\s*=` / `^'([^']*)'\s*=` // for TOML's legal quoted-key forms, normalized to the same key name — means // `model_verbosity` / `model_reasoning_effort` never satisfy a `model` probe, // and vice versa; `#`-prefixed lines (after trimming leading whitespace) are // treated as comments, never live pins. Shared by both scanTomlLines (the // lenient reader, boolean-only for reasoning effort) and parseCodexAgentToml // (the strict writer, which also needs the effort's value and both keys' line // indices so stripModel/stripReasoningEffort can remove exactly one line). function scanHeaderLines(lines: string[], blockStart: number, blockEnd: number): HeaderLineInfo { let model: string | null = null; let modelLineIndex: number | null = null; let reasoningEffort: string | null = null; let reasoningEffortLineIndex: number | null = null; for (let i = 0; i < lines.length; i++) { if (blockStart !== -1 && i >= blockStart && i <= blockEnd) continue; const trimmed = lines[i].trim(); if (trimmed === '' || trimmed.startsWith('#')) continue; const match = trimmed.match(/^(?:"([^"]*)"|'([^']*)'|([A-Za-z_][\w]*))\s*=\s*(.*)$/); if (!match) continue; const key = match[1] ?? match[2] ?? match[3]; const rawValue = match[4]; if (key === 'model') { model = unquoteTomlValue(rawValue); modelLineIndex = i; } else if (key === 'model_reasoning_effort') { reasoningEffort = unquoteTomlValue(rawValue); reasoningEffortLineIndex = i; } } return { model, modelLineIndex, reasoningEffort, reasoningEffortLineIndex }; } /** * The LENIENT reader entry point (Phase 2, moved verbatim in behavior). Never * fails: an unterminated block falls back to "rest of file is inside the * block" via {@link findDeveloperInstructionsBlockRange}'s own fallback. * `content` is expected already BOM-stripped (callers pass `stripBOM(raw)`). */ export function scanTomlLines(content: string): HeaderScanResult { const lines = content.split(/\r?\n/); const { start, end } = findDeveloperInstructionsBlockRange(lines); const { model, reasoningEffort } = scanHeaderLines(lines, start, end); return { model, hasReasoningEffort: reasoningEffort !== null }; } // Splits `content` into `{lines, terminators}` where `terminators[i]` is the // terminator that FOLLOWS `lines[i]` (`'\r\n'`, `'\r'`, `'\n'`, or `''` for a // line with none — only possible as the file's last line). The two arrays are // always the same length and there is NEVER a phantom trailing entry: a // source ending in a terminator (the common case) yields exactly as many // lines as it has content lines, not one more. `render` is then a plain // `lines[i] + terminators[i]` concatenation with no special-casing of "the // last line" — see `renderCodexAgentToml`. // // `String#split` with a capturing group interleaves the delimiters into the // result array — `"a\r\nb\nc".split(/(\r\n|\r|\n)/)` yields // `["a","\r\n","b","\n","c"]` — so even indices are line content and odd // indices are that line's terminator. When `content` ends WITH a terminator, // `split` appends one extra empty-string element after the last real // terminator (e.g. `"a\n".split(...)` → `["a","\n",""]`); that trailing `""` // is not a real line, it is `split`'s "nothing after the last delimiter" // marker, so the loop below stops before consuming it instead of recording it // as a phantom empty final line (the defect this replaced — see A29: a doc // with a phantom last element made every removal rule reason about the wrong // element for any trailing-newline-terminated file, the common case). `\r\n` // is tried before the bare `\r` alternative so a CRLF is never misread as a // lone-CR line followed by an empty LF-terminated line. function splitPreservingTerminators(content: string): { lines: string[]; terminators: string[] } { if (content === '') return { lines: [], terminators: [] }; const parts = content.split(/(\r\n|\r|\n)/); const lastIndex = parts.length - 1; const lines: string[] = []; const terminators: string[] = []; for (let i = 0; i < parts.length; i += 2) { if (i === lastIndex && parts[i] === '') break; // split's post-terminator marker, not a real line lines.push(parts[i]); terminators.push(parts[i + 1] ?? ''); } return { lines, terminators }; } /** * The STRICT parse entry point (Phase 3, the writer's half of the * reconciliation). Returns `{ok:false, reason:UNTERMINATED_BLOCK}` rather than * guessing when the `developer_instructions` block is opened but never closed. * On success, `doc` carries enough (the original `lines`/`terminators`, BOM/ * trailing-newline flags, and the two resolved values with their line indices) * for {@link renderCodexAgentToml} to reproduce the source byte-identically — * including a source with mixed line-ending styles — and for * {@link stripModel}/{@link stripReasoningEffort} to remove exactly one line * and its own terminator. */ export function parseCodexAgentToml(content: string): ParseCodexAgentTomlResult { const hadBOM = content.charCodeAt(0) === 0xfeff; const stripped = stripBOM(content); // Informational only — see CodexAgentDoc.eol's docstring. Never used by // renderCodexAgentToml. const eol: '\n' | '\r\n' = stripped.includes('\r\n') ? '\r\n' : '\n'; const trailingNewline = /(\r\n|\r|\n)$/.test(stripped); const { lines, terminators } = splitPreservingTerminators(stripped); const { start, end, terminated } = findDeveloperInstructionsBlockRange(lines); if (start !== -1 && !terminated) { return { ok: false, reason: PARSE_REASON.UNTERMINATED_BLOCK }; } const { model, modelLineIndex, reasoningEffort, reasoningEffortLineIndex } = scanHeaderLines(lines, start, end); const doc: CodexAgentDoc = { lines, terminators, eol, hadBOM, trailingNewline, blockRange: { start, end }, model, modelLineIndex, reasoningEffort, reasoningEffortLineIndex, }; return { ok: true, doc }; } /** * Renders `doc` back to a string. For an unmodified doc this is * byte-identical to the original `parseCodexAgentToml` input (matrix row * A14) — it never re-derives line content, only rejoins each line with its * OWN recorded terminator (`terminators[i]`, never the whole-file `eol`) and * re-prepends a BOM if one was present. This is a plain concatenation of the * surviving `[line, terminator]` pieces, so a source with mixed `\r\n`/`\n`/ * lone-`\r` line endings round-trips exactly, and a strip * ({@link stripModel}/{@link stripReasoningEffort}) removes only the target * line and its own terminator — every other line's ending is untouched. */ export function renderCodexAgentToml(doc: CodexAgentDoc): string { let body = ''; for (let i = 0; i < doc.lines.length; i++) { body += doc.lines[i] + (doc.terminators[i] ?? ''); } return doc.hadBOM ? BOM_CHAR + body : body; } // Removes exactly one line (by index) — AND its own terminator — from // `doc.lines`/`doc.terminators`, re-indexing the block range and the OTHER // key's line index so a subsequent strip/render still sees a consistent doc. // Never touches any other line's content or terminator. // // The one exception is when `index` names the file's LAST line: a plain // slice-out would drop the removed line's terminator but leave the // *previous* line's terminator standing in its place, which silently // invents (or drops) a trailing newline the source never had — a middle-line // removal never has this problem because the terminator that survives (the // one that WAS between the previous line and the removed one) is exactly the // terminator the new neighbors should have between them. For a last-line // removal, the file's trailing-newline-or-not status lives in whether the // REMOVED line's own terminator was empty (that is what `trailingNewline` // was computed from) — so the new last line inherits the removed line's // EMPTINESS only: if the removed terminator was `''`, the new last line's // terminator is cleared to `''` too. If the removed terminator was // non-empty, the source already ended with a newline and the new last line // already has the right one (its OWN, unchanged) — overwriting it with the // removed line's terminator would silently change the new last line's own // ending style on a mixed-EOL source (see A26). Removing the only remaining // line is the degenerate case: there is no new last line, so the result is // the empty document. function removeLine(doc: CodexAgentDoc, index: number, which: 'model' | 'reasoningEffort'): CodexAgentDoc { const isLastLine = index === doc.lines.length - 1; let lines: string[]; let terminators: string[]; if (doc.lines.length === 1) { lines = []; terminators = []; } else if (isLastLine) { lines = doc.lines.slice(0, index); terminators = doc.terminators.slice(0, index); // Inherit the removed line's EMPTINESS, never its STYLE: if the removed // line had no terminator (the source had no trailing newline), the new // last line's terminator becomes '' too. Otherwise the source DID end // with a newline, and the new last line already has the right one — its // OWN terminator (already carried over by the slice above), which may // differ in style from the removed line's (a mixed-EOL source) — so it is // left unchanged rather than overwritten. if (doc.terminators[index] === '') { terminators[terminators.length - 1] = ''; } } else { lines = doc.lines.slice(0, index).concat(doc.lines.slice(index + 1)); terminators = doc.terminators.slice(0, index).concat(doc.terminators.slice(index + 1)); } const reindex = (i: number | null): number | null => (i === null ? null : i > index ? i - 1 : i); const blockRange = { ...doc.blockRange }; if (blockRange.start !== -1) { if (blockRange.start > index) blockRange.start -= 1; if (blockRange.end > index) blockRange.end -= 1; } return { ...doc, lines, terminators, blockRange, model: which === 'model' ? null : doc.model, modelLineIndex: which === 'model' ? null : reindex(doc.modelLineIndex), reasoningEffort: which === 'reasoningEffort' ? null : doc.reasoningEffort, reasoningEffortLineIndex: which === 'reasoningEffort' ? null : reindex(doc.reasoningEffortLineIndex), }; } /** * Returns a new doc with the `model` line removed (a no-op copy if there was * no `model` line). Every other byte — comments, other keys, the * `developer_instructions` block, line endings, BOM — is untouched. */ export function stripModel(doc: CodexAgentDoc): CodexAgentDoc { if (doc.modelLineIndex === null) return doc; return removeLine(doc, doc.modelLineIndex, 'model'); } /** * Returns a new doc with the `model_reasoning_effort` line removed (a no-op * copy if there was none). Every other byte is untouched. */ export function stripReasoningEffort(doc: CodexAgentDoc): CodexAgentDoc { if (doc.reasoningEffortLineIndex === null) return doc; return removeLine(doc, doc.reasoningEffortLineIndex, 'reasoningEffort'); }