Files
msd-core/tests/review-parallel-lanes.test.cjs
Tom Boucher ec22377a9d fix(#4351): run-scope preserved review evidence (#4713)
* fix(#4351): run-scope preserved review evidence

The preserve block copied lane output into a flat .review-diagnostics/
using each file's source basename. A lane slug is stable across runs, so
the destination was stable across runs too -- and cp over an existing
file is a success, so a second review of the same phase destroyed the
first run's evidence with no error and no warning, in the one directory
that exists to outlive the rm -rf beside it.

Copy into one subdirectory per run instead. $RUN_DIR is mktemp -d, so
its basename is already unique per run by construction; the UTC stamp in
front is only a sort key and is omitted if date fails. Destination-only:
nothing inside $RUN_DIR is renamed, because both
prepare_trimmed_prompt_for_reviewer and the lane invocation resolver
depend on those exact basenames.

Verified by extracting the real fence and running it twice against one
phase dir: before, one report survived and it was run 2's; after, both.

Emitted-Drift-Ack-Growth: review.md — per-run diagnostics subdirectory plus the comment explaining why uniqueness cannot come from the clock
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* test(#4351): cover repeated runs, resolve through the run subdir

Adds the two-run rows the issue asks for: both runs' reports recoverable,
and a lane failing identically twice leaving two stubs. Both assert on
CONTENT, not a file count -- a clobber producing the same number of
files would pass a count-only check, and the defect is that run 1's
bytes were replaced.

runWriteReviewsFlow gains an optional phaseDir so a caller can run the
flow twice against one phase directory, which is the only arrangement
that can observe the overwrite. Omitted, it mints a fresh one as before.

The existing rows asserted a flat readdir of the diagnostics root, which
the fix makes stale. They now resolve through preservedPath/preservedNames
so they keep asserting WHICH files were preserved rather than silently
becoming assertions about the layout; the layout is pinned once,
explicitly, by oneRunSubdirectoryPerRun_4351.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* chore(#4351): add changeset fragment

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* chore(#4351): name the run, not the clock, in the changeset

Adversarial review: the fragment said each run gets its own "timestamped
subdirectory", which reads as though the timestamp provides the
separation. It does not -- collision-safety is mktemp's random basename,
and the stamp is a sort key that is dropped entirely when date fails.
The code comment already said so; the user-facing text now agrees.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* chore(#4351): backfill the changeset PR number

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

---------

Co-authored-by: sim <sim@local>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
2026-09-13 23:58:38 -04:00

1265 lines
55 KiB
JavaScript

'use strict';
/**
* Failing-first tests for #3034 (opt-in parallel reviewer lanes).
*
* Design: .gsd/phase/feat-3034-parallel-reviewer-lanes/40-design.md
* Test matrix: .gsd/phase/feat-3034-parallel-reviewer-lanes/50-test-matrix.md
*
* The unit under test is the real, shipped `<step name="invoke_reviewers">`
* fenced bash block in gsd-core/workflows/review.md — extracted and EXECUTED
* (never re-typed), with the single I/O seam `gsd_run` replaced by a shell
* function stub. The feature this file exercises (an opt-in
* `review.parallel_lanes` config key that backgrounds lane dispatch and
* joins before aggregation) does not exist yet, so several tests below are
* expected to be RED against the current shipped block.
*/
const { test, describe } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const {
createTempDir,
createTempProject,
cleanup,
readFileNormalized,
runGsdTools,
} = require('./helpers.cjs');
const { runHook } = require('./helpers/process-seam.cjs');
const { HOOK_FANOUT_TIMEOUT_MS } = require('./helpers/timeouts.cjs');
const { toPosixPath } = require('../gsd-core/bin/lib/shell-command-projection.cjs');
const { scanFencedBlocks } = require('../gsd-core/bin/lib/markdown-sectionizer.cjs');
const REPO_ROOT = path.join(__dirname, '..');
const REVIEW_MD_PATH = path.join(REPO_ROOT, 'gsd-core', 'workflows', 'review.md');
const SELECTED_3 = 'codex,gemini,claude';
// ─── extraction (source-text-is-the-product) ──────────────────────────────
/**
* Reads review.md and extracts the fenced bash block inside
* `<step name="invoke_reviewers">`. Mirrors extractCwdGuardBash() in
* tests/worktree-cleanup.test.cjs: readFileNormalized() strips CRLF at the
* read boundary (what actually makes the captured body safe to hand to
* bash); the `\r?\n` in the fence regex below is redundant on that
* already-normalized input but kept anyway because a bare `\n` in a
* markdown-fence-shaped regex trips the local/no-crlf-fragile-split rule.
*/
function extractInvokeReviewersBash() {
const content = readFileNormalized(REVIEW_MD_PATH);
const stepMarker = '<step name="invoke_reviewers">';
const stepIdx = content.indexOf(stepMarker);
if (stepIdx === -1) {
throw new Error(`extractInvokeReviewersBash: could not find "${stepMarker}" in ${REVIEW_MD_PATH}`);
}
const afterStep = content.slice(stepIdx + stepMarker.length);
const endMarker = '</step>';
const endIdx = afterStep.indexOf(endMarker);
if (endIdx === -1) {
throw new Error(`extractInvokeReviewersBash: could not find closing "${endMarker}" after invoke_reviewers in ${REVIEW_MD_PATH}`);
}
const stepBody = afterStep.slice(0, endIdx);
const stepBodyLines = stepBody.split(/\r?\n/);
const fenced = scanFencedBlocks(stepBodyLines).find(
(b) => b.closeLineIdx !== -1 && ['bash', 'sh'].includes((b.infoString || '').trim()),
);
if (!fenced) {
throw new Error(`extractInvokeReviewersBash: no \`\`\`bash fence found inside invoke_reviewers step in ${REVIEW_MD_PATH}`);
}
const block = stepBodyLines.slice(fenced.openLineIdx + 1, fenced.closeLineIdx).join('\n');
if (!block.trim()) {
throw new Error('extractInvokeReviewersBash: extracted bash block is empty');
}
if (!block.includes('review-lane invoke')) {
throw new Error('extractInvokeReviewersBash: extracted block does not contain "review-lane invoke" — anchor may have drifted');
}
if (!block.includes('SELECTED_REVIEWERS')) {
throw new Error('extractInvokeReviewersBash: extracted block does not contain "SELECTED_REVIEWERS" — anchor may have drifted');
}
return block;
}
// ─── stub preamble ─────────────────────────────────────────────────────────
//
// The ONLY I/O seam the extracted block calls is `gsd_run`. Everything else
// executed by runDispatch() is the real shipped shell text. `barrier` is the
// positive, deterministic concurrency proof this suite uses in place of any
// elapsed-time assertion (CLAUDE.md bans wall-clock assertions).
const STUB_PREAMBLE = [
'arg_after() {',
' local flag="$1"; shift',
' while [ $# -gt 0 ]; do',
' if [ "$1" = "$flag" ]; then',
' printf %s "$2"',
' return 0',
' fi',
' shift',
' done',
'}',
'',
'in_list() {',
' local needle="$1" list="$2" item old_ifs="$IFS"',
' IFS=","',
' for item in $list; do',
' if [ "$item" = "$needle" ]; then',
' IFS="$old_ifs"',
' return 0',
' fi',
' done',
' IFS="$old_ifs"',
' return 1',
'}',
'',
'dep_of() {',
' local slug="$1" pair old_ifs="$IFS"',
' IFS=","',
' for pair in $STUB_DEPS; do',
' case "$pair" in',
' "$slug="*)',
' printf %s "${pair#*=}"',
' IFS="$old_ifs"',
' return 0',
' ;;',
' esac',
' done',
' IFS="$old_ifs"',
'}',
'',
'wait_for_file() {',
' local target="$1" i=0',
' while [ ! -f "$target" ] && [ "$i" -lt 200 ]; do',
' sleep 0.05',
' i=$((i + 1))',
' done',
'}',
'',
'barrier() {',
' local slug="$1" i=0 count',
' touch "$BARRIER_DIR/$slug"',
' count=$(ls -1 "$BARRIER_DIR" | wc -l)',
' while [ "$count" -lt "$LANE_COUNT" ] && [ "$i" -lt 200 ]; do',
' sleep 0.05',
' i=$((i + 1))',
' count=$(ls -1 "$BARRIER_DIR" | wc -l)',
' done',
' if [ "$count" -lt "$LANE_COUNT" ]; then',
' echo "barrier-timeout:$slug" >> "$TRACE"',
' return 1',
' fi',
' return 0',
'}',
'',
'gsd_run() {',
' if [ "$1" = "query" ] && [ "$2" = "config-get" ] && [ "$3" = "review.parallel_lanes" ]; then',
' if [ "$STUB_CONFIG_GET_FAILS" = "1" ]; then',
' return 1',
' fi',
' printf %s "$STUB_PARALLEL"',
' return 0',
' fi',
'',
' if [ "$1" = "query" ] && [ "$2" = "review-lane" ] && [ "$3" = "plan" ]; then',
' shift 3',
' local sel',
' sel="$(arg_after --selected "$@")"',
' if in_list "$sel" "$STUB_BUDGET_FAIL"; then',
' printf %s \'{"promptBudget": 10}\'',
' else',
' printf %s \'{"promptBudget": -1}\'',
' fi',
' return 0',
' fi',
'',
' if [ "$1" = "query" ] && [ "$2" = "prompt-budget" ]; then',
' shift 2',
' local out base slug',
' out="$(arg_after --output-prompt "$@")"',
' base="$(basename "$out")"',
' slug="${base#gsd-review-prompt-}"',
' slug="${slug%.md}"',
' if in_list "$slug" "$STUB_BUDGET_FAIL"; then',
' return 2',
' fi',
' : > "$out"',
' return 0',
' fi',
'',
' if [ "$1" = "query" ] && [ "$2" = "review-lane" ] && [ "$3" = "invoke" ]; then',
' shift 3',
' local slug dep pad',
' slug="$(arg_after --slug "$@")"',
' echo "start:$slug" >> "$TRACE"',
' if [ "$STUB_BARRIER" = "1" ]; then',
' barrier "$slug" || true',
' fi',
' dep="$(dep_of "$slug")"',
' if [ -n "$dep" ]; then',
' wait_for_file "$RUN_DIR/done-$dep"',
' fi',
' echo "stub review body for $slug" > "$RUN_DIR/gsd-review-$slug.md"',
' pad=""',
' if [ "$STUB_PAD_BYTES" -gt 0 ] 2>/dev/null; then',
' pad="$(head -c "$STUB_PAD_BYTES" /dev/zero | tr "\\0" "x")"',
' fi',
' if ! in_list "$slug" "$STUB_SILENT"; then',
' printf \'{"slug":"%s","pad":"%s"}\\n\' "$slug" "$pad"',
' fi',
// #3689: the done-file is a cross-process happens-before edge — a
// dependent lane unblocks the instant this file appears (wait_for_file
// above just polls for its existence), so everything a dependent may
// observe (the "end:$slug" trace line) must be written BEFORE the file
// that releases it. touch-then-echo let a descheduled upstream lose the
// race to its own dependent, inverting the #3034 completion-order trace.
' echo "end:$slug" >> "$TRACE"',
' touch "$RUN_DIR/done-$slug"',
' if in_list "$slug" "$STUB_FAIL"; then',
' return 1',
' fi',
' return 0',
' fi',
'',
' return 0',
'}',
].join('\n');
// ─── dispatch runner ───────────────────────────────────────────────────────
/**
* Build the stub env for a runDispatch() call from `opts`. Every key is
* always present (never omitted) so the generated `set -u` script never
* dereferences an unset variable — `opts.parallel === null` deliberately
* maps to the empty string, which is exactly how an unset config key reads
* back through `config-get --raw`.
*/
function buildEnv(opts, runDir, tracePath, barrierDir) {
const selected = opts.selected;
const laneCount = selected.split(',').filter((s) => s.length > 0).length;
const deps = opts.deps || {};
return {
...process.env,
SELECTED_REVIEWERS: selected,
EXPLICIT_FLAG: '',
STUB_PARALLEL: opts.parallel === null || opts.parallel === undefined ? '' : opts.parallel,
STUB_CONFIG_GET_FAILS: opts.configGetFails ? '1' : '0',
STUB_FAIL: (opts.failSlugs || []).join(','),
STUB_BUDGET_FAIL: (opts.budgetFailSlugs || []).join(','),
STUB_SILENT: (opts.silentSlugs || []).join(','),
STUB_BARRIER: opts.barrier ? '1' : '0',
STUB_DEPS: Object.entries(deps).map(([k, v]) => `${k}=${v}`).join(','),
STUB_PAD_BYTES: String(opts.padBytes || 0),
LANE_COUNT: String(laneCount),
TRACE: tracePath,
BARRIER_DIR: barrierDir,
};
}
/**
* Runs the real extracted invoke_reviewers bash block with the gsd_run stub
* spliced in front of it. `{run_dir}` is replaced globally with a real temp
* directory. Returns the dispatch outcome plus the artifacts it produced.
*/
function runDispatch(t, opts) {
const scriptDir = createTempDir('gsd-3034-script-');
const runDir = createTempDir('gsd-3034-rundir-');
const barrierDir = createTempDir('gsd-3034-barrier-');
t.after(() => {
cleanup(scriptDir);
cleanup(runDir);
cleanup(barrierDir);
});
const block = extractInvokeReviewersBash();
const tracePath = path.join(scriptDir, 'trace.log');
const env = buildEnv(opts, runDir, tracePath, barrierDir);
const scriptPath = path.join(scriptDir, 'dispatch.sh');
const script = [
'#!/usr/bin/env bash',
'set -u',
STUB_PREAMBLE,
block.split('{run_dir}').join(runDir),
].join('\n');
fs.writeFileSync(scriptPath, script, { mode: 0o755 });
const result = runHook(scriptPath, [], {
interpreter: 'bash',
cwd: runDir,
env,
timeoutMs: HOOK_FANOUT_TIMEOUT_MS,
});
const jsonlPath = path.join(runDir, 'gsd-review-lane-results.jsonl');
const jsonl = fs.existsSync(jsonlPath) ? fs.readFileSync(jsonlPath, 'utf-8') : '';
const lines = jsonl.split('\n').filter((l) => l.trim() !== '');
const trace = fs.existsSync(tracePath)
? readFileNormalized(tracePath).split('\n').filter((l) => l.trim() !== '')
: [];
return {
outcome: result.outcome,
exitCode: result.exitCode,
stderr: result.stderr,
jsonl,
lines,
trace,
runDir,
};
}
/** Parsed slug order from JSONL lines — never assert on raw JSONL text. */
function slugOrder(lines) {
return lines.map((l) => JSON.parse(l).slug);
}
function serialTrace(slugs) {
return slugs.flatMap((s) => [`start:${s}`, `end:${s}`]);
}
// ─── #1/#2 — default and explicit-disabled serial dispatch ───────────────
describe('#3034 default and explicit-disabled dispatch stays serial', () => {
test('defaultsToSerialDispatchWhenKeyUnset', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: null });
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']));
});
test('staysSerialWhenExplicitlyDisabled', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'false' });
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']));
});
});
// ─── #3/#4 — opt-in concurrency and join-before-aggregate ─────────────────
describe('#3034 opt-in concurrency', () => {
test('dispatchesLanesConcurrentlyWhenEnabled', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', barrier: true });
const timeouts = result.trace.filter((l) => l.startsWith('barrier-timeout:'));
assert.deepEqual(timeouts, [], 'no lane should hit the barrier timeout when lanes run concurrently');
});
test('joinsAllLanesBeforeAggregation', (t) => {
// barrier:true is load-bearing, not decoration. Without it the stub lanes
// finish instantly and a missing `wait` could still race to 3 lines,
// making this pass intermittently. Held at the barrier, a missing join
// deterministically aggregates ZERO lines.
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', barrier: true });
assert.equal(result.lines.length, 3, 'dispatch must return only after every lane wrote its result');
});
});
// ─── #5/#6 — selection-order preservation ─────────────────────────────────
describe('#3034 JSONL preserves selection order, not completion order', () => {
test('preservesSelectionOrderSerial', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: null });
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini', 'claude']);
});
test('preservesSelectionOrderParallelDespiteCompletionOrder', (t) => {
// codex waits on gemini; gemini waits on claude -> forces reverse
// completion order (claude, gemini, codex) while selection order stays
// codex, gemini, claude.
const result = runDispatch(t, {
selected: SELECTED_3,
parallel: 'true',
deps: { codex: 'gemini', gemini: 'claude' },
});
const endMarkers = result.trace.filter((l) => l.startsWith('end:'));
// Pin the fixture actually forced reverse completion first — otherwise
// the selection-order assertion below would pass vacuously.
assert.deepEqual(endMarkers, ['end:claude', 'end:gemini', 'end:codex']);
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini', 'claude']);
});
});
// ─── #7/#8 — PIPE_BUF boundary triple (plus one clearly-oversized case) ───
describe('#3034 oversized lane results stay intact under concurrency', () => {
test('keepsOversizedLaneResultsIntactUnderConcurrency', async (t) => {
for (const padBytes of [4095, 4096, 4097, 8192]) {
await t.test(`padBytes=${padBytes}`, (t2) => {
const result = runDispatch(t2, { selected: SELECTED_3, parallel: 'true', padBytes });
assert.equal(result.lines.length, 3);
for (const line of result.lines) {
assert.doesNotThrow(() => JSON.parse(line), `line failed to parse at padBytes=${padBytes}: ${line.slice(0, 80)}...`);
}
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini', 'claude']);
});
}
});
});
// ─── #9 — single-lane boundary ─────────────────────────────────────────────
describe('#3034 single-lane boundary', () => {
test('singleLaneParallelMatchesSerial', (t) => {
const serial = runDispatch(t, { selected: 'codex', parallel: null });
const parallel = runDispatch(t, { selected: 'codex', parallel: 'true' });
assert.equal(serial.lines.length, 1);
assert.equal(parallel.lines.length, 1);
assert.deepEqual(slugOrder(serial.lines), slugOrder(parallel.lines));
const serialMd = fs.readFileSync(path.join(serial.runDir, 'gsd-review-codex.md'), 'utf-8');
const parallelMd = fs.readFileSync(path.join(parallel.runDir, 'gsd-review-codex.md'), 'utf-8');
assert.equal(serialMd, parallelMd);
});
});
// ─── #10 — empty selection is a no-op ──────────────────────────────────────
describe('#3034 empty selection', () => {
test('emptySelectionIsANoOp', (t) => {
const result = runDispatch(t, { selected: '', parallel: 'true' });
assert.equal(result.outcome, 'exited');
assert.equal(result.exitCode, 0);
assert.deepEqual(result.lines, []);
assert.deepEqual(result.trace, []);
});
});
describe('#3034 duplicate slug in the selection', () => {
test('duplicateSlugDispatchesOnceAndWritesOneLine', (t) => {
// A slug repeated in SELECTED_REVIEWERS would otherwise put two
// concurrent background jobs on the same `>`-truncated per-slug result
// file, corrupting whichever one finishes last. DISPATCH_SLUGS
// de-duplicates before dispatch, so codex must run exactly once.
const result = runDispatch(t, { selected: 'codex,gemini,codex', parallel: 'true' });
assert.equal(result.outcome, 'exited');
assert.equal(result.trace.filter((l) => l === 'start:codex').length, 1);
assert.equal(result.trace.filter((l) => l === 'end:codex').length, 1);
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini']);
});
});
// ─── #11/#12 — lane failure does not abort siblings ────────────────────────
describe('#3034 lane failure does not abort sibling lanes', () => {
test('laneFailureDoesNotAbortSiblingLanes', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', failSlugs: ['gemini'] });
assert.equal(result.lines.length, 3, 'a failing lane still contributes its stub result line');
assert.ok(
fs.existsSync(path.join(result.runDir, 'gsd-review-gemini.md')),
'the failing lane\'s diagnostic stub .md must be preserved',
);
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini', 'claude']);
});
test('laneFailureSerialUnchanged', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: null, failSlugs: ['gemini'] });
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']));
assert.equal(result.lines.length, 3);
});
});
// ─── #13 — budget-too-small skip emits no result line ─────────────────────
describe('#3034 budget-too-small skip', () => {
test('budgetSkipEmitsNoResultLine', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', budgetFailSlugs: ['gemini'] });
assert.deepEqual(slugOrder(result.lines), ['codex', 'claude'], 'the budget-skipped lane contributes no result line');
const stub = fs.readFileSync(path.join(result.runDir, 'gsd-review-gemini.md'), 'utf-8');
assert.match(stub, /skipped/);
assert.match(stub, /prompt budget/);
});
});
// ─── #14/#15 — silent / absent lane contributes no line ───────────────────
describe('#3034 silent lane contributes nothing, not a blank line', () => {
test('silentLaneContributesNoLine', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', silentSlugs: ['gemini'] });
assert.deepEqual(slugOrder(result.lines), ['codex', 'claude']);
// Folded row #15 (absent result file): budgetSkipEmitsNoResultLine above
// already exercises the "invoke never started, no file at all" shape via
// its `continue`. This assertion pins the sibling shape: a lane that DID
// run and wrote nothing must not leave a stray blank JSONL line.
assert.ok(!result.jsonl.includes('\n\n'), 'an empty lane result must not leave a blank JSONL line');
});
});
// ─── #16/#17 — non-canonical truthy values stay serial ────────────────────
describe('#3034 non-canonical truthy config values stay serial', () => {
test('nonCanonicalTruthyValuesStaySerial', async (t) => {
const nearMisses = ['TRUE', 'True', '1', 'yes', 'on', ' true', 'true '];
for (const value of nearMisses) {
await t.test(`parallel_lanes="${value}"`, (t2) => {
const result = runDispatch(t2, { selected: SELECTED_3, parallel: value });
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']), `value "${value}" must not opt into parallel dispatch`);
});
}
});
});
// ─── #18 — config-get failure fails safe to serial ─────────────────────────
describe('#3034 broken config tooling fails safe to serial', () => {
test('configGetFailureFallsBackToSerial', (t) => {
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', configGetFails: true });
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']));
});
});
// ─── #19/#20/#21 — config-set registers review.parallel_lanes ─────────────
describe('#3034 review.parallel_lanes config key', () => {
test('configSetAcceptsAndPersistsParallelLanes', (t) => {
const tmpDir = createTempProject();
t.after(() => cleanup(tmpDir));
const setResult = runGsdTools('config-set review.parallel_lanes true', tmpDir);
assert.ok(setResult.success, `config-set failed: ${setResult.error}`);
const configPath = path.join(tmpDir, '.planning', 'config.json');
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
assert.equal(config.review?.parallel_lanes, true);
assert.equal(typeof config.review?.parallel_lanes, 'boolean');
const getResult = runGsdTools('config-get review.parallel_lanes --raw', tmpDir);
assert.ok(getResult.success, `config-get failed: ${getResult.error}`);
assert.equal((getResult.output || '').trim(), 'true');
});
test('configSetPersistsBooleanFalse', (t) => {
const tmpDir = createTempProject();
t.after(() => cleanup(tmpDir));
const setResult = runGsdTools('config-set review.parallel_lanes false', tmpDir);
assert.ok(setResult.success, `config-set failed: ${setResult.error}`);
const configPath = path.join(tmpDir, '.planning', 'config.json');
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
assert.equal(config.review?.parallel_lanes, false);
assert.equal(typeof config.review?.parallel_lanes, 'boolean');
});
test('rejectsUnregisteredNeighbouringKey', (t) => {
const tmpDir = createTempProject();
t.after(() => cleanup(tmpDir));
// Missing trailing "s" — proves the whitelist is load-bearing and the
// two tests above are not vacuous (they'd pass even for an unregistered
// key if config-set accepted anything).
const result = runGsdTools('config-set review.parallel_lane true', tmpDir);
assert.equal(result.success, false, 'an unregistered near-miss key must be rejected');
});
});
// ─── #22 — serial/parallel artifact equivalence ────────────────────────────
describe('#3034 serial and parallel dispatch produce equivalent artifacts', () => {
test('serialAndParallelProduceEquivalentArtifacts', (t) => {
const serial = runDispatch(t, { selected: SELECTED_3, parallel: null });
const parallel = runDispatch(t, { selected: SELECTED_3, parallel: 'true' });
assert.deepEqual(slugOrder(serial.lines), slugOrder(parallel.lines));
assert.deepEqual(
serial.lines.map((l) => JSON.parse(l)),
parallel.lines.map((l) => JSON.parse(l)),
);
for (const slug of ['codex', 'gemini', 'claude']) {
const serialMd = fs.readFileSync(path.join(serial.runDir, `gsd-review-${slug}.md`), 'utf-8');
const parallelMd = fs.readFileSync(path.join(parallel.runDir, `gsd-review-${slug}.md`), 'utf-8');
assert.equal(serialMd, parallelMd, `gsd-review-${slug}.md must be byte-identical between the two paths`);
}
});
});
// ─────────────────────────────────────────────────────────────────────────────
// #3352 (ADR-3473 §8.5, phase #3885 item 3) — `write_reviews` must not emit
// REVIEWS.md from failed inputs, and `present_results` must preserve
// per-lane evidence before `rm -rf "{run_dir}"` destroys it.
//
// SEAM CHOICE: `write_reviews`/`present_results` are markdown PROSE
// instructing an LLM agent, not a single executable program — "do not write
// REVIEWS.md" and "display this message" are natural-language directives to
// the agent, not `if`/`fi` around a `Write` call this suite could invoke.
// Asserting on that prose text directly would be source-grep
// (`local/no-source-grep`), which this repo bans. What IS literal,
// deterministic bash in both steps — and is exactly what R2's gate condition
// depends on — is:
// (1) `write_reviews`'s new gate block, which computes
// `TOTAL_LANE_FAILURE`/`ALL_LANES_SKIPPED` from the aggregate JSONL
// and the lane stub files (the ONLY inputs the prose instructions key
// off of to decide whether to write REVIEWS.md at all);
// (2) `write_reviews`'s commit block (unchanged text, still real bash);
// (3) `present_results`'s new preserve-evidence block and its `rm -rf`
// cleanup block.
// Each is extracted and EXECUTED for real. To observe the documented
// consequence of the gate flags (whether REVIEWS.md and the commit actually
// happen), the driver below applies the step's OWN written contract —
// "If ALL_LANES_SKIPPED=true or TOTAL_LANE_FAILURE=true: do NOT write
// REVIEWS.md and do NOT run the commit. Otherwise: proceed" (review.md, the
// `write_reviews` step, verbatim) — as harness scaffolding around the real
// extracted blocks, the same way STUB_PREAMBLE above supplies scaffolding
// (`arg_after`/`in_list`/...) the extracted `invoke_reviewers` block calls
// into. This is not a re-typed copy of the logic under test: the FLAG
// computation is 100% the real fenced bash; only the "then do the file I/O"
// half — which the real workflow leaves to the LLM's own tool calls — is
// harness glue.
//
// R46 (`preservedEvidenceIsNotSweptIntoTheCommit`) is a regression guard for
// N6, not failing-first evidence: the commit fence's `--files` argument is
// byte-identical before and after #3352 (only the surrounding prose gained
// gating language), so this exact assertion also holds against the pre-fix
// text. It is retained because the invariant it pins (the commit step must
// never widen from the single REVIEWS.md path to a directory glob that
// would sweep `.review-diagnostics/` into a commit) is real and worth a
// permanent pin, and because it only runs at all through this same
// full-flow driver, whose surrounding gate/preserve blocks did not exist
// pre-fix (see the STEP 2 report for the exact pre-fix observation).
function extractStepBody(stepName) {
const content = readFileNormalized(REVIEW_MD_PATH);
const stepMarker = `<step name="${stepName}">`;
const stepIdx = content.indexOf(stepMarker);
if (stepIdx === -1) {
throw new Error(`extractStepBody: could not find "${stepMarker}" in ${REVIEW_MD_PATH}`);
}
const afterStep = content.slice(stepIdx + stepMarker.length);
const endIdx = afterStep.indexOf('</step>');
if (endIdx === -1) {
throw new Error(`extractStepBody: could not find closing "</step>" after ${stepName} in ${REVIEW_MD_PATH}`);
}
return afterStep.slice(0, endIdx);
}
/** Finds the first ```bash/```sh fence in `stepBody` whose text contains every string in `mustInclude`. */
function extractBashFenceContaining(stepBody, mustInclude, label) {
const stepBodyLines = stepBody.split(/\r?\n/);
for (const fenced of scanFencedBlocks(stepBodyLines)) {
if (fenced.closeLineIdx === -1) continue;
if (!['bash', 'sh'].includes((fenced.infoString || '').trim())) continue;
const block = stepBodyLines.slice(fenced.openLineIdx + 1, fenced.closeLineIdx).join('\n');
if (mustInclude.every((s) => block.includes(s))) return block;
}
throw new Error(`extractBashFenceContaining: no fence matching ${label} found (looked for ${JSON.stringify(mustInclude)})`);
}
function extractWriteReviewsGateBash() {
return extractBashFenceContaining(
extractStepBody('write_reviews'),
['TOTAL_LANE_FAILURE', 'ALL_LANES_SKIPPED', 'DISPATCH_SLUGS'],
'write_reviews #3352 gate block',
);
}
function extractWriteReviewsCommitBash() {
return extractBashFenceContaining(
extractStepBody('write_reviews'),
['gsd_run query commit', 'REVIEWS.md'],
'write_reviews commit block',
);
}
// #3885: preserve-evidence and cleanup are ONE fenced block (a shell variable
// cannot survive across separate fences), gated on `_PRESERVE_OK` so a failed
// `mkdir -p`/`cp` skips the `rm -rf` and leaves `{run_dir}` intact instead of
// destroying the only copy of the evidence it exists to protect.
function extractPresentResultsPreserveAndCleanupBash() {
return extractBashFenceContaining(
extractStepBody('present_results'),
['DIAG_DIR', 'nullglob', 'rm -rf', '{run_dir}', '_PRESERVE_OK'],
'present_results #3352/#3885 preserve+cleanup block',
);
}
/**
* Runs the real extracted write_reviews gate+commit blocks and the real
* present_results preserve+cleanup block, in the documented order, against
* a fixture run dir. `opts.jsonlLines` seeds the aggregate JSONL (omit/empty
* for "every lane failed"); `opts.lanes` seeds per-slug `gsd-review-<slug>.md`
* / `.md`'s sibling `.err`. See the seam-choice comment above this describe
* block for what is real bash vs. harness glue.
*
* `opts.blockDiagDirWithFile`: #3885 root-safe failure injection. Root
* bypasses `chmod 0o000` entirely (a Docker/CI default), so permission bits
* cannot induce a `mkdir -p`/`cp` failure deterministically. A filesystem
* TYPE conflict is root-safe instead: pre-creating a plain FILE at the exact
* path `mkdir -p` needs to create as a DIRECTORY makes `mkdir -p` fail with
* "not a directory" for every caller, root included, because it is not a
* permissions check at all.
*/
// #4351: preserved evidence now lands in a PER-RUN SUBDIRECTORY under
// `.review-diagnostics/`. A flat copy used each file's SOURCE basename, and a
// lane slug is stable across runs, so a second review of the same phase silently
// overwrote the first run's evidence.
//
// These helpers resolve THROUGH that subdirectory layer on purpose. Rows that
// assert WHICH files were preserved should keep asserting exactly that; without
// them each such row would quietly become an assertion about the directory
// LAYOUT instead, and would have to be rewritten again the next time the layout
// moves. The layout itself is pinned once, explicitly, by
// `oneRunSubdirectoryPerRun_4351`.
function preservedRunDirs(result) {
if (!fs.existsSync(result.diagDir)) return [];
return fs.readdirSync(result.diagDir)
.map((name) => path.join(result.diagDir, name))
.filter((p) => fs.statSync(p).isDirectory())
.sort();
}
function preservedNames(result) {
return preservedRunDirs(result)
.flatMap((dir) => fs.readdirSync(dir))
.sort();
}
// Resolves a preserved basename to its real path. When the file was NOT
// preserved this deliberately returns a path that does not exist, so a negative
// assertion (`existsSync(...) === false`) still reads naturally at the call site.
function preservedPath(result, name) {
for (const dir of preservedRunDirs(result)) {
const candidate = path.join(dir, name);
if (fs.existsSync(candidate)) return candidate;
}
return path.join(result.diagDir, name);
}
function runWriteReviewsFlow(t, opts) {
const scriptDir = createTempDir('gsd-3352-script-');
const runDir = createTempDir('gsd-3352-rundir-');
// #4351: an explicit phaseDir lets a caller run this flow TWICE against the
// SAME phase directory, which is the only way to observe a second run
// clobbering the first run's preserved evidence. Omitted (every pre-existing
// caller) it mints a fresh one exactly as before.
const phaseDir = opts.phaseDir || createTempDir('gsd-3352-phasedir-');
t.after(() => {
cleanup(scriptDir);
cleanup(runDir);
// A caller-supplied phaseDir is owned by the caller — cleaning it here would
// destroy run 1's evidence before run 2 could be asserted against it.
if (!opts.phaseDir) cleanup(phaseDir);
});
if (opts.blockDiagDirWithFile) {
fs.writeFileSync(path.join(phaseDir, '.review-diagnostics'), 'blocking file, not a directory\n');
}
// #4097: stage the run's OWN input copies exactly as prompt assembly
// (review.md's section-copy fence) writes them, so the preserve+cleanup
// block can be observed deciding input vs. evidence. `name` is a bare
// basename written into the fixture run dir.
for (const [name, content] of Object.entries(opts.seedFiles || {})) {
fs.writeFileSync(path.join(runDir, name), content);
}
if (opts.jsonlLines && opts.jsonlLines.length > 0) {
fs.writeFileSync(
path.join(runDir, 'gsd-review-lane-results.jsonl'),
opts.jsonlLines.map((l) => JSON.stringify(l)).join('\n') + '\n',
);
}
for (const [slug, files] of Object.entries(opts.lanes || {})) {
if (files.md !== undefined && files.md !== null) {
fs.writeFileSync(path.join(runDir, `gsd-review-${slug}.md`), files.md);
}
if (files.err !== undefined && files.err !== null) {
fs.writeFileSync(path.join(runDir, `gsd-review-${slug}.err`), files.err);
}
}
const gateBlock = extractWriteReviewsGateBash();
const commitBlock = extractWriteReviewsCommitBash();
const preserveAndCleanupBlock = extractPresentResultsPreserveAndCleanupBash();
const tracePath = path.join(scriptDir, 'commit-trace.log');
const reviewsMdPath = path.join(phaseDir, '03-REVIEWS.md');
// #3885 Windows fix: splice POSIX-form paths into the bash source text, not
// the OS-native ones `createTempDir()` returns. This is a fixture fix, not
// a product one — the real workflow never hits this seam with a backslash
// path in the first place: `{run_dir}` is created by `mktemp -d` running
// INSIDE the bash block itself (always POSIX-style, even under Git Bash on
// Windows), and `{phase_dir}` is the `phase_dir` field from
// `gsd_run query init.review`, which gsd-core/bin/lib/init.cjs already
// pipes through this exact same `toPosixPath()` before it is ever
// serialized (see the `phase_dir: ... toPosixPath(...)` call sites there).
// Splicing a native `C:\Users\...` string directly into unquoted bash
// source (e.g. the pre-existing, unquoted
// `--files {phase_dir}/{padded_phase}-REVIEWS.md` commit line) hits bash's
// own unquoted-backslash removal and silently drops every separator — a
// test-harness-only failure mode that following the fixture's own
// production analogue eliminates.
const runDirPosix = toPosixPath(runDir);
const phaseDirPosix = toPosixPath(phaseDir);
const substitute = (block) => block
.split('{run_dir}').join(runDirPosix)
.split('{phase_dir}').join(phaseDirPosix)
.split('{padded_phase}').join('03')
.split('{N}').join('3');
const script = [
'#!/usr/bin/env bash',
'set -u',
`SELECTED_REVIEWERS='${opts.selected}'`,
`TRACE='${tracePath}'`,
'gsd_run() { printf "%s\\n" "$*" >> "$TRACE"; }',
substitute(gateBlock),
'echo "GATE:TOTAL_LANE_FAILURE=$TOTAL_LANE_FAILURE"',
'echo "GATE:ALL_LANES_SKIPPED=$ALL_LANES_SKIPPED"',
'if [ "$TOTAL_LANE_FAILURE" = "false" ] && [ "$ALL_LANES_SKIPPED" = "false" ]; then',
// Harness glue standing in for the agent's own Write tool call (see the
// seam-choice comment above): the prose says "combine ... into
// REVIEWS.md" only when the gate above did not fire.
` touch '${reviewsMdPath}'`,
substitute(commitBlock),
'fi',
substitute(preserveAndCleanupBlock),
].join('\n');
fs.writeFileSync(path.join(scriptDir, 'flow.sh'), script, { mode: 0o755 });
// cwd is deliberately scriptDir, NOT runDir: every path this flow touches
// ($RUN_DIR, $DIAG_DIR, {phase_dir}) is already absolute in the extracted
// bash text, so the child's cwd is not load-bearing for anything the
// blocks do — but on Windows a process cannot delete a directory that is
// its OWN current working directory (unlike POSIX, where rm -rf on your
// cwd succeeds). Setting cwd to runDir here was a harness artifact with no
// production analogue (review.md never `cd`s into $RUN_DIR — see
// gsd-core/workflows/review.md's use of $RUN_DIR, always by absolute
// path), and it silently defeated `rm -rf "$RUN_DIR"` under Git-Bash on
// windows-latest: the directory's *contents* were removed but the
// still-open-as-cwd directory entry itself survived, so
// `fs.existsSync(runDir)` kept reporting true. This was the actual defect
// behind the two Windows-only failures — not a path-separator mismatch.
const result = runHook(path.join(scriptDir, 'flow.sh'), [], {
interpreter: 'bash',
cwd: scriptDir,
timeoutMs: HOOK_FANOUT_TIMEOUT_MS,
});
const stdout = result.stdout || '';
const diagDir = path.join(phaseDir, '.review-diagnostics');
const commitTrace = fs.existsSync(tracePath)
? readFileNormalized(tracePath).split('\n').filter((l) => l.trim() !== '')
: [];
return {
outcome: result.outcome,
exitCode: result.exitCode,
stderr: result.stderr,
totalLaneFailure: /GATE:TOTAL_LANE_FAILURE=(\S+)/.exec(stdout)?.[1],
allLanesSkipped: /GATE:ALL_LANES_SKIPPED=(\S+)/.exec(stdout)?.[1],
reviewsMdExists: fs.existsSync(reviewsMdPath),
runDirExists: fs.existsSync(runDir),
diagDir,
diagDirExists: fs.existsSync(diagDir),
commitTrace,
runDir,
phaseDir,
};
}
const BUDGET_SKIP_MD = (slug) => `${slug} review skipped: prompt budget (500 tokens) too small for the minimum review set.`;
const LANE_FAILURE_MD = (slug) => `${slug} review failed: exit 1`;
describe('#3352 every lane failed writes no REVIEWS.md', () => {
test('totalLaneFailureWritesNoReviewsMd_3352', (t) => {
const result = runWriteReviewsFlow(t, {
selected: 'codex,gemini,claude',
jsonlLines: [],
lanes: {
codex: { md: LANE_FAILURE_MD('codex'), err: 'stack trace: codex crashed\n' },
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'stack trace: gemini crashed\n' },
claude: { md: LANE_FAILURE_MD('claude'), err: 'stack trace: claude crashed\n' },
},
});
assert.equal(result.outcome, 'exited');
assert.equal(result.totalLaneFailure, 'true', `expected TOTAL_LANE_FAILURE=true; stderr=${result.stderr}`);
assert.equal(result.allLanesSkipped, 'false');
assert.equal(result.reviewsMdExists, false, 'no REVIEWS.md may be written when every lane failed');
assert.deepEqual(result.commitTrace, [], 'the commit must not run when every lane failed');
});
});
describe('#3352 one successful lane still writes REVIEWS.md (R1, must stay green)', () => {
test('oneSuccessfulLaneStillWritesReviews', (t) => {
const result = runWriteReviewsFlow(t, {
selected: 'codex,gemini,claude',
jsonlLines: [{ slug: 'codex' }],
lanes: {
codex: { md: '# Codex review\nlooks good\n' },
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'stack trace: gemini crashed\n' },
claude: { md: LANE_FAILURE_MD('claude'), err: 'stack trace: claude crashed\n' },
},
});
assert.equal(result.totalLaneFailure, 'false');
assert.equal(result.allLanesSkipped, 'false');
assert.equal(result.reviewsMdExists, true, 'at least one lane succeeded — REVIEWS.md must still be written');
assert.equal(result.commitTrace.length, 1, 'the commit must run exactly once');
});
});
describe('#3352 a budget-skipped lane is not a failure (N5)', () => {
test('budgetSkippedLanesAreNotFailures', (t) => {
const result = runWriteReviewsFlow(t, {
selected: 'codex,gemini,claude',
jsonlLines: [],
lanes: {
codex: { md: BUDGET_SKIP_MD('codex') },
gemini: { md: BUDGET_SKIP_MD('gemini') },
claude: { md: BUDGET_SKIP_MD('claude') },
},
});
assert.equal(result.allLanesSkipped, 'true', `every lane was budget-skipped, not failed; stderr=${result.stderr}`);
assert.equal(result.totalLaneFailure, 'false', 'a budget skip must never be classified as a failure');
assert.equal(result.reviewsMdExists, false, 'nothing to review — REVIEWS.md still must not be written');
assert.deepEqual(result.commitTrace, []);
});
});
describe('#3352 successful-lane count boundary (0/1/2)', () => {
test('successfulLaneCountBoundary', (t) => {
const zero = runWriteReviewsFlow(t, {
selected: 'codex,gemini',
jsonlLines: [],
lanes: {
codex: { md: LANE_FAILURE_MD('codex'), err: 'boom\n' },
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'boom\n' },
},
});
assert.equal(zero.totalLaneFailure, 'true');
assert.equal(zero.reviewsMdExists, false, '0 successful lanes: no REVIEWS.md');
const one = runWriteReviewsFlow(t, {
selected: 'codex,gemini',
jsonlLines: [{ slug: 'codex' }],
lanes: {
codex: { md: '# Codex\nok\n' },
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'boom\n' },
},
});
assert.equal(one.totalLaneFailure, 'false');
assert.equal(one.reviewsMdExists, true, '1 successful lane: REVIEWS.md is written');
const two = runWriteReviewsFlow(t, {
selected: 'codex,gemini',
jsonlLines: [{ slug: 'codex' }, { slug: 'gemini' }],
lanes: {
codex: { md: '# Codex\nok\n' },
gemini: { md: '# Gemini\nok\n' },
},
});
assert.equal(two.totalLaneFailure, 'false');
assert.equal(two.reviewsMdExists, true, '2 successful lanes: REVIEWS.md is written');
});
});
describe('#3352 per-lane evidence survives cleanup (R3)', () => {
test('laneEvidenceSurvivesCleanup_3352', (t) => {
const result = runWriteReviewsFlow(t, {
selected: 'codex,gemini,claude',
jsonlLines: [],
lanes: {
codex: { md: LANE_FAILURE_MD('codex'), err: 'stack trace: codex crashed\n' },
gemini: { md: '', err: '' }, // ran, wrote nothing, no error either — nothing to preserve for this slug's .err
claude: { md: LANE_FAILURE_MD('claude'), err: 'stack trace: claude crashed\n' },
},
});
assert.equal(result.runDirExists, false, 'the run dir must still be destroyed');
assert.equal(result.diagDirExists, true, 'diagnostics must have been preserved somewhere under phase_dir');
const codexMd = preservedPath(result, 'gsd-review-codex.md');
const codexErr = preservedPath(result, 'gsd-review-codex.err');
const claudeErr = preservedPath(result, 'gsd-review-claude.err');
assert.ok(fs.existsSync(codexMd), 'codex .md evidence must be preserved');
assert.equal(fs.readFileSync(codexMd, 'utf-8'), LANE_FAILURE_MD('codex'));
assert.ok(fs.existsSync(codexErr), 'codex non-empty .err evidence must be preserved');
assert.ok(fs.existsSync(claudeErr), 'claude non-empty .err evidence must be preserved');
const geminiErr = preservedPath(result, 'gsd-review-gemini.err');
assert.equal(fs.existsSync(geminiErr), false, 'an empty .err must not be copied as if it were real evidence');
});
});
describe('#3885 nothing to preserve still cleans up (must stay green)', () => {
test('nothingToPreserveStillCleansUp_3885', (t) => {
// No .md/.err fixtures written at all: `_DIAG_MD`/`_DIAG_ERR` are both
// empty, so this is the "nothing to preserve" branch, not a failure —
// `_PRESERVE_OK` starts (and stays) `true`, so cleanup proceeds exactly
// as if preservation had succeeded.
const result = runWriteReviewsFlow(t, {
selected: 'codex,gemini,claude',
jsonlLines: [],
lanes: {},
});
assert.equal(result.runDirExists, false, 'nothing to preserve is not a failure — run dir must still be removed');
assert.equal(result.diagDirExists, false, 'no diagnostics directory should be created when there is nothing to copy');
});
});
describe('#3885 failed preservation leaves run_dir intact (no silent swallow)', () => {
test('failedPreservationLeavesRunDirIntact_3885', (t) => {
// Root-safe failure injection: pre-create a plain FILE at the exact path
// `mkdir -p "$DIAG_DIR"` needs to create as a directory. This is a
// filesystem TYPE conflict, not a permission check, so it fails `mkdir -p`
// even when the test runs as root in Docker/CI (where `chmod 0o000`
// would be silently bypassed and this test would pass with zero real
// coverage — see #3885's brief).
const result = runWriteReviewsFlow(t, {
selected: 'codex,gemini,claude',
jsonlLines: [],
blockDiagDirWithFile: true,
lanes: {
codex: { md: LANE_FAILURE_MD('codex'), err: 'stack trace: codex crashed\n' },
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'stack trace: gemini crashed\n' },
claude: { md: LANE_FAILURE_MD('claude'), err: 'stack trace: claude crashed\n' },
},
});
assert.equal(
result.runDirExists,
true,
'a failed mkdir -p on DIAG_DIR must skip rm -rf and leave run_dir intact — this is the #3885 regression guard',
);
// The warning is emitted by the substituted bash script, which names
// $RUN_DIR in its POSIX-spliced form (see the #3885 comment in
// runWriteReviewsFlow) — compare against that same form rather than the
// OS-native `result.runDir`.
assert.ok(
result.stderr.includes(toPosixPath(result.runDir)),
`the failure warning must name the intact run_dir holding the un-preserved evidence; got stderr: ${result.stderr}`,
);
// The original per-lane evidence is still readable at its original
// location, uncorrupted, because it was never moved or destroyed.
assert.equal(
fs.readFileSync(path.join(result.runDir, 'gsd-review-codex.md'), 'utf-8'),
LANE_FAILURE_MD('codex'),
);
});
});
describe('#4097 the run\'s own input copies are not preserved as diagnostics', () => {
// The preserve+cleanup glob assumes every `gsd-review-*.md` in RUN_DIR is
// lane OUTPUT. But the workflow itself writes the run's assembled INPUTS
// into RUN_DIR under the same prefix at prompt-assembly time (review.md's
// section-copy fence): the combined prompt, instructions, roadmap, one copy
// of every plan under review, project/context/research/requirements
// sections, and the per-lane trimmed prompts. On a multi-plan phase those
// byte-identical `.planning/` duplicates bury the actual evidence (a
// handful of reviewer reports and `.err` sidecars) and grow the phase
// directory on every review run (#4097).
const INPUT_COPIES_4097 = {
'gsd-review-prompt.md': 'combined reviewer prompt\n',
'gsd-review-instructions.md': 'instructions section\n',
'gsd-review-roadmap.md': 'roadmap section\n',
'gsd-review-project.md': 'PROJECT.md copy\n',
'gsd-review-context.md': 'CONTEXT.md concatenation\n',
'gsd-review-research.md': 'RESEARCH.md concatenation\n',
'gsd-review-requirements.md': 'REQUIREMENTS.md copy\n',
'gsd-review-plan-12.6-01.md': 'plan 12.6-01 duplicate\n',
'gsd-review-plan-12.6-02.md': 'plan 12.6-02 duplicate\n',
// Per-lane trimmed prompt written by prepare_trimmed_prompt_for_reviewer
// — also a run input, also matched by the issue's `gsd-review-prompt*`
// exclusion prefix.
'gsd-review-prompt-claude.md': 'budget-trimmed prompt for lane claude\n',
};
test('inputCopiesAreNotSweptIntoDiagnostics_4097', (t) => {
const result = runWriteReviewsFlow(t, {
selected: 'claude',
jsonlLines: [{ slug: 'claude' }],
seedFiles: INPUT_COPIES_4097,
lanes: {
claude: { md: '# Claude review\nok\n', err: 'mild stderr warning\n' },
},
});
assert.equal(result.outcome, 'exited');
assert.equal(result.runDirExists, false, 'successful preservation must still clean up the run dir');
assert.equal(result.diagDirExists, true, 'real lane evidence must still be preserved');
const preserved = preservedNames(result);
// Only the lane's own output belongs in the diagnostics folder.
assert.deepEqual(
preserved,
['gsd-review-claude.err', 'gsd-review-claude.md'],
`diagnostics must hold exactly the lane report and stderr sidecar, not the run's input copies; got: ${preserved.join(', ')}`,
);
assert.equal(
fs.readFileSync(preservedPath(result, 'gsd-review-claude.md'), 'utf-8'),
'# Claude review\nok\n',
'the lane report must be preserved byte-identically',
);
for (const inputName of Object.keys(INPUT_COPIES_4097)) {
assert.equal(
fs.existsSync(preservedPath(result, inputName)),
false,
`input copy ${inputName} must NOT be swept into .review-diagnostics/ (#4097)`,
);
}
});
test('inputsOnlyRunLeavesNoDiagnosticsDir_4097', (t) => {
// Inputs only, no lane artifacts at all: inputs are not evidence, so
// this is the "nothing to preserve" branch — no diagnostics directory is
// created and cleanup proceeds (#4097 narrowing of #3352's glob).
const result = runWriteReviewsFlow(t, {
selected: 'claude',
jsonlLines: [],
seedFiles: INPUT_COPIES_4097,
lanes: {},
});
assert.equal(result.runDirExists, false, 'nothing to preserve is not a failure — run dir must still be removed');
assert.equal(result.diagDirExists, false, 'a run that produced only input copies has no evidence to preserve');
});
});
describe('#3352 preserved evidence is never swept into the commit (N6)', () => {
test('preservedEvidenceIsNotSweptIntoTheCommit', (t) => {
const result = runWriteReviewsFlow(t, {
selected: 'codex',
jsonlLines: [{ slug: 'codex' }],
lanes: {
codex: { md: '# Codex\nok\n', err: '' },
},
});
assert.equal(result.commitTrace.length, 1, 'exactly one commit call must run');
const commitArgs = result.commitTrace[0];
// The commit fence splices `{phase_dir}` into unquoted bash source as a
// POSIX-form path (see the #3885 comment in runWriteReviewsFlow) — build
// the expected string the same way rather than via `path.join`, which on
// win32 would re-insert native backslashes the substituted script never
// produces.
assert.ok(
commitArgs.includes(`${toPosixPath(result.phaseDir)}/03-REVIEWS.md`),
`commit must name the single REVIEWS.md file; got: ${commitArgs}`,
);
assert.ok(
!commitArgs.includes('.review-diagnostics'),
`commit must never name the diagnostics directory; got: ${commitArgs}`,
);
// The commit step's --files value is a single path, never a glob: a
// directory glob would expand to MULTIPLE argv tokens by the time
// `gsd_run` sees them, so more than one path after "--files" is itself
// the defect this row guards against.
const filesIdx = commitArgs.indexOf('--files ');
const afterFiles = commitArgs.slice(filesIdx + '--files '.length).trim();
assert.equal(afterFiles.split(/\s+/).length, 1, `--files must carry exactly one path; got: ${afterFiles}`);
});
});
// ─── #4351: repeated runs must accumulate evidence, never clobber it ────────
//
// The preserve block copies lane output into `.review-diagnostics/` using each
// file's SOURCE basename (`gsd-review-<slug>.md` / `.err`). A lane slug is
// stable across runs, so before this fix the destination path was stable across
// runs too — and `cp` over an existing file is a SUCCESS, so a second review of
// the same phase destroyed the first run's evidence with no error and no
// warning, in the one directory that exists to outlive `rm -rf "$RUN_DIR"`.
//
// These rows run the REAL extracted fence twice against ONE phase directory,
// which is the only arrangement that can observe the overwrite.
describe('#4351 repeated review runs accumulate evidence instead of clobbering it', () => {
test('twoRunsOnOnePhaseBothPreserved_4351', (t) => {
const sharedPhaseDir = createTempDir('gsd-4351-phasedir-');
t.after(() => cleanup(sharedPhaseDir));
const first = runWriteReviewsFlow(t, {
phaseDir: sharedPhaseDir,
selected: 'codex',
jsonlLines: [{ slug: 'codex' }],
lanes: { codex: { md: '# run one report\n', err: 'run one stderr\n' } },
});
assert.equal(first.outcome, 'exited');
assert.equal(first.diagDirExists, true, 'run 1 must preserve its evidence');
const second = runWriteReviewsFlow(t, {
phaseDir: sharedPhaseDir,
selected: 'codex',
jsonlLines: [{ slug: 'codex' }],
lanes: { codex: { md: '# run two report\n', err: 'run two stderr\n' } },
});
assert.equal(second.outcome, 'exited');
// Assert on CONTENT, not on a file count: a clobber that happened to produce
// the same number of files would pass a count-only check, and the whole
// defect is that run 1's bytes were replaced by run 2's.
const runDirs = preservedRunDirs(second);
assert.equal(
runDirs.length, 2,
`each run must own a distinct subdirectory; got ${runDirs.length}: ${runDirs.map((d) => path.basename(d)).join(', ')}`,
);
const reports = runDirs
.map((dir) => path.join(dir, 'gsd-review-codex.md'))
.filter((p) => fs.existsSync(p))
.map((p) => readFileNormalized(p));
assert.deepEqual(
reports.sort(),
['# run one report\n', '# run two report\n'].sort(),
'BOTH runs\' reports must be recoverable — before #4351 only the second survived',
);
});
test('identicalLaneFailureTwiceLeavesTwoStubs_4351', (t) => {
// The 2nd acceptance criterion, and the worst case for a flat layout: the
// SAME lane failing the SAME way twice produces byte-identical basenames.
const sharedPhaseDir = createTempDir('gsd-4351-phasedir-err-');
t.after(() => cleanup(sharedPhaseDir));
const opts = {
phaseDir: sharedPhaseDir,
selected: 'claude',
jsonlLines: [],
lanes: { claude: { md: '# lane failed\n', err: 'identical stack trace\n' } },
};
runWriteReviewsFlow(t, opts);
const second = runWriteReviewsFlow(t, opts);
const stubs = preservedRunDirs(second)
.map((dir) => path.join(dir, 'gsd-review-claude.err'))
.filter((p) => fs.existsSync(p));
assert.equal(
stubs.length, 2,
'two identical failures must leave two independently recoverable stubs',
);
for (const stub of stubs) {
assert.equal(readFileNormalized(stub), 'identical stack trace\n');
}
});
test('oneRunSubdirectoryPerRun_4351', (t) => {
// Pins the LAYOUT itself, once, so the helpers above stay honest: a single
// run creates exactly one subdirectory, the lane basenames are unchanged
// INSIDE it (nothing in $RUN_DIR was renamed — two consumers depend on those
// names), and no evidence is left sitting flat in the diagnostics root.
const result = runWriteReviewsFlow(t, {
selected: 'codex',
jsonlLines: [{ slug: 'codex' }],
lanes: { codex: { md: '# only run\n', err: 'only stderr\n' } },
});
const runDirs = preservedRunDirs(result);
assert.equal(runDirs.length, 1, 'one run must create exactly one subdirectory');
assert.deepEqual(
fs.readdirSync(runDirs[0]).sort(),
['gsd-review-codex.err', 'gsd-review-codex.md'],
'lane basenames must be unchanged inside the run subdirectory',
);
const flat = fs.readdirSync(result.diagDir)
.filter((name) => fs.statSync(path.join(result.diagDir, name)).isFile());
assert.deepEqual(flat, [], 'no evidence may be left flat in the diagnostics root');
});
});