2f920c5af3
* test(#4097): failing-first regression — run input copies swept into .review-diagnostics The present_results preserve+cleanup glob `_DIAG_MD=( "$RUN_DIR"/gsd-review-*.md )` matches both lane outputs and the run's own assembled input copies (prompt, instructions, roadmap, per-plan copies, project/context/research/requirements, per-lane trimmed prompts) because both share the gsd-review- prefix. Seed the full input-copy set into the existing runWriteReviewsFlow fixture and assert (a) the diagnostics dir holds exactly the lane report + .err sidecar and no input basenames, and (b) an inputs-only run creates no diagnostics dir at all and still cleans up. Both RED against the shipped block. * fix(#4097): preserve lane output only — exclude the run's input copies from evidence The preserve+cleanup glob treated every gsd-review-*.md in RUN_DIR as lane evidence, but the workflow itself writes the run's assembled INPUTS there under the same prefix (prompt, instructions, roadmap, per-plan copies, project/ context/research/requirements, per-lane trimmed prompts). Filter _DIAG_MD by basename against that closed input set instead. Direct glob iteration + case filter — identical under bash and zsh (#4099/ #4109), nullglob-safe (#2962), and the exclusion list is closed and owned in this step: a future input basename cannot silently rejoin the evidence set. Lane reports, diagnostic stubs and non-empty .err sidecars are unchanged. Emitted-Drift-Ack-Growth: review.md — #4097 narrows the present_results evidence-preservation glob: the closed input-basename exclusion list (case filter) plus its rationale note are deliberate additions so a future input basename cannot silently rejoin the evidence set. * changeset(#4097): fixed — review diagnostics no longer sweep run input copies * changeset(#4097): backfill PR number 4329 --------- Co-authored-by: sim <sim@local>
1120 lines
48 KiB
JavaScript
1120 lines
48 KiB
JavaScript
'use strict';
|
|
|
|
/**
|
|
* Failing-first tests for #3034 (opt-in parallel reviewer lanes).
|
|
*
|
|
* Design: .gsd/phase/feat-3034-parallel-reviewer-lanes/40-design.md
|
|
* Test matrix: .gsd/phase/feat-3034-parallel-reviewer-lanes/50-test-matrix.md
|
|
*
|
|
* The unit under test is the real, shipped `<step name="invoke_reviewers">`
|
|
* fenced bash block in gsd-core/workflows/review.md — extracted and EXECUTED
|
|
* (never re-typed), with the single I/O seam `gsd_run` replaced by a shell
|
|
* function stub. The feature this file exercises (an opt-in
|
|
* `review.parallel_lanes` config key that backgrounds lane dispatch and
|
|
* joins before aggregation) does not exist yet, so several tests below are
|
|
* expected to be RED against the current shipped block.
|
|
*/
|
|
|
|
const { test, describe } = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
|
|
const {
|
|
createTempDir,
|
|
createTempProject,
|
|
cleanup,
|
|
readFileNormalized,
|
|
runGsdTools,
|
|
} = require('./helpers.cjs');
|
|
const { runHook } = require('./helpers/process-seam.cjs');
|
|
const { HOOK_FANOUT_TIMEOUT_MS } = require('./helpers/timeouts.cjs');
|
|
const { toPosixPath } = require('../gsd-core/bin/lib/shell-command-projection.cjs');
|
|
const { scanFencedBlocks } = require('../gsd-core/bin/lib/markdown-sectionizer.cjs');
|
|
|
|
const REPO_ROOT = path.join(__dirname, '..');
|
|
const REVIEW_MD_PATH = path.join(REPO_ROOT, 'gsd-core', 'workflows', 'review.md');
|
|
|
|
const SELECTED_3 = 'codex,gemini,claude';
|
|
|
|
// ─── extraction (source-text-is-the-product) ──────────────────────────────
|
|
|
|
/**
|
|
* Reads review.md and extracts the fenced bash block inside
|
|
* `<step name="invoke_reviewers">`. Mirrors extractCwdGuardBash() in
|
|
* tests/worktree-cleanup.test.cjs: readFileNormalized() strips CRLF at the
|
|
* read boundary (what actually makes the captured body safe to hand to
|
|
* bash); the `\r?\n` in the fence regex below is redundant on that
|
|
* already-normalized input but kept anyway because a bare `\n` in a
|
|
* markdown-fence-shaped regex trips the local/no-crlf-fragile-split rule.
|
|
*/
|
|
function extractInvokeReviewersBash() {
|
|
const content = readFileNormalized(REVIEW_MD_PATH);
|
|
|
|
const stepMarker = '<step name="invoke_reviewers">';
|
|
const stepIdx = content.indexOf(stepMarker);
|
|
if (stepIdx === -1) {
|
|
throw new Error(`extractInvokeReviewersBash: could not find "${stepMarker}" in ${REVIEW_MD_PATH}`);
|
|
}
|
|
const afterStep = content.slice(stepIdx + stepMarker.length);
|
|
|
|
const endMarker = '</step>';
|
|
const endIdx = afterStep.indexOf(endMarker);
|
|
if (endIdx === -1) {
|
|
throw new Error(`extractInvokeReviewersBash: could not find closing "${endMarker}" after invoke_reviewers in ${REVIEW_MD_PATH}`);
|
|
}
|
|
const stepBody = afterStep.slice(0, endIdx);
|
|
|
|
const stepBodyLines = stepBody.split(/\r?\n/);
|
|
const fenced = scanFencedBlocks(stepBodyLines).find(
|
|
(b) => b.closeLineIdx !== -1 && ['bash', 'sh'].includes((b.infoString || '').trim()),
|
|
);
|
|
if (!fenced) {
|
|
throw new Error(`extractInvokeReviewersBash: no \`\`\`bash fence found inside invoke_reviewers step in ${REVIEW_MD_PATH}`);
|
|
}
|
|
const block = stepBodyLines.slice(fenced.openLineIdx + 1, fenced.closeLineIdx).join('\n');
|
|
|
|
if (!block.trim()) {
|
|
throw new Error('extractInvokeReviewersBash: extracted bash block is empty');
|
|
}
|
|
if (!block.includes('review-lane invoke')) {
|
|
throw new Error('extractInvokeReviewersBash: extracted block does not contain "review-lane invoke" — anchor may have drifted');
|
|
}
|
|
if (!block.includes('SELECTED_REVIEWERS')) {
|
|
throw new Error('extractInvokeReviewersBash: extracted block does not contain "SELECTED_REVIEWERS" — anchor may have drifted');
|
|
}
|
|
|
|
return block;
|
|
}
|
|
|
|
// ─── stub preamble ─────────────────────────────────────────────────────────
|
|
//
|
|
// The ONLY I/O seam the extracted block calls is `gsd_run`. Everything else
|
|
// executed by runDispatch() is the real shipped shell text. `barrier` is the
|
|
// positive, deterministic concurrency proof this suite uses in place of any
|
|
// elapsed-time assertion (CLAUDE.md bans wall-clock assertions).
|
|
|
|
const STUB_PREAMBLE = [
|
|
'arg_after() {',
|
|
' local flag="$1"; shift',
|
|
' while [ $# -gt 0 ]; do',
|
|
' if [ "$1" = "$flag" ]; then',
|
|
' printf %s "$2"',
|
|
' return 0',
|
|
' fi',
|
|
' shift',
|
|
' done',
|
|
'}',
|
|
'',
|
|
'in_list() {',
|
|
' local needle="$1" list="$2" item old_ifs="$IFS"',
|
|
' IFS=","',
|
|
' for item in $list; do',
|
|
' if [ "$item" = "$needle" ]; then',
|
|
' IFS="$old_ifs"',
|
|
' return 0',
|
|
' fi',
|
|
' done',
|
|
' IFS="$old_ifs"',
|
|
' return 1',
|
|
'}',
|
|
'',
|
|
'dep_of() {',
|
|
' local slug="$1" pair old_ifs="$IFS"',
|
|
' IFS=","',
|
|
' for pair in $STUB_DEPS; do',
|
|
' case "$pair" in',
|
|
' "$slug="*)',
|
|
' printf %s "${pair#*=}"',
|
|
' IFS="$old_ifs"',
|
|
' return 0',
|
|
' ;;',
|
|
' esac',
|
|
' done',
|
|
' IFS="$old_ifs"',
|
|
'}',
|
|
'',
|
|
'wait_for_file() {',
|
|
' local target="$1" i=0',
|
|
' while [ ! -f "$target" ] && [ "$i" -lt 200 ]; do',
|
|
' sleep 0.05',
|
|
' i=$((i + 1))',
|
|
' done',
|
|
'}',
|
|
'',
|
|
'barrier() {',
|
|
' local slug="$1" i=0 count',
|
|
' touch "$BARRIER_DIR/$slug"',
|
|
' count=$(ls -1 "$BARRIER_DIR" | wc -l)',
|
|
' while [ "$count" -lt "$LANE_COUNT" ] && [ "$i" -lt 200 ]; do',
|
|
' sleep 0.05',
|
|
' i=$((i + 1))',
|
|
' count=$(ls -1 "$BARRIER_DIR" | wc -l)',
|
|
' done',
|
|
' if [ "$count" -lt "$LANE_COUNT" ]; then',
|
|
' echo "barrier-timeout:$slug" >> "$TRACE"',
|
|
' return 1',
|
|
' fi',
|
|
' return 0',
|
|
'}',
|
|
'',
|
|
'gsd_run() {',
|
|
' if [ "$1" = "query" ] && [ "$2" = "config-get" ] && [ "$3" = "review.parallel_lanes" ]; then',
|
|
' if [ "$STUB_CONFIG_GET_FAILS" = "1" ]; then',
|
|
' return 1',
|
|
' fi',
|
|
' printf %s "$STUB_PARALLEL"',
|
|
' return 0',
|
|
' fi',
|
|
'',
|
|
' if [ "$1" = "query" ] && [ "$2" = "review-lane" ] && [ "$3" = "plan" ]; then',
|
|
' shift 3',
|
|
' local sel',
|
|
' sel="$(arg_after --selected "$@")"',
|
|
' if in_list "$sel" "$STUB_BUDGET_FAIL"; then',
|
|
' printf %s \'{"promptBudget": 10}\'',
|
|
' else',
|
|
' printf %s \'{"promptBudget": -1}\'',
|
|
' fi',
|
|
' return 0',
|
|
' fi',
|
|
'',
|
|
' if [ "$1" = "query" ] && [ "$2" = "prompt-budget" ]; then',
|
|
' shift 2',
|
|
' local out base slug',
|
|
' out="$(arg_after --output-prompt "$@")"',
|
|
' base="$(basename "$out")"',
|
|
' slug="${base#gsd-review-prompt-}"',
|
|
' slug="${slug%.md}"',
|
|
' if in_list "$slug" "$STUB_BUDGET_FAIL"; then',
|
|
' return 2',
|
|
' fi',
|
|
' : > "$out"',
|
|
' return 0',
|
|
' fi',
|
|
'',
|
|
' if [ "$1" = "query" ] && [ "$2" = "review-lane" ] && [ "$3" = "invoke" ]; then',
|
|
' shift 3',
|
|
' local slug dep pad',
|
|
' slug="$(arg_after --slug "$@")"',
|
|
' echo "start:$slug" >> "$TRACE"',
|
|
' if [ "$STUB_BARRIER" = "1" ]; then',
|
|
' barrier "$slug" || true',
|
|
' fi',
|
|
' dep="$(dep_of "$slug")"',
|
|
' if [ -n "$dep" ]; then',
|
|
' wait_for_file "$RUN_DIR/done-$dep"',
|
|
' fi',
|
|
' echo "stub review body for $slug" > "$RUN_DIR/gsd-review-$slug.md"',
|
|
' pad=""',
|
|
' if [ "$STUB_PAD_BYTES" -gt 0 ] 2>/dev/null; then',
|
|
' pad="$(head -c "$STUB_PAD_BYTES" /dev/zero | tr "\\0" "x")"',
|
|
' fi',
|
|
' if ! in_list "$slug" "$STUB_SILENT"; then',
|
|
' printf \'{"slug":"%s","pad":"%s"}\\n\' "$slug" "$pad"',
|
|
' fi',
|
|
// #3689: the done-file is a cross-process happens-before edge — a
|
|
// dependent lane unblocks the instant this file appears (wait_for_file
|
|
// above just polls for its existence), so everything a dependent may
|
|
// observe (the "end:$slug" trace line) must be written BEFORE the file
|
|
// that releases it. touch-then-echo let a descheduled upstream lose the
|
|
// race to its own dependent, inverting the #3034 completion-order trace.
|
|
' echo "end:$slug" >> "$TRACE"',
|
|
' touch "$RUN_DIR/done-$slug"',
|
|
' if in_list "$slug" "$STUB_FAIL"; then',
|
|
' return 1',
|
|
' fi',
|
|
' return 0',
|
|
' fi',
|
|
'',
|
|
' return 0',
|
|
'}',
|
|
].join('\n');
|
|
|
|
// ─── dispatch runner ───────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Build the stub env for a runDispatch() call from `opts`. Every key is
|
|
* always present (never omitted) so the generated `set -u` script never
|
|
* dereferences an unset variable — `opts.parallel === null` deliberately
|
|
* maps to the empty string, which is exactly how an unset config key reads
|
|
* back through `config-get --raw`.
|
|
*/
|
|
function buildEnv(opts, runDir, tracePath, barrierDir) {
|
|
const selected = opts.selected;
|
|
const laneCount = selected.split(',').filter((s) => s.length > 0).length;
|
|
const deps = opts.deps || {};
|
|
return {
|
|
...process.env,
|
|
SELECTED_REVIEWERS: selected,
|
|
EXPLICIT_FLAG: '',
|
|
STUB_PARALLEL: opts.parallel === null || opts.parallel === undefined ? '' : opts.parallel,
|
|
STUB_CONFIG_GET_FAILS: opts.configGetFails ? '1' : '0',
|
|
STUB_FAIL: (opts.failSlugs || []).join(','),
|
|
STUB_BUDGET_FAIL: (opts.budgetFailSlugs || []).join(','),
|
|
STUB_SILENT: (opts.silentSlugs || []).join(','),
|
|
STUB_BARRIER: opts.barrier ? '1' : '0',
|
|
STUB_DEPS: Object.entries(deps).map(([k, v]) => `${k}=${v}`).join(','),
|
|
STUB_PAD_BYTES: String(opts.padBytes || 0),
|
|
LANE_COUNT: String(laneCount),
|
|
TRACE: tracePath,
|
|
BARRIER_DIR: barrierDir,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Runs the real extracted invoke_reviewers bash block with the gsd_run stub
|
|
* spliced in front of it. `{run_dir}` is replaced globally with a real temp
|
|
* directory. Returns the dispatch outcome plus the artifacts it produced.
|
|
*/
|
|
function runDispatch(t, opts) {
|
|
const scriptDir = createTempDir('gsd-3034-script-');
|
|
const runDir = createTempDir('gsd-3034-rundir-');
|
|
const barrierDir = createTempDir('gsd-3034-barrier-');
|
|
t.after(() => {
|
|
cleanup(scriptDir);
|
|
cleanup(runDir);
|
|
cleanup(barrierDir);
|
|
});
|
|
|
|
const block = extractInvokeReviewersBash();
|
|
const tracePath = path.join(scriptDir, 'trace.log');
|
|
const env = buildEnv(opts, runDir, tracePath, barrierDir);
|
|
|
|
const scriptPath = path.join(scriptDir, 'dispatch.sh');
|
|
const script = [
|
|
'#!/usr/bin/env bash',
|
|
'set -u',
|
|
STUB_PREAMBLE,
|
|
block.split('{run_dir}').join(runDir),
|
|
].join('\n');
|
|
fs.writeFileSync(scriptPath, script, { mode: 0o755 });
|
|
|
|
const result = runHook(scriptPath, [], {
|
|
interpreter: 'bash',
|
|
cwd: runDir,
|
|
env,
|
|
timeoutMs: HOOK_FANOUT_TIMEOUT_MS,
|
|
});
|
|
|
|
const jsonlPath = path.join(runDir, 'gsd-review-lane-results.jsonl');
|
|
const jsonl = fs.existsSync(jsonlPath) ? fs.readFileSync(jsonlPath, 'utf-8') : '';
|
|
const lines = jsonl.split('\n').filter((l) => l.trim() !== '');
|
|
const trace = fs.existsSync(tracePath)
|
|
? readFileNormalized(tracePath).split('\n').filter((l) => l.trim() !== '')
|
|
: [];
|
|
|
|
return {
|
|
outcome: result.outcome,
|
|
exitCode: result.exitCode,
|
|
stderr: result.stderr,
|
|
jsonl,
|
|
lines,
|
|
trace,
|
|
runDir,
|
|
};
|
|
}
|
|
|
|
/** Parsed slug order from JSONL lines — never assert on raw JSONL text. */
|
|
function slugOrder(lines) {
|
|
return lines.map((l) => JSON.parse(l).slug);
|
|
}
|
|
|
|
function serialTrace(slugs) {
|
|
return slugs.flatMap((s) => [`start:${s}`, `end:${s}`]);
|
|
}
|
|
|
|
// ─── #1/#2 — default and explicit-disabled serial dispatch ───────────────
|
|
|
|
describe('#3034 default and explicit-disabled dispatch stays serial', () => {
|
|
test('defaultsToSerialDispatchWhenKeyUnset', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: null });
|
|
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']));
|
|
});
|
|
|
|
test('staysSerialWhenExplicitlyDisabled', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'false' });
|
|
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']));
|
|
});
|
|
});
|
|
|
|
// ─── #3/#4 — opt-in concurrency and join-before-aggregate ─────────────────
|
|
|
|
describe('#3034 opt-in concurrency', () => {
|
|
test('dispatchesLanesConcurrentlyWhenEnabled', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', barrier: true });
|
|
const timeouts = result.trace.filter((l) => l.startsWith('barrier-timeout:'));
|
|
assert.deepEqual(timeouts, [], 'no lane should hit the barrier timeout when lanes run concurrently');
|
|
});
|
|
|
|
test('joinsAllLanesBeforeAggregation', (t) => {
|
|
// barrier:true is load-bearing, not decoration. Without it the stub lanes
|
|
// finish instantly and a missing `wait` could still race to 3 lines,
|
|
// making this pass intermittently. Held at the barrier, a missing join
|
|
// deterministically aggregates ZERO lines.
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', barrier: true });
|
|
assert.equal(result.lines.length, 3, 'dispatch must return only after every lane wrote its result');
|
|
});
|
|
});
|
|
|
|
// ─── #5/#6 — selection-order preservation ─────────────────────────────────
|
|
|
|
describe('#3034 JSONL preserves selection order, not completion order', () => {
|
|
test('preservesSelectionOrderSerial', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: null });
|
|
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini', 'claude']);
|
|
});
|
|
|
|
test('preservesSelectionOrderParallelDespiteCompletionOrder', (t) => {
|
|
// codex waits on gemini; gemini waits on claude -> forces reverse
|
|
// completion order (claude, gemini, codex) while selection order stays
|
|
// codex, gemini, claude.
|
|
const result = runDispatch(t, {
|
|
selected: SELECTED_3,
|
|
parallel: 'true',
|
|
deps: { codex: 'gemini', gemini: 'claude' },
|
|
});
|
|
|
|
const endMarkers = result.trace.filter((l) => l.startsWith('end:'));
|
|
// Pin the fixture actually forced reverse completion first — otherwise
|
|
// the selection-order assertion below would pass vacuously.
|
|
assert.deepEqual(endMarkers, ['end:claude', 'end:gemini', 'end:codex']);
|
|
|
|
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini', 'claude']);
|
|
});
|
|
});
|
|
|
|
// ─── #7/#8 — PIPE_BUF boundary triple (plus one clearly-oversized case) ───
|
|
|
|
describe('#3034 oversized lane results stay intact under concurrency', () => {
|
|
test('keepsOversizedLaneResultsIntactUnderConcurrency', async (t) => {
|
|
for (const padBytes of [4095, 4096, 4097, 8192]) {
|
|
await t.test(`padBytes=${padBytes}`, (t2) => {
|
|
const result = runDispatch(t2, { selected: SELECTED_3, parallel: 'true', padBytes });
|
|
assert.equal(result.lines.length, 3);
|
|
for (const line of result.lines) {
|
|
assert.doesNotThrow(() => JSON.parse(line), `line failed to parse at padBytes=${padBytes}: ${line.slice(0, 80)}...`);
|
|
}
|
|
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini', 'claude']);
|
|
});
|
|
}
|
|
});
|
|
});
|
|
|
|
// ─── #9 — single-lane boundary ─────────────────────────────────────────────
|
|
|
|
describe('#3034 single-lane boundary', () => {
|
|
test('singleLaneParallelMatchesSerial', (t) => {
|
|
const serial = runDispatch(t, { selected: 'codex', parallel: null });
|
|
const parallel = runDispatch(t, { selected: 'codex', parallel: 'true' });
|
|
|
|
assert.equal(serial.lines.length, 1);
|
|
assert.equal(parallel.lines.length, 1);
|
|
assert.deepEqual(slugOrder(serial.lines), slugOrder(parallel.lines));
|
|
|
|
const serialMd = fs.readFileSync(path.join(serial.runDir, 'gsd-review-codex.md'), 'utf-8');
|
|
const parallelMd = fs.readFileSync(path.join(parallel.runDir, 'gsd-review-codex.md'), 'utf-8');
|
|
assert.equal(serialMd, parallelMd);
|
|
});
|
|
});
|
|
|
|
// ─── #10 — empty selection is a no-op ──────────────────────────────────────
|
|
|
|
describe('#3034 empty selection', () => {
|
|
test('emptySelectionIsANoOp', (t) => {
|
|
const result = runDispatch(t, { selected: '', parallel: 'true' });
|
|
assert.equal(result.outcome, 'exited');
|
|
assert.equal(result.exitCode, 0);
|
|
assert.deepEqual(result.lines, []);
|
|
assert.deepEqual(result.trace, []);
|
|
});
|
|
});
|
|
|
|
describe('#3034 duplicate slug in the selection', () => {
|
|
test('duplicateSlugDispatchesOnceAndWritesOneLine', (t) => {
|
|
// A slug repeated in SELECTED_REVIEWERS would otherwise put two
|
|
// concurrent background jobs on the same `>`-truncated per-slug result
|
|
// file, corrupting whichever one finishes last. DISPATCH_SLUGS
|
|
// de-duplicates before dispatch, so codex must run exactly once.
|
|
const result = runDispatch(t, { selected: 'codex,gemini,codex', parallel: 'true' });
|
|
assert.equal(result.outcome, 'exited');
|
|
assert.equal(result.trace.filter((l) => l === 'start:codex').length, 1);
|
|
assert.equal(result.trace.filter((l) => l === 'end:codex').length, 1);
|
|
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini']);
|
|
});
|
|
});
|
|
|
|
// ─── #11/#12 — lane failure does not abort siblings ────────────────────────
|
|
|
|
describe('#3034 lane failure does not abort sibling lanes', () => {
|
|
test('laneFailureDoesNotAbortSiblingLanes', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', failSlugs: ['gemini'] });
|
|
assert.equal(result.lines.length, 3, 'a failing lane still contributes its stub result line');
|
|
assert.ok(
|
|
fs.existsSync(path.join(result.runDir, 'gsd-review-gemini.md')),
|
|
'the failing lane\'s diagnostic stub .md must be preserved',
|
|
);
|
|
assert.deepEqual(slugOrder(result.lines), ['codex', 'gemini', 'claude']);
|
|
});
|
|
|
|
test('laneFailureSerialUnchanged', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: null, failSlugs: ['gemini'] });
|
|
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']));
|
|
assert.equal(result.lines.length, 3);
|
|
});
|
|
});
|
|
|
|
// ─── #13 — budget-too-small skip emits no result line ─────────────────────
|
|
|
|
describe('#3034 budget-too-small skip', () => {
|
|
test('budgetSkipEmitsNoResultLine', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', budgetFailSlugs: ['gemini'] });
|
|
assert.deepEqual(slugOrder(result.lines), ['codex', 'claude'], 'the budget-skipped lane contributes no result line');
|
|
const stub = fs.readFileSync(path.join(result.runDir, 'gsd-review-gemini.md'), 'utf-8');
|
|
assert.match(stub, /skipped/);
|
|
assert.match(stub, /prompt budget/);
|
|
});
|
|
});
|
|
|
|
// ─── #14/#15 — silent / absent lane contributes no line ───────────────────
|
|
|
|
describe('#3034 silent lane contributes nothing, not a blank line', () => {
|
|
test('silentLaneContributesNoLine', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', silentSlugs: ['gemini'] });
|
|
assert.deepEqual(slugOrder(result.lines), ['codex', 'claude']);
|
|
// Folded row #15 (absent result file): budgetSkipEmitsNoResultLine above
|
|
// already exercises the "invoke never started, no file at all" shape via
|
|
// its `continue`. This assertion pins the sibling shape: a lane that DID
|
|
// run and wrote nothing must not leave a stray blank JSONL line.
|
|
assert.ok(!result.jsonl.includes('\n\n'), 'an empty lane result must not leave a blank JSONL line');
|
|
});
|
|
});
|
|
|
|
// ─── #16/#17 — non-canonical truthy values stay serial ────────────────────
|
|
|
|
describe('#3034 non-canonical truthy config values stay serial', () => {
|
|
test('nonCanonicalTruthyValuesStaySerial', async (t) => {
|
|
const nearMisses = ['TRUE', 'True', '1', 'yes', 'on', ' true', 'true '];
|
|
for (const value of nearMisses) {
|
|
await t.test(`parallel_lanes="${value}"`, (t2) => {
|
|
const result = runDispatch(t2, { selected: SELECTED_3, parallel: value });
|
|
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']), `value "${value}" must not opt into parallel dispatch`);
|
|
});
|
|
}
|
|
});
|
|
});
|
|
|
|
// ─── #18 — config-get failure fails safe to serial ─────────────────────────
|
|
|
|
describe('#3034 broken config tooling fails safe to serial', () => {
|
|
test('configGetFailureFallsBackToSerial', (t) => {
|
|
const result = runDispatch(t, { selected: SELECTED_3, parallel: 'true', configGetFails: true });
|
|
assert.deepEqual(result.trace, serialTrace(['codex', 'gemini', 'claude']));
|
|
});
|
|
});
|
|
|
|
// ─── #19/#20/#21 — config-set registers review.parallel_lanes ─────────────
|
|
|
|
describe('#3034 review.parallel_lanes config key', () => {
|
|
test('configSetAcceptsAndPersistsParallelLanes', (t) => {
|
|
const tmpDir = createTempProject();
|
|
t.after(() => cleanup(tmpDir));
|
|
|
|
const setResult = runGsdTools('config-set review.parallel_lanes true', tmpDir);
|
|
assert.ok(setResult.success, `config-set failed: ${setResult.error}`);
|
|
|
|
const configPath = path.join(tmpDir, '.planning', 'config.json');
|
|
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
|
assert.equal(config.review?.parallel_lanes, true);
|
|
assert.equal(typeof config.review?.parallel_lanes, 'boolean');
|
|
|
|
const getResult = runGsdTools('config-get review.parallel_lanes --raw', tmpDir);
|
|
assert.ok(getResult.success, `config-get failed: ${getResult.error}`);
|
|
assert.equal((getResult.output || '').trim(), 'true');
|
|
});
|
|
|
|
test('configSetPersistsBooleanFalse', (t) => {
|
|
const tmpDir = createTempProject();
|
|
t.after(() => cleanup(tmpDir));
|
|
|
|
const setResult = runGsdTools('config-set review.parallel_lanes false', tmpDir);
|
|
assert.ok(setResult.success, `config-set failed: ${setResult.error}`);
|
|
|
|
const configPath = path.join(tmpDir, '.planning', 'config.json');
|
|
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
|
assert.equal(config.review?.parallel_lanes, false);
|
|
assert.equal(typeof config.review?.parallel_lanes, 'boolean');
|
|
});
|
|
|
|
test('rejectsUnregisteredNeighbouringKey', (t) => {
|
|
const tmpDir = createTempProject();
|
|
t.after(() => cleanup(tmpDir));
|
|
|
|
// Missing trailing "s" — proves the whitelist is load-bearing and the
|
|
// two tests above are not vacuous (they'd pass even for an unregistered
|
|
// key if config-set accepted anything).
|
|
const result = runGsdTools('config-set review.parallel_lane true', tmpDir);
|
|
assert.equal(result.success, false, 'an unregistered near-miss key must be rejected');
|
|
});
|
|
});
|
|
|
|
// ─── #22 — serial/parallel artifact equivalence ────────────────────────────
|
|
|
|
describe('#3034 serial and parallel dispatch produce equivalent artifacts', () => {
|
|
test('serialAndParallelProduceEquivalentArtifacts', (t) => {
|
|
const serial = runDispatch(t, { selected: SELECTED_3, parallel: null });
|
|
const parallel = runDispatch(t, { selected: SELECTED_3, parallel: 'true' });
|
|
|
|
assert.deepEqual(slugOrder(serial.lines), slugOrder(parallel.lines));
|
|
assert.deepEqual(
|
|
serial.lines.map((l) => JSON.parse(l)),
|
|
parallel.lines.map((l) => JSON.parse(l)),
|
|
);
|
|
|
|
for (const slug of ['codex', 'gemini', 'claude']) {
|
|
const serialMd = fs.readFileSync(path.join(serial.runDir, `gsd-review-${slug}.md`), 'utf-8');
|
|
const parallelMd = fs.readFileSync(path.join(parallel.runDir, `gsd-review-${slug}.md`), 'utf-8');
|
|
assert.equal(serialMd, parallelMd, `gsd-review-${slug}.md must be byte-identical between the two paths`);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// #3352 (ADR-3473 §8.5, phase #3885 item 3) — `write_reviews` must not emit
|
|
// REVIEWS.md from failed inputs, and `present_results` must preserve
|
|
// per-lane evidence before `rm -rf "{run_dir}"` destroys it.
|
|
//
|
|
// SEAM CHOICE: `write_reviews`/`present_results` are markdown PROSE
|
|
// instructing an LLM agent, not a single executable program — "do not write
|
|
// REVIEWS.md" and "display this message" are natural-language directives to
|
|
// the agent, not `if`/`fi` around a `Write` call this suite could invoke.
|
|
// Asserting on that prose text directly would be source-grep
|
|
// (`local/no-source-grep`), which this repo bans. What IS literal,
|
|
// deterministic bash in both steps — and is exactly what R2's gate condition
|
|
// depends on — is:
|
|
// (1) `write_reviews`'s new gate block, which computes
|
|
// `TOTAL_LANE_FAILURE`/`ALL_LANES_SKIPPED` from the aggregate JSONL
|
|
// and the lane stub files (the ONLY inputs the prose instructions key
|
|
// off of to decide whether to write REVIEWS.md at all);
|
|
// (2) `write_reviews`'s commit block (unchanged text, still real bash);
|
|
// (3) `present_results`'s new preserve-evidence block and its `rm -rf`
|
|
// cleanup block.
|
|
// Each is extracted and EXECUTED for real. To observe the documented
|
|
// consequence of the gate flags (whether REVIEWS.md and the commit actually
|
|
// happen), the driver below applies the step's OWN written contract —
|
|
// "If ALL_LANES_SKIPPED=true or TOTAL_LANE_FAILURE=true: do NOT write
|
|
// REVIEWS.md and do NOT run the commit. Otherwise: proceed" (review.md, the
|
|
// `write_reviews` step, verbatim) — as harness scaffolding around the real
|
|
// extracted blocks, the same way STUB_PREAMBLE above supplies scaffolding
|
|
// (`arg_after`/`in_list`/...) the extracted `invoke_reviewers` block calls
|
|
// into. This is not a re-typed copy of the logic under test: the FLAG
|
|
// computation is 100% the real fenced bash; only the "then do the file I/O"
|
|
// half — which the real workflow leaves to the LLM's own tool calls — is
|
|
// harness glue.
|
|
//
|
|
// R46 (`preservedEvidenceIsNotSweptIntoTheCommit`) is a regression guard for
|
|
// N6, not failing-first evidence: the commit fence's `--files` argument is
|
|
// byte-identical before and after #3352 (only the surrounding prose gained
|
|
// gating language), so this exact assertion also holds against the pre-fix
|
|
// text. It is retained because the invariant it pins (the commit step must
|
|
// never widen from the single REVIEWS.md path to a directory glob that
|
|
// would sweep `.review-diagnostics/` into a commit) is real and worth a
|
|
// permanent pin, and because it only runs at all through this same
|
|
// full-flow driver, whose surrounding gate/preserve blocks did not exist
|
|
// pre-fix (see the STEP 2 report for the exact pre-fix observation).
|
|
|
|
function extractStepBody(stepName) {
|
|
const content = readFileNormalized(REVIEW_MD_PATH);
|
|
const stepMarker = `<step name="${stepName}">`;
|
|
const stepIdx = content.indexOf(stepMarker);
|
|
if (stepIdx === -1) {
|
|
throw new Error(`extractStepBody: could not find "${stepMarker}" in ${REVIEW_MD_PATH}`);
|
|
}
|
|
const afterStep = content.slice(stepIdx + stepMarker.length);
|
|
const endIdx = afterStep.indexOf('</step>');
|
|
if (endIdx === -1) {
|
|
throw new Error(`extractStepBody: could not find closing "</step>" after ${stepName} in ${REVIEW_MD_PATH}`);
|
|
}
|
|
return afterStep.slice(0, endIdx);
|
|
}
|
|
|
|
/** Finds the first ```bash/```sh fence in `stepBody` whose text contains every string in `mustInclude`. */
|
|
function extractBashFenceContaining(stepBody, mustInclude, label) {
|
|
const stepBodyLines = stepBody.split(/\r?\n/);
|
|
for (const fenced of scanFencedBlocks(stepBodyLines)) {
|
|
if (fenced.closeLineIdx === -1) continue;
|
|
if (!['bash', 'sh'].includes((fenced.infoString || '').trim())) continue;
|
|
const block = stepBodyLines.slice(fenced.openLineIdx + 1, fenced.closeLineIdx).join('\n');
|
|
if (mustInclude.every((s) => block.includes(s))) return block;
|
|
}
|
|
throw new Error(`extractBashFenceContaining: no fence matching ${label} found (looked for ${JSON.stringify(mustInclude)})`);
|
|
}
|
|
|
|
function extractWriteReviewsGateBash() {
|
|
return extractBashFenceContaining(
|
|
extractStepBody('write_reviews'),
|
|
['TOTAL_LANE_FAILURE', 'ALL_LANES_SKIPPED', 'DISPATCH_SLUGS'],
|
|
'write_reviews #3352 gate block',
|
|
);
|
|
}
|
|
|
|
function extractWriteReviewsCommitBash() {
|
|
return extractBashFenceContaining(
|
|
extractStepBody('write_reviews'),
|
|
['gsd_run query commit', 'REVIEWS.md'],
|
|
'write_reviews commit block',
|
|
);
|
|
}
|
|
|
|
// #3885: preserve-evidence and cleanup are ONE fenced block (a shell variable
|
|
// cannot survive across separate fences), gated on `_PRESERVE_OK` so a failed
|
|
// `mkdir -p`/`cp` skips the `rm -rf` and leaves `{run_dir}` intact instead of
|
|
// destroying the only copy of the evidence it exists to protect.
|
|
function extractPresentResultsPreserveAndCleanupBash() {
|
|
return extractBashFenceContaining(
|
|
extractStepBody('present_results'),
|
|
['DIAG_DIR', 'nullglob', 'rm -rf', '{run_dir}', '_PRESERVE_OK'],
|
|
'present_results #3352/#3885 preserve+cleanup block',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Runs the real extracted write_reviews gate+commit blocks and the real
|
|
* present_results preserve+cleanup block, in the documented order, against
|
|
* a fixture run dir. `opts.jsonlLines` seeds the aggregate JSONL (omit/empty
|
|
* for "every lane failed"); `opts.lanes` seeds per-slug `gsd-review-<slug>.md`
|
|
* / `.md`'s sibling `.err`. See the seam-choice comment above this describe
|
|
* block for what is real bash vs. harness glue.
|
|
*
|
|
* `opts.blockDiagDirWithFile`: #3885 root-safe failure injection. Root
|
|
* bypasses `chmod 0o000` entirely (a Docker/CI default), so permission bits
|
|
* cannot induce a `mkdir -p`/`cp` failure deterministically. A filesystem
|
|
* TYPE conflict is root-safe instead: pre-creating a plain FILE at the exact
|
|
* path `mkdir -p` needs to create as a DIRECTORY makes `mkdir -p` fail with
|
|
* "not a directory" for every caller, root included, because it is not a
|
|
* permissions check at all.
|
|
*/
|
|
function runWriteReviewsFlow(t, opts) {
|
|
const scriptDir = createTempDir('gsd-3352-script-');
|
|
const runDir = createTempDir('gsd-3352-rundir-');
|
|
const phaseDir = createTempDir('gsd-3352-phasedir-');
|
|
t.after(() => {
|
|
cleanup(scriptDir);
|
|
cleanup(runDir);
|
|
cleanup(phaseDir);
|
|
});
|
|
|
|
if (opts.blockDiagDirWithFile) {
|
|
fs.writeFileSync(path.join(phaseDir, '.review-diagnostics'), 'blocking file, not a directory\n');
|
|
}
|
|
|
|
// #4097: stage the run's OWN input copies exactly as prompt assembly
|
|
// (review.md's section-copy fence) writes them, so the preserve+cleanup
|
|
// block can be observed deciding input vs. evidence. `name` is a bare
|
|
// basename written into the fixture run dir.
|
|
for (const [name, content] of Object.entries(opts.seedFiles || {})) {
|
|
fs.writeFileSync(path.join(runDir, name), content);
|
|
}
|
|
|
|
if (opts.jsonlLines && opts.jsonlLines.length > 0) {
|
|
fs.writeFileSync(
|
|
path.join(runDir, 'gsd-review-lane-results.jsonl'),
|
|
opts.jsonlLines.map((l) => JSON.stringify(l)).join('\n') + '\n',
|
|
);
|
|
}
|
|
for (const [slug, files] of Object.entries(opts.lanes || {})) {
|
|
if (files.md !== undefined && files.md !== null) {
|
|
fs.writeFileSync(path.join(runDir, `gsd-review-${slug}.md`), files.md);
|
|
}
|
|
if (files.err !== undefined && files.err !== null) {
|
|
fs.writeFileSync(path.join(runDir, `gsd-review-${slug}.err`), files.err);
|
|
}
|
|
}
|
|
|
|
const gateBlock = extractWriteReviewsGateBash();
|
|
const commitBlock = extractWriteReviewsCommitBash();
|
|
const preserveAndCleanupBlock = extractPresentResultsPreserveAndCleanupBash();
|
|
|
|
const tracePath = path.join(scriptDir, 'commit-trace.log');
|
|
const reviewsMdPath = path.join(phaseDir, '03-REVIEWS.md');
|
|
|
|
// #3885 Windows fix: splice POSIX-form paths into the bash source text, not
|
|
// the OS-native ones `createTempDir()` returns. This is a fixture fix, not
|
|
// a product one — the real workflow never hits this seam with a backslash
|
|
// path in the first place: `{run_dir}` is created by `mktemp -d` running
|
|
// INSIDE the bash block itself (always POSIX-style, even under Git Bash on
|
|
// Windows), and `{phase_dir}` is the `phase_dir` field from
|
|
// `gsd_run query init.review`, which gsd-core/bin/lib/init.cjs already
|
|
// pipes through this exact same `toPosixPath()` before it is ever
|
|
// serialized (see the `phase_dir: ... toPosixPath(...)` call sites there).
|
|
// Splicing a native `C:\Users\...` string directly into unquoted bash
|
|
// source (e.g. the pre-existing, unquoted
|
|
// `--files {phase_dir}/{padded_phase}-REVIEWS.md` commit line) hits bash's
|
|
// own unquoted-backslash removal and silently drops every separator — a
|
|
// test-harness-only failure mode that following the fixture's own
|
|
// production analogue eliminates.
|
|
const runDirPosix = toPosixPath(runDir);
|
|
const phaseDirPosix = toPosixPath(phaseDir);
|
|
|
|
const substitute = (block) => block
|
|
.split('{run_dir}').join(runDirPosix)
|
|
.split('{phase_dir}').join(phaseDirPosix)
|
|
.split('{padded_phase}').join('03')
|
|
.split('{N}').join('3');
|
|
|
|
const script = [
|
|
'#!/usr/bin/env bash',
|
|
'set -u',
|
|
`SELECTED_REVIEWERS='${opts.selected}'`,
|
|
`TRACE='${tracePath}'`,
|
|
'gsd_run() { printf "%s\\n" "$*" >> "$TRACE"; }',
|
|
substitute(gateBlock),
|
|
'echo "GATE:TOTAL_LANE_FAILURE=$TOTAL_LANE_FAILURE"',
|
|
'echo "GATE:ALL_LANES_SKIPPED=$ALL_LANES_SKIPPED"',
|
|
'if [ "$TOTAL_LANE_FAILURE" = "false" ] && [ "$ALL_LANES_SKIPPED" = "false" ]; then',
|
|
// Harness glue standing in for the agent's own Write tool call (see the
|
|
// seam-choice comment above): the prose says "combine ... into
|
|
// REVIEWS.md" only when the gate above did not fire.
|
|
` touch '${reviewsMdPath}'`,
|
|
substitute(commitBlock),
|
|
'fi',
|
|
substitute(preserveAndCleanupBlock),
|
|
].join('\n');
|
|
|
|
fs.writeFileSync(path.join(scriptDir, 'flow.sh'), script, { mode: 0o755 });
|
|
|
|
// cwd is deliberately scriptDir, NOT runDir: every path this flow touches
|
|
// ($RUN_DIR, $DIAG_DIR, {phase_dir}) is already absolute in the extracted
|
|
// bash text, so the child's cwd is not load-bearing for anything the
|
|
// blocks do — but on Windows a process cannot delete a directory that is
|
|
// its OWN current working directory (unlike POSIX, where rm -rf on your
|
|
// cwd succeeds). Setting cwd to runDir here was a harness artifact with no
|
|
// production analogue (review.md never `cd`s into $RUN_DIR — see
|
|
// gsd-core/workflows/review.md's use of $RUN_DIR, always by absolute
|
|
// path), and it silently defeated `rm -rf "$RUN_DIR"` under Git-Bash on
|
|
// windows-latest: the directory's *contents* were removed but the
|
|
// still-open-as-cwd directory entry itself survived, so
|
|
// `fs.existsSync(runDir)` kept reporting true. This was the actual defect
|
|
// behind the two Windows-only failures — not a path-separator mismatch.
|
|
const result = runHook(path.join(scriptDir, 'flow.sh'), [], {
|
|
interpreter: 'bash',
|
|
cwd: scriptDir,
|
|
timeoutMs: HOOK_FANOUT_TIMEOUT_MS,
|
|
});
|
|
|
|
const stdout = result.stdout || '';
|
|
const diagDir = path.join(phaseDir, '.review-diagnostics');
|
|
const commitTrace = fs.existsSync(tracePath)
|
|
? readFileNormalized(tracePath).split('\n').filter((l) => l.trim() !== '')
|
|
: [];
|
|
|
|
return {
|
|
outcome: result.outcome,
|
|
exitCode: result.exitCode,
|
|
stderr: result.stderr,
|
|
totalLaneFailure: /GATE:TOTAL_LANE_FAILURE=(\S+)/.exec(stdout)?.[1],
|
|
allLanesSkipped: /GATE:ALL_LANES_SKIPPED=(\S+)/.exec(stdout)?.[1],
|
|
reviewsMdExists: fs.existsSync(reviewsMdPath),
|
|
runDirExists: fs.existsSync(runDir),
|
|
diagDir,
|
|
diagDirExists: fs.existsSync(diagDir),
|
|
commitTrace,
|
|
runDir,
|
|
phaseDir,
|
|
};
|
|
}
|
|
|
|
const BUDGET_SKIP_MD = (slug) => `${slug} review skipped: prompt budget (500 tokens) too small for the minimum review set.`;
|
|
const LANE_FAILURE_MD = (slug) => `${slug} review failed: exit 1`;
|
|
|
|
describe('#3352 every lane failed writes no REVIEWS.md', () => {
|
|
test('totalLaneFailureWritesNoReviewsMd_3352', (t) => {
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini,claude',
|
|
jsonlLines: [],
|
|
lanes: {
|
|
codex: { md: LANE_FAILURE_MD('codex'), err: 'stack trace: codex crashed\n' },
|
|
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'stack trace: gemini crashed\n' },
|
|
claude: { md: LANE_FAILURE_MD('claude'), err: 'stack trace: claude crashed\n' },
|
|
},
|
|
});
|
|
assert.equal(result.outcome, 'exited');
|
|
assert.equal(result.totalLaneFailure, 'true', `expected TOTAL_LANE_FAILURE=true; stderr=${result.stderr}`);
|
|
assert.equal(result.allLanesSkipped, 'false');
|
|
assert.equal(result.reviewsMdExists, false, 'no REVIEWS.md may be written when every lane failed');
|
|
assert.deepEqual(result.commitTrace, [], 'the commit must not run when every lane failed');
|
|
});
|
|
});
|
|
|
|
describe('#3352 one successful lane still writes REVIEWS.md (R1, must stay green)', () => {
|
|
test('oneSuccessfulLaneStillWritesReviews', (t) => {
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini,claude',
|
|
jsonlLines: [{ slug: 'codex' }],
|
|
lanes: {
|
|
codex: { md: '# Codex review\nlooks good\n' },
|
|
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'stack trace: gemini crashed\n' },
|
|
claude: { md: LANE_FAILURE_MD('claude'), err: 'stack trace: claude crashed\n' },
|
|
},
|
|
});
|
|
assert.equal(result.totalLaneFailure, 'false');
|
|
assert.equal(result.allLanesSkipped, 'false');
|
|
assert.equal(result.reviewsMdExists, true, 'at least one lane succeeded — REVIEWS.md must still be written');
|
|
assert.equal(result.commitTrace.length, 1, 'the commit must run exactly once');
|
|
});
|
|
});
|
|
|
|
describe('#3352 a budget-skipped lane is not a failure (N5)', () => {
|
|
test('budgetSkippedLanesAreNotFailures', (t) => {
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini,claude',
|
|
jsonlLines: [],
|
|
lanes: {
|
|
codex: { md: BUDGET_SKIP_MD('codex') },
|
|
gemini: { md: BUDGET_SKIP_MD('gemini') },
|
|
claude: { md: BUDGET_SKIP_MD('claude') },
|
|
},
|
|
});
|
|
assert.equal(result.allLanesSkipped, 'true', `every lane was budget-skipped, not failed; stderr=${result.stderr}`);
|
|
assert.equal(result.totalLaneFailure, 'false', 'a budget skip must never be classified as a failure');
|
|
assert.equal(result.reviewsMdExists, false, 'nothing to review — REVIEWS.md still must not be written');
|
|
assert.deepEqual(result.commitTrace, []);
|
|
});
|
|
});
|
|
|
|
describe('#3352 successful-lane count boundary (0/1/2)', () => {
|
|
test('successfulLaneCountBoundary', (t) => {
|
|
const zero = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini',
|
|
jsonlLines: [],
|
|
lanes: {
|
|
codex: { md: LANE_FAILURE_MD('codex'), err: 'boom\n' },
|
|
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'boom\n' },
|
|
},
|
|
});
|
|
assert.equal(zero.totalLaneFailure, 'true');
|
|
assert.equal(zero.reviewsMdExists, false, '0 successful lanes: no REVIEWS.md');
|
|
|
|
const one = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini',
|
|
jsonlLines: [{ slug: 'codex' }],
|
|
lanes: {
|
|
codex: { md: '# Codex\nok\n' },
|
|
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'boom\n' },
|
|
},
|
|
});
|
|
assert.equal(one.totalLaneFailure, 'false');
|
|
assert.equal(one.reviewsMdExists, true, '1 successful lane: REVIEWS.md is written');
|
|
|
|
const two = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini',
|
|
jsonlLines: [{ slug: 'codex' }, { slug: 'gemini' }],
|
|
lanes: {
|
|
codex: { md: '# Codex\nok\n' },
|
|
gemini: { md: '# Gemini\nok\n' },
|
|
},
|
|
});
|
|
assert.equal(two.totalLaneFailure, 'false');
|
|
assert.equal(two.reviewsMdExists, true, '2 successful lanes: REVIEWS.md is written');
|
|
});
|
|
});
|
|
|
|
describe('#3352 per-lane evidence survives cleanup (R3)', () => {
|
|
test('laneEvidenceSurvivesCleanup_3352', (t) => {
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini,claude',
|
|
jsonlLines: [],
|
|
lanes: {
|
|
codex: { md: LANE_FAILURE_MD('codex'), err: 'stack trace: codex crashed\n' },
|
|
gemini: { md: '', err: '' }, // ran, wrote nothing, no error either — nothing to preserve for this slug's .err
|
|
claude: { md: LANE_FAILURE_MD('claude'), err: 'stack trace: claude crashed\n' },
|
|
},
|
|
});
|
|
|
|
assert.equal(result.runDirExists, false, 'the run dir must still be destroyed');
|
|
assert.equal(result.diagDirExists, true, 'diagnostics must have been preserved somewhere under phase_dir');
|
|
|
|
const codexMd = path.join(result.diagDir, 'gsd-review-codex.md');
|
|
const codexErr = path.join(result.diagDir, 'gsd-review-codex.err');
|
|
const claudeErr = path.join(result.diagDir, 'gsd-review-claude.err');
|
|
assert.ok(fs.existsSync(codexMd), 'codex .md evidence must be preserved');
|
|
assert.equal(fs.readFileSync(codexMd, 'utf-8'), LANE_FAILURE_MD('codex'));
|
|
assert.ok(fs.existsSync(codexErr), 'codex non-empty .err evidence must be preserved');
|
|
assert.ok(fs.existsSync(claudeErr), 'claude non-empty .err evidence must be preserved');
|
|
|
|
const geminiErr = path.join(result.diagDir, 'gsd-review-gemini.err');
|
|
assert.equal(fs.existsSync(geminiErr), false, 'an empty .err must not be copied as if it were real evidence');
|
|
});
|
|
});
|
|
|
|
describe('#3885 nothing to preserve still cleans up (must stay green)', () => {
|
|
test('nothingToPreserveStillCleansUp_3885', (t) => {
|
|
// No .md/.err fixtures written at all: `_DIAG_MD`/`_DIAG_ERR` are both
|
|
// empty, so this is the "nothing to preserve" branch, not a failure —
|
|
// `_PRESERVE_OK` starts (and stays) `true`, so cleanup proceeds exactly
|
|
// as if preservation had succeeded.
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini,claude',
|
|
jsonlLines: [],
|
|
lanes: {},
|
|
});
|
|
|
|
assert.equal(result.runDirExists, false, 'nothing to preserve is not a failure — run dir must still be removed');
|
|
assert.equal(result.diagDirExists, false, 'no diagnostics directory should be created when there is nothing to copy');
|
|
});
|
|
});
|
|
|
|
describe('#3885 failed preservation leaves run_dir intact (no silent swallow)', () => {
|
|
test('failedPreservationLeavesRunDirIntact_3885', (t) => {
|
|
// Root-safe failure injection: pre-create a plain FILE at the exact path
|
|
// `mkdir -p "$DIAG_DIR"` needs to create as a directory. This is a
|
|
// filesystem TYPE conflict, not a permission check, so it fails `mkdir -p`
|
|
// even when the test runs as root in Docker/CI (where `chmod 0o000`
|
|
// would be silently bypassed and this test would pass with zero real
|
|
// coverage — see #3885's brief).
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'codex,gemini,claude',
|
|
jsonlLines: [],
|
|
blockDiagDirWithFile: true,
|
|
lanes: {
|
|
codex: { md: LANE_FAILURE_MD('codex'), err: 'stack trace: codex crashed\n' },
|
|
gemini: { md: LANE_FAILURE_MD('gemini'), err: 'stack trace: gemini crashed\n' },
|
|
claude: { md: LANE_FAILURE_MD('claude'), err: 'stack trace: claude crashed\n' },
|
|
},
|
|
});
|
|
|
|
assert.equal(
|
|
result.runDirExists,
|
|
true,
|
|
'a failed mkdir -p on DIAG_DIR must skip rm -rf and leave run_dir intact — this is the #3885 regression guard',
|
|
);
|
|
// The warning is emitted by the substituted bash script, which names
|
|
// $RUN_DIR in its POSIX-spliced form (see the #3885 comment in
|
|
// runWriteReviewsFlow) — compare against that same form rather than the
|
|
// OS-native `result.runDir`.
|
|
assert.ok(
|
|
result.stderr.includes(toPosixPath(result.runDir)),
|
|
`the failure warning must name the intact run_dir holding the un-preserved evidence; got stderr: ${result.stderr}`,
|
|
);
|
|
// The original per-lane evidence is still readable at its original
|
|
// location, uncorrupted, because it was never moved or destroyed.
|
|
assert.equal(
|
|
fs.readFileSync(path.join(result.runDir, 'gsd-review-codex.md'), 'utf-8'),
|
|
LANE_FAILURE_MD('codex'),
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('#4097 the run\'s own input copies are not preserved as diagnostics', () => {
|
|
// The preserve+cleanup glob assumes every `gsd-review-*.md` in RUN_DIR is
|
|
// lane OUTPUT. But the workflow itself writes the run's assembled INPUTS
|
|
// into RUN_DIR under the same prefix at prompt-assembly time (review.md's
|
|
// section-copy fence): the combined prompt, instructions, roadmap, one copy
|
|
// of every plan under review, project/context/research/requirements
|
|
// sections, and the per-lane trimmed prompts. On a multi-plan phase those
|
|
// byte-identical `.planning/` duplicates bury the actual evidence (a
|
|
// handful of reviewer reports and `.err` sidecars) and grow the phase
|
|
// directory on every review run (#4097).
|
|
const INPUT_COPIES_4097 = {
|
|
'gsd-review-prompt.md': 'combined reviewer prompt\n',
|
|
'gsd-review-instructions.md': 'instructions section\n',
|
|
'gsd-review-roadmap.md': 'roadmap section\n',
|
|
'gsd-review-project.md': 'PROJECT.md copy\n',
|
|
'gsd-review-context.md': 'CONTEXT.md concatenation\n',
|
|
'gsd-review-research.md': 'RESEARCH.md concatenation\n',
|
|
'gsd-review-requirements.md': 'REQUIREMENTS.md copy\n',
|
|
'gsd-review-plan-12.6-01.md': 'plan 12.6-01 duplicate\n',
|
|
'gsd-review-plan-12.6-02.md': 'plan 12.6-02 duplicate\n',
|
|
// Per-lane trimmed prompt written by prepare_trimmed_prompt_for_reviewer
|
|
// — also a run input, also matched by the issue's `gsd-review-prompt*`
|
|
// exclusion prefix.
|
|
'gsd-review-prompt-claude.md': 'budget-trimmed prompt for lane claude\n',
|
|
};
|
|
|
|
test('inputCopiesAreNotSweptIntoDiagnostics_4097', (t) => {
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'claude',
|
|
jsonlLines: [{ slug: 'claude' }],
|
|
seedFiles: INPUT_COPIES_4097,
|
|
lanes: {
|
|
claude: { md: '# Claude review\nok\n', err: 'mild stderr warning\n' },
|
|
},
|
|
});
|
|
|
|
assert.equal(result.outcome, 'exited');
|
|
assert.equal(result.runDirExists, false, 'successful preservation must still clean up the run dir');
|
|
assert.equal(result.diagDirExists, true, 'real lane evidence must still be preserved');
|
|
|
|
const preserved = fs.existsSync(result.diagDir)
|
|
? fs.readdirSync(result.diagDir).sort()
|
|
: [];
|
|
// Only the lane's own output belongs in the diagnostics folder.
|
|
assert.deepEqual(
|
|
preserved,
|
|
['gsd-review-claude.err', 'gsd-review-claude.md'],
|
|
`diagnostics must hold exactly the lane report and stderr sidecar, not the run's input copies; got: ${preserved.join(', ')}`,
|
|
);
|
|
assert.equal(
|
|
fs.readFileSync(path.join(result.diagDir, 'gsd-review-claude.md'), 'utf-8'),
|
|
'# Claude review\nok\n',
|
|
'the lane report must be preserved byte-identically',
|
|
);
|
|
for (const inputName of Object.keys(INPUT_COPIES_4097)) {
|
|
assert.equal(
|
|
fs.existsSync(path.join(result.diagDir, inputName)),
|
|
false,
|
|
`input copy ${inputName} must NOT be swept into .review-diagnostics/ (#4097)`,
|
|
);
|
|
}
|
|
});
|
|
|
|
test('inputsOnlyRunLeavesNoDiagnosticsDir_4097', (t) => {
|
|
// Inputs only, no lane artifacts at all: inputs are not evidence, so
|
|
// this is the "nothing to preserve" branch — no diagnostics directory is
|
|
// created and cleanup proceeds (#4097 narrowing of #3352's glob).
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'claude',
|
|
jsonlLines: [],
|
|
seedFiles: INPUT_COPIES_4097,
|
|
lanes: {},
|
|
});
|
|
|
|
assert.equal(result.runDirExists, false, 'nothing to preserve is not a failure — run dir must still be removed');
|
|
assert.equal(result.diagDirExists, false, 'a run that produced only input copies has no evidence to preserve');
|
|
});
|
|
});
|
|
|
|
describe('#3352 preserved evidence is never swept into the commit (N6)', () => {
|
|
test('preservedEvidenceIsNotSweptIntoTheCommit', (t) => {
|
|
const result = runWriteReviewsFlow(t, {
|
|
selected: 'codex',
|
|
jsonlLines: [{ slug: 'codex' }],
|
|
lanes: {
|
|
codex: { md: '# Codex\nok\n', err: '' },
|
|
},
|
|
});
|
|
assert.equal(result.commitTrace.length, 1, 'exactly one commit call must run');
|
|
const commitArgs = result.commitTrace[0];
|
|
// The commit fence splices `{phase_dir}` into unquoted bash source as a
|
|
// POSIX-form path (see the #3885 comment in runWriteReviewsFlow) — build
|
|
// the expected string the same way rather than via `path.join`, which on
|
|
// win32 would re-insert native backslashes the substituted script never
|
|
// produces.
|
|
assert.ok(
|
|
commitArgs.includes(`${toPosixPath(result.phaseDir)}/03-REVIEWS.md`),
|
|
`commit must name the single REVIEWS.md file; got: ${commitArgs}`,
|
|
);
|
|
assert.ok(
|
|
!commitArgs.includes('.review-diagnostics'),
|
|
`commit must never name the diagnostics directory; got: ${commitArgs}`,
|
|
);
|
|
// The commit step's --files value is a single path, never a glob: a
|
|
// directory glob would expand to MULTIPLE argv tokens by the time
|
|
// `gsd_run` sees them, so more than one path after "--files" is itself
|
|
// the defect this row guards against.
|
|
const filesIdx = commitArgs.indexOf('--files ');
|
|
const afterFiles = commitArgs.slice(filesIdx + '--files '.length).trim();
|
|
assert.equal(afterFiles.split(/\s+/).length, 1, `--files must carry exactly one path; got: ${afterFiles}`);
|
|
});
|
|
});
|