'use strict'; /** * The capability fragments' probe fallbacks must be honest (#3909, ADR-3889 P5). * * Two capability fragments carry a shell snippet that runs a detector and * captures its JSON. Those snippets used `… 2>/dev/null || echo '{"detected":false}'`, * which FABRICATES a negative verdict whenever the probe exits non-zero. That is * wrong three separate ways, and all three are covered here: * * 1. `||` fires on exit 1 — which ADR-3889 P3 made the LEGITIMATE negative — * so a correct "no integration" answer got a second object appended to it. * 2. `$( )` captures the whole compound's stdout, so the fallback APPENDS * rather than replaces: an honest `{"skipped":true}` was immediately * contradicted by a fabricated `{"detected":false}` in the same string. * 3. A probe that genuinely could not run produced a clean, confident, * wrong `detected:false`. * * BEHAVIORAL, not source-grep: each test extracts the fragment's own fenced * bash block, executes it under `bash` with the surrounding contract stubbed * (`gsd_run`, `PHASE_DIR`, `PHASE`), and asserts on the captured variable. */ const { describe, test, afterEach } = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const { cleanup } = require('./helpers.cjs'); const { runNode, OUTCOME } = require('./helpers/process-seam.cjs'); const { splitLines } = require('../gsd-core/bin/lib/text-lines.cjs'); const REPO_ROOT = path.join(__dirname, '..'); const API_FRAGMENT = path.join( REPO_ROOT, 'capabilities', 'ai-integration', 'fragments', 'api-coverage-plan-pre.md'); const DELTA_FRAGMENT = path.join( REPO_ROOT, 'capabilities', 'assumption-delta', 'fragments', 'plan-pre.md'); const TOOLS_PATH = path.join(REPO_ROOT, 'gsd-core', 'bin', 'gsd-tools.cjs'); /** * The OUTER runNode() timeoutMs bound for a node -e wrapper that itself * spawns ONE bash subprocess to execute an extracted capability-fragment * probe snippet (a two-level nested spawn). Distinct from * HOOK_FANOUT_TIMEOUT_MS (a git-hook-shaped multi-spawn fan-out, ~4 spawns) * and LOOP_HOOK_POINT_CLI_TIMEOUT_MS (specifically gsd-tools.cjs, not a * generic node -e wrapper) despite the coincidentally-matching value. Does * NOT touch the inner `timeout: 60000` inside the node-e script's own * string literal a few lines above -- that is JS text content passed to a * spawned child, not a real object-literal property in this file's AST. */ const FRAGMENT_PROBE_SNIPPET_TIMEOUT_MS = 60000; /** * Pull the fragment's probe snippet out of its markdown: the first fenced * ```bash block that assigns `varName`. The block is returned verbatim so the * test executes exactly the bytes the planner is handed. */ function extractProbeBlock(fragmentPath, varName) { const md = splitLines(fs.readFileSync(fragmentPath, 'utf8')); const fences = []; let current = null; for (const line of md) { if (current === null) { if (line.trim() === '```bash') { current = []; } continue; } if (line.trim() === '```') { fences.push(current.join('\n')); current = null; continue; } current.push(line); } const block = fences.find((body) => body.includes(`${varName}=`)); assert.ok( block, `${path.basename(fragmentPath)} must contain a fenced bash block assigning ${varName}`, ); return block; } /** * Run a fragment snippet under bash and return the captured variable's value. * `prelude` stubs the surrounding workflow contract; `cwd` decides whether the * detector module is reachable (an unreachable one is how "the probe could not * launch" is simulated — no chmod, no monkeypatch, just a different cwd). */ function runSnippet({ block, varName, prelude, cwd }) { const script = `set -u\n${prelude}\n${block}\nprintf '%s' "\${${varName}}"\n`; const scriptFile = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-frag-')), 'run.sh'); fs.writeFileSync(scriptFile, script, 'utf8'); try { const r = runNode(['-e', ` const { spawnSync } = require('node:child_process'); const r = spawnSync('bash', [process.argv[1]], { cwd: process.argv[2], encoding: 'utf8', timeout: 60000 }); process.stdout.write(r.stdout || ''); `, scriptFile, cwd], { cwd, timeoutMs: FRAGMENT_PROBE_SNIPPET_TIMEOUT_MS }); assert.strictEqual(r.outcome, OUTCOME.EXITED, `snippet runner outcome: ${r.outcome}`); return r.stdout; } finally { cleanup(path.dirname(scriptFile)); } } /** Assert the captured value is exactly ONE JSON object, and return it. */ function parseSingleObject(captured, what) { assert.notStrictEqual(captured.trim(), '', `${what}: the fragment captured nothing at all`); let parsed; try { parsed = JSON.parse(captured); } catch (err) { assert.fail( `${what}: the fragment produced text that is not a single JSON object — ` + `a concatenated fallback is exactly this failure. Captured: ${JSON.stringify(captured)} ` + `(${err.message})`, ); } assert.strictEqual(typeof parsed, 'object', `${what}: payload must be an object`); assert.notStrictEqual(parsed, null, `${what}: payload must not be null`); return parsed; } // ─── api-coverage fragment ──────────────────────────────────────────────────── describe('api-coverage fragment probe fallback is honest (#3909)', () => { let tmpDir; afterEach(() => { if (tmpDir) { cleanup(tmpDir); tmpDir = null; } }); // The fragment's snippet builds SCOPE from `${PHASE_DIR}/*-PLAN.md` plus a // `gsd_run query roadmap.get-phase` call. Both are stubbed so the test // controls exactly what reaches the detector. function preludeFor(planBody) { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-apifrag-')); if (planBody !== null) { fs.writeFileSync(path.join(tmpDir, '01-PLAN.md'), planBody, 'utf8'); } return [ `PHASE_DIR=${JSON.stringify(tmpDir)}`, 'PHASE=01', 'gsd_run() { return 0; }', ].join('\n'); } function capture(planBody, { cwd = REPO_ROOT } = {}) { return runSnippet({ block: extractProbeBlock(API_FRAGMENT, 'API_COVERAGE_JSON'), varName: 'API_COVERAGE_JSON', prelude: preludeFor(planBody), cwd, }); } test('CONTROL: a scope with API vocabulary captures a detected verdict', () => { const j = parseSingleObject( capture('# Plan\nIntegrate the Stripe API and wrap its SDK.'), 'detected case'); assert.strictEqual(j.detected, true); assert.strictEqual(j.skipped, undefined); }); test('a LEGITIMATE negative (probe exit 1) is captured as ONE valid object', () => { // Regression: the detector exits 1 for a real negative, so `|| echo …` // fired on the success path and appended a second object. const j = parseSingleObject( capture('# Plan\nRefactor the internal state machine.'), 'legit negative'); assert.strictEqual(j.detected, false, 'a real negative must survive intact'); assert.strictEqual(j.skipped, undefined, 'a real negative is not a skip'); }); test('an HONEST skip (empty scope) is not contradicted by a fabricated verdict', () => { const j = parseSingleObject(capture(null), 'empty scope'); assert.strictEqual(j.skipped, true, 'an unexamined scope must report skipped'); assert.strictEqual( j.detected, undefined, 'the skip must not carry a detected key — that contradiction is the defect', ); }); test('a probe that CANNOT LAUNCH reports skipped, never detected:false', () => { // cwd without the module → node fails, stdout empty. The fragment must not // manufacture a verdict from that. const away = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-nomodule-')); try { const j = parseSingleObject( capture('# Plan\nIntegrate the Stripe API.', { cwd: away }), 'probe unavailable'); assert.strictEqual(j.skipped, true, 'a probe that could not run must report skipped'); assert.strictEqual(j.reason, 'probe_unavailable'); assert.strictEqual( j.detected, undefined, 'asserting detected:false from a probe that never ran is the bug this closes', ); } finally { cleanup(away); } }); }); // ─── assumption-delta fragment ──────────────────────────────────────────────── describe('assumption-delta fragment probe fallback is honest (#3909)', () => { let tmpDir; afterEach(() => { if (tmpDir) { cleanup(tmpDir); tmpDir = null; } }); function projectWithRoadmap(body) { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-deltafrag-')); fs.mkdirSync(path.join(tmpDir, '.planning'), { recursive: true }); if (body !== null) { fs.writeFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), body, 'utf8'); } return tmpDir; } // `gsd_run` is the workflow launcher's shell function; stub it to the real // CLI so the snippet exercises the genuine query route. function realGsdRunPrelude(projectDir) { return [ 'PHASE=01', `gsd_run() { ( cd ${JSON.stringify(projectDir)} && ` + `node ${JSON.stringify(TOOLS_PATH)} "$@" ); }`, ].join('\n'); } function capture(prelude) { return runSnippet({ block: extractProbeBlock(DELTA_FRAGMENT, 'ASSUMPTION_DELTA_JSON'), varName: 'ASSUMPTION_DELTA_JSON', prelude, cwd: REPO_ROOT, }); } test('CONTROL: a resolved section with a cue captures a detected verdict', () => { const dir = projectWithRoadmap( '# Roadmap\n\n### Phase 01: Auth\n\nAdd a second authentication method.\n'); const j = parseSingleObject(capture(realGsdRunPrelude(dir)), 'detected case'); assert.strictEqual(j.detected, true); assert.strictEqual(j.skipped, undefined); }); test('CONTROL: a resolved section with no cue captures ONE valid negative', () => { const dir = projectWithRoadmap( '# Roadmap\n\n### Phase 01: Cleanup\n\nRefactor the internal state machine.\n'); const j = parseSingleObject(capture(realGsdRunPrelude(dir)), 'legit negative'); assert.strictEqual(j.detected, false); assert.strictEqual(j.skipped, undefined); }); test('an unresolvable phase captures skipped, not a fabricated negative', () => { const dir = projectWithRoadmap(null); const j = parseSingleObject(capture(realGsdRunPrelude(dir)), 'unresolved phase'); assert.strictEqual(j.skipped, true); assert.strictEqual(j.detected, undefined); }); test('a probe that CANNOT LAUNCH reports skipped, never detected:false', () => { const j = parseSingleObject( capture(['PHASE=01', 'gsd_run() { return 127; }'].join('\n')), 'probe unavailable'); assert.strictEqual(j.skipped, true, 'a launcher that failed must not yield a verdict'); assert.strictEqual(j.reason, 'probe_unavailable'); assert.strictEqual(j.detected, undefined); }); }); // ─── Parity: one vocabulary across both fragments ───────────────────────────── describe('both fragments share one skipped-with-reason vocabulary (#3909)', () => { // Generative-fix divergence guard: two surfaces adopting one convention must // fail this test the moment they drift apart. test('both fragments emit the same probe-unavailable reason token', () => { const results = [ { name: 'api-coverage', block: extractProbeBlock(API_FRAGMENT, 'API_COVERAGE_JSON') }, { name: 'assumption-delta', block: extractProbeBlock(DELTA_FRAGMENT, 'ASSUMPTION_DELTA_JSON') }, ].map(({ name, block }) => { const varName = name === 'api-coverage' ? 'API_COVERAGE_JSON' : 'ASSUMPTION_DELTA_JSON'; const captured = runSnippet({ block, varName, // Force the unavailable path for both: no PHASE_DIR contents, and a // launcher/cwd that cannot produce output. prelude: [ 'PHASE_DIR=/nonexistent-phase-dir-3909', 'PHASE=01', 'gsd_run() { return 127; }', ].join('\n'), cwd: os.tmpdir(), }); return { name, payload: parseSingleObject(captured, name) }; }); for (const { name, payload } of results) { assert.strictEqual(payload.skipped, true, `${name} must report skipped when the probe cannot run`); } assert.strictEqual( results[0].payload.reason, results[1].payload.reason, 'the two fragments must not invent different reason tokens for the same condition', ); }); });