Files
msd-core/tests/enh-2447-roadmap-wave-deps.test.cjs
Rezolv 18995380ce feat(#1154): honest verifier — abstain (insufficient_spec) on non-inferable backstop truths (#1738)
* feat(verify-phase): honest verifier — abstain (insufficient_spec) on non-inferable backstop truths (#1154)

Carry the edge-probe's existing `backstop` (non-inferable) tier through the
plan-phase projection as a structured flat-scalar marker instead of a prose
parenthetical, and make verify-phase abstain -> human_needed (never silent-pass)
on a backstop truth it cannot confirm with explicit evidence. Truth-axis mirror
of #644's prohibition judgment-tier (ADR-550 D4).

Engine (deterministic, CI-tested per ADR-550 D5 — never the LLM verdict):
- src/probe-core.cts: truthStatement/truthVerification normalizers, projectTruths
  (conservative serializer), dispositionForUnverifiableTruth (backstop+no-evidence
  -> unverified/flagged/insufficient_spec; backstop+evidence -> green; inferable
  -> green, the over-abstention guard).
- src/roadmap.cts: coerceTruthToString now reads `statement` first so an object-form
  backstop truth is surfaced, not dropped (Hyrum backward-compat for truth-readers).

Workflow/agent/docs: plan-phase emits the structured marker (flat scalar, ADR-550
#1278); verify-phase + gsd-verifier add the abstain arm; new references/honest-verifier.md;
FEATURES/COMMANDS document insufficient_spec; ADR-550 amended (truth-axis D4 mirror).

Decisions adopted (trek-e review): insufficient_spec feeds existing human_needed with a
distinguishable reason (no new VERIFIER_STATUS); changeset Changed; round-trip parity
test; abstain-on-unconfirmed-backstop regression test red-first.

Implementation notes (deviations from the issue's proposed file list, verified live):
- frontmatter.cts needs no change — its flat parser already round-trips object-form truths.
- verify.cts needs no change — it grades artifacts/key_links structurally; truths are
  LLM-graded at the workflow layer, so consumption lives there + the deterministic helper.
- No CJS<->SDK hand-sync — the SDK seam was retired (ADR-0174); src/*.cts is sole source.

Regenerated artifacts: golden-install-parity fixtures, INVENTORY-MANIFEST, size baselines.

* chore(#1154): add changeset (Changed) for honest verifier

User-facing changelog fragment for #1738. Typed `Changed` (not `Added`) per
trek-e review condition 3 — the verify behavior shifts for backstop-bearing specs
(a confident silent `passed` becomes `human_needed`), which is user-visible even
though the schema marker is additive.

* docs(#1154): score-formula also excludes abstained insufficient_spec truths (review nit-1)

trek-e review nit: the verify-phase score sentence said PRESENT_BEHAVIOR_UNVERIFIED
truths were "the only ones excluded" from verified_truths. Post-#1154 an abstained
`insufficient_spec` backstop truth is also excluded (it is not ✓ VERIFIED and routes
to human_needed). Behavior was already correct; this tightens the wording.
Regenerated golden-install-parity fixtures + workflow-size baseline for the touched
verify-phase.md. (Nit-2 — a dedicated insufficient_spec_items frontmatter list — is
intentionally not taken: the current design is ADR-550-D4-conformant, the abstain
cause rides as a distinguishable report reason, and adding it would exceed the
approved scope.)

---------

Co-authored-by: Tom Boucher <trekkie@nomorestars.com>
2026-06-29 00:14:32 -04:00

297 lines
12 KiB
JavaScript

'use strict';
// allow-test-rule: source-text-is-the-product
// Reads .md/.json/.yml product files whose deployed text IS what the
// runtime loads — testing text content tests the deployed contract.
/**
* Tests for ROADMAP wave dependency surfacing (#2447).
*/
const { test, describe, afterEach } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const { runGsdTools, createTempProject, cleanup } = require('./helpers.cjs');
const PLAN_TEMPLATE = (wave, truths = []) => `---
phase: "1"
plan: "01-0${wave}"
type: standard
wave: ${wave}
depends_on: []
files_modified: []
autonomous: true
requirements: []
must_haves:
truths:
${truths.map(t => ` - ${t}`).join('\n') || ' - (none)'}
artifacts: []
key_links: []
---
<objective>
Plan ${wave} objective
</objective>
`;
function makePlanProject(files = {}) {
const dir = createTempProject();
fs.writeFileSync(path.join(dir, '.planning', 'ROADMAP.md'), '');
fs.mkdirSync(path.join(dir, '.planning', 'phases', '01-foundation'), { recursive: true });
for (const [rel, content] of Object.entries(files)) {
const abs = path.join(dir, rel);
fs.mkdirSync(path.dirname(abs), { recursive: true });
fs.writeFileSync(abs, content, 'utf-8');
}
return dir;
}
describe('roadmap annotate-dependencies', () => {
let tmpDir;
afterEach(() => cleanup(tmpDir));
test('inserts wave headers for multi-wave plan set', () => {
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap
### Phase 1: Foundation
**Goal:** Set up project
**Plans:** 2 plans
Plans:
- [ ] 01-01-PLAN.md — Set up DB
- [ ] 01-02-PLAN.md — Build API
`,
'.planning/phases/01-foundation/01-01-PLAN.md': PLAN_TEMPLATE(1, ['DB schema is correct']),
'.planning/phases/01-foundation/01-02-PLAN.md': PLAN_TEMPLATE(2, ['API returns 200']),
});
const result = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(result.success, `Command failed: ${result.error}`);
const out = JSON.parse(result.output);
assert.strictEqual(out.updated, true);
assert.strictEqual(out.waves, 2);
const roadmap = fs.readFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), 'utf-8');
assert.ok(roadmap.includes('**Wave 1**'), 'Wave 1 header present');
assert.ok(roadmap.includes('**Wave 2**'), 'Wave 2 header present');
assert.ok(roadmap.includes('blocked on Wave 1'), 'Wave 2 blocked-on note present');
});
test('does not insert wave headers for single-wave plan set', () => {
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap
### Phase 1: Foundation
**Goal:** Set up project
**Plans:** 2 plans
Plans:
- [ ] 01-01-PLAN.md — Set up DB
- [ ] 01-02-PLAN.md — Build API
`,
'.planning/phases/01-foundation/01-01-PLAN.md': PLAN_TEMPLATE(1, ['DB schema is correct']),
'.planning/phases/01-foundation/01-02-PLAN.md': PLAN_TEMPLATE(1, ['API returns 200']),
});
const result = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(result.success, `Command failed: ${result.error}`);
const roadmap = fs.readFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), 'utf-8');
assert.ok(!roadmap.includes('**Wave 1**'), 'no Wave header for single-wave set');
assert.ok(!roadmap.includes('blocked on'), 'no blocked-on note for single wave');
});
test('surfaces cross-cutting constraints when truths appear in 2+ plans', () => {
const sharedTruth = 'All endpoints require auth';
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap
### Phase 1: Foundation
**Goal:** Set up project
**Plans:** 2 plans
Plans:
- [ ] 01-01-PLAN.md — Set up DB
- [ ] 01-02-PLAN.md — Build API
`,
'.planning/phases/01-foundation/01-01-PLAN.md': PLAN_TEMPLATE(1, [sharedTruth, 'DB schema is correct']),
'.planning/phases/01-foundation/01-02-PLAN.md': PLAN_TEMPLATE(2, [sharedTruth, 'API returns 200']),
});
const result = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(result.success, `Command failed: ${result.error}`);
const out = JSON.parse(result.output);
assert.strictEqual(out.cross_cutting_constraints, 1);
const roadmap = fs.readFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), 'utf-8');
assert.ok(roadmap.includes('Cross-cutting constraints:'), 'constraints subsection present');
assert.ok(roadmap.includes(sharedTruth), 'shared truth listed');
});
test('#1154: surfaces a cross-cutting backstop (object-form) truth by its statement, not dropped', () => {
// An object-form backstop truth `{ statement, verification: backstop }` (the #1154 non-inferable
// marker on must_haves.truths) shared across 2 plans must be coerced by its `statement` — the
// Hyrum backward-compat guard: a truth-reader must tolerate the new object form, never drop it.
const backstopTruth = 'statement: Adjacent touching intervals merge\n verification: backstop';
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap
### Phase 1: Foundation
**Goal:** Set up project
**Plans:** 2 plans
Plans:
- [ ] 01-01-PLAN.md — Set up DB
- [ ] 01-02-PLAN.md — Build API
`,
'.planning/phases/01-foundation/01-01-PLAN.md': PLAN_TEMPLATE(1, [backstopTruth, 'DB schema is correct']),
'.planning/phases/01-foundation/01-02-PLAN.md': PLAN_TEMPLATE(2, [backstopTruth, 'API returns 200']),
});
const result = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(result.success, `Command failed: ${result.error}`);
const out = JSON.parse(result.output);
assert.strictEqual(out.cross_cutting_constraints, 1, 'the shared backstop truth is surfaced, not dropped');
const roadmap = fs.readFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), 'utf-8');
assert.ok(roadmap.includes('Adjacent touching intervals merge'), 'surfaced by its statement text, not [object Object]');
});
test('does not surface constraints that appear in only one plan', () => {
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap
### Phase 1: Foundation
**Goal:** Set up project
**Plans:** 2 plans
Plans:
- [ ] 01-01-PLAN.md — Set up DB
- [ ] 01-02-PLAN.md — Build API
`,
'.planning/phases/01-foundation/01-01-PLAN.md': PLAN_TEMPLATE(1, ['Only in plan 1']),
'.planning/phases/01-foundation/01-02-PLAN.md': PLAN_TEMPLATE(2, ['Only in plan 2']),
});
const result = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(result.success, `Command failed: ${result.error}`);
const out = JSON.parse(result.output);
assert.strictEqual(out.cross_cutting_constraints, 0);
const roadmap = fs.readFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), 'utf-8');
assert.ok(!roadmap.includes('Cross-cutting constraints:'), 'no constraints section when none are cross-cutting');
});
test('is idempotent — running twice does not double-insert wave headers', () => {
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap
### Phase 1: Foundation
**Goal:** Set up project
**Plans:** 2 plans
Plans:
- [ ] 01-01-PLAN.md — Set up DB
- [ ] 01-02-PLAN.md — Build API
`,
'.planning/phases/01-foundation/01-01-PLAN.md': PLAN_TEMPLATE(1),
'.planning/phases/01-foundation/01-02-PLAN.md': PLAN_TEMPLATE(2),
});
runGsdTools('roadmap annotate-dependencies 1', tmpDir);
const secondResult = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(secondResult.success);
const out = JSON.parse(secondResult.output);
assert.strictEqual(out.updated, false, 'second run should be no-op');
const roadmap = fs.readFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), 'utf-8');
const waveMatches = roadmap.match(/\*\*Wave \d+\*\*/g) || [];
assert.strictEqual(waveMatches.length, 2, 'exactly 2 wave headers (not doubled)');
});
test('returns no-op when phase has no plans', () => {
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap\n\n### Phase 1: Foundation\n**Goal:** Set up project\n`,
});
const result = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(result.success);
const out = JSON.parse(result.output);
assert.strictEqual(out.updated, false);
});
test('#2757: truths containing colons do not crash annotate-dependencies', () => {
// Unquoted truths with colons (Rails idioms: db:seed, /foo/:id, Class::Method)
// caused parseMustHavesBlock to return {} instead of a string, then t.trim() threw.
const colonTruths = [
'GET /foo/:id resolves to controller#show',
'Class::Method is idempotent',
'"Quoted truth with colon: inside"',
];
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap\n\n### Phase 1: Foundation\n**Goal:** Set up project\n**Plans:** 1 plan\n\nPlans:\n- [ ] 01-01-PLAN.md — Repro plan\n`,
'.planning/phases/01-foundation/01-01-PLAN.md': PLAN_TEMPLATE(1, colonTruths),
});
const result = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(result.success, `Command threw on colon-containing truths: ${result.error}`);
const out = JSON.parse(result.output);
assert.ok(typeof out.updated === 'boolean', 'should return a valid result object');
});
test('#314 map-lookup: found-path uses plan wave, miss-path defaults to wave 1', () => {
// Behavior lock for the O(1) Map swap: asserts BOTH branches of the lookup.
// - 01-01-PLAN.md is in planData (wave 2) → checklist line must land under Wave 2.
// - 01-99-PLAN.md is NOT in planData → null-on-miss → defaults to wave 1.
tmpDir = makePlanProject({
'.planning/ROADMAP.md': `# Roadmap
### Phase 1: Foundation
**Goal:** Set up project
**Plans:** 2 plans
Plans:
- [ ] 01-01-PLAN.md — Known plan
- [ ] 01-99-PLAN.md — Unknown plan (no PLAN.md)
`,
'.planning/phases/01-foundation/01-01-PLAN.md': PLAN_TEMPLATE(2),
// 01-99-PLAN.md intentionally absent — simulates a checklist entry with no backing plan file
});
const result = runGsdTools('roadmap annotate-dependencies 1', tmpDir);
assert.ok(result.success, `Command failed: ${result.error}`);
const roadmap = fs.readFileSync(path.join(tmpDir, '.planning', 'ROADMAP.md'), 'utf-8');
// Both waves must be present (wave 1 from the miss, wave 2 from the found entry)
assert.ok(roadmap.includes('**Wave 1**'), 'Wave 1 header present (miss-path default)');
assert.ok(roadmap.includes('**Wave 2**'), 'Wave 2 header present (found-path)');
// Known plan (wave 2) must appear AFTER Wave 2 header
const wave2Idx = roadmap.indexOf('**Wave 2**');
const knownLineIdx = roadmap.indexOf('01-01-PLAN.md');
assert.ok(knownLineIdx > wave2Idx, 'known plan line grouped under Wave 2');
// Unknown plan (wave 1 default) must appear AFTER Wave 1 header and BEFORE Wave 2 header
const wave1Idx = roadmap.indexOf('**Wave 1**');
const unknownLineIdx = roadmap.indexOf('01-99-PLAN.md');
assert.ok(unknownLineIdx > wave1Idx, 'unknown plan line grouped under Wave 1');
assert.ok(unknownLineIdx < wave2Idx, 'unknown plan line appears before Wave 2 section');
});
test('plan-phase.md documents annotate-dependencies step', () => {
const planPhase = fs.readFileSync(
path.join(__dirname, '../gsd-core/workflows/plan-phase.md'), 'utf-8'
);
assert.ok(planPhase.includes('annotate-dependencies'), 'plan-phase.md references annotate-dependencies command');
assert.ok(planPhase.includes('13d'), 'plan-phase.md has step 13d');
assert.ok(planPhase.includes('Cross-cutting constraints'), 'plan-phase.md documents cross-cutting constraints');
});
});