6cfa0c55d2
Removes kilo, kimi, kimi-code, copilot, windsurf, augment, trae, qwen, hermes, cline, codebuddy and pi end to end: capability descriptors, installer branches and converters (bin/install.js 14.9k -> 11.2k lines), TypeScript converters, hook surfaces and runtime homes, review lanes qwen/kimi-code, the two pi migrations, Kimi payload normalization in the hook guards, dead hostBehaviors vocabulary, launcher home probes, fixtures, runtime-specific tests and the prose that presented them as supported. Installer output for the six kept runtimes is byte-identical to before the prune. The Kimi tool-vocabulary tests in workflow-guard, read-guard and read-injection-scanner are left in place pending a decision.
417 lines
20 KiB
JavaScript
417 lines
20 KiB
JavaScript
// allow-test-rule: source-text-is-the-product
|
|
// The autonomous command and workflow markdown are runtime-loaded contracts.
|
|
// Checking their text verifies the shipped slash-command behavior.
|
|
|
|
'use strict';
|
|
|
|
const { describe, test } = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const { runNode } = require('./helpers/process-seam.cjs');
|
|
const { throwIfFailed } = require('./helpers/git-fixture.cjs');
|
|
const { PROBE_TIMEOUT_MS } = require('./helpers/timeouts.cjs');
|
|
|
|
const REPO_ROOT = path.join(__dirname, '..');
|
|
const COMMAND_PATH = path.join(REPO_ROOT, 'commands', 'msd', 'autonomous.md');
|
|
const WORKFLOW_PATH = path.join(REPO_ROOT, 'msd-core', 'workflows', 'autonomous.md');
|
|
const COMMANDS_DOC_PATH = path.join(REPO_ROOT, 'docs', 'COMMANDS.md');
|
|
const HOW_TO_PATH = path.join(REPO_ROOT, 'docs', 'how-to', 'run-phases-autonomously.md');
|
|
const TOOLS = path.join(REPO_ROOT, 'msd-core', 'bin', 'msd-tools.cjs');
|
|
// #2994: fragmentization moved the five converge-gated regions out of the host
|
|
// autonomous.md into dedicated step files (state:plan-strategy-converge) —
|
|
// see docs/reference/workflow-fragments.md. Tests that assert on this moved
|
|
// content read the step file directly rather than the host.
|
|
const STEP_FAIL_FAST_PATH = path.join(REPO_ROOT, 'msd-core', 'workflows', 'autonomous', 'steps', 'converge-fail-fast.md');
|
|
const STEP_DISPATCH_BG_PATH = path.join(REPO_ROOT, 'msd-core', 'workflows', 'autonomous', 'steps', 'converge-dispatch-bg.md');
|
|
const STEP_DISPATCH_INLINE_PATH = path.join(REPO_ROOT, 'msd-core', 'workflows', 'autonomous', 'steps', 'converge-dispatch-inline.md');
|
|
const STEP_LOOP_PATH = path.join(REPO_ROOT, 'msd-core', 'workflows', 'autonomous', 'steps', 'converge-loop.md');
|
|
const CONVERGENCE_WORKFLOW_PATH = path.join(REPO_ROOT, 'msd-core', 'workflows', 'plan-review-convergence.md');
|
|
|
|
function read(filePath) {
|
|
return fs.readFileSync(filePath, 'utf8');
|
|
}
|
|
|
|
describe('autonomous --converge flag (#711)', () => {
|
|
test('command advertises --converge and documents --cross-ai as alias', () => {
|
|
const command = read(COMMAND_PATH);
|
|
|
|
assert.match(
|
|
command,
|
|
/^argument-hint:.*--converge/m,
|
|
'autonomous command should advertise --converge in argument-hint',
|
|
);
|
|
assert.match(command, /--cross-ai/, 'autonomous command should document --cross-ai alias');
|
|
assert.match(
|
|
command,
|
|
/workflow\.plan_review_convergence=true/,
|
|
'autonomous command should mention the existing convergence feature gate',
|
|
);
|
|
});
|
|
|
|
test('workflow parses converge aliases into a plan strategy', () => {
|
|
const workflow = read(WORKFLOW_PATH);
|
|
|
|
assert.match(workflow, /PLAN_STRATEGY="local"/, 'workflow should default to local planning');
|
|
assert.match(workflow, /PLAN_STRATEGY="converge"/, 'workflow should opt into converge planning');
|
|
assert.match(workflow, /converge\|cross-ai/, 'workflow should accept --converge and --cross-ai');
|
|
});
|
|
|
|
test('explicit --converge overrides the config gate (#4600)', () => {
|
|
// #2994: this contract lives in the converge-fail-fast step file now
|
|
// (state:plan-strategy-converge) — the host only carries the gated
|
|
// conditional-read stub.
|
|
// #4600: an explicit `--converge`/`--cross-ai` (PLAN_STRATEGY=converge is
|
|
// set by nothing else) must WIN over `workflow.plan_review_convergence`
|
|
// — the config is the default for non-flag invocation, not a veto over an
|
|
// explicit operator request. The step must therefore not gate on the
|
|
// config at all, and must state the precedence so a runtime agent
|
|
// executes it as written.
|
|
const workflow = read(WORKFLOW_PATH);
|
|
const step = read(STEP_FAIL_FAST_PATH);
|
|
|
|
assert.match(
|
|
workflow,
|
|
/msd:section id="converge-fail-fast" when="state:plan-strategy-converge"/,
|
|
'workflow should keep the step behind state:plan-strategy-converge',
|
|
);
|
|
assert.doesNotMatch(
|
|
step,
|
|
/config-get workflow\.plan_review_convergence/,
|
|
'the step must not gate an explicit flag on workflow.plan_review_convergence (#4600)',
|
|
);
|
|
assert.doesNotMatch(step, /exit 1/, 'the step must not stop an explicit-flag run');
|
|
assert.match(
|
|
step,
|
|
/OVERRIDES the `workflow\.plan_review_convergence` config gate/,
|
|
'the step must state that the explicit flag overrides the config gate',
|
|
);
|
|
assert.match(
|
|
step,
|
|
/invocation carries `--override-gate`/,
|
|
'the step must state that the dispatched convergence run carries the override flag (#4600)',
|
|
);
|
|
assert.match(
|
|
step,
|
|
/without the flag, `PLAN_STRATEGY` is `local`/,
|
|
'the step must state that the config is not consulted on the non-flag autonomous path',
|
|
);
|
|
});
|
|
|
|
test('the dispatched convergence run bypasses the config gate (#4600)', () => {
|
|
// End-to-end contract: the fail-fast step proceeding is not enough — the
|
|
// dispatched msd-plan-review-convergence workflow has its own §1.5 gate
|
|
// that would veto the same run one step later. The autonomous dispatch
|
|
// must carry an explicit override the gate honors; the veto itself stays
|
|
// for standalone invocation, where the config gate is documented behavior.
|
|
const workflow = read(WORKFLOW_PATH);
|
|
const convergence = read(CONVERGENCE_WORKFLOW_PATH);
|
|
const howTo = read(HOW_TO_PATH);
|
|
|
|
assert.match(
|
|
workflow,
|
|
/--override-gate/,
|
|
'the autonomous converge dispatch must carry --override-gate so the dispatched run cannot be vetoed by the config gate (#4600)',
|
|
);
|
|
assert.match(
|
|
convergence,
|
|
/--override-gate/,
|
|
'plan-review-convergence must honor a --override-gate dispatch instead of failing fast (#4600)',
|
|
);
|
|
assert.match(
|
|
convergence,
|
|
/msd-plan-review-convergence is disabled \(workflow\.plan_review_convergence=false\)/,
|
|
'the §1.5 veto must remain for standalone invocation, where the config gate decides (#4600)',
|
|
);
|
|
assert.doesNotMatch(
|
|
workflow,
|
|
/fail fast unless the existing convergence feature gate/,
|
|
'the host must not instruct a runtime agent to fail fast on the gate before the converge step (#4600)',
|
|
);
|
|
assert.match(
|
|
howTo,
|
|
/overrides the gate for that run/,
|
|
'the how-to must document that an explicit --converge overrides the gate on /msd-autonomous (#4600)',
|
|
);
|
|
// Security-review constraint: the override must be appended conditionally on the converge
|
|
// strategy (never ride on local-strategy runs) and parsed token-anchored by §1.5.
|
|
// Search from the conditional: the host prose at the precedence sentence also names the
|
|
// flag, so a bare indexOf would resolve there and the ordering check could never pass.
|
|
const conditionalAt = workflow.indexOf('if [ "${PLAN_STRATEGY}" = "converge" ]; then');
|
|
const overrideAt = workflow.indexOf('--override-gate', conditionalAt);
|
|
assert.ok(
|
|
conditionalAt !== -1 && overrideAt !== -1,
|
|
'the PLAN_STRATEGY=converge conditional must exist and append --override-gate (#4600)',
|
|
);
|
|
assert.ok(
|
|
convergence.includes('(^|[[:space:]])--override-gate([[:space:]]|$)'),
|
|
'§1.5 must match --override-gate token-anchored so no other argument can carry it (#4600)',
|
|
);
|
|
});
|
|
|
|
test('workflow routes planning through plan-review-convergence when enabled', () => {
|
|
// #2994: the converge dispatch/loop bodies live in dedicated step files
|
|
// now (state:plan-strategy-converge) — only the local-planning fallback
|
|
// remains inline in the host.
|
|
const workflow = read(WORKFLOW_PATH);
|
|
const dispatchInline = read(STEP_DISPATCH_INLINE_PATH);
|
|
const loop = read(STEP_LOOP_PATH);
|
|
const dispatchBg = read(STEP_DISPATCH_BG_PATH);
|
|
|
|
assert.match(
|
|
dispatchInline,
|
|
/Skill\(skill="msd-plan-review-convergence", args="\$\{PHASE_NUM\} \$\{CONVERGENCE_ARGS\}"\)/,
|
|
'inline converge dispatch step should call msd-plan-review-convergence',
|
|
);
|
|
assert.match(
|
|
loop,
|
|
/Skill\(skill="msd-plan-review-convergence", args="\$\{PHASE_NUM\} \$\{CONVERGENCE_ARGS\}"\)/,
|
|
'default converge loop step should call msd-plan-review-convergence',
|
|
);
|
|
assert.match(
|
|
dispatchBg,
|
|
/Run plan convergence for phase \$\{PHASE_NUM\}: Skill\(skill=\\"msd-plan-review-convergence\\"/,
|
|
'interactive converge mode should dispatch plan convergence in the background agent',
|
|
);
|
|
assert.match(
|
|
workflow,
|
|
/Skill\(skill="msd-plan-phase", args="\$\{PHASE_NUM\}"\)/,
|
|
'local planning path should remain available for default autonomous runs',
|
|
);
|
|
});
|
|
|
|
test('workflow forwards reviewer flags and max cycles to convergence', () => {
|
|
const workflow = read(WORKFLOW_PATH);
|
|
// Non-lane convergence controls remain hand-written literals in the workflow.
|
|
const convergenceControls = ['--all', '--text'];
|
|
// Reviewer lane flags that were formerly hand-enumerated in the workflow text.
|
|
// They must now be DERIVED at runtime via `msd_run review-lane flags`, not listed.
|
|
const formerlyHardcodedLaneFlags = [
|
|
'--codex',
|
|
'--claude',
|
|
'--opencode',
|
|
'--ollama',
|
|
'--lm-studio',
|
|
'--llama-cpp',
|
|
];
|
|
// The literal-absence guard below excludes '--claude': the runtime-launcher
|
|
// preamble legitimately contains an unrelated "npx ... --claude --local"
|
|
// install-runtime flag, so a substring match on '--claude' would false-positive
|
|
// against that literal, not against a re-added reviewer-flag list.
|
|
const antiParityLaneFlags = formerlyHardcodedLaneFlags.filter((flag) => flag !== '--claude');
|
|
|
|
assert.match(workflow, /CONVERGENCE_ARGS/, 'workflow should build convergence pass-through args');
|
|
assert.match(
|
|
workflow,
|
|
/msd_run review-lane flags/,
|
|
'workflow should derive reviewer flags from the review-lane roster instead of hand-listing them',
|
|
);
|
|
for (const flag of convergenceControls) {
|
|
assert.ok(workflow.includes(flag), `workflow should pass through ${flag}`);
|
|
}
|
|
assert.match(workflow, /--max-cycles/, 'workflow should pass through --max-cycles N');
|
|
|
|
// Anti-parity guard (deliberately inverted polarity): the whole point of the
|
|
// review-lane-flags derivation is that reviewer lane flags are declared ONCE
|
|
// (in the review-lane roster) and never hand-listed again in workflow prose.
|
|
// If a future edit re-adds a hardcoded reviewer-flag list here, that is the
|
|
// regression this test exists to catch — so this assertion must FAIL when
|
|
// any of these flags reappear as literals in the workflow text.
|
|
for (const flag of antiParityLaneFlags) {
|
|
assert.ok(
|
|
!workflow.includes(flag),
|
|
`workflow should NOT hand-enumerate reviewer lane flag ${flag}; it must be derived via review-lane flags`,
|
|
);
|
|
}
|
|
|
|
// Behavioral coverage: prove the roster the workflow derives from actually
|
|
// yields the flags this test used to hardcode, so the derivation is not vacuous.
|
|
const laneFlagsResult = runNode([TOOLS, 'review-lane', 'flags'], { timeoutMs: PROBE_TIMEOUT_MS });
|
|
throwIfFailed(laneFlagsResult, `node ${TOOLS} review-lane flags`);
|
|
const laneFlags = laneFlagsResult.stdout.split('\n').filter(Boolean);
|
|
for (const flag of formerlyHardcodedLaneFlags) {
|
|
assert.ok(laneFlags.includes(flag), `review-lane flags should include ${flag}`);
|
|
}
|
|
});
|
|
|
|
test('docs show autonomous convergence usage', () => {
|
|
const commandsDoc = read(COMMANDS_DOC_PATH);
|
|
const howTo = read(HOW_TO_PATH);
|
|
|
|
assert.match(commandsDoc, /--converge/, 'COMMANDS.md should document --converge');
|
|
assert.match(commandsDoc, /--cross-ai/, 'COMMANDS.md should document --cross-ai alias');
|
|
assert.match(howTo, /\/msd-autonomous --only 4 --converge/, 'how-to should show single-phase converge usage');
|
|
});
|
|
});
|
|
|
|
describe('autonomous verification deferral contract', () => {
|
|
test('workflow records explicit deferred states instead of silently advancing (#1525)', () => {
|
|
const workflow = read(WORKFLOW_PATH);
|
|
|
|
assert.match(workflow, /verification_deferred_human/);
|
|
assert.match(workflow, /verification_deferred_gaps/);
|
|
assert.match(workflow, /Deferred Verification/);
|
|
assert.match(workflow, /msd:verify-work \$\{PHASE_NUM\}/);
|
|
assert.match(workflow, /msd:plan-phase \$\{PHASE_NUM\} --gaps/);
|
|
assert.match(
|
|
workflow,
|
|
/\| \$\{PHASE_NUM\} \| verification_deferred_human \| \/msd:verify-work \$\{PHASE_NUM\} \|/,
|
|
'human deferral must persist the exact deferred STATE row',
|
|
);
|
|
assert.match(
|
|
workflow,
|
|
/\| \$\{PHASE_NUM\} \| verification_deferred_gaps \| \/msd:plan-phase \$\{PHASE_NUM\} --gaps \|/,
|
|
'gap deferral must persist the exact deferred STATE row',
|
|
);
|
|
assert.doesNotMatch(
|
|
workflow,
|
|
/Human validation deferred` and proceed to iterate step/,
|
|
'human-needed deferral must not silently proceed to the next phase',
|
|
);
|
|
assert.doesNotMatch(
|
|
workflow,
|
|
/Gaps deferred` and proceed to iterate step/,
|
|
'gap deferral must not silently proceed to the next phase',
|
|
);
|
|
assert.match(
|
|
workflow,
|
|
/Skip deferred phases on autonomous re-entry/,
|
|
'reruns must explicitly skip deferred verification phases',
|
|
);
|
|
assert.match(
|
|
workflow,
|
|
/Deferred Verification \(Skipped on Re-entry\)/,
|
|
'workflow should surface skipped deferred phases and their resume commands',
|
|
);
|
|
});
|
|
|
|
test('workflow runs normal transition post-processing after passed verification (#1526)', () => {
|
|
const workflow = read(WORKFLOW_PATH);
|
|
const passedIdx = workflow.indexOf('**If `passed`:**');
|
|
const transitionIdx = workflow.indexOf('transition.md', passedIdx);
|
|
const iterateIdx = workflow.indexOf('Proceed to iterate step', passedIdx);
|
|
|
|
assert.ok(transitionIdx > passedIdx, 'passed verification must invoke transition.md');
|
|
assert.ok(
|
|
transitionIdx < iterateIdx,
|
|
'normal transition post-processing must run before autonomous iterates',
|
|
);
|
|
});
|
|
|
|
test('workflow reads canonical verification status before human-needed promotion (#1522)', () => {
|
|
const workflow = read(WORKFLOW_PATH);
|
|
const waitIdx = workflow.indexOf('After execute, read canonical verification');
|
|
const humanNeededIdx = workflow.indexOf('**If `human_needed`:**', waitIdx);
|
|
const promoteIdx = workflow.indexOf('set VERIFICATION frontmatter `status: passed`', humanNeededIdx);
|
|
const section = workflow.slice(waitIdx, humanNeededIdx);
|
|
|
|
assert.ok(waitIdx !== -1, 'workflow must document the post-execution verification read');
|
|
assert.ok(humanNeededIdx > waitIdx, 'human_needed branch must follow verification status read');
|
|
assert.ok(promoteIdx > humanNeededIdx, 'human_needed branch must contain the promotion action');
|
|
// #2589: the verification read uses the native --pick flag (no jq dependency).
|
|
// String-based check (not a regex literal) so the assertion stays robust to
|
|
// shell metacharacters in the snippet and parses cleanly under espree.
|
|
assert.ok(
|
|
section.includes('VERIFY_STATUS=$(msd_run query verification.status "${PHASE_DIR}" --pick status 2>/dev/null || true)'),
|
|
'autonomous must route human validation through canonical verification.status via the native --pick flag',
|
|
);
|
|
assert.doesNotMatch(
|
|
section,
|
|
/grep "\^status:"/,
|
|
'autonomous must not route stale human_needed reports from raw frontmatter',
|
|
);
|
|
});
|
|
|
|
test('workflow discovers incomplete phases from canonical verification projection (#1522)', () => {
|
|
const workflow = read(WORKFLOW_PATH);
|
|
const discoverStart = workflow.indexOf('<step name="discover_phases">');
|
|
const discoverEnd = workflow.indexOf('</step>', discoverStart);
|
|
const iterateStart = workflow.indexOf('<step name="iterate">');
|
|
const iterateEnd = workflow.indexOf('</step>', iterateStart);
|
|
const discoverStep = workflow.slice(discoverStart, discoverEnd);
|
|
const iterateStep = workflow.slice(iterateStart, iterateEnd);
|
|
|
|
assert.match(discoverStep, /INIT_MANAGER=\$\(msd_run query init\.manager\)/);
|
|
assert.ok(
|
|
discoverStep.includes('if [[ "$INIT_MANAGER" == @file:* ]]; then INIT_MANAGER=$(cat "${INIT_MANAGER#@file:}"); fi'),
|
|
'autonomous discovery must dereference large init.manager payloads before parsing',
|
|
);
|
|
assert.match(discoverStep, /phase_complete !== true/);
|
|
assert.match(discoverStep, /verification_status !== "passed"/);
|
|
// #4455: STATE.md is read through the workstream-resolved path from
|
|
// init.manager (state_path), not a hardcoded .planning/STATE.md literal —
|
|
// a MSD_WORKSTREAM run must read its own workstream's STATE.md.
|
|
assert.ok(
|
|
discoverStep.includes('STATE_PATH=$(_msd_field "$INIT_MANAGER" state_path)'),
|
|
'autonomous discovery must resolve STATE.md through init.manager, not a hardcoded path',
|
|
);
|
|
assert.match(discoverStep, /STATE_CONTENT=\$\(cat "\$STATE_PATH" 2>\/dev\/null \|\| true\)/);
|
|
assert.doesNotMatch(
|
|
discoverStep,
|
|
/STATE_CONTENT=\$\(cat \.planning\/STATE\.md 2>\/dev\/null \|\| true\)/,
|
|
'autonomous discovery must not regress to a hardcoded root .planning/STATE.md read (#4455)',
|
|
);
|
|
assert.match(discoverStep, /drop any phase whose number appears in the deferred-phase map/);
|
|
assert.doesNotMatch(discoverStep, /ROADMAP=\$\(msd_run query roadmap\.analyze\)/);
|
|
assert.doesNotMatch(discoverStep, /disk_status !== "complete"/);
|
|
|
|
assert.match(iterateStep, /INIT_MANAGER=\$\(msd_run query init\.manager\)/);
|
|
assert.ok(
|
|
iterateStep.includes('if [[ "$INIT_MANAGER" == @file:* ]]; then INIT_MANAGER=$(cat "${INIT_MANAGER#@file:}"); fi'),
|
|
'autonomous iteration must dereference large init.manager payloads before parsing',
|
|
);
|
|
assert.match(iterateStep, /phase_complete !== true/);
|
|
assert.match(iterateStep, /verification_status !== "passed"/);
|
|
assert.ok(
|
|
iterateStep.includes('STATE_PATH=$(_msd_field "$INIT_MANAGER" state_path)'),
|
|
'autonomous iteration must resolve STATE.md through init.manager, not a hardcoded path',
|
|
);
|
|
assert.match(iterateStep, /STATE_CONTENT=\$\(cat "\$STATE_PATH" 2>\/dev\/null \|\| true\)/);
|
|
assert.doesNotMatch(
|
|
iterateStep,
|
|
/STATE_CONTENT=\$\(cat \.planning\/STATE\.md 2>\/dev\/null \|\| true\)/,
|
|
'autonomous iteration must not regress to a hardcoded root .planning/STATE.md read (#4455)',
|
|
);
|
|
assert.match(iterateStep, /drop deferred phases from the autonomous queue/);
|
|
});
|
|
});
|
|
|
|
// ─── Issue #3210: bounded blocker retries, needs_human escalation ────────────
|
|
//
|
|
// handle_blocker's "Fix and retry" path had no attempt ceiling across
|
|
// invocations and no automatic escalation to a terminal needs_human state, so
|
|
// a non-converging blocker (e.g. an operator gate the executor cannot satisfy)
|
|
// looped indefinitely. Regression coverage lives here because this file owns
|
|
// the autonomous.md host-workflow contract.
|
|
|
|
describe('issue #3210: autonomous handle_blocker has a retry ceiling with needs_human escalation', () => {
|
|
function stepOf(content, name) {
|
|
const open = `<step name="${name}">`;
|
|
const from = content.indexOf(open);
|
|
assert.ok(from !== -1, `step "${name}" not found`);
|
|
const to = content.indexOf('</step>', from);
|
|
assert.ok(to !== -1, `step "${name}" has no closing tag`);
|
|
return content.slice(from, to);
|
|
}
|
|
|
|
test('handle_blocker bounds "Fix and retry" attempts per phase step', () => {
|
|
const step = stepOf(read(WORKFLOW_PATH), 'handle_blocker');
|
|
assert.match(
|
|
step,
|
|
/\b3\b.*retr|\bretr.*\b3\b|RETRY_COUNT|retry (ceiling|limit|count)/i,
|
|
'handle_blocker must track a bounded retry count for the same phase step instead of ' +
|
|
're-presenting "Fix and retry" indefinitely (#3210)'
|
|
);
|
|
});
|
|
|
|
test('handle_blocker auto-escalates to a terminal needs_human halt once the ceiling is exceeded', () => {
|
|
const step = stepOf(read(WORKFLOW_PATH), 'handle_blocker');
|
|
assert.match(
|
|
step,
|
|
/needs_human/,
|
|
'once the retry ceiling is exceeded, handle_blocker must halt autonomously in a terminal ' +
|
|
'needs_human state (surfacing the unmet items) instead of looping or asking again (#3210)'
|
|
);
|
|
});
|
|
});
|