fix(sdk): verify outcome gates advance correctly + regression tests

Address review findings from #1454:

1. runVerifyStep now returns success:false when gaps persist after
   exhausting retries (was always returning success:true)
2. human_needed + callback accept correctly sets outcome to passed
3. retryOnce skips retry for verification outcomes (gaps_found,
   human_needed) which have their own internal retry logic
4. Updated 3 existing tests to expect success:false on exhausted gaps
5. Added 3 regression tests:
   - persistent gaps_found does NOT append Advance step
   - persistent gaps_found does NOT call phaseComplete
   - verifier disabled still advances normally
This commit is contained in:
odmrs
2026-03-28 14:13:49 -03:00
parent 39d8688245
commit c5e4fea697
2 changed files with 80 additions and 7 deletions

View File

@@ -577,9 +577,9 @@ describe('PhaseRunner', () => {
// 1 initial + 1 retry = 2 calls (not 3)
expect(verifyCallCount).toBe(2);
// Verify step still succeeds (gap closure exhausted → proceed)
// Verify step fails when gaps persist after exhausting retries
const verifyStep = result.steps.find(s => s.step === PhaseStepType.Verify);
expect(verifyStep!.success).toBe(true);
expect(verifyStep!.success).toBe(false);
});
it('gaps_found triggers plan → execute → re-verify cycle', async () => {
@@ -659,9 +659,9 @@ describe('PhaseRunner', () => {
expect(afterVerify).not.toContain(PhaseStepType.Plan);
expect(afterVerify.filter(s => s === PhaseStepType.Execute)).toHaveLength(0);
// Verify step still reports success (exhausted retries → proceed)
// Verify step fails when gaps persist (no retries allowed)
const verifyStep = result.steps.find(s => s.step === PhaseStepType.Verify);
expect(verifyStep!.success).toBe(true);
expect(verifyStep!.success).toBe(false);
});
it('gap closure plan step failure proceeds to re-verify without executing', async () => {
@@ -724,8 +724,9 @@ describe('PhaseRunner', () => {
// 1 initial + 3 retries = 4 verify calls
expect(verifyCallCount).toBe(4);
// Verify step fails when gaps persist after all retries exhausted
const verifyStep = result.steps.find(s => s.step === PhaseStepType.Verify);
expect(verifyStep!.success).toBe(true);
expect(verifyStep!.success).toBe(false);
});
it('gap closure results are included in the final verify step planResults', async () => {
@@ -774,6 +775,69 @@ describe('PhaseRunner', () => {
});
});
// ─── Advance gate on persistent gaps ──────────────────────────────────
describe('advance gate on persistent gaps', () => {
it('persistent gaps_found does NOT append Advance step', async () => {
const phaseOp = makePhaseOp({ has_context: true, has_plans: true, plan_count: 1 });
const config = makeConfig({ workflow: { research: false, skip_discuss: true, plan_check: false } as any });
const deps = makeDeps({ config });
(deps.tools.initPhaseOp as ReturnType<typeof vi.fn>).mockResolvedValue(phaseOp);
mockRunPhaseStepSession.mockImplementation(async (_prompt, step) => {
if (step === PhaseStepType.Verify) {
return makePlanResult({
success: false,
error: { subtype: 'verification_failed', messages: ['Gaps persist'] },
});
}
return makePlanResult();
});
const runner = new PhaseRunner(deps);
const result = await runner.run('1');
const stepTypes = result.steps.map(s => s.step);
expect(stepTypes).not.toContain(PhaseStepType.Advance);
});
it('persistent gaps_found does NOT call phaseComplete', async () => {
const phaseOp = makePhaseOp({ has_context: true, has_plans: true, plan_count: 1 });
const config = makeConfig({ workflow: { research: false, skip_discuss: true, plan_check: false } as any });
const deps = makeDeps({ config });
(deps.tools.initPhaseOp as ReturnType<typeof vi.fn>).mockResolvedValue(phaseOp);
mockRunPhaseStepSession.mockImplementation(async (_prompt, step) => {
if (step === PhaseStepType.Verify) {
return makePlanResult({
success: false,
error: { subtype: 'verification_failed', messages: ['Gaps persist'] },
});
}
return makePlanResult();
});
const runner = new PhaseRunner(deps);
await runner.run('1');
expect(deps.tools.phaseComplete).not.toHaveBeenCalled();
});
it('verifier disabled still advances normally', async () => {
const phaseOp = makePhaseOp({ has_context: true, has_plans: true, plan_count: 1 });
const config = makeConfig({ workflow: { research: false, verifier: false, skip_discuss: true, plan_check: false } as any });
const deps = makeDeps({ config });
(deps.tools.initPhaseOp as ReturnType<typeof vi.fn>).mockResolvedValue(phaseOp);
const runner = new PhaseRunner(deps);
const result = await runner.run('1');
const stepTypes = result.steps.map(s => s.step);
expect(stepTypes).toContain(PhaseStepType.Advance);
expect(result.success).toBe(true);
});
});
// ─── Phase lifecycle events ────────────────────────────────────────────
describe('phase lifecycle events', () => {

View File

@@ -239,6 +239,8 @@ export class PhaseRunner {
if (!this.config.workflow.verifier) {
this.logger?.debug('Skipping verify: config.workflow.verifier=false');
} else {
// Verify has its own internal retry logic (gap closure). retryOnce only
// retries on unexpected session throws, not on verification outcomes like gaps_found.
const verifyResult = await this.retryOnce('verify', () => this.runVerifyStep(phaseNumber, sessionOpts, callbacks, options));
steps.push(verifyResult);
@@ -300,6 +302,9 @@ export class PhaseRunner {
const result = await fn();
if (result.success) return result;
// Don't retry verify outcomes (gaps_found, human_needed) — they have their own retry logic.
if (result.error?.startsWith('verification_')) return result;
this.logger?.warn(`Step "${label}" failed, retrying once...`);
return fn();
}
@@ -842,6 +847,7 @@ export class PhaseRunner {
});
if (decision === 'accept') {
outcome = 'passed';
break; // Treat as passed
} else if (decision === 'retry' && gapRetryCount < maxGapRetries) {
gapRetryCount++;
@@ -916,6 +922,7 @@ export class PhaseRunner {
}
const durationMs = Date.now() - stepStart;
const verifySuccess = outcome === 'passed';
this.eventStream.emitEvent({
type: GSDEventType.PhaseStepComplete,
@@ -923,15 +930,17 @@ export class PhaseRunner {
sessionId: lastResult?.sessionId ?? '',
phaseNumber,
step: PhaseStepType.Verify,
success: true,
success: verifySuccess,
durationMs,
...(!verifySuccess && { error: `verification_${outcome}` }),
});
return {
step: PhaseStepType.Verify,
success: true,
success: verifySuccess,
durationMs,
planResults: allPlanResults,
...(!verifySuccess && { error: `verification_${outcome}` }),
};
}