feat(execute-phase): classify quota/rate-limit failures across runtimes (#3095) (#3490)

* feat(execute-phase): classify quota/rate-limit failures across runtimes (#3095)

Dispatched executor subagents that die from provider quota or rate-limit
errors currently look identical to a crashed agent to the orchestrator —
so step 7's recovery prompt offers "retry now" when the right action is
"wait for reset and resume". This adds a runtime-agnostic classifier and
wires execute-phase step 7 to it.

- `agent.classify-failure` SDK query returns
  `{class: 'quota-exceeded' | 'classify-handoff-bug' | 'unknown-failure',
    sentinel?, retryAfterSeconds?}`. Sentinels cover Claude Code
  (`usage limit`, `429`), Copilot CLI (`rate_limit`,
  `user_weekly_rate_limited`), Codex (`usage_limit_reached`,
  `too many requests`), and Gemini (`RESOURCE_EXHAUSTED`,
  `exceeded your`).
- `execute-phase.md` step 7 now branches on the class. Quota-exceeded
  presents a wait-for-reset prompt and points at the safe-resume gate
  landing in #3212 instead of re-dispatching a fresh executor.
- `docs/research/provider-rate-limit-signals.md` records the proactive
  (header / SDK event) signals each provider exposes and the upstream
  Claude Code / Copilot / Codex issues blocking hook-side detection —
  the forward path once host runtimes surface them.

Resume-from-partial-worktree and context-load metrics from the original
report are deliberately out of scope; they overlap #3212's
`state.verify-against-disk` work already in flight.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(execute): render quota retry hint and refresh alias artifacts

* fix(workflow): restore slash namespace and execute-phase size budget

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

sdk/src/query/agent-failure-classifier.test.ts

@@ -0,0 +1,157 @@
+/**
+ * Unit tests for `classifyAgentFailure` (#3095 — execute-phase quota-kill classification).
+ *
+ * The orchestrator dispatches executor subagents and receives a free-text return body
+ * when an agent finishes or fails. Today the body is parsed only for the
+ * `classifyHandoffIfNeeded is not defined` Claude Code runtime bug (treated as
+ * possible-success), and everything else falls through to a generic
+ * "real failure" branch. Quota / rate-limit terminations look identical to a
+ * crashed agent to the orchestrator, so the recovery prompt is wrong
+ * (offers "retry now" when the right action is "wait and resume").
+ *
+ * This classifier returns a structured class plus the matched sentinel so the
+ * workflow can route to a quota-distinct recovery prompt.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { classifyAgentFailure } from './agent-failure-classifier.js';
+
+describe('classifyAgentFailure', () => {
+  describe('quota-exceeded class', () => {
+    it('matches the exact org-monthly-limit message from the #3095 report', () => {
+      const body = "You've hit your org's monthly usage limit";
+      const result = classifyAgentFailure(body);
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.sentinel).toBe('usage limit');
+    });
+
+    it('matches "rate limit" sentinel', () => {
+      const result = classifyAgentFailure('Anthropic API rate limit hit; retry later.');
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.sentinel).toBe('rate limit');
+    });
+
+    it('matches "quota" sentinel', () => {
+      const result = classifyAgentFailure('Your monthly quota has been exhausted.');
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.sentinel).toBe('quota');
+    });
+
+    it('matches HTTP 429 sentinel', () => {
+      const result = classifyAgentFailure('Request failed: HTTP 429 Too Many Requests');
+      expect(result.class).toBe('quota-exceeded');
+      // 429 wins over the secondary "too many requests" tag (more specific).
+      expect(result.sentinel).toBe('429');
+    });
+
+    it('matches "too many requests" when 429 not present (Codex/OpenAI variant)', () => {
+      const result = classifyAgentFailure('Too Many Requests — please retry later');
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.sentinel).toBe('too many requests');
+    });
+
+    it('matches "usage_limit_reached" (Codex CLI sentinel)', () => {
+      const result = classifyAgentFailure('agent failed: usage_limit_reached');
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.sentinel).toBe('usage_limit_reached');
+    });
+
+    it('matches "RESOURCE_EXHAUSTED" (Gemini CLI sentinel)', () => {
+      const result = classifyAgentFailure(
+        'Error: RESOURCE_EXHAUSTED — You exceeded your current quota',
+      );
+      expect(result.class).toBe('quota-exceeded');
+      // First-match precedence per SENTINEL_ORDER — quota wins here.
+      expect(['quota', 'resource_exhausted']).toContain(result.sentinel);
+    });
+
+    it('matches "user_weekly_rate_limited" (Copilot CLI sentinel)', () => {
+      const result = classifyAgentFailure('Server Error: user_weekly_rate_limited');
+      expect(result.class).toBe('quota-exceeded');
+      // The high-priority "rate limit" tag also appears as a substring here;
+      // either is acceptable as long as classification is quota-exceeded.
+      expect(result.sentinel).toBeDefined();
+    });
+
+    it('matches "exceeded your" generic quota phrasing (OpenAI / Gemini)', () => {
+      const result = classifyAgentFailure(
+        "You exceeded your current quota, please check your plan and billing details.",
+      );
+      expect(result.class).toBe('quota-exceeded');
+    });
+
+    it('matches case-insensitively', () => {
+      const result = classifyAgentFailure('USAGE LIMIT REACHED');
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.sentinel).toBe('usage limit');
+    });
+
+    it('extracts retry-after seconds when present in the body', () => {
+      const body = 'rate_limit_error: retry-after: 3600';
+      const result = classifyAgentFailure(body);
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.retryAfterSeconds).toBe(3600);
+    });
+
+    it('extracts retry-after seconds from header-style header value', () => {
+      // Some runtimes echo the raw Retry-After header value.
+      const body = 'Failed: 429 Too Many Requests (Retry-After: 60)';
+      const result = classifyAgentFailure(body);
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.retryAfterSeconds).toBe(60);
+    });
+
+    it('does not parse retry-after when token is embedded in another word', () => {
+      const body = 'noretry-after: 3600 quota exceeded';
+      const result = classifyAgentFailure(body);
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.retryAfterSeconds).toBeUndefined();
+    });
+
+    it('returns undefined retryAfterSeconds when no hint present', () => {
+      const result = classifyAgentFailure("You've hit your org's monthly usage limit");
+      expect(result.class).toBe('quota-exceeded');
+      expect(result.retryAfterSeconds).toBeUndefined();
+    });
+  });
+
+  describe('classify-handoff-bug class', () => {
+    it('preserves the existing classifyHandoffIfNeeded pass-through', () => {
+      // Keep the existing Claude Code runtime bug path callable through the
+      // same classifier so step 7 has a single dispatch point.
+      const body = 'ReferenceError: classifyHandoffIfNeeded is not defined';
+      const result = classifyAgentFailure(body);
+      expect(result.class).toBe('classify-handoff-bug');
+      expect(result.sentinel).toBe('classifyhandoffifneeded is not defined');
+    });
+  });
+
+  describe('unknown-failure class (fallback)', () => {
+    it('returns unknown-failure for a generic crash', () => {
+      const result = classifyAgentFailure('Error: something blew up');
+      expect(result.class).toBe('unknown-failure');
+      expect(result.sentinel).toBeUndefined();
+    });
+
+    it('returns unknown-failure for empty input', () => {
+      const result = classifyAgentFailure('');
+      expect(result.class).toBe('unknown-failure');
+    });
+
+    it('returns unknown-failure for whitespace-only input', () => {
+      const result = classifyAgentFailure('   \n\t  ');
+      expect(result.class).toBe('unknown-failure');
+    });
+  });
+
+  describe('precedence', () => {
+    it('quota sentinel wins over classify-handoff sentinel when both present', () => {
+      // The runtime bug is a post-completion handler crash; if the underlying
+      // cause was quota, that is the actionable signal — surface it.
+      const body =
+        "You've hit your org's monthly usage limit\nReferenceError: classifyHandoffIfNeeded is not defined";
+      const result = classifyAgentFailure(body);
+      expect(result.class).toBe('quota-exceeded');
+    });
+  });
+});

sdk/src/query/agent-failure-classifier.ts

@@ -0,0 +1,105 @@
+/**
+ * Classify the free-text return body from a dispatched executor subagent into
+ * an actionable failure class for the orchestrator's recovery router.
+ *
+ * Context (#3095): the orchestrator currently treats every non-success agent
+ * return as a generic "real failure" except for the Claude Code
+ * `classifyHandoffIfNeeded is not defined` runtime bug. Quota / rate-limit
+ * terminations look identical to a crashed agent — but the right user
+ * response is "wait for reset and resume", not "retry now or abort".
+ *
+ * Sentinel coverage spans the runtimes GSD supports as executor targets:
+ *   - Anthropic / Claude Code  — "usage limit", "rate limit", "quota", "429", "retry-after"
+ *   - GitHub Copilot CLI       — "rate limit", "rate_limited", "user_weekly_rate_limited"
+ *   - OpenAI Codex CLI         — "429", "usage_limit_reached", "too many requests"
+ *   - Google Gemini CLI        — "RESOURCE_EXHAUSTED", "exceeded your", "quota"
+ *
+ * See docs/research/provider-rate-limit-signals.md for the proactive (header
+ * / SDK event) signals the orchestrator could use once the host runtime
+ * (Claude Code, Copilot, Codex) exposes them to hooks.
+ */
+
+export type AgentFailureClass =
+  | 'quota-exceeded'
+  | 'classify-handoff-bug'
+  | 'unknown-failure';
+
+export interface AgentFailureClassification {
+  class: AgentFailureClass;
+  /** Lower-cased substring that matched, if any. Useful for log lines. */
+  sentinel?: string;
+  /** Seconds the runtime asked us to wait, parsed from "retry-after: N". */
+  retryAfterSeconds?: number;
+}
+
+// Order matters: the first match wins, so list the most specific / most
+// actionable sentinels first. "429" beats "too many requests"; "quota" beats
+// "RESOURCE_EXHAUSTED" because "quota" is the universal token. All matches
+// are case-insensitive (sentinel value below is the lower-cased form).
+const QUOTA_SENTINELS: ReadonlyArray<string> = [
+  '429',
+  'usage_limit_reached',
+  'usage limit',
+  'rate limit',
+  'rate-limited',
+  // `rate_limit` (stem) covers `rate_limited`, `rate_limit_error`,
+  // `rate_limit_exceeded`, and Copilot's `user_weekly_rate_limited`.
+  'rate_limit',
+  'resource_exhausted',
+  'quota',
+  'too many requests',
+  'exceeded your',
+];
+
+const CLASSIFY_HANDOFF_SENTINEL = 'classifyhandoffifneeded is not defined';
+
+function parseRetryAfter(body: string): number | undefined {
+  // Match "retry-after: N" or "Retry-After: N" with optional surrounding
+  // punctuation. Captures integer seconds only — HTTP-date form is rare in
+  // agent return bodies and not worth the surface area.
+  const match = body.match(/\bretry[-_ ]after[:\s]+(\d+)\b/i);
+  if (!match) return undefined;
+  const seconds = Number.parseInt(match[1]!, 10);
+  return Number.isFinite(seconds) ? seconds : undefined;
+}
+
+/**
+ * Query-handler wrapper for `agent.classify-failure`. Reads the body to
+ * classify from the joined positional args (typed via `--`) so workflow
+ * shell snippets can pass it as `gsd-sdk query agent.classify-failure -- "$BODY"`.
+ */
+export async function agentClassifyFailure(
+  args: string[],
+): Promise<{ data: AgentFailureClassification }> {
+  const body = args.join(' ');
+  return { data: classifyAgentFailure(body) };
+}
+
+export function classifyAgentFailure(body: string): AgentFailureClassification {
+  const normalized = body.toLowerCase();
+
+  if (normalized.trim() === '') {
+    return { class: 'unknown-failure' };
+  }
+
+  // Quota sentinels take precedence over the classifyHandoff runtime bug:
+  // a quota-kill that *also* crashed the completion handler is still a quota
+  // event, and the recovery path differs (wait vs. spot-check-and-treat-as-OK).
+  for (const sentinel of QUOTA_SENTINELS) {
+    if (normalized.includes(sentinel)) {
+      const retryAfterSeconds = parseRetryAfter(body);
+      return retryAfterSeconds === undefined
+        ? { class: 'quota-exceeded', sentinel }
+        : { class: 'quota-exceeded', sentinel, retryAfterSeconds };
+    }
+  }
+
+  if (normalized.includes(CLASSIFY_HANDOFF_SENTINEL)) {
+    return {
+      class: 'classify-handoff-bug',
+      sentinel: CLASSIFY_HANDOFF_SENTINEL,
+    };
+  }
+
+  return { class: 'unknown-failure' };
+}

sdk/src/query/command-aliases.generated.ts

@@ -104,6 +104,7 @@ export interface NonFamilyCommandAlias {
 }
 
 export const NON_FAMILY_COMMAND_ALIASES: readonly NonFamilyCommandAlias[] = [
+  { canonical: 'agent.classify-failure', aliases: ['agent classify-failure'], mutation: false },
   { canonical: 'check-commit', aliases: [], mutation: true },
   { canonical: 'check.decision-coverage-plan', aliases: ['check decision-coverage-plan'], mutation: false },
   { canonical: 'check.decision-coverage-verify', aliases: ['check decision-coverage-verify'], mutation: false },

sdk/src/query/command-manifest.non-family.ts

@@ -80,4 +80,6 @@ export const NON_FAMILY_COMMAND_MANIFEST: readonly NonFamilyCommandManifestEntry
   { canonical: 'generate-claude-md', aliases: [], mutation: true, outputMode: 'json' },
 
   { canonical: 'verify-summary', aliases: ['verify.summary', 'verify summary'], mutation: false, outputMode: 'raw' },
+
+  { canonical: 'agent.classify-failure', aliases: ['agent classify-failure'], mutation: false, outputMode: 'json' },
 ] as const;

sdk/src/query/command-static-catalog-foundation.ts

@@ -24,6 +24,7 @@ import { checkCompletion } from './check-completion.js';
 import { checkGates } from './check-gates.js';
 import { checkVerificationStatus } from './check-verification-status.js';
 import { checkShipReady } from './check-ship-ready.js';
+import { agentClassifyFailure } from './agent-failure-classifier.js';
 
 export const FOUNDATION_STATIC_CATALOG: ReadonlyArray<readonly [string, QueryHandler]> = [
   ['generate-slug', generateSlug],
@@ -97,4 +98,6 @@ export const DECISION_ROUTING_STATIC_CATALOG: ReadonlyArray<readonly [string, Qu
   ['check.ship-ready', checkShipReady],
   ['check ship-ready', checkShipReady],
   ['commands', commandsList],
+  ['agent.classify-failure', agentClassifyFailure],
+  ['agent classify-failure', agentClassifyFailure],
 ] as const;