From b621d556f0e603be98c6fe00caf40bc2d221da37 Mon Sep 17 00:00:00 2001 From: benzntech Date: Fri, 20 Mar 2026 14:53:46 +0530 Subject: [PATCH] feat: add Exa and Firecrawl MCP support for research agents Integrate Exa (semantic search) and Firecrawl (deep web scraping) as MCP-based research tools, following the existing Brave Search pattern. - Add tool declarations to all 3 researcher agents - Add tool strategy sections with usage guidance and priority - Add config detection for FIRECRAWL_API_KEY and EXA_API_KEY env vars - Add firecrawl/exa_search config keys to core defaults and init output - Update source priority hierarchy across all researchers --- agents/gsd-phase-researcher.md | 29 +++++++++++++++++++++++++++-- agents/gsd-project-researcher.md | 29 +++++++++++++++++++++++++++-- agents/gsd-ui-researcher.md | 8 ++++++-- get-shit-done/bin/lib/config.cjs | 12 +++++++++++- get-shit-done/bin/lib/core.cjs | 4 ++++ get-shit-done/bin/lib/init.cjs | 12 ++++++++++++ 6 files changed, 87 insertions(+), 7 deletions(-) diff --git a/agents/gsd-phase-researcher.md b/agents/gsd-phase-researcher.md index 1a767b9c8..4eb0386f8 100644 --- a/agents/gsd-phase-researcher.md +++ b/agents/gsd-phase-researcher.md @@ -1,7 +1,7 @@ --- name: gsd-phase-researcher description: Researches how to implement a phase before planning. Produces RESEARCH.md consumed by gsd-planner. Spawned by /gsd:plan-phase orchestrator. -tools: Read, Write, Bash, Grep, Glob, WebSearch, WebFetch, mcp__context7__* +tools: Read, Write, Bash, Grep, Glob, WebSearch, WebFetch, mcp__context7__*, mcp__firecrawl__*, mcp__exa__* color: cyan # hooks: # PostToolUse: @@ -137,6 +137,31 @@ If `brave_search: false` (or not set), use built-in WebSearch tool instead. Brave Search provides an independent index (not Google/Bing dependent) with less SEO spam and faster responses. +### Exa Semantic Search (MCP) + +Check `exa_search` from init context. If `true`, use Exa for semantic, research-heavy queries: + +``` +mcp__exa__web_search_exa with query: "your semantic query" +``` + +**Best for:** Research questions where keyword search fails — "best approaches to X", finding technical/academic content, discovering niche libraries. Returns semantically relevant results. + +If `exa_search: false` (or not set), fall back to WebSearch or Brave Search. + +### Firecrawl Deep Scraping (MCP) + +Check `firecrawl` from init context. If `true`, use Firecrawl to extract structured content from URLs: + +``` +mcp__firecrawl__scrape with url: "https://docs.example.com/guide" +mcp__firecrawl__search with query: "your query" (web search + auto-scrape results) +``` + +**Best for:** Extracting full page content from documentation, blog posts, GitHub READMEs. Use after finding a URL from Exa, WebSearch, or known docs. Returns clean markdown. + +If `firecrawl: false` (or not set), fall back to WebFetch. + ## Verification Protocol **WebSearch findings MUST be verified:** @@ -161,7 +186,7 @@ For each WebSearch finding: | MEDIUM | WebSearch verified with official source, multiple credible sources | State with attribution | | LOW | WebSearch only, single source, unverified | Flag as needing validation | -Priority: Context7 > Official Docs > Official GitHub > Verified WebSearch > Unverified WebSearch +Priority: Context7 > Exa (verified) > Firecrawl (official docs) > Official GitHub > Brave/WebSearch (verified) > WebSearch (unverified) diff --git a/agents/gsd-project-researcher.md b/agents/gsd-project-researcher.md index 5f0f6afba..d58bced09 100644 --- a/agents/gsd-project-researcher.md +++ b/agents/gsd-project-researcher.md @@ -1,7 +1,7 @@ --- name: gsd-project-researcher description: Researches domain ecosystem before roadmap creation. Produces files in .planning/research/ consumed during roadmap creation. Spawned by /gsd:new-project or /gsd:new-milestone orchestrators. -tools: Read, Write, Bash, Grep, Glob, WebSearch, WebFetch, mcp__context7__* +tools: Read, Write, Bash, Grep, Glob, WebSearch, WebFetch, mcp__context7__*, mcp__firecrawl__*, mcp__exa__* color: cyan # hooks: # PostToolUse: @@ -116,6 +116,31 @@ If `brave_search: false` (or not set), use built-in WebSearch tool instead. Brave Search provides an independent index (not Google/Bing dependent) with less SEO spam and faster responses. +### Exa Semantic Search (MCP) + +Check `exa_search` from orchestrator context. If `true`, use Exa for research-heavy, semantic queries: + +``` +mcp__exa__web_search_exa with query: "your semantic query" +``` + +**Best for:** Research questions where keyword search fails — "best approaches to X", finding technical/academic content, discovering niche libraries, ecosystem exploration. Returns semantically relevant results rather than keyword matches. + +If `exa_search: false` (or not set), fall back to WebSearch or Brave Search. + +### Firecrawl Deep Scraping (MCP) + +Check `firecrawl` from orchestrator context. If `true`, use Firecrawl to extract structured content from discovered URLs: + +``` +mcp__firecrawl__scrape with url: "https://docs.example.com/guide" +mcp__firecrawl__search with query: "your query" (web search + auto-scrape results) +``` + +**Best for:** Extracting full page content from documentation, blog posts, GitHub READMEs, comparison articles. Use after finding a relevant URL from Exa, WebSearch, or known docs. Returns clean markdown instead of raw HTML. + +If `firecrawl: false` (or not set), fall back to WebFetch. + ## Verification Protocol **WebSearch findings must be verified:** @@ -138,7 +163,7 @@ Never present LOW confidence findings as authoritative. | MEDIUM | WebSearch verified with official source, multiple credible sources agree | State with attribution | | LOW | WebSearch only, single source, unverified | Flag as needing validation | -**Source priority:** Context7 → Official Docs → Official GitHub → WebSearch (verified) → WebSearch (unverified) +**Source priority:** Context7 → Exa (verified) → Firecrawl (official docs) → Official GitHub → Brave/WebSearch (verified) → WebSearch (unverified) diff --git a/agents/gsd-ui-researcher.md b/agents/gsd-ui-researcher.md index 940b0ff93..ab13ed3e7 100644 --- a/agents/gsd-ui-researcher.md +++ b/agents/gsd-ui-researcher.md @@ -1,7 +1,7 @@ --- name: gsd-ui-researcher description: Produces UI-SPEC.md design contract for frontend phases. Reads upstream artifacts, detects design system state, asks only unanswered questions. Spawned by /gsd:ui-phase orchestrator. -tools: Read, Write, Bash, Grep, Glob, WebSearch, WebFetch, mcp__context7__* +tools: Read, Write, Bash, Grep, Glob, WebSearch, WebFetch, mcp__context7__*, mcp__firecrawl__*, mcp__exa__* color: "#E879F9" # hooks: # PostToolUse: @@ -89,7 +89,11 @@ Your UI-SPEC.md is consumed by: |----------|------|---------|-------------| | 1st | Codebase Grep/Glob | Existing tokens, components, styles, config files | HIGH | | 2nd | Context7 | Component library API docs, shadcn preset format | HIGH | -| 3rd | WebSearch | Design pattern references, accessibility standards | Needs verification | +| 3rd | Exa (MCP) | Design pattern references, accessibility standards, semantic research | MEDIUM (verify) | +| 4th | Firecrawl (MCP) | Deep scrape component library docs, design system references | HIGH (content depends on source) | +| 5th | WebSearch | Fallback keyword search for ecosystem discovery | Needs verification | + +**Exa/Firecrawl:** Check `exa_search` and `firecrawl` from orchestrator context. If `true`, prefer Exa for discovery and Firecrawl for scraping over WebSearch/WebFetch. **Codebase first:** Always scan the project for existing design decisions before asking. diff --git a/get-shit-done/bin/lib/config.cjs b/get-shit-done/bin/lib/config.cjs index d7bc44df8..f8e16308b 100644 --- a/get-shit-done/bin/lib/config.cjs +++ b/get-shit-done/bin/lib/config.cjs @@ -13,7 +13,7 @@ const { const VALID_CONFIG_KEYS = new Set([ 'mode', 'granularity', 'parallelization', 'commit_docs', 'model_profile', - 'search_gitignored', 'brave_search', + 'search_gitignored', 'brave_search', 'firecrawl', 'exa_search', 'workflow.research', 'workflow.plan_check', 'workflow.verifier', 'workflow.nyquist_validation', 'workflow.ui_phase', 'workflow.ui_safety_gate', 'workflow.text_mode', @@ -64,6 +64,14 @@ function ensureConfigFile(cwd) { const braveKeyFile = path.join(homedir, '.gsd', 'brave_api_key'); const hasBraveSearch = !!(process.env.BRAVE_API_KEY || fs.existsSync(braveKeyFile)); + // Detect Firecrawl API key availability + const firecrawlKeyFile = path.join(homedir, '.gsd', 'firecrawl_api_key'); + const hasFirecrawl = !!(process.env.FIRECRAWL_API_KEY || fs.existsSync(firecrawlKeyFile)); + + // Detect Exa API key availability + const exaKeyFile = path.join(homedir, '.gsd', 'exa_api_key'); + const hasExaSearch = !!(process.env.EXA_API_KEY || fs.existsSync(exaKeyFile)); + // Load user-level defaults from ~/.gsd/defaults.json if available const globalDefaultsPath = path.join(homedir, '.gsd', 'defaults.json'); let userDefaults = {}; @@ -101,6 +109,8 @@ function ensureConfigFile(cwd) { }, parallelization: true, brave_search: hasBraveSearch, + firecrawl: hasFirecrawl, + exa_search: hasExaSearch, }; const defaults = { ...hardcoded, diff --git a/get-shit-done/bin/lib/core.cjs b/get-shit-done/bin/lib/core.cjs index edcf95e3c..28ccb4d03 100644 --- a/get-shit-done/bin/lib/core.cjs +++ b/get-shit-done/bin/lib/core.cjs @@ -161,6 +161,8 @@ function loadConfig(cwd) { nyquist_validation: true, parallelization: true, brave_search: false, + firecrawl: false, + exa_search: false, text_mode: false, // when true, use plain-text numbered lists instead of AskUserQuestion menus sub_repos: [], resolve_model_ids: false, // when true, resolve aliases (opus/sonnet/haiku) to full model IDs @@ -242,6 +244,8 @@ function loadConfig(cwd) { nyquist_validation: get('nyquist_validation', { section: 'workflow', field: 'nyquist_validation' }) ?? defaults.nyquist_validation, parallelization, brave_search: get('brave_search') ?? defaults.brave_search, + firecrawl: get('firecrawl') ?? defaults.firecrawl, + exa_search: get('exa_search') ?? defaults.exa_search, text_mode: get('text_mode', { section: 'workflow', field: 'text_mode' }) ?? defaults.text_mode, sub_repos: get('sub_repos', { section: 'planning', field: 'sub_repos' }) ?? defaults.sub_repos, resolve_model_ids: get('resolve_model_ids') ?? defaults.resolve_model_ids, diff --git a/get-shit-done/bin/lib/init.cjs b/get-shit-done/bin/lib/init.cjs index 6083dd908..185e1d9ec 100644 --- a/get-shit-done/bin/lib/init.cjs +++ b/get-shit-done/bin/lib/init.cjs @@ -196,6 +196,14 @@ function cmdInitNewProject(cwd, raw) { const braveKeyFile = path.join(homedir, '.gsd', 'brave_api_key'); const hasBraveSearch = !!(process.env.BRAVE_API_KEY || fs.existsSync(braveKeyFile)); + // Detect Firecrawl API key availability + const firecrawlKeyFile = path.join(homedir, '.gsd', 'firecrawl_api_key'); + const hasFirecrawl = !!(process.env.FIRECRAWL_API_KEY || fs.existsSync(firecrawlKeyFile)); + + // Detect Exa API key availability + const exaKeyFile = path.join(homedir, '.gsd', 'exa_api_key'); + const hasExaSearch = !!(process.env.EXA_API_KEY || fs.existsSync(exaKeyFile)); + // Detect existing code (cross-platform — no Unix `find` dependency) let hasCode = false; let hasPackageFile = false; @@ -248,6 +256,8 @@ function cmdInitNewProject(cwd, raw) { // Enhanced search brave_search_available: hasBraveSearch, + firecrawl_available: hasFirecrawl, + exa_search_available: hasExaSearch, // File paths project_path: '.planning/PROJECT.md', @@ -474,6 +484,8 @@ function cmdInitPhaseOp(cwd, phase, raw) { // Config commit_docs: config.commit_docs, brave_search: config.brave_search, + firecrawl: config.firecrawl, + exa_search: config.exa_search, // Phase info phase_found: !!phaseInfo,