'use strict'; /** * Package Legitimacy Gate — structural contract tests (#2827) * * Verifies that the three agents (researcher, planner, executor) contain the * interlocking instruction text that forms the slopsquatting defence gate. */ const { describe, test, before } = require('node:test'); const assert = require('node:assert/strict'); const fs = require('fs'); const path = require('path'); const AGENTS = path.join(__dirname, '..', 'agents'); const RESEARCHER = path.join(AGENTS, 'gsd-phase-researcher.md'); const PLANNER = path.join(AGENTS, 'gsd-planner.md'); const EXECUTOR = path.join(AGENTS, 'gsd-executor.md'); function parseSections(md) { const lines = md.split('\n'); const sections = []; let current = { heading: '__preamble__', body: [] }; let inFence = false; for (const line of lines) { if (line.trimStart().startsWith('```')) inFence = !inFence; if (!inFence && /^#{1,3} /.test(line)) { sections.push(current); current = { heading: line.replace(/^#+\s*/, '').trim(), body: [] }; continue; } current.body.push(line); } sections.push(current); return sections; } function extractCodeBlocks(text) { const blocks = []; const lines = text.split('\n'); let inside = false; let buf = []; for (const line of lines) { if (line.trimStart().startsWith('```')) { if (inside) { blocks.push(buf.join('\n')); buf = []; } inside = !inside; continue; } if (inside) buf.push(line); } return blocks; } function extractResearchTemplate(content) { const lines = content.split('\n'); let inside = false; let isMarkdownFence = false; let buf = []; for (const line of lines) { if (!inside && line.startsWith('```markdown')) { inside = true; isMarkdownFence = true; buf = []; continue; } if (inside && line.startsWith('```') && isMarkdownFence) { const candidate = buf.join('\n'); if (/^\s*#\s+Phase\b/m.test(candidate)) return candidate; inside = false; isMarkdownFence = false; continue; } if (inside) buf.push(line); } return ''; } function extractPlanTemplate(content) { const blocks = extractCodeBlocks(content); for (const block of blocks) { if (/^\s*/m.test(block)) return block; } return ''; } function extractXmlElement(text, tag) { const start = text.indexOf(`<${tag}>`); const end = text.indexOf(``); if (start === -1 || end === -1) return ''; return text.slice(start, end + tag.length + 3); } function normalizeTokens(text) { return text .toLowerCase() .replace(/https?:\/\//g, ' ') .replace(/[[\]]/g, '') .replace(/[^a-z0-9{}:_-]+/g, ' ') .trim() .split(/\s+/) .filter(Boolean); } function hasAllTokens(text, required) { const tokenSet = new Set(normalizeTokens(text)); return required.every((token) => tokenSet.has(token.toLowerCase())); } function anyLineHasAll(lines, required) { return lines.some((line) => hasAllTokens(line, required)); } function parseMarkdownTable(lines) { const tableLines = lines.filter((line) => /^\s*\|/.test(line)); if (tableLines.length < 2) return null; const toCells = (line) => line .trim() .replace(/^\|/, '') .replace(/\|$/, '') .split('|') .map((cell) => cell.trim()); const headers = toCells(tableLines[0]); const rows = tableLines .slice(2) .map(toCells) .filter((cells) => cells.length === headers.length) .map((cells) => ({ cells, fields: Object.fromEntries(headers.map((h, i) => [h, cells[i]])), })); return { headers, rows }; } function parseMarkdownTables(lines) { const groups = []; let current = []; for (const line of lines) { if (/^\s*\|/.test(line)) { current.push(line); continue; } if (current.length > 0) { groups.push(current); current = []; } } if (current.length > 0) groups.push(current); return groups .map((group) => parseMarkdownTable(group)) .filter(Boolean); } function lineIndexes(lines, predicate) { const indexes = []; for (let i = 0; i < lines.length; i += 1) { if (predicate(lines[i], i)) indexes.push(i); } return indexes; } function inNearbyWindow(sourceIndexes, targetIndexes, distance) { return sourceIndexes.some((src) => targetIndexes.some((dst) => Math.abs(src - dst) <= distance)); } function readModel(filePath) { const text = fs.readFileSync(filePath, 'utf-8'); return { text, lines: text.split('\n'), sections: parseSections(text), codeBlocks: extractCodeBlocks(text), }; } // allow-test-rule: source-text-is-the-product // Agent .md files — their text IS what the runtime loads. // Testing text content tests the deployed contract. // Per CONTRIBUTING.md exception matrix. describe('gsd-phase-researcher.md — package-legitimacy seam invocation', () => { let model; before(() => { model = readModel(RESEARCHER); }); test('invokes gsd-tools query package-legitimacy check inside a fenced code block', () => { const found = model.codeBlocks.some((block) => hasAllTokens(block, ['package-legitimacy', 'check']) ); assert.ok(found, 'researcher must invoke package-legitimacy check inside a fenced code block'); }); test('package-legitimacy invocation includes --ecosystem flag', () => { const found = model.codeBlocks.some((block) => hasAllTokens(block, ['package-legitimacy', 'check']) && hasAllTokens(block, ['--ecosystem']) ); assert.ok(found, 'package-legitimacy check must include --ecosystem flag'); }); test('documents SLOP, SUS, OK verdict interpretation', () => { const hasSLOP = anyLineHasAll(model.lines, ['slop']); const hasSUS = anyLineHasAll(model.lines, ['sus']); const hasOK = anyLineHasAll(model.lines, ['ok']); assert.ok(hasSLOP && hasSUS && hasOK, 'researcher must document SLOP, SUS, OK verdict interpretation'); }); test('documents [ASSUMED] tag for WebSearch-discovered packages not verified against authoritative source', () => { const hasAssumedLine = anyLineHasAll(model.lines, ['assumed']); const hasWebSearchOrTraining = model.lines.some((line) => hasAllTokens(line, ['websearch']) || hasAllTokens(line, ['training']) ); assert.ok( hasAssumedLine && hasWebSearchOrTraining, 'researcher must document [ASSUMED] tag for packages from non-authoritative sources' ); }); }); describe('gsd-phase-researcher.md — Package Legitimacy Audit section in template', () => { let templateSections; before(() => { const model = readModel(RESEARCHER); const template = extractResearchTemplate(model.text); templateSections = parseSections(template); }); test('RESEARCH.md template contains Package Legitimacy Audit section', () => { const section = templateSections.find((s) => s.heading === 'Package Legitimacy Audit'); assert.ok(section, 'RESEARCH.md template must include a Package Legitimacy Audit section'); }); test('Package Legitimacy Audit table has required columns', () => { const section = templateSections.find((s) => s.heading === 'Package Legitimacy Audit'); assert.ok(section, 'Package Legitimacy Audit section must exist'); const table = parseMarkdownTable(section.body); assert.ok(table, 'Package Legitimacy Audit section must include a markdown table'); // 'slopcheck' column renamed to 'Verdict' to reflect the code seam (gsd-tools query package-legitimacy) const expected = ['Package', 'Registry', 'Age', 'Downloads', 'Verdict', 'Disposition']; for (const column of expected) { assert.ok(table.headers.includes(column), `audit table must have "${column}" column`); } }); test('audit section documents [SLOP], [SUS], and [OK] dispositions', () => { const section = templateSections.find((s) => s.heading === 'Package Legitimacy Audit'); assert.ok(section, 'Package Legitimacy Audit section must exist'); const table = parseMarkdownTable(section.body); assert.ok(table, 'Package Legitimacy Audit section must include a markdown table'); const rowTexts = table.rows.map((row) => row.cells.join(' ')); const slop = rowTexts.some((value) => hasAllTokens(value, ['slop'])); const sus = rowTexts.some((value) => hasAllTokens(value, ['sus'])); const ok = rowTexts.some((value) => hasAllTokens(value, ['ok'])); assert.ok(slop, 'audit section must document [SLOP] disposition'); assert.ok(sus, 'audit section must document [SUS] disposition'); assert.ok(ok, 'audit section must document [OK] disposition'); }); }); describe('gsd-phase-researcher.md — ecosystem-specific package verification', () => { let model; before(() => { model = readModel(RESEARCHER); }); test('documents pip index versions for Python phases', () => { assert.ok(anyLineHasAll(model.lines, ['pip', 'index', 'versions']), 'researcher must document pip index versions'); }); test('documents cargo search for Rust phases', () => { assert.ok(anyLineHasAll(model.lines, ['cargo', 'search']), 'researcher must document cargo search'); }); }); describe('gsd-phase-researcher.md — no npx --yes auto-download', () => { let model; before(() => { model = readModel(RESEARCHER); }); test('does not invoke npx --yes inside a code block', () => { const found = model.codeBlocks.some((block) => hasAllTokens(block, ['npx', '--yes'])); assert.equal(found, false, 'researcher must not invoke npx --yes in any code block'); }); test('context7 is accessed via mcp__context7__ tools (not raw CLI)', () => { // The research-plan seam routes context7 queries; the agent calls MCP tools directly. // Verify the provider table references mcp__context7__ rather than a raw ctx7 CLI invocation. const hasMcpContext7 = anyLineHasAll(model.lines, ['mcp__context7__']); assert.ok(hasMcpContext7, 'researcher must reference mcp__context7__ tools for context7 access'); }); }); describe('gsd-phase-researcher.md — WebSearch-origin package tagging', () => { let model; before(() => { model = readModel(RESEARCHER); }); test('packages discovered via WebSearch are tagged [ASSUMED]', () => { const webSearchLines = lineIndexes(model.lines, (line) => hasAllTokens(line, ['websearch'])); const assumedLines = lineIndexes(model.lines, (line) => hasAllTokens(line, ['assumed'])); assert.ok(webSearchLines.length > 0, 'researcher file must mention WebSearch'); assert.ok(assumedLines.length > 0, 'researcher file must mention [ASSUMED]'); assert.ok( inNearbyWindow(webSearchLines, assumedLines, 25), 'researcher must instruct WebSearch-discovered packages are tagged [ASSUMED] in nearby guidance' ); }); }); describe('gsd-planner.md — checkpoint gate for [ASSUMED]/[SUS] packages', () => { let model; before(() => { model = readModel(PLANNER); }); test('checkpoint:human-verify guidance references [ASSUMED] and [SUS]', () => { const hasCheckpoint = anyLineHasAll(model.lines, ['checkpoint:human-verify']); const hasAssumed = anyLineHasAll(model.lines, ['assumed']); const hasSus = anyLineHasAll(model.lines, ['sus']); assert.ok(hasCheckpoint && hasAssumed, 'planner must gate [ASSUMED] packages behind checkpoint:human-verify'); assert.ok(hasCheckpoint && hasSus, 'planner must gate [SUS] packages behind checkpoint:human-verify'); }); test('package-legitimacy checkpoint uses blocking-human gate and non-auto-approvable language', () => { const hasBlockingHumanGate = anyLineHasAll(model.lines, ['checkpoint:human-verify', 'blocking-human']); const hasNeverAutoApproveRule = model.lines.some((line) => hasAllTokens(line, ['never', 'auto-approvable']) || hasAllTokens(line, ['never', 'auto', 'approvable']) ); assert.ok(hasBlockingHumanGate, 'planner legitimacy checkpoint must use gate="blocking-human"'); assert.ok(hasNeverAutoApproveRule, 'planner must state legitimacy checkpoints are never auto-approvable'); }); test('package verification checkpoint includes registry URL guidance', () => { const hasRegistryGuidance = model.lines.some((line) => hasAllTokens(line, ['npmjs', 'package']) || hasAllTokens(line, ['pypi', 'project']) || hasAllTokens(line, ['crates', 'crates']) ); assert.ok(hasRegistryGuidance, 'planner package-verify checkpoint must include registry URL examples'); }); }); describe('gsd-planner.md — supply-chain row in threat_model template', () => { let planTemplate; let threatModelBlock; before(() => { const model = readModel(PLANNER); planTemplate = extractPlanTemplate(model.text); threatModelBlock = extractXmlElement(planTemplate, 'threat_model'); }); test('PLAN.md template contains threat_model element', () => { assert.ok(/^\s*/m.test(planTemplate), 'PLAN.md template must include '); }); test('threat_model template includes supply-chain row with mitigate disposition', () => { const tables = parseMarkdownTables(threatModelBlock.split('\n')); const strideTable = tables.find((table) => table.headers.includes('Threat ID')); assert.ok(strideTable, 'threat_model must include STRIDE threat register table'); const supplyChainRow = strideTable.rows.find((row) => hasAllTokens(row.cells[0] || '', ['t-{phase}-sc'])); assert.ok(supplyChainRow, 'threat_model must include T-{phase}-SC supply-chain row'); const disposition = supplyChainRow.cells[3] || ''; assert.ok(hasAllTokens(disposition, ['mitigate']), 'supply-chain threat disposition must be mitigate'); }); }); describe('gsd-planner.md — no npx --yes auto-download', () => { test('does not invoke npx --yes inside a code block', () => { const model = readModel(PLANNER); const found = model.codeBlocks.some((block) => hasAllTokens(block, ['npx', '--yes'])); assert.equal(found, false, 'planner must not invoke npx --yes in any code block'); }); }); describe('gsd-executor.md — package installs excluded from RULE 3 auto-fix', () => { let model; before(() => { model = readModel(EXECUTOR); }); test('does not invoke npx --yes inside a code block', () => { const found = model.codeBlocks.some((block) => hasAllTokens(block, ['npx', '--yes'])); assert.equal(found, false, 'executor must not invoke npx --yes in any code block'); }); test('RULE 3 section explicitly excludes package-manager installs', () => { const rule3Line = lineIndexes(model.lines, (line) => hasAllTokens(line, ['rule', '3']))[0]; assert.notEqual(rule3Line, undefined, 'executor must contain RULE 3 section'); const window = model.lines.slice(rule3Line, rule3Line + 35); const hasInstallCommands = anyLineHasAll(window, ['npm', 'install']) || anyLineHasAll(window, ['pip', 'install']) || anyLineHasAll(window, ['cargo', 'add']); const hasExclusionLanguage = anyLineHasAll(window, ['excluded']) || anyLineHasAll(window, ['not', 'auto-fixable']) || anyLineHasAll(window, ['do', 'not']); assert.ok(hasInstallCommands && hasExclusionLanguage, 'RULE 3 must explicitly exclude package-manager installs'); }); test('failed package installs surface checkpoint:human-verify', () => { const rule3Line = lineIndexes(model.lines, (line) => hasAllTokens(line, ['rule', '3']))[0]; assert.notEqual(rule3Line, undefined, 'executor must contain RULE 3 section'); const window = model.lines.slice(rule3Line, rule3Line + 50); const hasFailureLanguage = anyLineHasAll(window, ['failed', 'install']) || anyLineHasAll(window, ['install', 'fails']) || anyLineHasAll(window, ['install', 'failed']); const hasCheckpoint = anyLineHasAll(window, ['checkpoint:human-verify']); assert.ok( hasFailureLanguage && hasCheckpoint, 'executor must emit checkpoint:human-verify when package install fails' ); }); test('auto mode does not auto-approve package-legitimacy checkpoints', () => { const autoModeLine = lineIndexes(model.lines, (line) => hasAllTokens(line, ['auto-mode', 'checkpoint', 'behavior']))[0]; assert.notEqual(autoModeLine, undefined, 'executor must define auto-mode checkpoint behavior'); const window = model.lines.slice(autoModeLine, autoModeLine + 25); const hasExceptionRule = anyLineHasAll(window, ['except', 'package-legitimacy', 'checkpoints']) || anyLineHasAll(window, ['do', 'not', 'auto-approve']) || anyLineHasAll(window, ['blocking-human']); assert.ok( hasExceptionRule, 'executor auto mode must explicitly block auto-approval for package-legitimacy checkpoints' ); }); });