test(#1976): consolidate 17 agent regression tests into agent-contract suites

Fold 17 issue-named agent-markdown regression files into their canonical agent
suites: agent-frontmatter (shared agent-body-contract catch-all: read-loop guards,
retired-slash-ref bans, sycophancy hardening, doc-writer/code-fixer/review-fix/mapper
contracts, write-truncation), planner-language-regression (planner directive/grep/phase
contract), executor-mvp-tdd-section, verifier-behavior-unverified, intel, and
research-agent-profiles. Verbatim block-scoped describe wrappers; 98 subtests conserved
1:1. All unit (markdown-content) tests — no CLI/env surface. No new files.

Regenerates regression-name allowlist (222->207), ratchets file-count allowlist (intel
entry removed, phase 4->3), makes 13 relocated allow-test-rule exemptions issue-ref-
compliant (ADR-456; prunes stale ids). No CONTEXT.md/ADR refs named the moved files.
lint:ci green.

Part of epic #1969. Closes #1976.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
2026-07-03 03:08:11 -04:00
parent 08f9956019
commit cb7ca4f3dc
26 changed files with 1964 additions and 1842 deletions

View File

@@ -127,3 +127,71 @@ test('shipped workflow flags behavior-unverified truths even on infrastructure p
test('standalone template per-truth guideline respects gaps_found precedence', () => {
assert.match(standalone, /becomes `human_needed`[\s\S]{0,80}?gaps_found/i);
});
// ────────────────────────────────────────────────────────────────────────
// Folded from tests/bug-3321-verifier-runs-probes.test.cjs — consolidation epic #1969 (B7 #1976)
// ────────────────────────────────────────────────────────────────────────
{
const { describe: __foldDescribe } = require('node:test');
__foldDescribe("folded:bug-3321-verifier-runs-probes (consolidation epic #1969 B7 #1976)", () => {
'use strict';
const { describe, test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const REPO_ROOT = path.join(__dirname, '..');
const VERIFIER_AGENT = path.join(REPO_ROOT, 'agents', 'gsd-verifier.md');
function verifierProbeContract(content) {
const sectionStart = content.indexOf('## Step 7c: Probe Execution');
const sectionEnd = content.indexOf('## Step 8:', sectionStart);
assert.notEqual(sectionStart, -1, 'verifier must define Step 7c');
assert.notEqual(sectionEnd, -1, 'verifier must close Step 7c before Step 8');
const section = content.slice(sectionStart, sectionEnd);
const codeBlocks = [...section.matchAll(/```bash\r?\n([\s\S]*?)\r?\n```/g)].map((match) => match[1].split(/\r?\n/).join('\n'));
const executionSteps = [...section.matchAll(/^\d+\.\s+(.+)$/gm)].map((match) => match[1]);
return {
title: 'Step 7c: Probe Execution',
conventionalDiscoveryCommand: codeBlocks[0]?.split('\n').find((line) => line.startsWith('find scripts')) || null,
declaredDiscoveryCommand: codeBlocks[0]?.split('\n').find((line) => line.startsWith('grep -R')) || null,
executionCommand: codeBlocks[1] || '',
executionSteps,
statusRows: [...section.matchAll(/^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|[^|]+\|\s*([^|]+)\|$/gm)]
.map((match) => ({ probe: match[1], command: match[2], statuses: match[3].trim() })),
summaryClaimsRejected: section.includes('SUMMARY.md probe pass claims are not evidence'),
};
}
describe('bug #3321: gsd-verifier runs probes instead of trusting SUMMARY claims', () => {
test('verifier prompt requires direct probe discovery and execution', () => {
const content = fs.readFileSync(VERIFIER_AGENT, 'utf8');
const contract = verifierProbeContract(content);
assert.equal(contract.title, 'Step 7c: Probe Execution');
assert.equal(contract.conventionalDiscoveryCommand, "find scripts -path '*/tests/probe-*.sh' -type f 2>/dev/null | sort");
assert.equal(
contract.declaredDiscoveryCommand,
"grep -R -n -E 'probe-[^[:space:]]+\\.sh|scripts/.*/tests/probe-.*\\.sh' \"$PHASE_DIR\"/*-PLAN.md \"$PHASE_DIR\"/*-SUMMARY.md 2>/dev/null",
);
assert.deepEqual(contract.executionSteps, [
'Build the `PROBES` list from explicit PLAN declarations first; include conventional `scripts/*/tests/probe-*.sh` when the phase is a migration/tooling phase or the success criteria mention probes.',
'For every documented probe path, if the file is missing or unreadable, mark `MISSING_PROBE` and set `status: gaps_found`. Do not require the executable bit because probes run through `bash "$probe"`.',
'Run each probe from the built `PROBES` list (declared + conventional) from the repository root:',
'Exit code 0 is PASS. Any non-zero exit is FAILED and must include stdout/stderr evidence in VERIFICATION.md.',
'Do not substitute executor narration, SUMMARY.md PASS-marker counts, or a different dry-run driver command for the probe result.',
]);
assert.equal(contract.executionCommand, 'for probe in "${PROBES[@]}"; do\n timeout 30s bash "$probe"\ndone');
assert.deepEqual(contract.statusRows, [{
probe: 'scripts/.../probe-name.sh',
command: 'bash "$probe"',
statuses: 'PASS / FAILED / MISSING_PROBE',
}]);
assert.equal(contract.summaryClaimsRejected, true);
});
});
});
}