aac6d3635b
bug-2801, bug-3072, bug-3321 each read .md workflow files from disk and
parse fenced code blocks with /\`\`\`bash\n(...)\`\`\`/ — on a Windows
checkout with autocrlf=true the file content is CRLF, the regex never
matches, code blocks come back empty, and the assertion fails with
"expected bash code blocks in workflow" (and similar).
Fix: accept optional \r before each literal \n in the fence delimiter
and split lines on /\r?\n/. Same pattern bug-2839 had with its local
parseFrontmatter copy (already migrated to the CRLF-aware shared helper
in 1be0e4e2). bug-2995 + tests/security-scan.test.cjs also match this
grep but use \n in literal-string fixtures (not file-content regexes);
not affected.
Validated: holodeck (ubuntu docker) 11224/0 pass.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
59 lines
3.3 KiB
JavaScript
59 lines
3.3 KiB
JavaScript
'use strict';
|
|
|
|
const { describe, test } = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
|
|
const REPO_ROOT = path.join(__dirname, '..');
|
|
const VERIFIER_AGENT = path.join(REPO_ROOT, 'agents', 'gsd-verifier.md');
|
|
|
|
function verifierProbeContract(content) {
|
|
const sectionStart = content.indexOf('## Step 7c: Probe Execution');
|
|
const sectionEnd = content.indexOf('## Step 8:', sectionStart);
|
|
assert.notEqual(sectionStart, -1, 'verifier must define Step 7c');
|
|
assert.notEqual(sectionEnd, -1, 'verifier must close Step 7c before Step 8');
|
|
|
|
const section = content.slice(sectionStart, sectionEnd);
|
|
const codeBlocks = [...section.matchAll(/```bash\r?\n([\s\S]*?)\r?\n```/g)].map((match) => match[1].split(/\r?\n/).join('\n'));
|
|
const executionSteps = [...section.matchAll(/^\d+\.\s+(.+)$/gm)].map((match) => match[1]);
|
|
return {
|
|
title: 'Step 7c: Probe Execution',
|
|
conventionalDiscoveryCommand: codeBlocks[0]?.split('\n').find((line) => line.startsWith('find scripts')) || null,
|
|
declaredDiscoveryCommand: codeBlocks[0]?.split('\n').find((line) => line.startsWith('grep -R')) || null,
|
|
executionCommand: codeBlocks[1] || '',
|
|
executionSteps,
|
|
statusRows: [...section.matchAll(/^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|[^|]+\|\s*([^|]+)\|$/gm)]
|
|
.map((match) => ({ probe: match[1], command: match[2], statuses: match[3].trim() })),
|
|
summaryClaimsRejected: section.includes('SUMMARY.md probe pass claims are not evidence'),
|
|
};
|
|
}
|
|
|
|
describe('bug #3321: gsd-verifier runs probes instead of trusting SUMMARY claims', () => {
|
|
test('verifier prompt requires direct probe discovery and execution', () => {
|
|
const content = fs.readFileSync(VERIFIER_AGENT, 'utf8');
|
|
const contract = verifierProbeContract(content);
|
|
|
|
assert.equal(contract.title, 'Step 7c: Probe Execution');
|
|
assert.equal(contract.conventionalDiscoveryCommand, "find scripts -path '*/tests/probe-*.sh' -type f 2>/dev/null | sort");
|
|
assert.equal(
|
|
contract.declaredDiscoveryCommand,
|
|
"grep -R -n -E 'probe-[^[:space:]]+\\.sh|scripts/.*/tests/probe-.*\\.sh' \"$PHASE_DIR\"/*-PLAN.md \"$PHASE_DIR\"/*-SUMMARY.md 2>/dev/null",
|
|
);
|
|
assert.deepEqual(contract.executionSteps, [
|
|
'Build the `PROBES` list from explicit PLAN declarations first; include conventional `scripts/*/tests/probe-*.sh` when the phase is a migration/tooling phase or the success criteria mention probes.',
|
|
'For every documented probe path, if the file is missing or unreadable, mark `MISSING_PROBE` and set `status: gaps_found`. Do not require the executable bit because probes run through `bash "$probe"`.',
|
|
'Run each probe from the built `PROBES` list (declared + conventional) from the repository root:',
|
|
'Exit code 0 is PASS. Any non-zero exit is FAILED and must include stdout/stderr evidence in VERIFICATION.md.',
|
|
'Do not substitute executor narration, SUMMARY.md PASS-marker counts, or a different dry-run driver command for the probe result.',
|
|
]);
|
|
assert.equal(contract.executionCommand, 'for probe in "${PROBES[@]}"; do\n timeout 30s bash "$probe"\ndone');
|
|
assert.deepEqual(contract.statusRows, [{
|
|
probe: 'scripts/.../probe-name.sh',
|
|
command: 'bash "$probe"',
|
|
statuses: 'PASS / FAILED / MISSING_PROBE',
|
|
}]);
|
|
assert.equal(contract.summaryClaimsRejected, true);
|
|
});
|
|
});
|