* test(#4724): add failing-first coverage for Surefire XML RED evidence * fix(#4724): classify Surefire/Failsafe XML as RED evidence check tdd-red-evidence parsed only node:test TAP, so a JVM project's genuine Maven red scored INVALID_RED while hand-written synthetic TAP scored RED_EVIDENCE_OK — the gate was passable only by fabricating its input (issue #4724's measured repro). classifyRedEvidence detects Surefire/Failsafe XML (a <testsuite> element) and parses it by TAG-BOUNDARY scanning: each <testcase> owns its own tag (self-closing) or the segment up to its </testcase> closer, so the issue's warned-about spanning trap (a lazy lazy match from a green self-closing case to the next closing tag) cannot misreport names. A <failure> or <error> child marks the case failing; the target matches at class granularity (exact classname, dotted-suffix, or method name). Any parse anomaly degrades to not-failing — the module stays fail-closed and PURE (no fs/clock; report freshness remains the workflow's run-start check per the issue's implementation notes). TAP classification is byte-identical: all existing fixtures stay green. * test(#4724): pin the scanner hardening — truncation, TAP-message flip, CDATA phantom * docs(#4724): backfill changeset PR number --------- Co-authored-by: sim <sim@local>
This commit is contained in:
Tom Boucher
5
.changeset/bold-deer-click.md
Normal file
5
.changeset/bold-deer-click.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
type: Fixed
|
||||
pr: 4825
|
||||
---
|
||||
**TDD red evidence accepts Surefire/Failsafe XML** — the tdd-red-evidence gate parsed only node:test TAP, so a JVM project's genuine Maven red scored INVALID_RED while hand-written synthetic TAP scored RED_EVIDENCE_OK (the gate was passable only by fabricating its input). Surefire/Failsafe XML reports now classify by tag-boundary scanning: a testcase with a <failure> or <error> child for the target class is a real red; self-closing passing cases are never spanned into failing names. (#4724)
|
||||
@@ -46,9 +46,12 @@ export interface RedEvidenceInput {
|
||||
command: unknown;
|
||||
/** The command's exit code. */
|
||||
exitCode: unknown;
|
||||
/** The command's combined stdout (TAP for node --test). */
|
||||
/** The command's combined stdout (TAP for node --test; Surefire/Failsafe XML for Maven). */
|
||||
output: unknown;
|
||||
/** Identity of the target test named by the plan (its `test('...')` name). */
|
||||
/**
|
||||
* Identity of the target the plan named: the `test('...')` name for
|
||||
* node:test runs, or the target CLASS for Surefire/Failsafe runs (#4724).
|
||||
*/
|
||||
targetTest: unknown;
|
||||
/** Path of the test file the target test lives in (file-named failures are crashes). */
|
||||
targetFile?: unknown;
|
||||
@@ -90,6 +93,59 @@ function baseOf(p: unknown): string {
|
||||
return typeof p === 'string' ? (p.split(/[\\/]/).pop() ?? p) : '';
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* #4724 — Surefire/Failsafe XML summary parsed by TAG-BOUNDARY scanning, NOT
|
||||
* by a lazy `(.*?)<\/testcase>` regex: Surefire writes PASSING cases as
|
||||
* self-closing `<testcase … />` elements, so a lazy span from a green case to
|
||||
* the next closing tag swallows every case between and misreports names beside
|
||||
* the right failure message. The scanner walks `<testcase` start tags; each
|
||||
* case's extent is its own tag (self-closing) or the segment up to its
|
||||
* `</testcase>` closer (testcases never nest). A `<failure` or `<error` child
|
||||
* marks the case failing; the reported name is `classname#name`. Any parse
|
||||
* anomaly degrades to "case not failing" — the module stays fail-closed.
|
||||
*/
|
||||
function parseSurefireSummary(output: string): {
|
||||
tests: number;
|
||||
pass: number;
|
||||
fail: number;
|
||||
failing_tests: string[];
|
||||
} {
|
||||
let tests = 0;
|
||||
let fail = 0;
|
||||
const failing_tests: string[] = [];
|
||||
let idx = output.indexOf('<testcase');
|
||||
while (idx !== -1) {
|
||||
const tagEnd = output.indexOf('>', idx);
|
||||
// #4724 review (HIGH): a `<testcase` with no `>` (truncated/stream-cut
|
||||
// output) must terminate the scan — resuming the search at -1 clamps to 0
|
||||
// and re-finds the same tag forever. Degrade to what was scanned so far
|
||||
// (fail-closed: an incomplete report proves nothing).
|
||||
if (tagEnd === -1) break;
|
||||
tests++;
|
||||
const tagText = output.slice(idx, tagEnd + 1);
|
||||
// Self-closing (`/>`): the case has NO body — a failing sibling after it
|
||||
// must never be attributed to this case (the #4724 spanning trap).
|
||||
const selfClosing = output[tagEnd - 1] === '/';
|
||||
const closeIdx = selfClosing ? -1 : output.indexOf('</testcase>', tagEnd);
|
||||
// CDATA sections are verbatim captured output (e.g. System.out echoing
|
||||
// test source) — their content must never be scanned for failure tags
|
||||
// (#4724 review: CDATA phantom failures).
|
||||
const body = (selfClosing
|
||||
? ''
|
||||
: output.slice(tagEnd + 1, closeIdx === -1 ? output.length : closeIdx)
|
||||
).replace(/<!\[CDATA\[[\s\S]*?\]\]>/g, '');
|
||||
const name = /name="([^"]*)"/.exec(tagText)?.[1] ?? '';
|
||||
const classname = /classname="([^"]*)"/.exec(tagText)?.[1] ?? '';
|
||||
if (body.includes('<failure') || body.includes('<error')) {
|
||||
fail++;
|
||||
failing_tests.push(classname ? `${classname}#${name}` : name);
|
||||
}
|
||||
idx = output.indexOf('<testcase', closeIdx === -1 ? tagEnd : closeIdx);
|
||||
}
|
||||
return { tests, pass: tests - fail, fail, failing_tests };
|
||||
}
|
||||
|
||||
/** Coerce and validate the raw record's scalar fields. Returns null exit_code only when absent/non-numeric. */
|
||||
function readInput(input: RedEvidenceInput): {
|
||||
command: string;
|
||||
@@ -131,8 +187,34 @@ export function classifyRedEvidence(input: RedEvidenceInput): RedEvidenceResult
|
||||
};
|
||||
}
|
||||
const { command, exitCode, output, targetTest } = parsed;
|
||||
const summary = parseNodeTestSummary(output);
|
||||
const failing = tapFailedTestNames(output);
|
||||
|
||||
// #4724: format detection. Surefire/Failsafe XML (a `<testsuite>` element) is
|
||||
// parsed by tag-boundary scanning; everything else stays on the proven TAP
|
||||
// primitives. Unknown formats keep the TAP parse — which finds nothing — and
|
||||
// fail closed below.
|
||||
// #4724 review: BOTH element names must appear — a TAP error message that
|
||||
// merely quotes "<testsuite>" (e.g. "expected <testsuite> was 2") must not
|
||||
// flip a genuine TAP red into the XML path.
|
||||
const isSurefireXml = output.includes('<testsuite') && output.includes('<testcase');
|
||||
let summary: { tests: number; pass: number; fail: number };
|
||||
let failing: string[];
|
||||
if (isSurefireXml) {
|
||||
const sf = parseSurefireSummary(output);
|
||||
summary = { tests: sf.tests, pass: sf.pass, fail: sf.fail };
|
||||
failing = sf.failing_tests;
|
||||
} else {
|
||||
summary = parseNodeTestSummary(output);
|
||||
failing = tapFailedTestNames(output);
|
||||
}
|
||||
// Surefire target matching is class-level (the plan names the class); the
|
||||
// entry's last classname segment or its test name must carry the target.
|
||||
const surefireTargetMatches = isSurefireXml
|
||||
? failing.some((entry: string) => {
|
||||
const cls = entry.split('#')[0] ?? '';
|
||||
const nm = entry.split('#')[1] ?? '';
|
||||
return cls === targetTest || cls.endsWith(`.${targetTest}`) || nm === targetTest;
|
||||
})
|
||||
: failing.includes(targetTest);
|
||||
const evidence = {
|
||||
command,
|
||||
exit_code: exitCode,
|
||||
@@ -153,14 +235,15 @@ export function classifyRedEvidence(input: RedEvidenceInput): RedEvidenceResult
|
||||
if (summary.tests === 0) {
|
||||
return { verdict: 'INVALID_RED', reason: 'zero_tests_discovered', evidence };
|
||||
}
|
||||
// Nonzero exit but TAP reports no failing test: harness/setup/parser crash
|
||||
// whose failure never reached a test assertion (or unparseable output).
|
||||
// Nonzero exit but the report shows no failing test: harness/setup/parser
|
||||
// crash whose failure never reached a test assertion (or unparseable output).
|
||||
if (summary.fail === 0 || failing.length === 0) {
|
||||
return { verdict: 'INVALID_RED', reason: 'nonzero_exit_without_test_failure', evidence };
|
||||
}
|
||||
// Fixture/load failure: every failing entry is named like the target FILE —
|
||||
// node reports a load-time crash (throw-on-require, syntax error, ENOENT
|
||||
// fixture) as a file-named `not ok 1 - <file>`, never the target test.
|
||||
// (TAP-only shape: Surefire failures carry class#method names.)
|
||||
const targetBase = baseOf(input?.targetFile ?? '');
|
||||
const distinctlyNamed = failing.filter((n) => (targetBase ? baseOf(n) !== targetBase : true));
|
||||
if (distinctlyNamed.length === 0) {
|
||||
@@ -168,7 +251,7 @@ export function classifyRedEvidence(input: RedEvidenceInput): RedEvidenceResult
|
||||
}
|
||||
// Unrelated failure: real tests ran and failed, but none is the target test
|
||||
// the plan named — an unrelated assertion must not authorize GREEN.
|
||||
if (!distinctlyNamed.includes(targetTest)) {
|
||||
if (!(isSurefireXml ? surefireTargetMatches : distinctlyNamed.includes(targetTest))) {
|
||||
return { verdict: 'INVALID_RED', reason: 'no_target_test_failure', evidence };
|
||||
}
|
||||
return { verdict: 'RED_EVIDENCE_OK', reason: 'target_test_failed', evidence };
|
||||
|
||||
@@ -247,3 +247,176 @@ describe('executor spec requires intentional RED evidence before GREEN (#3770)',
|
||||
'execute-mvp-tdd.md must halt GREEN on INVALID_RED');
|
||||
});
|
||||
});
|
||||
|
||||
// ── #4724 — Surefire/Failsafe XML RED evidence ────────────────────────────────
|
||||
// A JVM project's genuine red is a Surefire/Failsafe XML report, not node:test
|
||||
// TAP. The gate used to parse only TAP, so a real Maven red scored
|
||||
// INVALID_RED while hand-written synthetic TAP scored RED_EVIDENCE_OK — the
|
||||
// gate was passable only by fabricating its input. The XML scanner walks
|
||||
// <testcase> tag boundaries (NOT a lazy spanning regex: Surefire writes
|
||||
// passing cases self-closing, so `(.*?)</testcase>` spans from a green case
|
||||
// to the next closing tag and reports wrong method names).
|
||||
|
||||
describe('#4724 — Surefire/Failsafe XML RED evidence', () => {
|
||||
const XML_HEAD = '<?xml version="1.0" encoding="UTF-8"?>\n';
|
||||
|
||||
function surefire(cases) {
|
||||
const body = cases.join('\n');
|
||||
return `${XML_HEAD}<testsuite name="com.example.AppTest" tests="${cases.length}" failures="1" errors="0">\n${body}\n</testsuite>`;
|
||||
}
|
||||
|
||||
const failingCase = (cls, name) =>
|
||||
`<testcase name="${name}" classname="${cls}" time="0.01"><failure message="expected 1 was 2">1 != 2</failure></testcase>`;
|
||||
const errorCase = (cls, name) =>
|
||||
`<testcase name="${name}" classname="${cls}" time="0.01"><error message="boom">NullPointerException</error></testcase>`;
|
||||
const greenCase = (cls, name) =>
|
||||
`<testcase name="${name}" classname="${cls}" time="0.01"/>`;
|
||||
|
||||
const INPUT = {
|
||||
command: 'mvn -Dtest=AppTest test',
|
||||
exitCode: 1,
|
||||
targetTest: 'AppTest',
|
||||
targetFile: 'src/test/java/com/example/AppTest.java',
|
||||
};
|
||||
|
||||
test('a genuine Surefire red with the target class failing classifies RED_EVIDENCE_OK (#4724)', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: surefire([failingCase('com.example.AppTest', 'divides_by_zero')]),
|
||||
});
|
||||
assert.equal(result.verdict, 'RED_EVIDENCE_OK');
|
||||
assert.equal(result.reason, 'target_test_failed');
|
||||
assert.equal(result.evidence.fail, 1);
|
||||
assert.ok(
|
||||
result.evidence.failing_tests.some((n) => n.includes('AppTest')),
|
||||
'failing_tests must carry the classname so the target matches',
|
||||
);
|
||||
});
|
||||
|
||||
test('an unrelated Surefire failure is not the target test', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: surefire([failingCase('com.other.UnrelatedTest', 'unrelated_case')]),
|
||||
});
|
||||
assert.equal(result.verdict, 'INVALID_RED');
|
||||
assert.equal(result.reason, 'no_target_test_failure');
|
||||
});
|
||||
|
||||
test('an all-green self-closing Surefire report is unexpected_green', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
exitCode: 0,
|
||||
output: surefire([greenCase('com.example.AppTest', 'passes'), greenCase('com.example.AppTest', 'passes2')]),
|
||||
});
|
||||
assert.equal(result.verdict, 'INVALID_RED');
|
||||
assert.equal(result.reason, 'unexpected_green');
|
||||
});
|
||||
|
||||
test('self-closing passing cases are not spanned into failing names (#4725-class trap)', () => {
|
||||
// The issue's trap: a lazy /(.*?)<\/testcase>/ regex starting at the green
|
||||
// self-closing case spans to the NEXT closing tag, misreporting the green
|
||||
// case's name beside the failing one. Boundary scanning must not.
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: surefire([
|
||||
greenCase('com.example.AppTest', 'green_before_failure'),
|
||||
failingCase('com.example.AppTest', 'the_real_failure'),
|
||||
]),
|
||||
});
|
||||
assert.equal(result.verdict, 'RED_EVIDENCE_OK');
|
||||
assert.deepEqual(
|
||||
result.evidence.failing_tests,
|
||||
['com.example.AppTest#the_real_failure'],
|
||||
'exactly the failing case is reported — no spanned green-case names',
|
||||
);
|
||||
});
|
||||
|
||||
test('an <error> child counts as a failing testcase', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: surefire([errorCase('com.example.AppTest', 'explodes')]),
|
||||
});
|
||||
assert.equal(result.verdict, 'RED_EVIDENCE_OK');
|
||||
assert.equal(result.evidence.fail, 1);
|
||||
});
|
||||
|
||||
test('a testcase-free Surefire report is zero_tests_discovered', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: `${XML_HEAD}<testsuite name="com.example.AppTest" tests="0" failures="0" errors="0"></testsuite>`,
|
||||
});
|
||||
assert.equal(result.verdict, 'INVALID_RED');
|
||||
assert.equal(result.reason, 'zero_tests_discovered');
|
||||
});
|
||||
|
||||
test('Failsafe XML (same schema) classifies like Surefire', () => {
|
||||
const result = classifyRedEvidence({
|
||||
command: 'mvn -Dtest=AppIT verify',
|
||||
exitCode: 1,
|
||||
targetTest: 'AppIT',
|
||||
targetFile: 'src/test/java/com/example/AppIT.java',
|
||||
output: `${XML_HEAD}<testsuite name="com.example.AppIT" tests="1" failures="1" errors="0">\n${failingCase('com.example.AppIT', 'integration_red')}\n</testsuite>`,
|
||||
});
|
||||
assert.equal(result.verdict, 'RED_EVIDENCE_OK');
|
||||
assert.equal(result.reason, 'target_test_failed');
|
||||
});
|
||||
});
|
||||
|
||||
// ── #4724 review hardening — scanner edge cases ──────────────────────────────
|
||||
|
||||
test('#4724: a <testcase with no > (truncated output) terminates and fails closed', () => {
|
||||
// Pre-hardening this hung forever: indexOf('<testcase', -1) clamps to 0 and
|
||||
// re-found the same tag. Degrade to what was scanned — an incomplete report
|
||||
// proves nothing.
|
||||
const result = classifyRedEvidence({
|
||||
command: 'mvn test',
|
||||
exitCode: 1,
|
||||
targetTest: 'AppTest',
|
||||
output: '<?xml version="1.0"?><testsuite><testcase name="x" classname="C"',
|
||||
});
|
||||
assert.equal(result.verdict, 'INVALID_RED');
|
||||
});
|
||||
|
||||
test('#4724: a TAP red whose message quotes <testsuite> stays on the TAP path', () => {
|
||||
// Format detection requires BOTH <testsuite and <testcase: a genuine TAP
|
||||
// red whose error message merely quotes "<testsuite>" must not flip to the
|
||||
// XML path (which would misread it as zero_tests_discovered).
|
||||
const tap = [
|
||||
'TAP version 13',
|
||||
'# Subtest: the test',
|
||||
'not ok 1 - expected <testsuite> was 2',
|
||||
' ---',
|
||||
' error: |-',
|
||||
' expected <testsuite> was 2',
|
||||
' ...',
|
||||
'# tests 1',
|
||||
'# pass 0',
|
||||
'# fail 1',
|
||||
].join('\n');
|
||||
const result = classifyRedEvidence({
|
||||
command: 'node --test',
|
||||
exitCode: 1,
|
||||
targetTest: 'the test',
|
||||
output: tap,
|
||||
});
|
||||
assert.equal(result.status ?? result.verdict, 'INVALID_RED');
|
||||
assert.equal(result.reason, 'no_target_test_failure',
|
||||
'the TAP path must classify it (fail=1), not the XML path (zero tests)');
|
||||
assert.equal(result.evidence.fail, 1);
|
||||
});
|
||||
|
||||
test('#4724: CDATA sections in a passing case are never scanned as failures', () => {
|
||||
// A passing case whose captured System.out (CDATA) echoes an <error .../>
|
||||
// literal must not count as failing — CDATA is verbatim content.
|
||||
const cd = '<testcase name="prints" classname="com.example.AppTest"><system-out><![CDATA[echo <error x/></system-out>]]></testcase>';
|
||||
const fl = '<testcase name="x" classname="com.other.Unrelated"><failure message="e">1 != 2</failure></testcase>';
|
||||
const result = classifyRedEvidence({
|
||||
command: 'mvn test',
|
||||
exitCode: 1,
|
||||
targetTest: 'UnrelatedTest',
|
||||
output: `<?xml version="1.0"?>\n<testsuite>\n${cd}\n${fl}\n</testsuite>`,
|
||||
});
|
||||
assert.equal(result.reason, 'no_target_test_failure',
|
||||
'the CDATA phantom must not flip the unrelated failure into a target match');
|
||||
assert.equal(result.evidence.fail, 1, 'only the real failing case counts');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user