* test(#4724): add failing-first coverage for Surefire XML RED evidence * fix(#4724): classify Surefire/Failsafe XML as RED evidence check tdd-red-evidence parsed only node:test TAP, so a JVM project's genuine Maven red scored INVALID_RED while hand-written synthetic TAP scored RED_EVIDENCE_OK — the gate was passable only by fabricating its input (issue #4724's measured repro). classifyRedEvidence detects Surefire/Failsafe XML (a <testsuite> element) and parses it by TAG-BOUNDARY scanning: each <testcase> owns its own tag (self-closing) or the segment up to its </testcase> closer, so the issue's warned-about spanning trap (a lazy lazy match from a green self-closing case to the next closing tag) cannot misreport names. A <failure> or <error> child marks the case failing; the target matches at class granularity (exact classname, dotted-suffix, or method name). Any parse anomaly degrades to not-failing — the module stays fail-closed and PURE (no fs/clock; report freshness remains the workflow's run-start check per the issue's implementation notes). TAP classification is byte-identical: all existing fixtures stay green. * test(#4724): pin the scanner hardening — truncation, TAP-message flip, CDATA phantom * docs(#4724): backfill changeset PR number --------- Co-authored-by: sim <sim@local>
This commit is contained in:
Tom Boucher
@@ -247,3 +247,176 @@ describe('executor spec requires intentional RED evidence before GREEN (#3770)',
|
||||
'execute-mvp-tdd.md must halt GREEN on INVALID_RED');
|
||||
});
|
||||
});
|
||||
|
||||
// ── #4724 — Surefire/Failsafe XML RED evidence ────────────────────────────────
|
||||
// A JVM project's genuine red is a Surefire/Failsafe XML report, not node:test
|
||||
// TAP. The gate used to parse only TAP, so a real Maven red scored
|
||||
// INVALID_RED while hand-written synthetic TAP scored RED_EVIDENCE_OK — the
|
||||
// gate was passable only by fabricating its input. The XML scanner walks
|
||||
// <testcase> tag boundaries (NOT a lazy spanning regex: Surefire writes
|
||||
// passing cases self-closing, so `(.*?)</testcase>` spans from a green case
|
||||
// to the next closing tag and reports wrong method names).
|
||||
|
||||
describe('#4724 — Surefire/Failsafe XML RED evidence', () => {
|
||||
const XML_HEAD = '<?xml version="1.0" encoding="UTF-8"?>\n';
|
||||
|
||||
function surefire(cases) {
|
||||
const body = cases.join('\n');
|
||||
return `${XML_HEAD}<testsuite name="com.example.AppTest" tests="${cases.length}" failures="1" errors="0">\n${body}\n</testsuite>`;
|
||||
}
|
||||
|
||||
const failingCase = (cls, name) =>
|
||||
`<testcase name="${name}" classname="${cls}" time="0.01"><failure message="expected 1 was 2">1 != 2</failure></testcase>`;
|
||||
const errorCase = (cls, name) =>
|
||||
`<testcase name="${name}" classname="${cls}" time="0.01"><error message="boom">NullPointerException</error></testcase>`;
|
||||
const greenCase = (cls, name) =>
|
||||
`<testcase name="${name}" classname="${cls}" time="0.01"/>`;
|
||||
|
||||
const INPUT = {
|
||||
command: 'mvn -Dtest=AppTest test',
|
||||
exitCode: 1,
|
||||
targetTest: 'AppTest',
|
||||
targetFile: 'src/test/java/com/example/AppTest.java',
|
||||
};
|
||||
|
||||
test('a genuine Surefire red with the target class failing classifies RED_EVIDENCE_OK (#4724)', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: surefire([failingCase('com.example.AppTest', 'divides_by_zero')]),
|
||||
});
|
||||
assert.equal(result.verdict, 'RED_EVIDENCE_OK');
|
||||
assert.equal(result.reason, 'target_test_failed');
|
||||
assert.equal(result.evidence.fail, 1);
|
||||
assert.ok(
|
||||
result.evidence.failing_tests.some((n) => n.includes('AppTest')),
|
||||
'failing_tests must carry the classname so the target matches',
|
||||
);
|
||||
});
|
||||
|
||||
test('an unrelated Surefire failure is not the target test', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: surefire([failingCase('com.other.UnrelatedTest', 'unrelated_case')]),
|
||||
});
|
||||
assert.equal(result.verdict, 'INVALID_RED');
|
||||
assert.equal(result.reason, 'no_target_test_failure');
|
||||
});
|
||||
|
||||
test('an all-green self-closing Surefire report is unexpected_green', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
exitCode: 0,
|
||||
output: surefire([greenCase('com.example.AppTest', 'passes'), greenCase('com.example.AppTest', 'passes2')]),
|
||||
});
|
||||
assert.equal(result.verdict, 'INVALID_RED');
|
||||
assert.equal(result.reason, 'unexpected_green');
|
||||
});
|
||||
|
||||
test('self-closing passing cases are not spanned into failing names (#4725-class trap)', () => {
|
||||
// The issue's trap: a lazy /(.*?)<\/testcase>/ regex starting at the green
|
||||
// self-closing case spans to the NEXT closing tag, misreporting the green
|
||||
// case's name beside the failing one. Boundary scanning must not.
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: surefire([
|
||||
greenCase('com.example.AppTest', 'green_before_failure'),
|
||||
failingCase('com.example.AppTest', 'the_real_failure'),
|
||||
]),
|
||||
});
|
||||
assert.equal(result.verdict, 'RED_EVIDENCE_OK');
|
||||
assert.deepEqual(
|
||||
result.evidence.failing_tests,
|
||||
['com.example.AppTest#the_real_failure'],
|
||||
'exactly the failing case is reported — no spanned green-case names',
|
||||
);
|
||||
});
|
||||
|
||||
test('an <error> child counts as a failing testcase', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: surefire([errorCase('com.example.AppTest', 'explodes')]),
|
||||
});
|
||||
assert.equal(result.verdict, 'RED_EVIDENCE_OK');
|
||||
assert.equal(result.evidence.fail, 1);
|
||||
});
|
||||
|
||||
test('a testcase-free Surefire report is zero_tests_discovered', () => {
|
||||
const result = classifyRedEvidence({
|
||||
...INPUT,
|
||||
output: `${XML_HEAD}<testsuite name="com.example.AppTest" tests="0" failures="0" errors="0"></testsuite>`,
|
||||
});
|
||||
assert.equal(result.verdict, 'INVALID_RED');
|
||||
assert.equal(result.reason, 'zero_tests_discovered');
|
||||
});
|
||||
|
||||
test('Failsafe XML (same schema) classifies like Surefire', () => {
|
||||
const result = classifyRedEvidence({
|
||||
command: 'mvn -Dtest=AppIT verify',
|
||||
exitCode: 1,
|
||||
targetTest: 'AppIT',
|
||||
targetFile: 'src/test/java/com/example/AppIT.java',
|
||||
output: `${XML_HEAD}<testsuite name="com.example.AppIT" tests="1" failures="1" errors="0">\n${failingCase('com.example.AppIT', 'integration_red')}\n</testsuite>`,
|
||||
});
|
||||
assert.equal(result.verdict, 'RED_EVIDENCE_OK');
|
||||
assert.equal(result.reason, 'target_test_failed');
|
||||
});
|
||||
});
|
||||
|
||||
// ── #4724 review hardening — scanner edge cases ──────────────────────────────
|
||||
|
||||
test('#4724: a <testcase with no > (truncated output) terminates and fails closed', () => {
|
||||
// Pre-hardening this hung forever: indexOf('<testcase', -1) clamps to 0 and
|
||||
// re-found the same tag. Degrade to what was scanned — an incomplete report
|
||||
// proves nothing.
|
||||
const result = classifyRedEvidence({
|
||||
command: 'mvn test',
|
||||
exitCode: 1,
|
||||
targetTest: 'AppTest',
|
||||
output: '<?xml version="1.0"?><testsuite><testcase name="x" classname="C"',
|
||||
});
|
||||
assert.equal(result.verdict, 'INVALID_RED');
|
||||
});
|
||||
|
||||
test('#4724: a TAP red whose message quotes <testsuite> stays on the TAP path', () => {
|
||||
// Format detection requires BOTH <testsuite and <testcase: a genuine TAP
|
||||
// red whose error message merely quotes "<testsuite>" must not flip to the
|
||||
// XML path (which would misread it as zero_tests_discovered).
|
||||
const tap = [
|
||||
'TAP version 13',
|
||||
'# Subtest: the test',
|
||||
'not ok 1 - expected <testsuite> was 2',
|
||||
' ---',
|
||||
' error: |-',
|
||||
' expected <testsuite> was 2',
|
||||
' ...',
|
||||
'# tests 1',
|
||||
'# pass 0',
|
||||
'# fail 1',
|
||||
].join('\n');
|
||||
const result = classifyRedEvidence({
|
||||
command: 'node --test',
|
||||
exitCode: 1,
|
||||
targetTest: 'the test',
|
||||
output: tap,
|
||||
});
|
||||
assert.equal(result.status ?? result.verdict, 'INVALID_RED');
|
||||
assert.equal(result.reason, 'no_target_test_failure',
|
||||
'the TAP path must classify it (fail=1), not the XML path (zero tests)');
|
||||
assert.equal(result.evidence.fail, 1);
|
||||
});
|
||||
|
||||
test('#4724: CDATA sections in a passing case are never scanned as failures', () => {
|
||||
// A passing case whose captured System.out (CDATA) echoes an <error .../>
|
||||
// literal must not count as failing — CDATA is verbatim content.
|
||||
const cd = '<testcase name="prints" classname="com.example.AppTest"><system-out><![CDATA[echo <error x/></system-out>]]></testcase>';
|
||||
const fl = '<testcase name="x" classname="com.other.Unrelated"><failure message="e">1 != 2</failure></testcase>';
|
||||
const result = classifyRedEvidence({
|
||||
command: 'mvn test',
|
||||
exitCode: 1,
|
||||
targetTest: 'UnrelatedTest',
|
||||
output: `<?xml version="1.0"?>\n<testsuite>\n${cd}\n${fl}\n</testsuite>`,
|
||||
});
|
||||
assert.equal(result.reason, 'no_target_test_failure',
|
||||
'the CDATA phantom must not flip the unrelated failure into a target match');
|
||||
assert.equal(result.evidence.fail, 1, 'only the real failing case counts');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user