golem15/msd-core
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(#4724): classify Surefire/Failsafe XML as RED evidence (#4825)

Browse Source 
* test(#4724): add failing-first coverage for Surefire XML RED evidence

* fix(#4724): classify Surefire/Failsafe XML as RED evidence

check tdd-red-evidence parsed only node:test TAP, so a JVM project's
genuine Maven red scored INVALID_RED while hand-written synthetic TAP
scored RED_EVIDENCE_OK — the gate was passable only by fabricating its
input (issue #4724's measured repro).

classifyRedEvidence detects Surefire/Failsafe XML (a <testsuite> element)
and parses it by TAG-BOUNDARY scanning: each <testcase> owns its own tag
(self-closing) or the segment up to its </testcase> closer, so the
issue's warned-about spanning trap (a lazy lazy match from a green
self-closing case to the next closing tag) cannot misreport names. A
<failure> or <error> child marks the case failing; the target matches at
class granularity (exact classname, dotted-suffix, or method name). Any
parse anomaly degrades to not-failing — the module stays fail-closed and
PURE (no fs/clock; report freshness remains the workflow's run-start
check per the issue's implementation notes).

TAP classification is byte-identical: all existing fixtures stay green.

* test(#4724): pin the scanner hardening — truncation, TAP-message flip, CDATA phantom

* docs(#4724): backfill changeset PR number

---------

Co-authored-by: sim <sim@local>
This commit is contained in:
Tom Boucher
2026-09-17 11:30:23 -04:00
committed by GitHub
parent 7d0c6339d0
commit fb3e228a0d
3 changed files with 268 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
src/tdd-red-evidence.cts
View File

@@ -46,9 +46,12 @@ export interface RedEvidenceInput {
command: unknown;
/** The command's exit code. */
exitCode: unknown;
/** The command's combined stdout (TAP for node --test). */
/** The command's combined stdout (TAP for node --test; Surefire/Failsafe XML for Maven). */
output: unknown;
/** Identity of the target test named by the plan (its `test('...')` name). */
/**
* Identity of the target the plan named: the `test('...')` name for
* node:test runs, or the target CLASS for Surefire/Failsafe runs (#4724).
*/
targetTest: unknown;
/** Path of the test file the target test lives in (file-named failures are crashes). */
targetFile?: unknown;
@@ -90,6 +93,59 @@ function baseOf(p: unknown): string {
return typeof p === 'string' ? (p.split(/[\\/]/).pop() ?? p) : '';
}
/**
* #4724 — Surefire/Failsafe XML summary parsed by TAG-BOUNDARY scanning, NOT
* by a lazy `(.*?)<\/testcase>` regex: Surefire writes PASSING cases as
* self-closing `<testcase … />` elements, so a lazy span from a green case to
* the next closing tag swallows every case between and misreports names beside
* the right failure message. The scanner walks `<testcase` start tags; each
* case's extent is its own tag (self-closing) or the segment up to its
* `</testcase>` closer (testcases never nest). A `<failure` or `<error` child
* marks the case failing; the reported name is `classname#name`. Any parse
* anomaly degrades to "case not failing" — the module stays fail-closed.
*/
function parseSurefireSummary(output: string): {
tests: number;
pass: number;
fail: number;
failing_tests: string[];
} {
let tests = 0;
let fail = 0;
const failing_tests: string[] = [];
let idx = output.indexOf('<testcase');
while (idx !== -1) {
const tagEnd = output.indexOf('>', idx);
// #4724 review (HIGH): a `<testcase` with no `>` (truncated/stream-cut
// output) must terminate the scan — resuming the search at -1 clamps to 0
// and re-finds the same tag forever. Degrade to what was scanned so far
// (fail-closed: an incomplete report proves nothing).
if (tagEnd === -1) break;
tests++;
const tagText = output.slice(idx, tagEnd + 1);
// Self-closing (`/>`): the case has NO body — a failing sibling after it
// must never be attributed to this case (the #4724 spanning trap).
const selfClosing = output[tagEnd - 1] === '/';
const closeIdx = selfClosing ? -1 : output.indexOf('</testcase>', tagEnd);
// CDATA sections are verbatim captured output (e.g. System.out echoing
// test source) — their content must never be scanned for failure tags
// (#4724 review: CDATA phantom failures).
const body = (selfClosing
? ''
: output.slice(tagEnd + 1, closeIdx === -1 ? output.length : closeIdx)
).replace(/<!\[CDATA\[[\s\S]*?\]\]>/g, '');
const name = /name="([^"]*)"/.exec(tagText)?.[1] ?? '';
const classname = /classname="([^"]*)"/.exec(tagText)?.[1] ?? '';
if (body.includes('<failure') || body.includes('<error')) {
fail++;
failing_tests.push(classname ? `${classname}#${name}` : name);
}
idx = output.indexOf('<testcase', closeIdx === -1 ? tagEnd : closeIdx);
}
return { tests, pass: tests - fail, fail, failing_tests };
}
/** Coerce and validate the raw record's scalar fields. Returns null exit_code only when absent/non-numeric. */
function readInput(input: RedEvidenceInput): {
command: string;
@@ -131,8 +187,34 @@ export function classifyRedEvidence(input: RedEvidenceInput): RedEvidenceResult
};
}
const { command, exitCode, output, targetTest } = parsed;
const summary = parseNodeTestSummary(output);
const failing = tapFailedTestNames(output);
// #4724: format detection. Surefire/Failsafe XML (a `<testsuite>` element) is
// parsed by tag-boundary scanning; everything else stays on the proven TAP
// primitives. Unknown formats keep the TAP parse — which finds nothing — and
// fail closed below.
// #4724 review: BOTH element names must appear — a TAP error message that
// merely quotes "<testsuite>" (e.g. "expected <testsuite> was 2") must not
// flip a genuine TAP red into the XML path.
const isSurefireXml = output.includes('<testsuite') && output.includes('<testcase');
let summary: { tests: number; pass: number; fail: number };
let failing: string[];
if (isSurefireXml) {
const sf = parseSurefireSummary(output);
summary = { tests: sf.tests, pass: sf.pass, fail: sf.fail };
failing = sf.failing_tests;
} else {
summary = parseNodeTestSummary(output);
failing = tapFailedTestNames(output);
}
// Surefire target matching is class-level (the plan names the class); the
// entry's last classname segment or its test name must carry the target.
const surefireTargetMatches = isSurefireXml
? failing.some((entry: string) => {
const cls = entry.split('#')[0] ?? '';
const nm = entry.split('#')[1] ?? '';
return cls === targetTest || cls.endsWith(`.${targetTest}`) || nm === targetTest;
})
: failing.includes(targetTest);
const evidence = {
command,
exit_code: exitCode,
@@ -153,14 +235,15 @@ export function classifyRedEvidence(input: RedEvidenceInput): RedEvidenceResult
if (summary.tests === 0) {
return { verdict: 'INVALID_RED', reason: 'zero_tests_discovered', evidence };
}
// Nonzero exit but TAP reports no failing test: harness/setup/parser crash
// whose failure never reached a test assertion (or unparseable output).
// Nonzero exit but the report shows no failing test: harness/setup/parser
// crash whose failure never reached a test assertion (or unparseable output).
if (summary.fail === 0 || failing.length === 0) {
return { verdict: 'INVALID_RED', reason: 'nonzero_exit_without_test_failure', evidence };
}
// Fixture/load failure: every failing entry is named like the target FILE —
// node reports a load-time crash (throw-on-require, syntax error, ENOENT
// fixture) as a file-named `not ok 1 - <file>`, never the target test.
// (TAP-only shape: Surefire failures carry class#method names.)
const targetBase = baseOf(input?.targetFile ?? '');
const distinctlyNamed = failing.filter((n) => (targetBase ? baseOf(n) !== targetBase : true));
if (distinctlyNamed.length === 0) {
@@ -168,7 +251,7 @@ export function classifyRedEvidence(input: RedEvidenceInput): RedEvidenceResult
}
// Unrelated failure: real tests ran and failed, but none is the target test
// the plan named — an unrelated assertion must not authorize GREEN.
if (!distinctlyNamed.includes(targetTest)) {
if (!(isSurefireXml ? surefireTargetMatches : distinctlyNamed.includes(targetTest))) {
return { verdict: 'INVALID_RED', reason: 'no_target_test_failure', evidence };
}
return { verdict: 'RED_EVIDENCE_OK', reason: 'target_test_failed', evidence };