enhance(#1279): harden node-test fail-first proof to require a NON-VACUOUS red

A violation fixture that crashes the negative test at load emits a file-named
# fail 1 — a crash, not the assertion firing red. Require a failing test named
distinctly from the file (isNonVacuousNodeTestRed), symmetric with the clean-pass
non-vacuity guard and the lint-rule specific-rule-id requirement. Closes the one
soundness asymmetry surfaced by adversarial review (code-review IN-01).
This commit is contained in:
Dave
2026-06-16 00:18:38 -04:00
parent 9aa5e88186
commit da52c53f34
2 changed files with 69 additions and 4 deletions

View File

@@ -248,6 +248,40 @@ export function isNonVacuousNodeTestPass(out: string, target: string): boolean {
return tapTestNames(out).some((n) => baseOf(n) !== tgtBase);
}
/** The names of FAILING (run) tests from TAP `not ok N - <name>` lines, excluding `# SKIP`/`# TODO`
* directives (a skipped/todo line never ran). The fail-first analog of `tapTestNames`. */
export function tapFailedTestNames(out: string): string[] {
if (typeof out !== 'string') return [];
const names: string[] = [];
const re = /^not ok \d+ - (.+)$/gm;
let m: RegExpExecArray | null;
while ((m = re.exec(out)) !== null) {
const rest = m[1];
if (/\s#\s*(?:SKIP|TODO)\b/i.test(rest)) continue; // skipped/todo did not run
names.push(rest.replace(/\s+#\s.*$/, '').trim());
}
return names;
}
/**
* A NON-VACUOUS node-test RED — the fail-first proof analog of `isNonVacuousNodeTestPass`. True iff
* the run reports `# fail >= 1` AND at least one FAILING test is named DISTINCTLY from the target file.
*
* Why the distinct-name guard: a violation fixture that makes the negative test CRASH at load
* (ENOENT / throw-on-require / syntax error) emits a FILE-NAMED `not ok 1 - <file>` with `# fail 1`.
* That is a crash, NOT the negative assertion firing red — so it must not "prove" the test is a
* regression guard (the RED-side mirror of the BL-01 vacuity hole on the pass side). Requiring a
* failing test named distinctly from the file closes that hole, symmetric with the clean-pass guard.
*
* KNOWN CONSTRAINT (fail-closed, not a hole): a negative test whose `test('...')` name is EXACTLY its
* own file basename is conservatively rejected — same benign authoring constraint, same safe direction.
*/
export function isNonVacuousNodeTestRed(out: string, target: string): boolean {
if (!isNodeTestRed(out)) return false; // no `# fail >= 1` summary -> not red (fail-closed)
const tgtBase = baseOf(target);
return tapFailedTestNames(out).some((n) => baseOf(n) !== tgtBase);
}
/** Number of file results in an eslint `--format json` report (0 if unparseable / not an array). */
export function eslintFileResultCount(jsonText: string): number {
try {
@@ -420,9 +454,11 @@ function defaultRunCheck(check: CheckDescriptor, cwd: string, timeoutMs?: number
* id to appear in the report (messages OR suppressedMessages — an inline-disabled violation still
* proves the rule has teeth, #1259 B1). Absent fixture / unresolvable eslint → not proven.
* - node-test: spawn the negative test (TAP) with `GSD_PROHIB_SUBJECT` set to the `violationFixture`
* — the CONVENTION (#1279) by which a negative test reads its subject-under-test — and require the
* run to go RED (`isNodeTestRed`: `# fail >= 1`). A toothless test that passes anyway → not proven.
* Absent fixture → not proven (fail-closed; NEVER falls back to attestation).
* — the CONVENTION (#1279) by which a negative test reads its subject-under-test — and require a
* NON-VACUOUS red (`isNonVacuousNodeTestRed`: `# fail >= 1` AND a failing test named distinctly
* from the file, so a load-CRASH on the bad subject is not mistaken for the assertion firing red).
* A toothless test that passes anyway → not proven. Absent fixture → not proven (fail-closed;
* NEVER falls back to attestation).
*/
function defaultProveFailFirst(check: CheckDescriptor, cwd: string, timeoutMs?: number): FailFirstProof {
try {
@@ -476,7 +512,7 @@ function defaultProveFailFirst(check: CheckDescriptor, cwd: string, timeoutMs?:
const stdout = e && typeof e === 'object' && 'stdout' in e ? (e as { stdout?: unknown }).stdout : '';
out = typeof stdout === 'string' ? stdout : '';
}
return { provenFailFirst: isNodeTestRed(out), method: 'violation-fixture' };
return { provenFailFirst: isNonVacuousNodeTestRed(out, check.target), method: 'violation-fixture' };
}
// Unknown kind — defensive; the LOCATE guard already rejects it.
return { provenFailFirst: false };

View File

@@ -385,6 +385,35 @@ describe('prohibition-enforcement real-runner helpers (#1259)', () => {
assert.equal(enforce.isNodeTestRed('no summary'), false, 'no parseable summary -> not RED (fail-closed for the prover)');
});
// ─── #1279 isNonVacuousNodeTestRed (FF-03 hardening) — a NON-VACUOUS red proof ───
// The fail-first PROOF must mirror the clean-pass non-vacuity discipline: a violation fixture that
// makes the negative test CRASH at load (ENOENT/throw/syntax) emits a FILE-NAMED `# fail 1` — the
// test never ran its assertion, so that is NOT proof the test is a regression guard. Require at
// least one FAILING test named DISTINCTLY from the target file (symmetric with isNonVacuousNodeTestPass).
test('isNonVacuousNodeTestRed: a file-named-only failure (a crash, not an assertion) does NOT prove fail-first', () => {
const enforce = require(ENFORCEMENT_LIB);
// node --test of a file that throws at load: `not ok 1 - <file>`, `# fail 1` — a crash, not a
// negative assertion firing red. Must NOT count as a non-vacuous red.
const crash = 'not ok 1 - neg.test.cjs\n# tests 1\n# pass 0\n# fail 1\n';
assert.equal(enforce.isNonVacuousNodeTestRed(crash, 'neg.test.cjs'), false,
'a file-named-only failure is a load crash, not a proven regression guard — fail-closed');
// BASENAME-NORMALIZED: node may report the file failure by an absolute/normalized path.
const crashAbs = 'not ok 1 - /tmp/x/neg.test.cjs\n# tests 1\n# pass 0\n# fail 1\n';
assert.equal(enforce.isNonVacuousNodeTestRed(crashAbs, 'neg.test.cjs'), false,
'an absolute-path file-named failure is still a crash (basename compare)');
// A genuine negative assertion firing red carries a descriptive name distinct from the file.
const realRed = 'not ok 1 - rejects the forbidden pattern\n# tests 1\n# pass 0\n# fail 1\n';
assert.equal(enforce.isNonVacuousNodeTestRed(realRed, 'neg.test.cjs'), true,
'a distinctly-named failing test is a genuine non-vacuous red — proves fail-first');
// No failure at all -> not red.
assert.equal(enforce.isNonVacuousNodeTestRed('ok 1 - guards\n# tests 1\n# pass 1\n# fail 0\n', 'neg.test.cjs'), false,
'# fail 0 is not red regardless of names');
// SKIP/TODO failing lines never ran -> excluded (mirror tapTestNames m1).
const skippedRed = 'not ok 1 - rejects the forbidden pattern # SKIP\n# tests 1\n# pass 0\n# fail 1\n';
assert.equal(enforce.isNonVacuousNodeTestRed(skippedRed, 'neg.test.cjs'), false,
'a SKIP/TODO failing line did not actually run -> not a proof');
});
test('tapTestNames EXCLUDES skipped/todo tests (they never ran, m1)', () => {
const enforce = require(ENFORCEMENT_LIB);
assert.deepEqual(enforce.tapTestNames('ok 1 - guards the must-NOT\nok 2 - other # SKIP\nok 3 - later # TODO\n'),