diff --git a/src/prohibition-enforcement.cts b/src/prohibition-enforcement.cts index 8249c1bb9..416bfe570 100644 --- a/src/prohibition-enforcement.cts +++ b/src/prohibition-enforcement.cts @@ -248,6 +248,40 @@ export function isNonVacuousNodeTestPass(out: string, target: string): boolean { return tapTestNames(out).some((n) => baseOf(n) !== tgtBase); } +/** The names of FAILING (run) tests from TAP `not ok N - ` lines, excluding `# SKIP`/`# TODO` + * directives (a skipped/todo line never ran). The fail-first analog of `tapTestNames`. */ +export function tapFailedTestNames(out: string): string[] { + if (typeof out !== 'string') return []; + const names: string[] = []; + const re = /^not ok \d+ - (.+)$/gm; + let m: RegExpExecArray | null; + while ((m = re.exec(out)) !== null) { + const rest = m[1]; + if (/\s#\s*(?:SKIP|TODO)\b/i.test(rest)) continue; // skipped/todo did not run + names.push(rest.replace(/\s+#\s.*$/, '').trim()); + } + return names; +} + +/** + * A NON-VACUOUS node-test RED — the fail-first proof analog of `isNonVacuousNodeTestPass`. True iff + * the run reports `# fail >= 1` AND at least one FAILING test is named DISTINCTLY from the target file. + * + * Why the distinct-name guard: a violation fixture that makes the negative test CRASH at load + * (ENOENT / throw-on-require / syntax error) emits a FILE-NAMED `not ok 1 - ` with `# fail 1`. + * That is a crash, NOT the negative assertion firing red — so it must not "prove" the test is a + * regression guard (the RED-side mirror of the BL-01 vacuity hole on the pass side). Requiring a + * failing test named distinctly from the file closes that hole, symmetric with the clean-pass guard. + * + * KNOWN CONSTRAINT (fail-closed, not a hole): a negative test whose `test('...')` name is EXACTLY its + * own file basename is conservatively rejected — same benign authoring constraint, same safe direction. + */ +export function isNonVacuousNodeTestRed(out: string, target: string): boolean { + if (!isNodeTestRed(out)) return false; // no `# fail >= 1` summary -> not red (fail-closed) + const tgtBase = baseOf(target); + return tapFailedTestNames(out).some((n) => baseOf(n) !== tgtBase); +} + /** Number of file results in an eslint `--format json` report (0 if unparseable / not an array). */ export function eslintFileResultCount(jsonText: string): number { try { @@ -420,9 +454,11 @@ function defaultRunCheck(check: CheckDescriptor, cwd: string, timeoutMs?: number * id to appear in the report (messages OR suppressedMessages — an inline-disabled violation still * proves the rule has teeth, #1259 B1). Absent fixture / unresolvable eslint → not proven. * - node-test: spawn the negative test (TAP) with `GSD_PROHIB_SUBJECT` set to the `violationFixture` - * — the CONVENTION (#1279) by which a negative test reads its subject-under-test — and require the - * run to go RED (`isNodeTestRed`: `# fail >= 1`). A toothless test that passes anyway → not proven. - * Absent fixture → not proven (fail-closed; NEVER falls back to attestation). + * — the CONVENTION (#1279) by which a negative test reads its subject-under-test — and require a + * NON-VACUOUS red (`isNonVacuousNodeTestRed`: `# fail >= 1` AND a failing test named distinctly + * from the file, so a load-CRASH on the bad subject is not mistaken for the assertion firing red). + * A toothless test that passes anyway → not proven. Absent fixture → not proven (fail-closed; + * NEVER falls back to attestation). */ function defaultProveFailFirst(check: CheckDescriptor, cwd: string, timeoutMs?: number): FailFirstProof { try { @@ -476,7 +512,7 @@ function defaultProveFailFirst(check: CheckDescriptor, cwd: string, timeoutMs?: const stdout = e && typeof e === 'object' && 'stdout' in e ? (e as { stdout?: unknown }).stdout : ''; out = typeof stdout === 'string' ? stdout : ''; } - return { provenFailFirst: isNodeTestRed(out), method: 'violation-fixture' }; + return { provenFailFirst: isNonVacuousNodeTestRed(out, check.target), method: 'violation-fixture' }; } // Unknown kind — defensive; the LOCATE guard already rejects it. return { provenFailFirst: false }; diff --git a/tests/prohibition-enforcement.test.cjs b/tests/prohibition-enforcement.test.cjs index 764e59e57..24e520a4a 100644 --- a/tests/prohibition-enforcement.test.cjs +++ b/tests/prohibition-enforcement.test.cjs @@ -385,6 +385,35 @@ describe('prohibition-enforcement real-runner helpers (#1259)', () => { assert.equal(enforce.isNodeTestRed('no summary'), false, 'no parseable summary -> not RED (fail-closed for the prover)'); }); + // ─── #1279 isNonVacuousNodeTestRed (FF-03 hardening) — a NON-VACUOUS red proof ─── + // The fail-first PROOF must mirror the clean-pass non-vacuity discipline: a violation fixture that + // makes the negative test CRASH at load (ENOENT/throw/syntax) emits a FILE-NAMED `# fail 1` — the + // test never ran its assertion, so that is NOT proof the test is a regression guard. Require at + // least one FAILING test named DISTINCTLY from the target file (symmetric with isNonVacuousNodeTestPass). + test('isNonVacuousNodeTestRed: a file-named-only failure (a crash, not an assertion) does NOT prove fail-first', () => { + const enforce = require(ENFORCEMENT_LIB); + // node --test of a file that throws at load: `not ok 1 - `, `# fail 1` — a crash, not a + // negative assertion firing red. Must NOT count as a non-vacuous red. + const crash = 'not ok 1 - neg.test.cjs\n# tests 1\n# pass 0\n# fail 1\n'; + assert.equal(enforce.isNonVacuousNodeTestRed(crash, 'neg.test.cjs'), false, + 'a file-named-only failure is a load crash, not a proven regression guard — fail-closed'); + // BASENAME-NORMALIZED: node may report the file failure by an absolute/normalized path. + const crashAbs = 'not ok 1 - /tmp/x/neg.test.cjs\n# tests 1\n# pass 0\n# fail 1\n'; + assert.equal(enforce.isNonVacuousNodeTestRed(crashAbs, 'neg.test.cjs'), false, + 'an absolute-path file-named failure is still a crash (basename compare)'); + // A genuine negative assertion firing red carries a descriptive name distinct from the file. + const realRed = 'not ok 1 - rejects the forbidden pattern\n# tests 1\n# pass 0\n# fail 1\n'; + assert.equal(enforce.isNonVacuousNodeTestRed(realRed, 'neg.test.cjs'), true, + 'a distinctly-named failing test is a genuine non-vacuous red — proves fail-first'); + // No failure at all -> not red. + assert.equal(enforce.isNonVacuousNodeTestRed('ok 1 - guards\n# tests 1\n# pass 1\n# fail 0\n', 'neg.test.cjs'), false, + '# fail 0 is not red regardless of names'); + // SKIP/TODO failing lines never ran -> excluded (mirror tapTestNames m1). + const skippedRed = 'not ok 1 - rejects the forbidden pattern # SKIP\n# tests 1\n# pass 0\n# fail 1\n'; + assert.equal(enforce.isNonVacuousNodeTestRed(skippedRed, 'neg.test.cjs'), false, + 'a SKIP/TODO failing line did not actually run -> not a proof'); + }); + test('tapTestNames EXCLUDES skipped/todo tests (they never ran, m1)', () => { const enforce = require(ENFORCEMENT_LIB); assert.deepEqual(enforce.tapTestNames('ok 1 - guards the must-NOT\nok 2 - other # SKIP\nok 3 - later # TODO\n'),