enhance(#1279): harden node-test fail-first proof to require a NON-VACUOUS red
A violation fixture that crashes the negative test at load emits a file-named # fail 1 — a crash, not the assertion firing red. Require a failing test named distinctly from the file (isNonVacuousNodeTestRed), symmetric with the clean-pass non-vacuity guard and the lint-rule specific-rule-id requirement. Closes the one soundness asymmetry surfaced by adversarial review (code-review IN-01).
This commit is contained in:
@@ -248,6 +248,40 @@ export function isNonVacuousNodeTestPass(out: string, target: string): boolean {
|
||||
return tapTestNames(out).some((n) => baseOf(n) !== tgtBase);
|
||||
}
|
||||
|
||||
/** The names of FAILING (run) tests from TAP `not ok N - <name>` lines, excluding `# SKIP`/`# TODO`
|
||||
* directives (a skipped/todo line never ran). The fail-first analog of `tapTestNames`. */
|
||||
export function tapFailedTestNames(out: string): string[] {
|
||||
if (typeof out !== 'string') return [];
|
||||
const names: string[] = [];
|
||||
const re = /^not ok \d+ - (.+)$/gm;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(out)) !== null) {
|
||||
const rest = m[1];
|
||||
if (/\s#\s*(?:SKIP|TODO)\b/i.test(rest)) continue; // skipped/todo did not run
|
||||
names.push(rest.replace(/\s+#\s.*$/, '').trim());
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* A NON-VACUOUS node-test RED — the fail-first proof analog of `isNonVacuousNodeTestPass`. True iff
|
||||
* the run reports `# fail >= 1` AND at least one FAILING test is named DISTINCTLY from the target file.
|
||||
*
|
||||
* Why the distinct-name guard: a violation fixture that makes the negative test CRASH at load
|
||||
* (ENOENT / throw-on-require / syntax error) emits a FILE-NAMED `not ok 1 - <file>` with `# fail 1`.
|
||||
* That is a crash, NOT the negative assertion firing red — so it must not "prove" the test is a
|
||||
* regression guard (the RED-side mirror of the BL-01 vacuity hole on the pass side). Requiring a
|
||||
* failing test named distinctly from the file closes that hole, symmetric with the clean-pass guard.
|
||||
*
|
||||
* KNOWN CONSTRAINT (fail-closed, not a hole): a negative test whose `test('...')` name is EXACTLY its
|
||||
* own file basename is conservatively rejected — same benign authoring constraint, same safe direction.
|
||||
*/
|
||||
export function isNonVacuousNodeTestRed(out: string, target: string): boolean {
|
||||
if (!isNodeTestRed(out)) return false; // no `# fail >= 1` summary -> not red (fail-closed)
|
||||
const tgtBase = baseOf(target);
|
||||
return tapFailedTestNames(out).some((n) => baseOf(n) !== tgtBase);
|
||||
}
|
||||
|
||||
/** Number of file results in an eslint `--format json` report (0 if unparseable / not an array). */
|
||||
export function eslintFileResultCount(jsonText: string): number {
|
||||
try {
|
||||
@@ -420,9 +454,11 @@ function defaultRunCheck(check: CheckDescriptor, cwd: string, timeoutMs?: number
|
||||
* id to appear in the report (messages OR suppressedMessages — an inline-disabled violation still
|
||||
* proves the rule has teeth, #1259 B1). Absent fixture / unresolvable eslint → not proven.
|
||||
* - node-test: spawn the negative test (TAP) with `GSD_PROHIB_SUBJECT` set to the `violationFixture`
|
||||
* — the CONVENTION (#1279) by which a negative test reads its subject-under-test — and require the
|
||||
* run to go RED (`isNodeTestRed`: `# fail >= 1`). A toothless test that passes anyway → not proven.
|
||||
* Absent fixture → not proven (fail-closed; NEVER falls back to attestation).
|
||||
* — the CONVENTION (#1279) by which a negative test reads its subject-under-test — and require a
|
||||
* NON-VACUOUS red (`isNonVacuousNodeTestRed`: `# fail >= 1` AND a failing test named distinctly
|
||||
* from the file, so a load-CRASH on the bad subject is not mistaken for the assertion firing red).
|
||||
* A toothless test that passes anyway → not proven. Absent fixture → not proven (fail-closed;
|
||||
* NEVER falls back to attestation).
|
||||
*/
|
||||
function defaultProveFailFirst(check: CheckDescriptor, cwd: string, timeoutMs?: number): FailFirstProof {
|
||||
try {
|
||||
@@ -476,7 +512,7 @@ function defaultProveFailFirst(check: CheckDescriptor, cwd: string, timeoutMs?:
|
||||
const stdout = e && typeof e === 'object' && 'stdout' in e ? (e as { stdout?: unknown }).stdout : '';
|
||||
out = typeof stdout === 'string' ? stdout : '';
|
||||
}
|
||||
return { provenFailFirst: isNodeTestRed(out), method: 'violation-fixture' };
|
||||
return { provenFailFirst: isNonVacuousNodeTestRed(out, check.target), method: 'violation-fixture' };
|
||||
}
|
||||
// Unknown kind — defensive; the LOCATE guard already rejects it.
|
||||
return { provenFailFirst: false };
|
||||
|
||||
@@ -385,6 +385,35 @@ describe('prohibition-enforcement real-runner helpers (#1259)', () => {
|
||||
assert.equal(enforce.isNodeTestRed('no summary'), false, 'no parseable summary -> not RED (fail-closed for the prover)');
|
||||
});
|
||||
|
||||
// ─── #1279 isNonVacuousNodeTestRed (FF-03 hardening) — a NON-VACUOUS red proof ───
|
||||
// The fail-first PROOF must mirror the clean-pass non-vacuity discipline: a violation fixture that
|
||||
// makes the negative test CRASH at load (ENOENT/throw/syntax) emits a FILE-NAMED `# fail 1` — the
|
||||
// test never ran its assertion, so that is NOT proof the test is a regression guard. Require at
|
||||
// least one FAILING test named DISTINCTLY from the target file (symmetric with isNonVacuousNodeTestPass).
|
||||
test('isNonVacuousNodeTestRed: a file-named-only failure (a crash, not an assertion) does NOT prove fail-first', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
// node --test of a file that throws at load: `not ok 1 - <file>`, `# fail 1` — a crash, not a
|
||||
// negative assertion firing red. Must NOT count as a non-vacuous red.
|
||||
const crash = 'not ok 1 - neg.test.cjs\n# tests 1\n# pass 0\n# fail 1\n';
|
||||
assert.equal(enforce.isNonVacuousNodeTestRed(crash, 'neg.test.cjs'), false,
|
||||
'a file-named-only failure is a load crash, not a proven regression guard — fail-closed');
|
||||
// BASENAME-NORMALIZED: node may report the file failure by an absolute/normalized path.
|
||||
const crashAbs = 'not ok 1 - /tmp/x/neg.test.cjs\n# tests 1\n# pass 0\n# fail 1\n';
|
||||
assert.equal(enforce.isNonVacuousNodeTestRed(crashAbs, 'neg.test.cjs'), false,
|
||||
'an absolute-path file-named failure is still a crash (basename compare)');
|
||||
// A genuine negative assertion firing red carries a descriptive name distinct from the file.
|
||||
const realRed = 'not ok 1 - rejects the forbidden pattern\n# tests 1\n# pass 0\n# fail 1\n';
|
||||
assert.equal(enforce.isNonVacuousNodeTestRed(realRed, 'neg.test.cjs'), true,
|
||||
'a distinctly-named failing test is a genuine non-vacuous red — proves fail-first');
|
||||
// No failure at all -> not red.
|
||||
assert.equal(enforce.isNonVacuousNodeTestRed('ok 1 - guards\n# tests 1\n# pass 1\n# fail 0\n', 'neg.test.cjs'), false,
|
||||
'# fail 0 is not red regardless of names');
|
||||
// SKIP/TODO failing lines never ran -> excluded (mirror tapTestNames m1).
|
||||
const skippedRed = 'not ok 1 - rejects the forbidden pattern # SKIP\n# tests 1\n# pass 0\n# fail 1\n';
|
||||
assert.equal(enforce.isNonVacuousNodeTestRed(skippedRed, 'neg.test.cjs'), false,
|
||||
'a SKIP/TODO failing line did not actually run -> not a proof');
|
||||
});
|
||||
|
||||
test('tapTestNames EXCLUDES skipped/todo tests (they never ran, m1)', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
assert.deepEqual(enforce.tapTestNames('ok 1 - guards the must-NOT\nok 2 - other # SKIP\nok 3 - later # TODO\n'),
|
||||
|
||||
Reference in New Issue
Block a user