fix(1259-01): address trek-e review — B1 (fatal/suppressed), B2 (bounded subprocess), M1/M2, minors
Maintainer CHANGES_REQUESTED (reviewede08667e5, pre-portability-fix): - B1: lint-rule no longer greens an unparseable target (eslintHasFatalError -> fail closed on any fatal/parse error) or an inline-suppressed violation (eslintJsonHasRule now scans suppressedMessages too). RED-first + real-runner repros. - B2: both child spawns get a bounded timeout (30s node / 60s eslint) + 16MiB maxBuffer; timeout fails closed. Injectable timeoutMs (positive-only — 0/negative can't disable the bound) enables a fast 1.5s hang test. - M1: scoped verify-phase.md — the check descriptor is author-supplied for now; filed #1278 for deterministic auto-locate of the descriptor (the locate half). - M2: added tests/prohibition-enforcement.property.test.cjs (fast-check fail-closed invariants; within the <=2-file budget). - m1: tapTestNames excludes # SKIP/# TODO; parseNodeTestSummary tracks # cancelled; isNonVacuousNodeTestPass requires cancelled===0. - m2: scoped the determinism claim to the decision/parse layer (real runner is env-dependent). - m3: filed #1279 for machine-proven fail-first (violation-fixture probe). - n1: -- before target in both arg builders (option-injection). n2: dropped dead token. - B3 (Windows npx) was already fixed in2af76306(pushed ~65s after the review). Verified node 22 + 24; size baseline regenerated for the verify-phase note.
This commit is contained in:
60
tests/prohibition-enforcement.property.test.cjs
Normal file
60
tests/prohibition-enforcement.property.test.cjs
Normal file
@@ -0,0 +1,60 @@
|
||||
// Property-based tests for the prohibition-enforcement parsing/transform helpers (#1259, ADR-550 D5d).
|
||||
// RULESET.TESTS.property-based-testing: the producer is a parsing module (parseNodeTestSummary,
|
||||
// tapTestNames, eslintJsonHasRule, eslintHasFatalError, eslintFileResultCount), so it carries
|
||||
// fast-check invariants — especially the fail-closed safety invariants of the verify-time gate.
|
||||
'use strict';
|
||||
process.env.GSD_TEST_MODE = '1';
|
||||
|
||||
const { test, describe } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const path = require('node:path');
|
||||
const fc = require('./helpers/fast-check-setup.cjs');
|
||||
|
||||
const ENFORCEMENT_LIB = path.join(__dirname, '..', 'gsd-core', 'bin', 'lib', 'prohibition-enforcement.cjs');
|
||||
|
||||
describe('prohibition-enforcement properties (#1259)', () => {
|
||||
test('parseNodeTestSummary never throws and returns non-negative integer counts', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
fc.assert(fc.property(fc.string(), (s) => {
|
||||
const r = enforce.parseNodeTestSummary(s);
|
||||
for (const k of ['tests', 'pass', 'fail', 'cancelled']) {
|
||||
assert.ok(Number.isInteger(r[k]) && r[k] >= 0, `${k} is a non-negative integer`);
|
||||
}
|
||||
}));
|
||||
});
|
||||
|
||||
test('isNonVacuousNodeTestPass FAIL-CLOSED: a run with any failure or cancellation never greens', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
fc.assert(fc.property(
|
||||
fc.nat({ max: 50 }), fc.integer({ min: 1, max: 50 }), fc.nat({ max: 50 }), fc.string(),
|
||||
(tests, failOrCancel, pass, name) => {
|
||||
// A summary with fail>=1 (and, separately, cancelled>=1) must NEVER be a non-vacuous pass,
|
||||
// regardless of the reported test name.
|
||||
const failing = `ok 1 - ${name}\n# tests ${tests + 1}\n# pass ${pass}\n# fail ${failOrCancel}\n# cancelled 0\n`;
|
||||
assert.equal(enforce.isNonVacuousNodeTestPass(failing, 'neg.test.cjs'), false);
|
||||
const cancelled = `ok 1 - ${name}\n# tests ${tests + 1}\n# pass ${pass}\n# fail 0\n# cancelled ${failOrCancel}\n`;
|
||||
assert.equal(enforce.isNonVacuousNodeTestPass(cancelled, 'neg.test.cjs'), false);
|
||||
},
|
||||
));
|
||||
});
|
||||
|
||||
test('eslintJsonHasRule / eslintHasFatalError FAIL-CLOSED on any non-JSON / non-array input', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
fc.assert(fc.property(fc.string(), (s) => {
|
||||
// Only exercise strings that are NOT a valid JSON array (the unreadable-report branch).
|
||||
let isArray = false;
|
||||
try { isArray = Array.isArray(JSON.parse(s)); } catch { isArray = false; }
|
||||
fc.pre(!isArray);
|
||||
assert.equal(enforce.eslintJsonHasRule(s, 'local/no-source-grep'), true, 'unreadable report -> violation (fail-closed)');
|
||||
assert.equal(enforce.eslintHasFatalError(s), true, 'unreadable report -> fatal (fail-closed)');
|
||||
}));
|
||||
});
|
||||
|
||||
test('eslintFileResultCount never throws and is non-negative', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
fc.assert(fc.property(fc.string(), (s) => {
|
||||
const n = enforce.eslintFileResultCount(s);
|
||||
assert.ok(Number.isInteger(n) && n >= 0);
|
||||
}));
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,3 @@
|
||||
// allow-test-rule: runtime-contract-is-the-product (#1259) — the test-tier enforcement PRODUCER is
|
||||
// the deployed verify-time gate; these assertions pin its deterministic locate/fail-first/run/
|
||||
// evidence-construction contract to the code (ADR-550 D5d).
|
||||
//
|
||||
// Behavioral tests for the deterministic prohibition-enforcement producer (#1259, ADR-550 D5d
|
||||
// "heavy half"). Requires the BUILT gsd-core/bin/lib/prohibition-enforcement.cjs — authored as
|
||||
// src/prohibition-enforcement.cts and compiled by `npm run build:lib` (mirrors how the verify-tier
|
||||
@@ -221,16 +217,32 @@ describe('prohibition-enforcement: deterministic test-tier producer (#1259 / ADR
|
||||
// weakens "non-vacuous pass" or the ruleId filter is caught — the contract the injected-runner tests
|
||||
// above deliberately bypass.
|
||||
describe('prohibition-enforcement real-runner helpers (#1259)', () => {
|
||||
test('parseNodeTestSummary extracts the TAP tests/pass/fail counts', () => {
|
||||
test('parseNodeTestSummary extracts the TAP tests/pass/fail/cancelled counts', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
assert.deepEqual(enforce.parseNodeTestSummary('# tests 3\n# pass 2\n# fail 1\n'), { tests: 3, pass: 2, fail: 1 });
|
||||
assert.deepEqual(enforce.parseNodeTestSummary('no summary here'), { tests: 0, pass: 0, fail: 0 });
|
||||
assert.deepEqual(enforce.parseNodeTestSummary('# tests 3\n# pass 2\n# fail 1\n# cancelled 1\n'),
|
||||
{ tests: 3, pass: 2, fail: 1, cancelled: 1 });
|
||||
assert.deepEqual(enforce.parseNodeTestSummary('no summary here'), { tests: 0, pass: 0, fail: 0, cancelled: 0 });
|
||||
});
|
||||
|
||||
test('tapTestNames extracts ok/not-ok names, stripping directives', () => {
|
||||
test('tapTestNames EXCLUDES skipped/todo tests (they never ran, m1)', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
assert.deepEqual(enforce.tapTestNames('ok 1 - guards the must-NOT\nnot ok 2 - other # SKIP\n'),
|
||||
['guards the must-NOT', 'other']);
|
||||
assert.deepEqual(enforce.tapTestNames('ok 1 - guards the must-NOT\nok 2 - other # SKIP\nok 3 - later # TODO\n'),
|
||||
['guards the must-NOT'], 'a # SKIP / # TODO test is not a real run and must not count');
|
||||
});
|
||||
|
||||
test('isNonVacuousNodeTestPass: a SKIPPED negative test (file wrapper passes) is NOT a pass (m1)', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
// file wrapper + a skipped negative test: pass>=1 but the only named test is skipped -> vacuous.
|
||||
const skipped = 'ok 1 - empty.test.cjs\nok 2 - the negative test # SKIP\n# tests 2\n# pass 2\n# fail 0\n# cancelled 0\n';
|
||||
assert.equal(enforce.isNonVacuousNodeTestPass(skipped, 'empty.test.cjs'), false,
|
||||
'a skipped negative test never executed -> must not green');
|
||||
});
|
||||
|
||||
test('isNonVacuousNodeTestPass: a CANCELLED run is not a pass (m1)', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
const cancelled = 'ok 1 - guards\n# tests 1\n# pass 1\n# fail 0\n# cancelled 1\n';
|
||||
assert.equal(enforce.isNonVacuousNodeTestPass(cancelled, 'neg.test.cjs'), false,
|
||||
'a cancelled run is not a clean pass');
|
||||
});
|
||||
|
||||
test('isNonVacuousNodeTestPass: an empty file (node names the test after the file) is NOT a pass (BL-01)', () => {
|
||||
@@ -271,6 +283,22 @@ describe('prohibition-enforcement real-runner helpers (#1259)', () => {
|
||||
assert.equal(enforce.eslintFileResultCount('[]'), 0);
|
||||
assert.equal(enforce.eslintFileResultCount('garbage'), 0);
|
||||
});
|
||||
|
||||
test('eslintHasFatalError: a parse/fatal error must fail closed (B1)', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
const fatal = JSON.stringify([{ messages: [{ ruleId: null, fatal: true, severity: 2, message: 'Parsing error' }], fatalErrorCount: 1 }]);
|
||||
assert.equal(enforce.eslintHasFatalError(fatal), true, 'a fatal/parse error means the rule never ran -> fail closed');
|
||||
const clean = JSON.stringify([{ messages: [], fatalErrorCount: 0 }]);
|
||||
assert.equal(enforce.eslintHasFatalError(clean), false, 'a clean lint has no fatal error');
|
||||
assert.equal(enforce.eslintHasFatalError('not json'), true, 'an unreadable report is treated as fatal (fail closed)');
|
||||
});
|
||||
|
||||
test('eslintJsonHasRule also reads suppressedMessages — an inline-disabled violation still counts (B1)', () => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
const suppressed = JSON.stringify([{ messages: [], suppressedMessages: [{ ruleId: 'local/no-source-grep' }] }]);
|
||||
assert.equal(enforce.eslintJsonHasRule(suppressed, 'local/no-source-grep'), true,
|
||||
'a violation suppressed via // eslint-disable must NOT be treated as clean');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Real runner end-to-end (NO injected runCheck; #1259 SF-02 / BL-01 / SF-01) ──
|
||||
@@ -296,6 +324,23 @@ describe('prohibition-enforcement REAL runner end-to-end (#1259)', () => {
|
||||
assert.equal(result.evidence.length, 1);
|
||||
});
|
||||
|
||||
test('a HANGING node-test fails closed via the bounded timeout (B2: no unbounded subprocess)', (t) => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
const dir = createTempDir('prohib-hang-');
|
||||
t.after(() => cleanup(dir));
|
||||
const tf = path.join(dir, 'hang.test.cjs');
|
||||
// A test that never returns; the bounded timeout must kill it and dispose non-green.
|
||||
fs.writeFileSync(tf,
|
||||
"const { test } = require('node:test');\ntest('hangs forever', () => { while (true) {} });\n");
|
||||
const result = enforce.runProhibitionEnforcement(
|
||||
TEST_TIER,
|
||||
{ kind: 'node-test', target: tf, failFirst: true },
|
||||
{ cwd: dir, timeoutMs: 1500 },
|
||||
);
|
||||
assert.notEqual(result.status, 'green', 'a hung check must be killed and fail closed — never hang verify or green');
|
||||
assert.equal(result.located, true);
|
||||
});
|
||||
|
||||
test('an EMPTY node-test file (exit 0, zero tests) does NOT green via the real runner (BL-01)', (t) => {
|
||||
const enforce = require(ENFORCEMENT_LIB);
|
||||
const dir = createTempDir('prohib-real-empty-');
|
||||
|
||||
@@ -85,6 +85,6 @@
|
||||
"undo.md": 10431,
|
||||
"update.md": 21053,
|
||||
"validate-phase.md": 10745,
|
||||
"verify-phase.md": 32334,
|
||||
"verify-phase.md": 33076,
|
||||
"verify-work.md": 31157
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user