feat(3597): split test suites and add Node 22/24/26 OS matrix

scripts/run-tests.cjs gains `--suite <name>` filtering using a filename
suffix convention (`*.security.test.cjs`, `*.integration.test.cjs`, …).
Files with no marker are `unit` (the default fast lane); files with a
marker land in the matching suite. No `--suite` flag preserves the prior
behavior of running every test (backcompat for `npm test` and
`npm run test:coverage`).

New package scripts wire the suites to stable entrypoints:
test:unit, test:integration, test:install, test:security, test:slow,
test:coverage:unit, test:coverage:all. Unknown suite → exit 2 with the
list of valid suites; empty suite → exit 0 with a stderr notice so empty
lanes (e.g. `security` before adversarial tests land) don't gate CI.

CI matrix grows from `ubuntu × {22,24}` + a single macOS lane to
`{ubuntu, macos, windows} × {22, 24, 26}`. `fail-fast: false` so one
lane failure doesn't cancel siblings. Node 26 is `continue-on-error`
until actions/setup-node stabilises that image. PR CI runs unit +
integration + security on every cell; `install` and `slow` only on
`main` push. A dedicated `coverage` job runs `test:coverage:unit` on
ubuntu/Node 24 and uploads the report.

Grouping policy lives in docs/TESTING-SUITES.md with a pointer from
CONTRIBUTING.md. New harness test covers arg parsing, filter selection,
empty-suite behavior, and failure propagation.

Closes #3597.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
2026-05-16 08:34:09 -04:00
parent beb7e05254
commit e82876fe45
7 changed files with 513 additions and 33 deletions

View File

@@ -2,32 +2,142 @@
// Cross-platform test runner — resolves test file globs via Node
// instead of relying on shell expansion (which fails on Windows PowerShell/cmd).
// Propagates NODE_V8_COVERAGE so c8 collects coverage from the child process.
//
// Suite filtering (issue #3597):
// node scripts/run-tests.cjs # default — runs ALL tests (backcompat)
// node scripts/run-tests.cjs --suite all # explicit "everything"
// node scripts/run-tests.cjs --suite unit # only files with no other suite marker
// node scripts/run-tests.cjs --suite security # *.security.test.cjs
// node scripts/run-tests.cjs --suite integration # *.integration.test.cjs
// node scripts/run-tests.cjs --suite install # *.install.test.cjs
// node scripts/run-tests.cjs --suite slow # *.slow.test.cjs
//
// Suite grouping convention: filename suffix marker before `.test.cjs`.
// A file named `foo.security.test.cjs` belongs to the `security` suite.
// A file named `foo.test.cjs` (no marker) belongs to the `unit` suite.
// See docs/TESTING-SUITES.md for full grouping policy.
'use strict';
const { readdirSync } = require('fs');
const { join } = require('path');
const { execFileSync } = require('child_process');
const testDir = join(__dirname, '..', 'tests');
const files = readdirSync(testDir)
.filter(f => f.endsWith('.test.cjs'))
.sort()
.map(f => join('tests', f));
const SUITES = ['all', 'unit', 'integration', 'install', 'security', 'slow'];
const MARKED_SUITES = ['integration', 'install', 'security', 'slow'];
if (files.length === 0) {
console.error('No test files found in tests/');
process.exit(1);
function parseArgs(argv) {
let suite = null;
let seen = false;
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === '--suite') {
if (seen) {
return { error: 'duplicate --suite flag' };
}
seen = true;
const v = argv[i + 1];
if (!v || v.startsWith('--')) {
return { error: '--suite requires a value' };
}
suite = v;
i++;
} else if (a.startsWith('--suite=')) {
if (seen) {
return { error: 'duplicate --suite flag' };
}
seen = true;
suite = a.slice('--suite='.length);
if (!suite) {
return { error: '--suite requires a value' };
}
} else {
return { error: `unknown argument: ${a}` };
}
}
return { suite };
}
const concurrency = process.env.TEST_CONCURRENCY
? `--test-concurrency=${process.env.TEST_CONCURRENCY}`
: '--test-concurrency=4';
try {
execFileSync(process.execPath, ['--test', concurrency, ...files], {
stdio: 'inherit',
env: { ...process.env },
});
} catch (err) {
process.exit(err.status || 1);
// Return the marked suite name embedded in a filename, or null if it's unmarked.
// foo.security.test.cjs -> "security"
// foo.test.cjs -> null (unit)
function suiteOf(filename) {
if (!filename.endsWith('.test.cjs')) return null;
const base = filename.slice(0, -'.test.cjs'.length);
const lastDot = base.lastIndexOf('.');
if (lastDot === -1) return null;
const marker = base.slice(lastDot + 1);
return MARKED_SUITES.includes(marker) ? marker : null;
}
function selectFiles(allFiles, suite) {
if (suite === null || suite === 'all') {
return allFiles;
}
if (suite === 'unit') {
return allFiles.filter(f => suiteOf(f) === null);
}
return allFiles.filter(f => suiteOf(f) === suite);
}
function main() {
const args = process.argv.slice(2);
const parsed = parseArgs(args);
if (parsed.error) {
console.error(`run-tests: ${parsed.error}`);
console.error(`Valid suites: ${SUITES.join(', ')}`);
process.exit(2);
}
const suite = parsed.suite;
if (suite !== null && !SUITES.includes(suite)) {
console.error(`run-tests: unknown suite "${suite}"`);
console.error(`Valid suites: ${SUITES.join(', ')}`);
process.exit(2);
}
const testDir = process.env.GSD_TEST_DIR
? process.env.GSD_TEST_DIR
: join(__dirname, '..', 'tests');
const allFiles = readdirSync(testDir)
.filter(f => f.endsWith('.test.cjs'))
.sort();
if (allFiles.length === 0) {
console.error('No test files found in tests/');
process.exit(1);
}
const selected = selectFiles(allFiles, suite).map(f => join(testDir, f));
if (selected.length === 0) {
// Empty suite: report and exit 0 so empty lanes (e.g. `security` before
// adversarial tests land) don't gate CI. CI consumers wanting strictness
// can grep stderr for "no tests in suite".
console.error(`run-tests: no tests in suite "${suite || 'all'}"`);
process.exit(0);
}
// Log selected files to stderr for CI / harness-test visibility.
// node:test default reporter doesn't echo filenames, so this gives
// operators a single stable line they can grep.
console.error(
`run-tests: suite="${suite || 'all'}" files=${selected.length}: ${selected
.map(f => f.split(/[\\/]/).pop())
.join(' ')}`,
);
const concurrency = process.env.TEST_CONCURRENCY
? `--test-concurrency=${process.env.TEST_CONCURRENCY}`
: '--test-concurrency=4';
try {
execFileSync(process.execPath, ['--test', concurrency, ...selected], {
stdio: 'inherit',
env: { ...process.env },
});
} catch (err) {
process.exit(err.status || 1);
}
}
main();