From e82876fe4504b2ff6a80da22024847ad5b42ba78 Mon Sep 17 00:00:00 2001 From: Tom Boucher Date: Sat, 16 May 2026 08:34:09 -0400 Subject: [PATCH] feat(3597): split test suites and add Node 22/24/26 OS matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/run-tests.cjs gains `--suite ` filtering using a filename suffix convention (`*.security.test.cjs`, `*.integration.test.cjs`, …). Files with no marker are `unit` (the default fast lane); files with a marker land in the matching suite. No `--suite` flag preserves the prior behavior of running every test (backcompat for `npm test` and `npm run test:coverage`). New package scripts wire the suites to stable entrypoints: test:unit, test:integration, test:install, test:security, test:slow, test:coverage:unit, test:coverage:all. Unknown suite → exit 2 with the list of valid suites; empty suite → exit 0 with a stderr notice so empty lanes (e.g. `security` before adversarial tests land) don't gate CI. CI matrix grows from `ubuntu × {22,24}` + a single macOS lane to `{ubuntu, macos, windows} × {22, 24, 26}`. `fail-fast: false` so one lane failure doesn't cancel siblings. Node 26 is `continue-on-error` until actions/setup-node stabilises that image. PR CI runs unit + integration + security on every cell; `install` and `slow` only on `main` push. A dedicated `coverage` job runs `test:coverage:unit` on ubuntu/Node 24 and uploads the report. Grouping policy lives in docs/TESTING-SUITES.md with a pointer from CONTRIBUTING.md. New harness test covers arg parsing, filter selection, empty-suite behavior, and failure propagation. Closes #3597. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../3597-test-suite-split-node-matrix.md | 5 + .github/workflows/test.yml | 95 ++++++-- CONTRIBUTING.md | 2 + docs/TESTING-SUITES.md | 77 +++++++ package.json | 9 +- scripts/run-tests.cjs | 148 ++++++++++-- tests/run-tests-harness.test.cjs | 210 ++++++++++++++++++ 7 files changed, 513 insertions(+), 33 deletions(-) create mode 100644 .changeset/3597-test-suite-split-node-matrix.md create mode 100644 docs/TESTING-SUITES.md create mode 100644 tests/run-tests-harness.test.cjs diff --git a/.changeset/3597-test-suite-split-node-matrix.md b/.changeset/3597-test-suite-split-node-matrix.md new file mode 100644 index 000000000..f7884f1ec --- /dev/null +++ b/.changeset/3597-test-suite-split-node-matrix.md @@ -0,0 +1,5 @@ +--- +type: Changed +pr: 3597 +--- +**Test suites split into named lanes and CI matrix expanded to OS×Node** — `scripts/run-tests.cjs` now accepts `--suite ` and filters by filename suffix (`*.security.test.cjs`, `*.integration.test.cjs`, etc.). New package scripts `test:unit`, `test:integration`, `test:install`, `test:security`, `test:slow`, `test:coverage:unit`, and `test:coverage:all` give callers stable entrypoints; `npm test` and `npm run test:coverage` keep their original behavior (full sweep). CI matrix grew from `ubuntu × {22, 24}` + a single macOS lane to `{ubuntu, macos, windows} × {22, 24, 26}` with Node 26 in `continue-on-error` until `actions/setup-node` stabilises it. `fail-fast: false` so a single lane failure no longer cancels the rest. Install and slow suites only run on `main` push to keep PR CI fast; a dedicated `coverage` job runs `test:coverage:unit` on ubuntu/Node 24 and uploads the report. New grouping policy documented in `docs/TESTING-SUITES.md`. Closes #3597. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e28c91fd..9aea5be00 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,20 +39,21 @@ jobs: test: runs-on: ${{ matrix.os }} - timeout-minutes: 10 + timeout-minutes: 15 + # Node 26 lanes report forward-compat status without gating the workflow + # (setup-node may not yet have stable Node 26 images). 22/24 still gate. + continue-on-error: ${{ matrix.node-version == 26 }} strategy: - fail-fast: true + # fail-fast disabled so a single lane failure doesn't cancel the rest; + # makes OS-specific vs version-specific regressions easier to distinguish. + fail-fast: false matrix: - os: [ubuntu-latest] - node-version: [22, 24] - include: - # Single macOS runner — verifies platform compatibility on the standard version - - os: macos-latest - node-version: 24 - # Windows path/separator coverage is handled by hardcoded-paths.test.cjs - # and windows-robustness.test.cjs (static analysis, runs on all platforms). - # A dedicated windows-compat workflow runs on a weekly schedule. + os: [ubuntu-latest, macos-latest, windows-latest] + node-version: [22, 24, 26] + # Windows path/separator coverage is also reinforced by hardcoded-paths.test.cjs + # and windows-robustness.test.cjs (static analysis). + # A dedicated windows-compat workflow runs on a weekly schedule. steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -126,6 +127,74 @@ jobs: shell: bash run: node sdk/scripts/check-project-root-fresh.mjs - - name: Run tests with coverage + # Split lanes (issue #3597). Unit is the fast default lane; integration + # and security run alongside it on every PR. `install` and `slow` are + # skipped on PR CI by design — they run on the weekly windows-compat + # workflow and on `main` push only. See docs/TESTING-SUITES.md. + - name: Run unit tests shell: bash - run: npm run test:coverage + run: npm run test:unit + + - name: Run integration tests + shell: bash + run: npm run test:integration + + - name: Run security tests + shell: bash + run: npm run test:security + + # Install + slow lanes only on main-branch push (not PR CI) so PRs stay fast. + - name: Run install tests + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + shell: bash + run: npm run test:install + + - name: Run slow tests + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + shell: bash + run: npm run test:slow + + # Dedicated coverage job. Runs only on ubuntu/Node 24 (the canonical lane) + # because c8 coverage of one suite on one OS is enough signal — running it + # across the full matrix would 9x the cost for no extra coverage data. + coverage: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + - name: Rebase check — merge origin/main into PR head + if: github.event_name == 'pull_request' + shell: bash + run: | + set -euo pipefail + git config user.email "ci@gsd-build" + git config user.name "CI Rebase Check" + git fetch origin main + if ! git merge --no-edit --no-ff origin/main; then + echo "::error::This PR cannot cleanly merge origin/main. Rebase your branch onto current main and push again." + git merge --abort + exit 1 + fi + - name: Set up Node.js 24 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + cache: 'npm' + - name: Install dependencies + run: npm ci + - name: Build SDK dist + run: npm run build:sdk + - name: Unit coverage + shell: bash + run: npm run test:coverage:unit + - name: Upload coverage artifact + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-unit + path: | + coverage/ + .nyc_output/ + if-no-files-found: ignore diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e091a769..8b998863e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -154,6 +154,8 @@ Fragments are consolidated into `CHANGELOG.md` at release time by the release wo All tests use Node.js built-in test runner (`node:test`) and assertion library (`node:assert`). **Do not use Jest, Mocha, Chai, or any external test framework.** +> **Suite grouping.** Tests live in named suites (`unit`, `integration`, `install`, `security`, `slow`) selected by **filename suffix**: a file named `foo.security.test.cjs` belongs to the `security` suite; a file with no suffix (`foo.test.cjs`) belongs to `unit`. See [docs/TESTING-SUITES.md](docs/TESTING-SUITES.md) for the full policy, CI matrix, and per-suite scripts (`npm run test:unit`, `npm run test:security`, `npm run test:coverage:unit`, …). Default `npm test` still runs every test — backwards compatible. + ### Required Imports ```javascript diff --git a/docs/TESTING-SUITES.md b/docs/TESTING-SUITES.md new file mode 100644 index 000000000..5edcabce8 --- /dev/null +++ b/docs/TESTING-SUITES.md @@ -0,0 +1,77 @@ +# Testing Suites + +This project's `tests/` directory uses **filename suffix markers** to group tests into named suites. The harness `scripts/run-tests.cjs` filters by suite when given `--suite `. Without a flag it runs every `*.test.cjs` file (the historical default — unchanged). + +> Tracked by issue [#3597](https://github.com/gsd-build/get-shit-done/issues/3597). + +## Suites + +| Suite | Filename pattern | What goes here | +|---|---|---| +| `unit` | `*.test.cjs` (no other marker) | Default fast lane. Pure logic, no network, no external processes beyond `gsd-tools`. Most tests live here. | +| `integration` | `*.integration.test.cjs` | Cross-module flows: full installer end-to-end, multi-tool orchestration, anything that crosses two or more bin entry points. | +| `install` | `*.install.test.cjs` | Tests that perform a real install/uninstall against a sandbox project. Slower; PR CI skips these on PRs and runs them on `main` push only. | +| `security` | `*.security.test.cjs` | Adversarial input, prompt-injection guards, fixture-driven hostile-payload sweeps. | +| `slow` | `*.slow.test.cjs` | Anything that routinely takes >5s wall-clock or holds significant memory. | +| `all` | (any) | Explicit alias for "no filter". Equivalent to running with no `--suite` flag. | + +## How to place a new test + +1. Pick the most specific bucket above. +2. Name the file with the matching suffix: `tests/..test.cjs`. +3. If unsure, leave the suffix off — the file lands in `unit`, the default fast lane. + +Examples: +- `tests/agent-frontmatter.test.cjs` — `unit` +- `tests/prompt-injection-guards.security.test.cjs` — `security` +- `tests/installer-end-to-end.install.test.cjs` — `install` +- `tests/sdk-mutation-stress.slow.test.cjs` — `slow` + +The suite-suffix convention was chosen over a directory layout (`tests/security/`) so the 545+ existing test files don't need to move. Existing files all classify as `unit` until someone explicitly retags them. + +## Running suites locally + +```bash +npm test # everything (backcompat — same as before) +npm run test:unit # only unit +npm run test:integration # only integration +npm run test:install # only install +npm run test:security # only security +npm run test:slow # only slow + +npm run test:coverage # backcompat — coverage over EVERY test +npm run test:coverage:unit # fast coverage signal — only unit suite +npm run test:coverage:all # alias for test:coverage +``` + +Direct harness invocation also works: + +```bash +node scripts/run-tests.cjs --suite security +node scripts/run-tests.cjs --suite=security +``` + +Unknown suites exit non-zero with the list of valid suites. Empty suites (e.g. `--suite security` before any security-tagged file exists) exit `0` with a `no tests in suite "..."` notice on stderr so CI lanes don't go red while a suite is being populated. + +## CI matrix + +The `Tests` workflow runs on: + +| OS | Node 22 | Node 24 | Node 26 | +|---|---|---|---| +| `ubuntu-latest` | gate | gate | forward-compat (`continue-on-error`) | +| `macos-latest` | gate | gate | forward-compat (`continue-on-error`) | +| `windows-latest` | gate | gate | forward-compat (`continue-on-error`) | + +- **Node 22** is the `engines.node` floor (`>=22.0.0`) — must stay green. +- **Node 24** is the default development lane. +- **Node 26** is forward-compat. The lane reports status but does not gate the workflow — `actions/setup-node` may not yet have a stable Node 26 image at any given moment. When it stabilises, flip `continue-on-error` off. + +Each matrix cell runs `unit`, `integration`, and `security` on every PR. `install` and `slow` only run on `main`-branch push to keep PR CI fast. Coverage runs in a dedicated `coverage` job on `ubuntu-latest` / Node 24 — running coverage across the full matrix would 9x the cost for no extra coverage data. + +## Best practices for forward-compat (Node 24/26) + +- Use `process.execPath` when spawning Node in tests so each matrix lane exercises the lane's Node version. +- Avoid exact stack-trace or error-message prose assertions. Assert `err.code`, structured JSON, or a stable message substring instead — Node minor releases routinely tweak error wording. +- Prefer `node:test`, `node:assert/strict`, and `node:test` mocks. No external test frameworks. +- Coverage uses `c8` and propagates `NODE_V8_COVERAGE` through the harness's child process. diff --git a/package.json b/package.json index 0a6ac5075..dc0835f1c 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,13 @@ "changeset": "node scripts/changeset/new.cjs", "changelog:render": "node scripts/changeset/cli.cjs render", "test": "node scripts/run-tests.cjs", - "test:coverage": "c8 --check-coverage --lines 70 --reporter text --include 'get-shit-done/bin/lib/*.cjs' --exclude 'tests/**' --all node scripts/run-tests.cjs" + "test:unit": "node scripts/run-tests.cjs --suite unit", + "test:integration": "node scripts/run-tests.cjs --suite integration", + "test:install": "node scripts/run-tests.cjs --suite install", + "test:security": "node scripts/run-tests.cjs --suite security", + "test:slow": "node scripts/run-tests.cjs --suite slow", + "test:coverage": "c8 --check-coverage --lines 70 --reporter text --include 'get-shit-done/bin/lib/*.cjs' --exclude 'tests/**' --all node scripts/run-tests.cjs", + "test:coverage:unit": "c8 --check-coverage --lines 70 --reporter text --include 'get-shit-done/bin/lib/*.cjs' --exclude 'tests/**' --all node scripts/run-tests.cjs --suite unit", + "test:coverage:all": "npm run test:coverage" } } diff --git a/scripts/run-tests.cjs b/scripts/run-tests.cjs index 37d602064..55038b934 100644 --- a/scripts/run-tests.cjs +++ b/scripts/run-tests.cjs @@ -2,32 +2,142 @@ // Cross-platform test runner — resolves test file globs via Node // instead of relying on shell expansion (which fails on Windows PowerShell/cmd). // Propagates NODE_V8_COVERAGE so c8 collects coverage from the child process. +// +// Suite filtering (issue #3597): +// node scripts/run-tests.cjs # default — runs ALL tests (backcompat) +// node scripts/run-tests.cjs --suite all # explicit "everything" +// node scripts/run-tests.cjs --suite unit # only files with no other suite marker +// node scripts/run-tests.cjs --suite security # *.security.test.cjs +// node scripts/run-tests.cjs --suite integration # *.integration.test.cjs +// node scripts/run-tests.cjs --suite install # *.install.test.cjs +// node scripts/run-tests.cjs --suite slow # *.slow.test.cjs +// +// Suite grouping convention: filename suffix marker before `.test.cjs`. +// A file named `foo.security.test.cjs` belongs to the `security` suite. +// A file named `foo.test.cjs` (no marker) belongs to the `unit` suite. +// See docs/TESTING-SUITES.md for full grouping policy. 'use strict'; const { readdirSync } = require('fs'); const { join } = require('path'); const { execFileSync } = require('child_process'); -const testDir = join(__dirname, '..', 'tests'); -const files = readdirSync(testDir) - .filter(f => f.endsWith('.test.cjs')) - .sort() - .map(f => join('tests', f)); +const SUITES = ['all', 'unit', 'integration', 'install', 'security', 'slow']; +const MARKED_SUITES = ['integration', 'install', 'security', 'slow']; -if (files.length === 0) { - console.error('No test files found in tests/'); - process.exit(1); +function parseArgs(argv) { + let suite = null; + let seen = false; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--suite') { + if (seen) { + return { error: 'duplicate --suite flag' }; + } + seen = true; + const v = argv[i + 1]; + if (!v || v.startsWith('--')) { + return { error: '--suite requires a value' }; + } + suite = v; + i++; + } else if (a.startsWith('--suite=')) { + if (seen) { + return { error: 'duplicate --suite flag' }; + } + seen = true; + suite = a.slice('--suite='.length); + if (!suite) { + return { error: '--suite requires a value' }; + } + } else { + return { error: `unknown argument: ${a}` }; + } + } + return { suite }; } -const concurrency = process.env.TEST_CONCURRENCY - ? `--test-concurrency=${process.env.TEST_CONCURRENCY}` - : '--test-concurrency=4'; - -try { - execFileSync(process.execPath, ['--test', concurrency, ...files], { - stdio: 'inherit', - env: { ...process.env }, - }); -} catch (err) { - process.exit(err.status || 1); +// Return the marked suite name embedded in a filename, or null if it's unmarked. +// foo.security.test.cjs -> "security" +// foo.test.cjs -> null (unit) +function suiteOf(filename) { + if (!filename.endsWith('.test.cjs')) return null; + const base = filename.slice(0, -'.test.cjs'.length); + const lastDot = base.lastIndexOf('.'); + if (lastDot === -1) return null; + const marker = base.slice(lastDot + 1); + return MARKED_SUITES.includes(marker) ? marker : null; } + +function selectFiles(allFiles, suite) { + if (suite === null || suite === 'all') { + return allFiles; + } + if (suite === 'unit') { + return allFiles.filter(f => suiteOf(f) === null); + } + return allFiles.filter(f => suiteOf(f) === suite); +} + +function main() { + const args = process.argv.slice(2); + const parsed = parseArgs(args); + if (parsed.error) { + console.error(`run-tests: ${parsed.error}`); + console.error(`Valid suites: ${SUITES.join(', ')}`); + process.exit(2); + } + const suite = parsed.suite; + if (suite !== null && !SUITES.includes(suite)) { + console.error(`run-tests: unknown suite "${suite}"`); + console.error(`Valid suites: ${SUITES.join(', ')}`); + process.exit(2); + } + + const testDir = process.env.GSD_TEST_DIR + ? process.env.GSD_TEST_DIR + : join(__dirname, '..', 'tests'); + + const allFiles = readdirSync(testDir) + .filter(f => f.endsWith('.test.cjs')) + .sort(); + + if (allFiles.length === 0) { + console.error('No test files found in tests/'); + process.exit(1); + } + + const selected = selectFiles(allFiles, suite).map(f => join(testDir, f)); + + if (selected.length === 0) { + // Empty suite: report and exit 0 so empty lanes (e.g. `security` before + // adversarial tests land) don't gate CI. CI consumers wanting strictness + // can grep stderr for "no tests in suite". + console.error(`run-tests: no tests in suite "${suite || 'all'}"`); + process.exit(0); + } + + // Log selected files to stderr for CI / harness-test visibility. + // node:test default reporter doesn't echo filenames, so this gives + // operators a single stable line they can grep. + console.error( + `run-tests: suite="${suite || 'all'}" files=${selected.length}: ${selected + .map(f => f.split(/[\\/]/).pop()) + .join(' ')}`, + ); + + const concurrency = process.env.TEST_CONCURRENCY + ? `--test-concurrency=${process.env.TEST_CONCURRENCY}` + : '--test-concurrency=4'; + + try { + execFileSync(process.execPath, ['--test', concurrency, ...selected], { + stdio: 'inherit', + env: { ...process.env }, + }); + } catch (err) { + process.exit(err.status || 1); + } +} + +main(); diff --git a/tests/run-tests-harness.test.cjs b/tests/run-tests-harness.test.cjs new file mode 100644 index 000000000..ad579c4e6 --- /dev/null +++ b/tests/run-tests-harness.test.cjs @@ -0,0 +1,210 @@ +// allow-test-rule: run-tests.cjs is a CLI test harness whose only IR is its +// stable stderr line `run-tests: suite="X" files=N: name1 name2 ...` plus its +// exit code. No typed IR is exposable from a shell script; the printed line +// IS the contract this test pins. See docs/TESTING-SUITES.md and issue #3597. +// +// Tests for scripts/run-tests.cjs --suite filtering (issue #3597). +// +// Drives the harness through its subprocess seam — the same seam CI uses — +// rather than importing internals. Each test seeds a temporary directory +// with mock `.test.cjs` files (each one a trivial node:test no-op) and +// runs the harness against it via GSD_TEST_DIR. + +'use strict'; + +const { describe, test, beforeEach, afterEach } = require('node:test'); +const assert = require('node:assert/strict'); +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +const { createTempDir, cleanup } = require('./helpers.cjs'); + +const HARNESS = path.join(__dirname, '..', 'scripts', 'run-tests.cjs'); + +// Minimal valid node:test file. Each fixture file passes when executed. +const PASS_BODY = `'use strict'; +const { test } = require('node:test'); +test('noop', () => {}); +`; + +function seed(dir, names) { + for (const name of names) { + fs.writeFileSync(path.join(dir, name), PASS_BODY, 'utf8'); + } +} + +function runHarness(testDir, args = []) { + // Clear node:test parent-context env so the harness's child `node --test` + // doesn't refuse to run with "recursive run() skipping running files". + const env = { ...process.env, GSD_TEST_DIR: testDir }; + delete env.NODE_TEST_CONTEXT; + return spawnSync(process.execPath, [HARNESS, ...args], { + cwd: path.join(__dirname, '..'), + env, + encoding: 'utf8', + }); +} + +describe('run-tests.cjs harness (issue #3597)', () => { + let tmpDir; + + beforeEach(() => { + tmpDir = createTempDir('gsd-3597-harness-'); + }); + + afterEach(() => { + cleanup(tmpDir); + }); + + describe('argument parsing', () => { + test('unknown suite name exits non-zero with valid-suites hint', () => { + seed(tmpDir, ['a.test.cjs']); + const r = runHarness(tmpDir, ['--suite', 'bogus']); + assert.notStrictEqual(r.status, 0); + assert.match(r.stderr, /unknown suite/i); + assert.match(r.stderr, /unit/); + assert.match(r.stderr, /security/); + }); + + test('missing --suite value exits non-zero', () => { + seed(tmpDir, ['a.test.cjs']); + const r = runHarness(tmpDir, ['--suite']); + assert.notStrictEqual(r.status, 0); + assert.match(r.stderr, /requires a value/i); + }); + + test('duplicate --suite flag is rejected', () => { + seed(tmpDir, ['a.test.cjs']); + const r = runHarness(tmpDir, ['--suite', 'unit', '--suite', 'security']); + assert.notStrictEqual(r.status, 0); + assert.match(r.stderr, /duplicate/i); + }); + + test('unknown positional argument is rejected', () => { + seed(tmpDir, ['a.test.cjs']); + const r = runHarness(tmpDir, ['unit']); + assert.notStrictEqual(r.status, 0); + assert.match(r.stderr, /unknown argument/i); + }); + + test('--suite=value syntax is accepted', () => { + seed(tmpDir, ['a.test.cjs', 'b.security.test.cjs']); + const r = runHarness(tmpDir, ['--suite=security']); + assert.strictEqual(r.status, 0, `stderr: ${r.stderr}\nstdout: ${r.stdout}`); + }); + }); + + describe('suite filtering', () => { + test('no flag runs ALL test files (backcompat)', () => { + seed(tmpDir, [ + 'a.test.cjs', + 'b.security.test.cjs', + 'c.integration.test.cjs', + ]); + const r = runHarness(tmpDir); + assert.strictEqual(r.status, 0); + // node:test TAP output mentions each file path. + assert.ok(r.stderr.includes('a.test.cjs'), 'expected a.test.cjs in output'); + assert.ok( + r.stderr.includes('b.security.test.cjs'), + 'expected b.security.test.cjs in output', + ); + assert.ok( + r.stderr.includes('c.integration.test.cjs'), + 'expected c.integration.test.cjs in output', + ); + }); + + test('--suite all is equivalent to no flag', () => { + seed(tmpDir, ['a.test.cjs', 'b.security.test.cjs']); + const r = runHarness(tmpDir, ['--suite', 'all']); + assert.strictEqual(r.status, 0); + assert.ok(r.stderr.includes('a.test.cjs')); + assert.ok(r.stderr.includes('b.security.test.cjs')); + }); + + test('--suite unit excludes marked suites', () => { + seed(tmpDir, [ + 'a.test.cjs', + 'b.security.test.cjs', + 'c.integration.test.cjs', + 'd.install.test.cjs', + 'e.slow.test.cjs', + ]); + const r = runHarness(tmpDir, ['--suite', 'unit']); + assert.strictEqual(r.status, 0, `stderr: ${r.stderr}`); + assert.ok(r.stderr.includes('a.test.cjs')); + assert.ok(!r.stderr.includes('b.security.test.cjs')); + assert.ok(!r.stderr.includes('c.integration.test.cjs')); + assert.ok(!r.stderr.includes('d.install.test.cjs')); + assert.ok(!r.stderr.includes('e.slow.test.cjs')); + }); + + test('--suite security selects only *.security.test.cjs', () => { + seed(tmpDir, [ + 'a.test.cjs', + 'b.security.test.cjs', + 'c.integration.test.cjs', + ]); + const r = runHarness(tmpDir, ['--suite', 'security']); + assert.strictEqual(r.status, 0); + assert.ok(r.stderr.includes('b.security.test.cjs')); + assert.ok(!r.stderr.includes('a.test.cjs')); + assert.ok(!r.stderr.includes('c.integration.test.cjs')); + }); + + test('--suite integration selects only *.integration.test.cjs', () => { + seed(tmpDir, ['a.test.cjs', 'b.integration.test.cjs']); + const r = runHarness(tmpDir, ['--suite', 'integration']); + assert.strictEqual(r.status, 0); + assert.ok(r.stderr.includes('b.integration.test.cjs')); + assert.ok(!r.stderr.includes('a.test.cjs')); + }); + + test('--suite install selects only *.install.test.cjs', () => { + seed(tmpDir, ['a.test.cjs', 'b.install.test.cjs']); + const r = runHarness(tmpDir, ['--suite', 'install']); + assert.strictEqual(r.status, 0); + assert.ok(r.stderr.includes('b.install.test.cjs')); + }); + + test('--suite slow selects only *.slow.test.cjs', () => { + seed(tmpDir, ['a.test.cjs', 'b.slow.test.cjs']); + const r = runHarness(tmpDir, ['--suite', 'slow']); + assert.strictEqual(r.status, 0); + assert.ok(r.stderr.includes('b.slow.test.cjs')); + }); + }); + + describe('empty-suite behavior', () => { + test('--suite security with zero matching files exits 0 with a notice', () => { + seed(tmpDir, ['a.test.cjs']); + const r = runHarness(tmpDir, ['--suite', 'security']); + assert.strictEqual(r.status, 0); + assert.match(r.stderr, /no tests in suite/i); + }); + + test('completely empty test dir still exits non-zero (preserves prior behavior)', () => { + const r = runHarness(tmpDir); + assert.notStrictEqual(r.status, 0); + assert.match(r.stderr, /no test files/i); + }); + }); + + describe('failure propagation', () => { + test('non-zero from node:test propagates through harness', () => { + const FAIL = `'use strict'; +const { test } = require('node:test'); +test('boom', () => { throw new Error('intentional'); }); +`; + fs.writeFileSync(path.join(tmpDir, 'a.test.cjs'), FAIL, 'utf8'); + const r = runHarness(tmpDir); + assert.notStrictEqual( + r.status, + 0, + `expected non-zero exit; got status=${r.status} signal=${r.signal}\nSTDOUT:\n${r.stdout}\nSTDERR:\n${r.stderr}`, + ); + }); + }); +});