From a647053dcf7a7088ab7e976e57509284742a2518 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 9 Jun 2026 23:00:46 -0400 Subject: [PATCH 1/7] =?UTF-8?q?ci(scope):=20narrow=20#494=20invariant=20?= =?UTF-8?q?=E2=80=94=20changed=20tests=20join=20windows=20lane,=20not=20fu?= =?UTF-8?q?ll=20matrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit full_matrix fired on 15/15 sampled PRs because any tests/** change forced it, costing ~25 runner-minutes each. A changed test file now always joins the scoped windows lane (covering the #482 OS-specific failure class per-file) and still runs on ubuntu 22/24 via targeted_tests; the residual macOS / windows-node-22 cross-product is covered on every push to next. Also narrows WINDOWS_HINTS from 6 substrings (102/633 files, a ~10-minute scoped lane) to windows/win32/shell/path — the dropped hints (workflow, install, hook) are either platform-independent lint tests or already covered by fullMatrix rules. Co-Authored-By: Claude Fable 5 --- scripts/ci-test-scope.cjs | 31 ++++++--- scripts/run-cross-platform-tests.cjs | 67 ------------------- tests/ci-test-scope.test.cjs | 34 +++++++--- ...cjs => graphify-auto-update.slow.test.cjs} | 0 ...er-migration-install.integration.test.cjs} | 0 ...> prompt-injection-scan.security.test.cjs} | 0 ... read-injection-scanner.security.test.cjs} | 0 tests/run-cross-platform-tests.test.cjs | 46 ------------- ...cjs => secret-scan-lint.security.test.cjs} | 0 ...curity-prompt-injection.security.test.cjs} | 0 ...st.cjs => security-scan.security.test.cjs} | 0 11 files changed, 47 insertions(+), 131 deletions(-) delete mode 100644 scripts/run-cross-platform-tests.cjs rename tests/{graphify-auto-update.test.cjs => graphify-auto-update.slow.test.cjs} (100%) rename tests/{installer-migration-install-integration.test.cjs => installer-migration-install.integration.test.cjs} (100%) rename tests/{prompt-injection-scan.test.cjs => prompt-injection-scan.security.test.cjs} (100%) rename tests/{read-injection-scanner.test.cjs => read-injection-scanner.security.test.cjs} (100%) delete mode 100644 tests/run-cross-platform-tests.test.cjs rename tests/{secret-scan-lint.test.cjs => secret-scan-lint.security.test.cjs} (100%) rename tests/{security-prompt-injection.test.cjs => security-prompt-injection.security.test.cjs} (100%) rename tests/{security-scan.test.cjs => security-scan.security.test.cjs} (100%) diff --git a/scripts/ci-test-scope.cjs b/scripts/ci-test-scope.cjs index c90ffb485..fa78344b4 100644 --- a/scripts/ci-test-scope.cjs +++ b/scripts/ci-test-scope.cjs @@ -169,11 +169,11 @@ const RULES = [ path.includes('prompt-injection-scan') || path.startsWith('tests/fixtures/adversarial/security/'), tests: [ - 'tests/secret-scan-lint.test.cjs', - 'tests/prompt-injection-scan.test.cjs', - 'tests/security-prompt-injection.test.cjs', - 'tests/read-injection-scanner.test.cjs', - 'tests/security-scan.test.cjs', + 'tests/secret-scan-lint.security.test.cjs', + 'tests/prompt-injection-scan.security.test.cjs', + 'tests/security-prompt-injection.security.test.cjs', + 'tests/read-injection-scanner.security.test.cjs', + 'tests/security-scan.security.test.cjs', ], }, { @@ -308,7 +308,13 @@ function addAll(set, values) { for (const value of values) set.add(value); } -const WINDOWS_HINTS = ['windows', 'path', 'shell', 'workflow', 'install', 'hook']; +// Windows-sensitive filename hints — deliberately narrow. 'workflow', +// 'install', and 'hook' were dropped from this list: workflow-lint tests are +// platform-independent YAML/policy checks, and the installer/hooks RULES set +// fullMatrix=true, so the full Windows lane already runs when those paths +// change. The old six-hint list pulled 102 of ~633 test files into the scoped +// windows lane, turning it into a ~10-minute job on every PR. +const WINDOWS_HINTS = ['windows', 'win32', 'shell', 'path']; const isWindowsHint = s => WINDOWS_HINTS.some(k => s.toLowerCase().includes(k)); function classify(files) { @@ -343,10 +349,15 @@ function classify(files) { if (file.startsWith('tests/') && file.endsWith('.test.cjs')) { targeted.add(file); - fullMatrix = true; - if (isWindowsHint(file)) { - windows.add(file); - } + // #494 invariant, narrowed: a changed test must still be exercised on + // the divergent OS before merge, but at per-file cost — it ALWAYS joins + // the scoped windows lane instead of triggering the three full parity + // lanes. (full_matrix fired on 15/15 sampled PRs because test-driven + // PRs always touch tests/, costing ~25 runner-minutes each.) Changed + // tests already run on ubuntu-22 and ubuntu-24 via targeted_tests; the + // residual macOS / windows-node-22 cross-product is covered by the full + // matrix on every push to next. + windows.add(file); } for (const rule of RULES) { diff --git a/scripts/run-cross-platform-tests.cjs b/scripts/run-cross-platform-tests.cjs deleted file mode 100644 index 31a8f4a8e..000000000 --- a/scripts/run-cross-platform-tests.cjs +++ /dev/null @@ -1,67 +0,0 @@ -'use strict'; - -const { spawnSync } = require('child_process'); -const { ExitError, runMain } = require('./lib/cli-exit.cjs'); - -const CROSS_PLATFORM_TEST_REASON = Object.freeze({ - PASS: 'pass', - TEST_FAILURE: 'test_failure', - INFRA_FAILURE: 'infra_failure', - UNKNOWN_FAILURE: 'unknown_failure', -}); - -function classify(exitCode, output) { - if (exitCode === 0) return CROSS_PLATFORM_TEST_REASON.PASS; - if (exitCode === 2 || /infrastructure failure|worktree\.Construct/i.test(output)) { - return CROSS_PLATFORM_TEST_REASON.INFRA_FAILURE; - } - if (/\bFAIL\b|\d+\s+failures?\)/i.test(output)) { - return CROSS_PLATFORM_TEST_REASON.TEST_FAILURE; - } - return CROSS_PLATFORM_TEST_REASON.UNKNOWN_FAILURE; -} - -function runCrossPlatformTests(options = {}, deps = {}) { - const { - base = 'next', - head = 'HEAD', - source = '.', - targets = 'linux,macos', - cwd = process.cwd(), - } = options; - const runner = deps.spawnSync || spawnSync; - - const args = ['--targets', targets, '--base', base, '--head', head, '--source', source]; - const result = runner('gsd-test', args, { cwd, encoding: 'utf8' }); - - const stdout = result.stdout || ''; - const stderr = result.stderr || ''; - const output = `${stdout}\n${stderr}`; - const exitCode = Number(result.status ?? 1); - const reason = classify(exitCode, output); - - return { - ok: exitCode === 0, - reason, - exitCode, - command: ['gsd-test', ...args].join(' '), - stdout, - stderr, - }; -} - -if (require.main === module) { - function main() { - const result = runCrossPlatformTests(); - const line = `[cross-platform-tests] reason=${result.reason} exit=${result.exitCode}`; - if (result.ok) { - process.stdout.write(`${line}\n`); - return 0; - } - process.stderr.write(`${line}\n`); - throw new ExitError(result.exitCode); - } - runMain(main); -} - -module.exports = { CROSS_PLATFORM_TEST_REASON, runCrossPlatformTests }; diff --git a/tests/ci-test-scope.test.cjs b/tests/ci-test-scope.test.cjs index 08c67f723..59c77a831 100644 --- a/tests/ci-test-scope.test.cjs +++ b/tests/ci-test-scope.test.cjs @@ -279,18 +279,36 @@ describe('ci-test-scope.cjs', () => { }); }); -describe('ci-test-scope superset invariant (#494)', () => { - // Facet A: any tests/** change → full_matrix === true - test('A1: a specific changed test file forces full_matrix', () => { +describe('ci-test-scope superset invariant (#494, narrowed)', () => { + // Facet A (narrowed): a changed test file no longer triggers the full + // parity matrix — instead it must ALWAYS run on the scoped windows lane, + // so OS-specific breakage in the changed test (the #482 class) is still + // exercised pre-merge. Ubuntu 22/24 coverage comes via targeted_tests. + test('A1: a changed test file joins the windows scoped lane without full_matrix', () => { const result = scopeFor(['tests/bug-1974-context-exhaustion-record.test.cjs']); - assert.strictEqual(result.full_matrix, true, - `expected full_matrix=true for tests/** change, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, false, + `expected full_matrix=false for a tests/**-only change, got: ${JSON.stringify(result)}`); + assert.ok(result.targeted_tests.includes('tests/bug-1974-context-exhaustion-record.test.cjs'), + `expected the changed test in targeted_tests, got: ${JSON.stringify(result.targeted_tests)}`); + assert.ok(result.windows_tests.includes('tests/bug-1974-context-exhaustion-record.test.cjs'), + `expected the changed test in windows_tests, got: ${JSON.stringify(result.windows_tests)}`); }); - test('A2: any tests/** path forces full_matrix', () => { + test('A2: a changed test file with no windows hint still joins the windows lane', () => { + // commands.test.cjs matches none of the WINDOWS_HINTS substrings — the + // unconditional changed-test → windows lane rule must include it anyway. + const result = scopeFor(['tests/commands.test.cjs']); + assert.strictEqual(result.full_matrix, false); + assert.ok(result.windows_tests.includes('tests/commands.test.cjs'), + `expected hint-less changed test in windows_tests, got: ${JSON.stringify(result.windows_tests)}`); + }); + + test('A3: a deleted/nonexistent test path falls back to the unit token, no full_matrix', () => { const result = scopeFor(['tests/some-new.test.cjs']); - assert.strictEqual(result.full_matrix, true, - `expected full_matrix=true for tests/** change, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, false); + // The nonexistent file is filtered by existingTests(); with nothing left, + // the #408 fallback applies so the targeted lane still runs something. + assert.deepStrictEqual(result.targeted_tests, ['unit']); }); // Facet B: commands/**, agents/** → code_changed AND docs-parity selected diff --git a/tests/graphify-auto-update.test.cjs b/tests/graphify-auto-update.slow.test.cjs similarity index 100% rename from tests/graphify-auto-update.test.cjs rename to tests/graphify-auto-update.slow.test.cjs diff --git a/tests/installer-migration-install-integration.test.cjs b/tests/installer-migration-install.integration.test.cjs similarity index 100% rename from tests/installer-migration-install-integration.test.cjs rename to tests/installer-migration-install.integration.test.cjs diff --git a/tests/prompt-injection-scan.test.cjs b/tests/prompt-injection-scan.security.test.cjs similarity index 100% rename from tests/prompt-injection-scan.test.cjs rename to tests/prompt-injection-scan.security.test.cjs diff --git a/tests/read-injection-scanner.test.cjs b/tests/read-injection-scanner.security.test.cjs similarity index 100% rename from tests/read-injection-scanner.test.cjs rename to tests/read-injection-scanner.security.test.cjs diff --git a/tests/run-cross-platform-tests.test.cjs b/tests/run-cross-platform-tests.test.cjs deleted file mode 100644 index d55bb30a8..000000000 --- a/tests/run-cross-platform-tests.test.cjs +++ /dev/null @@ -1,46 +0,0 @@ -const { test, describe } = require('node:test'); -const assert = require('node:assert/strict'); - -const { CROSS_PLATFORM_TEST_REASON, runCrossPlatformTests } = require('../scripts/run-cross-platform-tests.cjs'); - -describe('run cross-platform tests module', () => { - test('returns pass reason on zero exit', () => { - const out = runCrossPlatformTests({}, { - spawnSync: () => ({ status: 0, stdout: 'ok', stderr: '' }), - }); - assert.strictEqual(out.ok, true); - assert.strictEqual(out.reason, CROSS_PLATFORM_TEST_REASON.PASS); - assert.ok(out.command.includes('--base next')); - }); - - test('classifies infrastructure failure', () => { - const out = runCrossPlatformTests({}, { - spawnSync: () => ({ status: 2, stdout: '', stderr: 'infrastructure failure' }), - }); - assert.strictEqual(out.ok, false); - assert.strictEqual(out.reason, CROSS_PLATFORM_TEST_REASON.INFRA_FAILURE); - }); - - test('classifies test failure from FAIL marker', () => { - const out = runCrossPlatformTests({}, { - spawnSync: () => ({ status: 1, stdout: 'linux FAIL 1/2 tests (1 failures)', stderr: '' }), - }); - assert.strictEqual(out.ok, false); - assert.strictEqual(out.reason, CROSS_PLATFORM_TEST_REASON.TEST_FAILURE); - }); - - test('passes options through to command args', () => { - let cmd = null; - let args = null; - runCrossPlatformTests({ base: 'main', head: 'abc123', source: '/tmp/x', targets: 'linux' }, { - spawnSync: (c, a) => { - cmd = c; - args = a; - return { status: 0, stdout: '', stderr: '' }; - }, - }); - - assert.strictEqual(cmd, 'gsd-test'); - assert.deepStrictEqual(args, ['--targets', 'linux', '--base', 'main', '--head', 'abc123', '--source', '/tmp/x']); - }); -}); diff --git a/tests/secret-scan-lint.test.cjs b/tests/secret-scan-lint.security.test.cjs similarity index 100% rename from tests/secret-scan-lint.test.cjs rename to tests/secret-scan-lint.security.test.cjs diff --git a/tests/security-prompt-injection.test.cjs b/tests/security-prompt-injection.security.test.cjs similarity index 100% rename from tests/security-prompt-injection.test.cjs rename to tests/security-prompt-injection.security.test.cjs diff --git a/tests/security-scan.test.cjs b/tests/security-scan.security.test.cjs similarity index 100% rename from tests/security-scan.test.cjs rename to tests/security-scan.security.test.cjs From cd5db1f8db0e759cdb71b64484f3f5c4306547ed Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 9 Jun 2026 23:01:00 -0400 Subject: [PATCH 2/7] test(suites): seed security/slow/integration suites via measured retags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames (git mv) with all references updated (ci-test-scope RULES, windows-parity allowlist, test-file-count allowlist, docs in 6 locales): - 5 scanner tests -> *.security.test.cjs — the 'Run security tests' CI step ran zero files since the suite taxonomy landed; it is now honest. - graphify-auto-update -> *.slow.test.cjs (36s, slowest file in the suite; e2e gsd-tools spawns) — runs on full-matrix lanes and push to next. - installer-migration-install-integration -> *.integration.test.cjs (13s; an integration test by its own name). Coverage gate measured after retags: 88.55% lines (gate 70%). Co-Authored-By: Claude Fable 5 --- CONTEXT.md | 2 +- docs/FEATURES.md | 2 +- docs/TESTING-SUITES.md | 48 ++++++++-- docs/USER-GUIDE.md | 2 +- docs/explanation/security-model.md | 2 +- docs/ja-JP/FEATURES.md | 2 +- docs/ja-JP/USER-GUIDE.md | 2 +- docs/ja-JP/explanation/security-model.md | 2 +- docs/ko-KR/FEATURES.md | 2 +- docs/ko-KR/USER-GUIDE.md | 2 +- docs/ko-KR/explanation/security-model.md | 2 +- docs/pt-BR/USER-GUIDE.md | 2 +- docs/pt-BR/explanation/security-model.md | 2 +- docs/zh-CN/FEATURES.md | 2 +- docs/zh-CN/USER-GUIDE.md | 2 +- docs/zh-CN/explanation/security-model.md | 2 +- scripts/lint-test-file-count.allowlist.json | 6 +- tests/fixtures/adversarial/security/README.md | 2 +- tests/lint-regression-test-names.test.cjs | 95 +++++++++++++++++++ tests/policy-lint-shallow-checkout.test.cjs | 2 +- tests/prompt-injection-scan.security.test.cjs | 2 +- tests/windows-test-parity-guard.test.cjs | 6 +- 22 files changed, 161 insertions(+), 30 deletions(-) create mode 100644 tests/lint-regression-test-names.test.cjs diff --git a/CONTEXT.md b/CONTEXT.md index bd78058ed..2f909f29a 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -521,7 +521,7 @@ The canonical lint infrastructure adopted in ADR 452 (`docs/adr/452-eslint-lint- `DEFECT.SUPERSEDED-CONCURRENT-PRS.fix-forward=close superseded PRs via gh api PATCH state=closed; do not comment on self-authored PRs (k101); the link to the merged PR makes supersession discoverable in PR history` `DEFECT.PROMPT-INJECTION-SCAN-COLLISION.symptom=custom XML element name in agent .md file matches scripts/scan-prompt-injection regex; legitimate agent vocabulary trips the security gate` -`DEFECT.PROMPT-INJECTION-SCAN-COLLISION.examples=#3309 added a bare 'human' element (angle-bracket-wrapped) for verify-block harvesting; tests/prompt-injection-scan.test.cjs flags angle-bracket-wrapped names matching system|assistant|human (open or close form)` +`DEFECT.PROMPT-INJECTION-SCAN-COLLISION.examples=#3309 added a bare 'human' element (angle-bracket-wrapped) for verify-block harvesting; tests/prompt-injection-scan.security.test.cjs flags angle-bracket-wrapped names matching system|assistant|human (open or close form)` `DEFECT.PROMPT-INJECTION-SCAN-COLLISION.detect=any new bare tag in agents/*.md` `DEFECT.PROMPT-INJECTION-SCAN-COLLISION.fix-forward=hyphenate the tag (, ) — scanner regex matches bare names only` diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 7b6b0cbfa..952cc4503 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -1293,7 +1293,7 @@ PreToolUse hook that scans Write/Edit calls targeting `.planning/` for injection **3. Workflow Guard Hook** (`gsd-workflow-guard.js`) PreToolUse hook that detects when Claude attempts file edits outside a GSD workflow context. Advises using `/gsd-quick` or `/gsd-fast` instead of direct edits. Configurable via `hooks.workflow_guard` (default: false). -**4. CI-Ready Injection Scanner** (`prompt-injection-scan.test.cjs`) +**4. CI-Ready Injection Scanner** (`prompt-injection-scan.security.test.cjs`) Test suite that scans all agent, workflow, and command files for embedded injection vectors. **Requirements:** diff --git a/docs/TESTING-SUITES.md b/docs/TESTING-SUITES.md index 8705a030e..b5f31e053 100644 --- a/docs/TESTING-SUITES.md +++ b/docs/TESTING-SUITES.md @@ -29,6 +29,23 @@ Examples: The suite-suffix convention was chosen over a directory layout (`tests/security/`) so the 545+ existing test files don't need to move. Existing files all classify as `unit` until someone explicitly retags them. +## Regression tests + +**Do not create new top-level `tests/bug-NNNN-*.test.cjs` files.** Add the +regression case to the owning module's main test file instead (e.g. a +`describe('regressions')` block in `tests/.test.cjs`). + +`node --test` spawns one child process per FILE, so file count — not test +count — is the unit of CI overhead, and it is worst on Windows lanes where +every spawn is Defender-scanned. The 2026-06 CI audit found 244 one-off +`bug-*` files (~38% of the suite). That population is grandfathered in +`scripts/lint-regression-test-names.allowlist.json` and enforced by an +identity ratchet (`npm run lint:regression-names`, part of `npm run lint:ci`): + +- A **new** `bug-*` file fails CI — fold it into the owning module's file. +- **Deleting/consolidating** a grandfathered file requires pruning its + allowlist entry, so the baseline only ever shrinks. + ## Running suites locally ```bash @@ -53,6 +70,12 @@ node scripts/run-tests.cjs --files "tests/command-contract.test.cjs tests/core.t node scripts/run-tests.cjs --files-from .ci-selected-tests.txt ``` +`npm run test:affected` (scripts/run-affected-tests.cjs) is a **local-only** +convenience that selects tests via the `require()` dependency graph of your +working-tree diff. CI does not use it — CI selection is the rule table in +`scripts/ci-test-scope.cjs`, which is the authoritative mapping. If the two +disagree, trust (and fix) the rule table. + Unknown suites exit non-zero with the list of valid suites. Empty suites (e.g. `--suite security` before any security-tagged file exists) exit `0` with a `no tests in suite "..."` notice on stderr so CI lanes don't go red while a suite is being populated. ## CI matrix @@ -72,14 +95,27 @@ The `Tests` workflow runs every PR through a scoped gate generated by smoke set. They are for confidence on the affected surface, not for counting tests. -The default PR gate runs the broad `unit`, `integration`, and `security` suites -once on Ubuntu / Node 24, scoped smoke on Ubuntu / Node 22, scoped -Windows/path/shell tests on Windows / Node 24, and unit coverage once on Ubuntu / -Node 24. PRs touching workflow, package, test-runner, install, release, or +The default PR gate runs the broad `unit` (under the c8 coverage gate), +`integration`, and `security` suites once on Ubuntu / Node 24, scoped smoke on +Ubuntu / Node 22, and scoped Windows-sensitive tests on Windows / Node 24. +**Every changed test file always joins the Windows scoped lane** (the #494 +invariant, narrowed): a modified test is exercised on the divergent OS before +merge at per-file cost, without paying for the three full parity lanes. + +PRs touching workflow, package, test-runner, install, release, or Windows-sensitive surfaces also run the full parity matrix on macOS and the older Windows runtime, plus `install` and `slow` on the primary Ubuntu lane. -Coverage stays single-lane because multiplying coverage across OS/runtime lanes -adds cost without improving the threshold signal. +Everything (including the full parity matrix) runs on every push to `next`, +which covers the residual macOS / Windows-Node-22 cross-product for scoped PRs. + +Coverage runs inside the Ubuntu / Node 24 full lane (not a separate job — that +duplicated the entire unit run) and stays single-lane because multiplying +coverage across OS/runtime lanes adds cost without improving the threshold +signal. Note the gate's deliberate blind spot: it measures +`gsd-core/bin/lib/*.cjs` only — `scripts/`, `hooks/`, and `bin/` are +unenforced, and `stryker.config.mjs` additionally excludes ~48% of lib lines +from mutation testing (see the UNMUTATED list there). Widening either gate is +tracked work, not an accident to "fix" silently by raising thresholds. To inspect the scope locally: diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index e83cfae8f..33e894224 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -386,7 +386,7 @@ GSD generates markdown files that become LLM system prompts. This means any user - `gsd-prompt-guard.js` — Scans Write/Edit calls to `.planning/` for injection patterns (always active, advisory-only) - `gsd-workflow-guard.js` — Warns on file edits outside GSD workflow context (opt-in via `hooks.workflow_guard`) -**CI Scanner:** `prompt-injection-scan.test.cjs` scans all agent, workflow, and command files for embedded injection vectors. +**CI Scanner:** `prompt-injection-scan.security.test.cjs` scans all agent, workflow, and command files for embedded injection vectors. --- diff --git a/docs/explanation/security-model.md b/docs/explanation/security-model.md index 53115c7ca..b55ef2395 100644 --- a/docs/explanation/security-model.md +++ b/docs/explanation/security-model.md @@ -158,7 +158,7 @@ injected instructions in untrusted content — catching cases where an attacker has embedded instructions in a file that GSD is about to incorporate into an agent's context. -**CI scanner.** `prompt-injection-scan.test.cjs` scans all agent, workflow, +**CI scanner.** `prompt-injection-scan.security.test.cjs` scans all agent, workflow, and command files for embedded injection vectors as part of the test suite. This catches injection attempts in the GSD source itself — for example, a supply-chain attack that modified a workflow file to add a role-override diff --git a/docs/ja-JP/FEATURES.md b/docs/ja-JP/FEATURES.md index eda9783a7..0109debc3 100644 --- a/docs/ja-JP/FEATURES.md +++ b/docs/ja-JP/FEATURES.md @@ -1227,7 +1227,7 @@ fix(03-01): correct auth token expiry **3. ワークフローガードフック**(`gsd-workflow-guard.js`) Claude が GSD ワークフローコンテキスト外でファイル編集を試行した際に検出する PreToolUse フック。直接編集の代わりに `/gsd-quick` や `/gsd-fast` の使用をアドバイスします。`hooks.workflow_guard`(デフォルト: false)で設定可能です。 -**4. CI 対応インジェクションスキャナー**(`prompt-injection-scan.test.cjs`) +**4. CI 対応インジェクションスキャナー**(`prompt-injection-scan.security.test.cjs`) すべてのエージェント、ワークフロー、コマンドファイルに埋め込まれたインジェクションベクターをスキャンするテストスイート。 **要件:** diff --git a/docs/ja-JP/USER-GUIDE.md b/docs/ja-JP/USER-GUIDE.md index 0982dce70..264953822 100644 --- a/docs/ja-JP/USER-GUIDE.md +++ b/docs/ja-JP/USER-GUIDE.md @@ -369,7 +369,7 @@ GSD は LLM のシステムプロンプトになるマークダウンファイ - `gsd-prompt-guard.js` — `.planning/` への Write/Edit 呼び出しでインジェクションパターンをスキャンする(常時有効、アドバイザリーのみ) - `gsd-workflow-guard.js` — GSD ワークフローコンテキスト外でのファイル編集を警告する(`hooks.workflow_guard` 経由でオプトイン) -**CI スキャナー:** `prompt-injection-scan.test.cjs` はすべてのエージェント、ワークフロー、コマンドファイルに埋め込まれたインジェクションベクターをスキャンします。 +**CI スキャナー:** `prompt-injection-scan.security.test.cjs` はすべてのエージェント、ワークフロー、コマンドファイルに埋め込まれたインジェクションベクターをスキャンします。 --- diff --git a/docs/ja-JP/explanation/security-model.md b/docs/ja-JP/explanation/security-model.md index afc5c3d93..be3014d07 100644 --- a/docs/ja-JP/explanation/security-model.md +++ b/docs/ja-JP/explanation/security-model.md @@ -73,7 +73,7 @@ GSD Core はプロンプトインジェクションを 3 つのレベルで対 **ランタイムフック:`gsd-read-injection-scanner.js`。** このフックはすべての Read ツール呼び出しの出力で発火します。GSD がエージェントのコンテキストに組み込もうとしているファイルの *読み取ったばかりのコンテンツ* をスキャンし、攻撃者が命令を埋め込んでいるケースをキャッチします。 -**CI スキャナー。** `prompt-injection-scan.test.cjs` はテストスイートの一部として、すべてのエージェント、ワークフロー、コマンドファイルに埋め込まれたインジェクションベクターをスキャンします。これは GSD ソース自体でのインジェクション試みをキャッチします——たとえば、ワークフローファイルにロールオーバーライド命令を追加するよう変更したサプライチェーン攻撃。 +**CI スキャナー。** `prompt-injection-scan.security.test.cjs` はテストスイートの一部として、すべてのエージェント、ワークフロー、コマンドファイルに埋め込まれたインジェクションベクターをスキャンします。これは GSD ソース自体でのインジェクション試みをキャッチします——たとえば、ワークフローファイルにロールオーバーライド命令を追加するよう変更したサプライチェーン攻撃。 ### Read Injection Scanner vs Prompt Guard diff --git a/docs/ko-KR/FEATURES.md b/docs/ko-KR/FEATURES.md index c393e8982..be2f5d268 100644 --- a/docs/ko-KR/FEATURES.md +++ b/docs/ko-KR/FEATURES.md @@ -1131,7 +1131,7 @@ fix(03-01): correct auth token expiry **3. 워크플로우 가드 훅** (`gsd-workflow-guard.js`) Claude가 GSD 워크플로우 컨텍스트 밖에서 파일 편집을 시도하는 것을 감지하는 PreToolUse 훅입니다. 직접 편집 대신 `/gsd-quick` 또는 `/gsd-fast` 사용을 권고합니다. `hooks.workflow_guard`로 구성 가능합니다(기본값: false). -**4. CI 준비 주입 스캐너** (`prompt-injection-scan.test.cjs`) +**4. CI 준비 주입 스캐너** (`prompt-injection-scan.security.test.cjs`) 모든 에이전트, 워크플로우, 명령어 파일에서 포함된 주입 벡터를 스캔하는 테스트 스위트입니다. **요구사항.** diff --git a/docs/ko-KR/USER-GUIDE.md b/docs/ko-KR/USER-GUIDE.md index 0370fd753..a2f8c01aa 100644 --- a/docs/ko-KR/USER-GUIDE.md +++ b/docs/ko-KR/USER-GUIDE.md @@ -369,7 +369,7 @@ GSD는 LLM 시스템 프롬프트가 되는 마크다운 파일을 생성합니 - `gsd-prompt-guard.js` — `.planning/`에 대한 Write/Edit 호출에서 인젝션 패턴 스캔 (항상 활성, 자문 전용) - `gsd-workflow-guard.js` — GSD 워크플로우 컨텍스트 외부에서 파일 편집 시 경고 (`hooks.workflow_guard`를 통한 옵트인) -**CI 스캐너:** `prompt-injection-scan.test.cjs`는 모든 에이전트, 워크플로우, 명령어 파일에서 삽입된 인젝션 벡터를 스캔합니다. +**CI 스캐너:** `prompt-injection-scan.security.test.cjs`는 모든 에이전트, 워크플로우, 명령어 파일에서 삽입된 인젝션 벡터를 스캔합니다. --- diff --git a/docs/ko-KR/explanation/security-model.md b/docs/ko-KR/explanation/security-model.md index 644e61a3e..4cbd1c2a6 100644 --- a/docs/ko-KR/explanation/security-model.md +++ b/docs/ko-KR/explanation/security-model.md @@ -79,7 +79,7 @@ GSD Core는 세 가지 수준에서 프롬프트 인젝션을 다룬다. **런타임 훅: `gsd-read-injection-scanner.js`.** 이 훅은 모든 Read 도구 호출의 출력에서 실행된다. 방금 읽은 *콘텐츠*를 신뢰할 수 없는 콘텐츠의 주입된 지시 사항으로 스캔한다 — 공격자가 GSD가 에이전트 컨텍스트에 통합하려는 파일에 지시 사항을 내장한 경우를 잡아낸다. -**CI 스캐너.** `prompt-injection-scan.test.cjs`는 테스트 스위트의 일부로 내장된 인젝션 벡터가 있는지 모든 에이전트, 워크플로우, 명령 파일을 스캔한다. 이는 GSD 소스 자체의 인젝션 시도를 잡아낸다 — 예를 들어 워크플로우 파일을 수정하여 역할 재정의 지시 사항을 추가하는 공급망 공격. +**CI 스캐너.** `prompt-injection-scan.security.test.cjs`는 테스트 스위트의 일부로 내장된 인젝션 벡터가 있는지 모든 에이전트, 워크플로우, 명령 파일을 스캔한다. 이는 GSD 소스 자체의 인젝션 시도를 잡아낸다 — 예를 들어 워크플로우 파일을 수정하여 역할 재정의 지시 사항을 추가하는 공급망 공격. ### 읽기 인젝션 스캐너 vs 프롬프트 가드 diff --git a/docs/pt-BR/USER-GUIDE.md b/docs/pt-BR/USER-GUIDE.md index 6b63cf518..e0b269607 100644 --- a/docs/pt-BR/USER-GUIDE.md +++ b/docs/pt-BR/USER-GUIDE.md @@ -369,7 +369,7 @@ O GSD gera arquivos markdown que se tornam prompts de sistema de LLM. Isso signi - `gsd-prompt-guard.js` — Verifica chamadas Write/Edit para `.planning/` em busca de padrões de injeção (sempre ativo, somente consultivo) - `gsd-workflow-guard.js` — Avisa sobre edições de arquivos fora do contexto do workflow GSD (opt-in via `hooks.workflow_guard`) -**Scanner de CI:** `prompt-injection-scan.test.cjs` verifica todos os arquivos de agentes, workflows e comandos em busca de vetores de injeção incorporados. +**Scanner de CI:** `prompt-injection-scan.security.test.cjs` verifica todos os arquivos de agentes, workflows e comandos em busca de vetores de injeção incorporados. --- diff --git a/docs/pt-BR/explanation/security-model.md b/docs/pt-BR/explanation/security-model.md index fa866c152..1b1308090 100644 --- a/docs/pt-BR/explanation/security-model.md +++ b/docs/pt-BR/explanation/security-model.md @@ -168,7 +168,7 @@ de ser lido* em busca de instruções injetadas em conteúdo não confiável — capturando casos em que um atacante incorporou instruções em um arquivo que o GSD está prestes a incorporar ao contexto de um agente. -**Scanner de CI.** `prompt-injection-scan.test.cjs` escaneia todos os arquivos +**Scanner de CI.** `prompt-injection-scan.security.test.cjs` escaneia todos os arquivos de agente, workflow e comando em busca de vetores de injeção embutidos como parte do conjunto de testes. Isso detecta tentativas de injeção no próprio código-fonte do GSD — por exemplo, um ataque de cadeia de suprimentos que diff --git a/docs/zh-CN/FEATURES.md b/docs/zh-CN/FEATURES.md index e72be9ca6..993df8f55 100644 --- a/docs/zh-CN/FEATURES.md +++ b/docs/zh-CN/FEATURES.md @@ -1244,7 +1244,7 @@ PreToolUse 钩子,扫描针对 `.planning/` 的 Write/Edit 调用中的注入 **3. 工作流守护钩子**(`gsd-workflow-guard.js`) PreToolUse 钩子,检测 Claude 在 GSD 工作流上下文之外尝试文件编辑的情况。建议使用 `/gsd-quick` 或 `/gsd-fast` 替代直接编辑。可通过 `hooks.workflow_guard` 配置(默认:false)。 -**4. CI 就绪注入扫描器**(`prompt-injection-scan.test.cjs`) +**4. CI 就绪注入扫描器**(`prompt-injection-scan.security.test.cjs`) 扫描所有智能体、工作流和命令文件中嵌入注入向量的测试套件。 **需求:** diff --git a/docs/zh-CN/USER-GUIDE.md b/docs/zh-CN/USER-GUIDE.md index 51610bf8b..3c7c0acea 100644 --- a/docs/zh-CN/USER-GUIDE.md +++ b/docs/zh-CN/USER-GUIDE.md @@ -368,7 +368,7 @@ GSD 生成的 Markdown 文件会成为 LLM 系统提示。这意味着流入规 - `gsd-prompt-guard.js` — 扫描写入 `.planning/` 的 Write/Edit 调用中的注入模式(始终活跃,仅建议) - `gsd-workflow-guard.js` — 对 GSD 工作流上下文之外的文件编辑发出警告(通过 `hooks.workflow_guard` 选择性启用) -**CI 扫描器:** `prompt-injection-scan.test.cjs` 扫描所有 agent、工作流和命令文件中的嵌入式注入向量。 +**CI 扫描器:** `prompt-injection-scan.security.test.cjs` 扫描所有 agent、工作流和命令文件中的嵌入式注入向量。 --- diff --git a/docs/zh-CN/explanation/security-model.md b/docs/zh-CN/explanation/security-model.md index 9250066bb..cc7c0810a 100644 --- a/docs/zh-CN/explanation/security-model.md +++ b/docs/zh-CN/explanation/security-model.md @@ -73,7 +73,7 @@ GSD Core 在三个层面应对提示注入。 **运行时钩子:`gsd-read-injection-scanner.js`。** 该钩子在每次 Read 工具调用的输出时触发。它扫描*刚刚读取的内容*中在不可信内容中注入的指令——捕获攻击者在 GSD 即将纳入代理上下文的文件中嵌入指令的情况。 -**CI 扫描器。** `prompt-injection-scan.test.cjs` 作为测试套件的一部分,扫描所有代理、工作流和命令文件中嵌入的注入向量。这能捕获 GSD 源代码本身的注入尝试——例如,修改工作流文件以添加角色覆盖指令的供应链攻击。 +**CI 扫描器。** `prompt-injection-scan.security.test.cjs` 作为测试套件的一部分,扫描所有代理、工作流和命令文件中嵌入的注入向量。这能捕获 GSD 源代码本身的注入尝试——例如,修改工作流文件以添加角色覆盖指令的供应链攻击。 ### 读取注入扫描器与提示守卫的对比 diff --git a/scripts/lint-test-file-count.allowlist.json b/scripts/lint-test-file-count.allowlist.json index 215bec00d..c61b9c64c 100644 --- a/scripts/lint-test-file-count.allowlist.json +++ b/scripts/lint-test-file-count.allowlist.json @@ -26,7 +26,7 @@ "graphify": { "files": [ "bug-622-graphify-optional-graph-html.test.cjs", - "graphify-auto-update.test.cjs", + "graphify-auto-update.slow.test.cjs", "graphify-query.test.cjs", "graphify-visualization.test.cjs", "graphify.test.cjs" @@ -82,8 +82,8 @@ }, "security": { "files": [ - "security-prompt-injection.test.cjs", - "security-scan.test.cjs", + "security-prompt-injection.security.test.cjs", + "security-scan.security.test.cjs", "security.test.cjs" ], "issue": "TBD" diff --git a/tests/fixtures/adversarial/security/README.md b/tests/fixtures/adversarial/security/README.md index ab1ac2a3e..23de998d1 100644 --- a/tests/fixtures/adversarial/security/README.md +++ b/tests/fixtures/adversarial/security/README.md @@ -1,7 +1,7 @@ # Adversarial security fixtures (#3596) Reusable hostile payloads consumed by -`tests/security-prompt-injection.test.cjs`. +`tests/security-prompt-injection.security.test.cjs`. The fixtures here are pure data — they are loaded by the test as input to the production code under test (hooks, validators, sanitizers, CLI). diff --git a/tests/lint-regression-test-names.test.cjs b/tests/lint-regression-test-names.test.cjs new file mode 100644 index 000000000..8f30cd816 --- /dev/null +++ b/tests/lint-regression-test-names.test.cjs @@ -0,0 +1,95 @@ +'use strict'; + +// Tests for scripts/lint-regression-test-names.cjs — the identity ratchet +// that bans NEW top-level bug-NNNN test files (2026-06 CI audit). Uses the +// script's env overrides to point at sandbox fixture dirs; never touches the +// real tests/ directory or allowlist. + +const { describe, test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const os = require('node:os'); +const path = require('path'); +const { cleanup } = require('./helpers.cjs'); + +const ROOT = path.join(__dirname, '..'); +const SCRIPT = path.join(ROOT, 'scripts', 'lint-regression-test-names.cjs'); + +let sandbox; + +function runLint({ files, allowlist }) { + const testsDir = path.join(sandbox, `tests-${Math.random().toString(36).slice(2)}`); + fs.mkdirSync(testsDir, { recursive: true }); + for (const f of files) fs.writeFileSync(path.join(testsDir, f), ''); + const allowlistPath = path.join(testsDir, 'allowlist.json'); + fs.writeFileSync(allowlistPath, JSON.stringify(allowlist)); + return spawnSync(process.execPath, [SCRIPT], { + cwd: ROOT, + encoding: 'utf8', + env: { + ...process.env, + GSD_LINT_REGRESSION_TESTS_DIR: testsDir, + GSD_LINT_REGRESSION_ALLOWLIST: allowlistPath, + }, + }); +} + +describe('lint-regression-test-names', () => { + before(() => { + sandbox = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-lint-regression-')); + }); + + after(() => { + cleanup(sandbox); + }); + + test('passes when every bug-* file is grandfathered', () => { + const r = runLint({ + files: ['bug-100-old.test.cjs', 'module.test.cjs'], + allowlist: ['bug-100-old.test.cjs'], + }); + assert.strictEqual(r.status, 0, `stderr: ${r.stderr}`); + }); + + test('fails on a novel bug-* file with fold-into-module guidance', () => { + const r = runLint({ + files: ['bug-100-old.test.cjs', 'bug-200-new.test.cjs'], + allowlist: ['bug-100-old.test.cjs'], + }); + assert.notStrictEqual(r.status, 0); + assert.match(r.stderr, /bug-200-new\.test\.cjs/); + assert.match(r.stderr, /owning module/); + }); + + test('fails on a stale allowlist entry (ratchet-down enforcement)', () => { + const r = runLint({ + files: ['bug-100-old.test.cjs'], + allowlist: ['bug-100-old.test.cjs', 'bug-300-gone.test.cjs'], + }); + assert.notStrictEqual(r.status, 0); + assert.match(r.stderr, /bug-300-gone\.test\.cjs/); + }); + + test('ignores non-bug test files and suite-marked non-bug names', () => { + const r = runLint({ + files: ['module.test.cjs', 'feature.integration.test.cjs', 'debug-1-not-a-bug.test.cjs'], + allowlist: [], + }); + assert.strictEqual(r.status, 0, `stderr: ${r.stderr}`); + }); + + test('catches a suite-marked bug-* file too (no marker escape hatch)', () => { + const r = runLint({ + files: ['bug-400-sneaky.security.test.cjs'], + allowlist: [], + }); + assert.notStrictEqual(r.status, 0); + assert.match(r.stderr, /bug-400-sneaky\.security\.test\.cjs/); + }); + + test('repo baseline passes (real tests/ dir against real allowlist)', () => { + const r = spawnSync(process.execPath, [SCRIPT], { cwd: ROOT, encoding: 'utf8' }); + assert.strictEqual(r.status, 0, `stderr: ${r.stderr}\nstdout: ${r.stdout}`); + }); +}); diff --git a/tests/policy-lint-shallow-checkout.test.cjs b/tests/policy-lint-shallow-checkout.test.cjs index 82a96b985..303d532f0 100644 --- a/tests/policy-lint-shallow-checkout.test.cjs +++ b/tests/policy-lint-shallow-checkout.test.cjs @@ -12,7 +12,7 @@ * deeper than 50, which is intentional. * * Note: security-scan.yml legitimately uses fetch-depth: 0 and is NOT covered - * by this test (see tests/security-scan.test.cjs). + * by this test (see tests/security-scan.security.test.cjs). */ const { describe, test } = require('node:test'); diff --git a/tests/prompt-injection-scan.security.test.cjs b/tests/prompt-injection-scan.security.test.cjs index 4027bbb04..c32d98c97 100644 --- a/tests/prompt-injection-scan.security.test.cjs +++ b/tests/prompt-injection-scan.security.test.cjs @@ -58,7 +58,7 @@ const ALLOWLIST = new Set([ 'hooks/gsd-prompt-guard.js', // The prompt guard hook 'hooks/gsd-read-injection-scanner.js', // The read injection scanner (contains patterns) 'tests/security.test.cjs', // Security tests - 'tests/prompt-injection-scan.test.cjs', // This file + 'tests/prompt-injection-scan.security.test.cjs', // This file ]); // Workflows that exceed the 50K strict-mode size threshold due to legitimate diff --git a/tests/windows-test-parity-guard.test.cjs b/tests/windows-test-parity-guard.test.cjs index ed1a8922a..fd8eb30f0 100644 --- a/tests/windows-test-parity-guard.test.cjs +++ b/tests/windows-test-parity-guard.test.cjs @@ -49,12 +49,12 @@ const SELF = path.basename(__filename); const KNOWN_OFFENDERS = Object.freeze({ splitNewlineOnFileContent: new Set([ 'release-coverage-scope.test.cjs', - 'secret-scan-lint.test.cjs', - 'security-scan.test.cjs', + 'secret-scan-lint.security.test.cjs', + 'security-scan.security.test.cjs', ]), fenceRegexLiteralNewline: new Set([ 'bug-2995-post-install-script-paths.test.cjs', - 'security-scan.test.cjs', + 'security-scan.security.test.cjs', ]), frontmatterAnchorLiteralNewline: new Set([ 'bug-1967-cache-invalidation.test.cjs', From 622e4be6d825b7d5fa09c9fd19bce38126b804fe Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 9 Jun 2026 23:03:14 -0400 Subject: [PATCH 3/7] test(ratchet): ban new top-level bug-NNNN test files via identity allowlist 244 one-off bug-* files (~38% of the suite) are grandfathered in lint-regression-test-names.allowlist.json; new ones fail lint with fold-into-module guidance, and deletions force allowlist pruning so the baseline only shrinks. Wired into npm run lint:ci (new single entry point for every CI lint). Policy documented in docs/TESTING-SUITES.md. Co-Authored-By: Claude Fable 5 --- package.json | 2 + .../lint-regression-test-names.allowlist.json | 246 ++++++++++++++++++ scripts/lint-regression-test-names.cjs | 78 ++++++ 3 files changed, 326 insertions(+) create mode 100644 scripts/lint-regression-test-names.allowlist.json create mode 100644 scripts/lint-regression-test-names.cjs diff --git a/package.json b/package.json index 050571a0e..57f6ce2ad 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,8 @@ "pretest:coverage": "npm run build:lib && npm run lint:skill-deps", "lint": "eslint . --cache --cache-location node_modules/.cache/eslint/", "lint:fix": "eslint . --fix", + "lint:ci": "eslint . && npm run lint:skill-deps && node scripts/lint-test-file-count.cjs && node scripts/lint-command-contract.cjs && node scripts/lint-pr-check-project-dir.cjs && npm run lint:legacy-name && node scripts/lint-regression-test-names.cjs", + "lint:regression-names": "node scripts/lint-regression-test-names.cjs", "lint:descriptions": "node scripts/lint-descriptions.cjs", "lint:skill-deps": "node scripts/lint-skill-deps.cjs", "lint:test-file-count": "node scripts/lint-test-file-count.cjs", diff --git a/scripts/lint-regression-test-names.allowlist.json b/scripts/lint-regression-test-names.allowlist.json new file mode 100644 index 000000000..f99f80a62 --- /dev/null +++ b/scripts/lint-regression-test-names.allowlist.json @@ -0,0 +1,246 @@ +[ + "bug-10-semver-policy-consolidation.test.cjs", + "bug-130-finishinstall-opencode-testmode.test.cjs", + "bug-131-release-tarball-smoke-explicit-home.test.cjs", + "bug-14-progress-auto-flag-dropped.test.cjs", + "bug-167-query-meta-command.test.cjs", + "bug-17-askuserquestion-option-cap.test.cjs", + "bug-170-workflow-fallback-install-hint.test.cjs", + "bug-1736-local-install-commands.test.cjs", + "bug-1754-js-hook-guard.test.cjs", + "bug-1817-sh-hook-guard.test.cjs", + "bug-1818-unknown-flags.test.cjs", + "bug-1826-phases-clear-confirm.test.cjs", + "bug-1829-inherit-model-profile.test.cjs", + "bug-1834-sh-hooks-installed.test.cjs", + "bug-1891-file-resolution.test.cjs", + "bug-190-bridge-collapse.test.cjs", + "bug-1906-hook-relative-paths.test.cjs", + "bug-1908-uninstall-manifest.test.cjs", + "bug-1924-preserve-user-artifacts.test.cjs", + "bug-1967-cache-invalidation.test.cjs", + "bug-1974-context-exhaustion-record.test.cjs", + "bug-2002-offer-next-context.test.cjs", + "bug-2004-pr-branch-milestone.test.cjs", + "bug-21-state-md-template-frontmatter.test.cjs", + "bug-211-launcher-home-fallback.test.cjs", + "bug-2136-sh-hook-version.test.cjs", + "bug-214-phase-researcher-write-truncation-contract.test.cjs", + "bug-214-writer-agents-write-truncation-contract.test.cjs", + "bug-222-research-synthesizer-write-contract.test.cjs", + "bug-224-pick-stdout-capture.test.cjs", + "bug-2248-local-install-statusline.test.cjs", + "bug-2256-model-overrides-transport.test.cjs", + "bug-2268-parallel-discuss.test.cjs", + "bug-2344-read-guard-claudecode-env.test.cjs", + "bug-2346-agent-read-loop-guards.test.cjs", + "bug-2351-intel-kilo-layout.test.cjs", + "bug-2376-opencode-windows-home-path.test.cjs", + "bug-2384-post-merge-deletion-audit.test.cjs", + "bug-2388-plan-phase-no-branch-rename.test.cjs", + "bug-2396-makefile-test-priority.test.cjs", + "bug-2399-commit-docs-plan-phase.test.cjs", + "bug-2410-stream-checkpoint-heartbeats.test.cjs", + "bug-2418-antigravity-bare-path.test.cjs", + "bug-2419-project-researcher-agent.test.cjs", + "bug-2421-planner-grep-gate-hygiene.test.cjs", + "bug-2424-reapply-patches-baseline-detection.test.cjs", + "bug-2432-quick-plan-predispatch-commit.test.cjs", + "bug-2451-context-monitor-over-report.test.cjs", + "bug-2470-update-md-claude-path.test.cjs", + "bug-2492-context-coverage-gate.test.cjs", + "bug-2501-resurrection-detection.test.cjs", + "bug-2502-insert-phase-state-update.test.cjs", + "bug-2504-uat-foundation-phases.test.cjs", + "bug-2506-settings-profile-nonclaude-warning.test.cjs", + "bug-2516-inherit-model-execute-phase.test.cjs", + "bug-2520-read-guard-hook-subprocess-env.test.cjs", + "bug-2523-quick-deferred-items.test.cjs", + "bug-2530-valid-config-keys.test.cjs", + "bug-2543-gsd-slash-namespace.test.cjs", + "bug-2545-copilot-unreplaced-paths.test.cjs", + "bug-2549-2550-2552-discuss-phase-context.test.cjs", + "bug-2554-decimal-phase-filter.test.cjs", + "bug-2557-gemini-local-hook-paths.test.cjs", + "bug-2559-stale-search-year.test.cjs", + "bug-260-worktree-path-guard.test.cjs", + "bug-2601-inherit-model-profile.test.cjs", + "bug-261-worktree-force-add-guard.test.cjs", + "bug-2630-state-frontmatter-milestone-switch.test.cjs", + "bug-2638-sub-repos-canonical-location.test.cjs", + "bug-2643-skill-frontmatter-name.test.cjs", + "bug-2659-audit-open-crash.test.cjs", + "bug-2660-one-liner-extraction.test.cjs", + "bug-2661-roadmap-sync-parallel.test.cjs", + "bug-2686-review-fix-worktree.test.cjs", + "bug-2698-crlf-install.test.cjs", + "bug-2760-codex-install-defensive.test.cjs", + "bug-2769-requirements-header-variants.test.cjs", + "bug-2770-annotate-deps-int-coerce.test.cjs", + "bug-2771-user-profile-manifest.test.cjs", + "bug-2772-gitmodules-path-intersection.test.cjs", + "bug-2784-update-cache-clear-path.test.cjs", + "bug-279-codex-agent-mapping.test.cjs", + "bug-2794-opencode-model-profile-overrides.test.cjs", + "bug-2798-context-window-config-key.test.cjs", + "bug-2801-ingest-docs-handler.test.cjs", + "bug-2808-skill-hyphen-name.test.cjs", + "bug-2831-opencode-home-path-prefix.test.cjs", + "bug-2836-audit-open-summary-uat-drift.test.cjs", + "bug-2838-summary-rescue-gitignored-planning.test.cjs", + "bug-2839-review-fix-transactional-cleanup.test.cjs", + "bug-2851-workflow-bare-gsd-tools.test.cjs", + "bug-2866-codex-strip-no-trailing-newline.test.cjs", + "bug-2876-skill-frontmatter-quote.test.cjs", + "bug-2911-audit-open-output-shape.test.cjs", + "bug-2912-progress-context-authority.test.cjs", + "bug-2916-handle-branching-default-base.test.cjs", + "bug-2942-detect-custom-skills.test.cjs", + "bug-2943-config-get-context-window-default.test.cjs", + "bug-2948-spike-wrap-up-dispatch.test.cjs", + "bug-2949-sketch-wrap-up-dispatch.test.cjs", + "bug-2950-stale-command-refs.test.cjs", + "bug-2954-help-md-slash-command-stubs.test.cjs", + "bug-2957-claude-global-postinstall-message.test.cjs", + "bug-2969-verify-reapply-patches.test.cjs", + "bug-2973-profile-user-skills-path.test.cjs", + "bug-2979-hook-absolute-node.test.cjs", + "bug-2986-config-schema-mutation-killers.test.cjs", + "bug-2990-code-fixer-worktree-branch.test.cjs", + "bug-2992-check-latest-version.test.cjs", + "bug-2994-verify-reapply-patches-installed-path.test.cjs", + "bug-2995-post-install-script-paths.test.cjs", + "bug-2998-pristine-dir-populated.test.cjs", + "bug-3017-codex-hook-absolute-node.test.cjs", + "bug-3018-codex-discuss-fallback.test.cjs", + "bug-3019-help-passthrough.test.cjs", + "bug-3037-gemini-duplicate-commands.test.cjs", + "bug-3050-update-backup-eacces-nonfatal.test.cjs", + "bug-3054-stale-gsd-next-references.test.cjs", + "bug-3072-optional-sketch-findings-guard.test.cjs", + "bug-3083-resume-route-clear.test.cjs", + "bug-3086-git-create-tag-config-gate.test.cjs", + "bug-3087-planner-directive-language.test.cjs", + "bug-3096-ai-integration-phase-parallel-race.test.cjs", + "bug-3097-3099-executor-worktree-path-safety.test.cjs", + "bug-3120-secure-phase-empty-register.test.cjs", + "bug-3126-global-skills-base-runtime-path.test.cjs", + "bug-3127-state-begin-phase-idempotent.test.cjs", + "bug-3128-roadmap-plan-count-slug-layout.test.cjs", + "bug-3129-validate-commit-git-bypass.test.cjs", + "bug-3130-update-npx-robust-invocation.test.cjs", + "bug-3135-capture-backlog-workflow.test.cjs", + "bug-3150-stats-json-decimal-phase-gaps.test.cjs", + "bug-3156-plan-phase-opencode-dispatch.test.cjs", + "bug-3163-codex-agents-md.test.cjs", + "bug-3168-task-to-agent-rename.test.cjs", + "bug-3181-node-cellar-path.test.cjs", + "bug-3195-quick-resurrection-guard.test.cjs", + "bug-3197-gsd-tools-config-whitelist.test.cjs", + "bug-321-config-defaults-clone-strategy.test.cjs", + "bug-3212-execute-phase-stall-safe-resume.test.cjs", + "bug-3227-config-set-model-overrides.test.cjs", + "bug-3236-capture-seed-one-shot.test.cjs", + "bug-3242-state-update-progress-trample.test.cjs", + "bug-3243-dotted-command-form.test.cjs", + "bug-3245-codex-toml-floats.test.cjs", + "bug-3257-nested-plans-undercount.test.cjs", + "bug-3258-no-stale-gsd-intel-references.test.cjs", + "bug-3275-fmstr-non-string-scalars.test.cjs", + "bug-3285-codex-hooks-state-allowed.test.cjs", + "bug-3286-state-write-routing.test.cjs", + "bug-3288-model-catalog-install-path.test.cjs", + "bug-3290-intel-updater-layout-block.test.cjs", + "bug-33-settings-model-profile-adaptive.test.cjs", + "bug-3320-planner-deep-work-rules.test.cjs", + "bug-3321-verifier-runs-probes.test.cjs", + "bug-3346-codex-aot-toml-key.test.cjs", + "bug-3357-codex-legacy-hooks-json-migration.test.cjs", + "bug-3360-codex-execute-phase-worktrees.test.cjs", + "bug-338-local-install-settings-local-json.test.cjs", + "bug-3381-verify-work-workstream.test.cjs", + "bug-3384-secondary-defects.test.cjs", + "bug-3407-pristine-stale-content.test.cjs", + "bug-3413-shell-command-projection.test.cjs", + "bug-3418-progress-flag-routing.test.cjs", + "bug-3426-codex-windows-hooks.test.cjs", + "bug-3427-3433-codex-install-shape.test.cjs", + "bug-3430-planner-phase-contract.test.cjs", + "bug-3431-debug-command-yaml.test.cjs", + "bug-3441-path-action-projection.test.cjs", + "bug-3442-codex-legacy-hooks-json-migration.test.cjs", + "bug-3442-shim-projection-drift-guard.test.cjs", + "bug-3446-resume-continue-here-discovery.test.cjs", + "bug-3454-state-dollar-backreference-growth.test.cjs", + "bug-3489-complete-phase-idempotent.test.cjs", + "bug-3491-nested-git-worktree.test.cjs", + "bug-3509-path-spaces.test.cjs", + "bug-3516-reapply-patches-gsd-update-filter.test.cjs", + "bug-3521-quick-cleanup-cwd-pin.test.cjs", + "bug-3523-cjs-loadconfig-branching-strategy-warning.test.cjs", + "bug-3537-padded-id-against-unpadded-roadmap.test.cjs", + "bug-3541-installer-migration-prompt-user-resolution.test.cjs", + "bug-3542-executor-git-stash-prohibition.test.cjs", + "bug-3562-codex-install-skill-surface.test.cjs", + "bug-3566-codex-hooks-feature-canonical-key.test.cjs", + "bug-3571-configuration-manifest-install-path.test.cjs", + "bug-3582-codex-skills-materialized.test.cjs", + "bug-3584-runtime-slash-emitters.test.cjs", + "bug-3584-runtime-slash-formatter.test.cjs", + "bug-3588-npm-audit-clean.test.cjs", + "bug-3599-roadmap-get-phase-project-code-prefix.test.cjs", + "bug-3605-stale-research-insert-phase-agent-refs.test.cjs", + "bug-3608-antigravity-update-runtime-classification.test.cjs", + "bug-3610-installer-migration-bundled-hooks-classification.test.cjs", + "bug-3628-bundled-hook-classifier-whitelist.test.cjs", + "bug-3631-router-raw-flag.test.cjs", + "bug-3657-verify-reapply-patches-pristine-drift.test.cjs", + "bug-3659-applysurface-prune-skill-dirs.test.cjs", + "bug-3668-workflow-runtime-resolution.test.cjs", + "bug-3670-cursor-local-install-migration-lock.test.cjs", + "bug-3677-agent-colon-namespace-leak.test.cjs", + "bug-3678-executor-commit-docs-respect.test.cjs", + "bug-3683-command-colon-namespace-leak.test.cjs", + "bug-3683-command-cross-reference-invariant.test.cjs", + "bug-3683-workflow-colon-namespace-leak.test.cjs", + "bug-3689-resume-glob-nomatch.test.cjs", + "bug-3691-annotate-deps-plans-block-variants.test.cjs", + "bug-3706-ui-safety-gate-false-positives.test.cjs", + "bug-3707-locked-worktree-cleanup.test.cjs", + "bug-3727-code-review-fix-flag-dispatch.test.cjs", + "bug-3735-profiles-core-includes-surface.test.cjs", + "bug-3739-gap-checker-padded-prefix-context.test.cjs", + "bug-376-claude-js-hook-gsd-rewriter.test.cjs", + "bug-378-update-check-scoped-name.test.cjs", + "bug-3784-gsd-settings-model-profile-ui-omits-adaptive.test.cjs", + "bug-3805-fast-md-log-to-state-schema.test.cjs", + "bug-3808-codex-adapter-text-mode-fallback.test.cjs", + "bug-384-agents-runtime-aware.test.cjs", + "bug-397-state-preserve-executor-authored.test.cjs", + "bug-410-install-defaults-test-mode-guard.test.cjs", + "bug-416-archive-dir-null.test.cjs", + "bug-442-config-dir-equals-in-path.test.cjs", + "bug-444-resolver-local-claude-install.test.cjs", + "bug-447-gap-analysis-phase-req-ids.test.cjs", + "bug-474-clock-seam-date-determinism.test.cjs", + "bug-492-effort-manifest-fallback.test.cjs", + "bug-500-planned-phase-progress-corruption.test.cjs", + "bug-501-flat-phase-details-milestone-leak.test.cjs", + "bug-503-update-agent-antigravity-detection.test.cjs", + "bug-505-remove-dead-sdk-verification.test.cjs", + "bug-549-total-phases-overcounts-with-phase-section-heading.test.cjs", + "bug-557-details-summary-milestone-strip.test.cjs", + "bug-570-codex-leak-scanner.test.cjs", + "bug-571-doc-writer-fix-mode-edit-only.test.cjs", + "bug-580-local-sh-hook-bash-wrapper.test.cjs", + "bug-619-codebase-drift-gate-shim.test.cjs", + "bug-621-plan-phase-gap-analysis-gsd-run.test.cjs", + "bug-622-graphify-optional-graph-html.test.cjs", + "bug-630-wave-cleanup-orchestrator-root.test.cjs", + "bug-637-workflow-no-hardcoded-home-tool.test.cjs", + "bug-641-files-from-suite-token.test.cjs", + "bug-663-redos-roadmap-phase-parsing.test.cjs", + "bug-685-windowshide-spawn.test.cjs", + "bug-687-agy-timeout.test.cjs", + "bug-704-codex-launcher-path-corruption.test.cjs" +] diff --git a/scripts/lint-regression-test-names.cjs b/scripts/lint-regression-test-names.cjs new file mode 100644 index 000000000..724f81b52 --- /dev/null +++ b/scripts/lint-regression-test-names.cjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node +'use strict'; + +/** + * lint-regression-test-names.cjs — ban NEW top-level bug-NNNN test files. + * + * ## Why + * + * The 2026-06 CI audit found 244 one-off `tests/bug-NNNN-*.test.cjs` files — + * ~38% of the suite. `node --test` spawns one child process per FILE, so file + * count (not test count) is the unit of CI overhead, and it is worst on the + * Windows lanes where every spawn is Defender-scanned. Each regression test + * belongs in the owning module's main test file as a regression case (e.g. a + * `describe('regressions')` block in `tests/.test.cjs`), where it + * costs zero additional processes. + * + * ## What this enforces + * + * Identity ratchet (scripts/lib/allowlist-ratchet.cjs) over basenames matching + * /^bug-\d+.*\.test\.cjs$/ in tests/: + * - A NEW bug-* file (not in the allowlist) fails: fold the regression into + * the owning module's test file instead. + * - A REMOVED bug-* file with a stale allowlist entry also fails: prune the + * entry from scripts/lint-regression-test-names.allowlist.json so the + * baseline only ever shrinks. + * + * See docs/TESTING-SUITES.md ("Regression tests") for the placement policy. + */ + +const fs = require('fs'); +const path = require('path'); +const { assertWithinAllowlist } = require('./lib/allowlist-ratchet.cjs'); +const { ExitError, runMain } = require('./lib/cli-exit.cjs'); + +const ROOT = path.join(__dirname, '..'); +// Env overrides exist for the lint's own tests only (sandbox fixture dirs). +const TESTS_DIR = process.env.GSD_LINT_REGRESSION_TESTS_DIR || path.join(ROOT, 'tests'); +const ALLOWLIST_PATH = + process.env.GSD_LINT_REGRESSION_ALLOWLIST || + path.join(__dirname, 'lint-regression-test-names.allowlist.json'); + +const BUG_FILE_RE = /^bug-\d+.*\.test\.cjs$/; + +function main() { + const current = fs + .readdirSync(TESTS_DIR) + .filter(f => BUG_FILE_RE.test(f)) + .sort(); + const known = JSON.parse(fs.readFileSync(ALLOWLIST_PATH, 'utf8')); + + const failures = []; + const { novel } = assertWithinAllowlist({ + label: 'regression-test-names', + current, + known, + fail: msg => failures.push(msg), + pruneHint: 'edit scripts/lint-regression-test-names.allowlist.json', + }); + + if (failures.length > 0) { + for (const msg of failures) console.error(msg); + if (novel.length > 0) { + console.error( + '\nNew bug-NNNN test files are no longer accepted. Add the regression ' + + "case to the owning module's test file (e.g. a describe('regressions') " + + 'block in tests/.test.cjs) instead of creating a new file. ' + + 'See docs/TESTING-SUITES.md.' + ); + } + throw new ExitError(1); + } + + console.log( + `ok lint-regression-test-names: ${current.length} grandfathered bug-* file(s), no novel offenders` + ); +} + +runMain(main); From a869df2acffb763200a6a08aaf2dfe9966a6c75d Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 9 Jun 2026 23:03:14 -0400 Subject: [PATCH 4/7] ci(test.yml): fold coverage into ubuntu-24 lane, add scripts/ floor, single lint step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Coverage gate (test:coverage:unit) now runs inside the ubuntu/24 full lane; the standalone coverage job duplicated that lane's entire unit run (~4 min of runner time per PR). required-tests gate updated accordingly. - New second-tier floor: c8 check-coverage --lines 55 over scripts/** re-slices the same V8 data (measured 65.95%) — the CI/release/lint tooling was previously enforced at 0%. - Coverage artifact now excludes coverage/tmp (>1 GB of raw V8 dumps). - lint-tests runs npm run lint:ci — one orchestrated step, identical set locally and in CI; drops the no-op eslint --cache flag (CI never restored the cache directory). - Delete unreferenced scripts/run-cross-platform-tests.cjs (+ its test); document the mutation UNMUTATED blind spot (~48% of lib lines) in stryker.config.mjs. Co-Authored-By: Claude Fable 5 --- .github/workflows/test.yml | 101 ++++++++++++++----------------------- stryker.config.mjs | 9 ++++ 2 files changed, 46 insertions(+), 64 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d95860e26..e217bd008 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -103,18 +103,12 @@ jobs: # lint scripts) require() the built modules — so build them explicitly. - name: Build runtime lib (required by lint scripts) run: npm run build:lib - - name: Lint — ESLint (source-grep + timing + no-only-tests + quality) - run: npx eslint . --cache --cache-location node_modules/.cache/eslint/ - - name: Lint — skill dependency graph - run: npm run lint:skill-deps - - name: Lint — test file count per module - run: node scripts/lint-test-file-count.cjs - - name: Lint — command contract (ADR-0002) - run: node scripts/lint-command-contract.cjs - - name: Lint — PR checks use projectDir - run: node scripts/lint-pr-check-project-dir.cjs - - name: Lint — legacy directory name guard (#604) - run: npm run lint:legacy-name + # Single orchestrated lint entry point (npm run lint:ci) so local and CI + # always run the identical set. ESLint's --cache flag is intentionally + # NOT used here: CI never restores node_modules/.cache, so the flag was + # a no-op that only implied caching existed. + - name: Lint — all (ESLint, skill deps, test-file count, command contract, PR checks, legacy name, regression-test names) + run: npm run lint:ci test: name: test (${{ matrix.os }}, ${{ matrix.node-version }}) @@ -196,9 +190,36 @@ jobs: if: matrix.scope != 'full' run: node scripts/run-tests.cjs --files-from .ci-selected-tests.txt - - name: Run unit tests + # The unit suite runs ONCE here, under c8 with the coverage gate — the + # former standalone `coverage` job duplicated this lane's entire unit + # run (~4 min of runner time per PR) just to collect the same numbers. + - name: Run unit tests (coverage gate ≥70% on gsd-core/bin/lib) if: matrix.scope == 'full' - run: npm run test:unit + env: + NODE_OPTIONS: --max-old-space-size=6144 + run: npm run test:coverage:unit + + # Second-tier floor over the CI/release/lint tooling itself. Re-slices + # the SAME V8 coverage data left in coverage/tmp by the run above — no + # extra suite execution. Audit 2026-06: scripts/ measured 65.95%; the + # 55% floor prevents a collapse to zero-coverage tooling while leaving + # headroom for variance. Raise deliberately, never lower. + - name: Coverage floor — scripts/ tooling (≥55%) + if: matrix.scope == 'full' + run: npx c8 check-coverage --lines 55 --include 'scripts/**/*.cjs' --exclude 'tests/**' --all + + - name: Upload coverage artifact + if: always() && matrix.scope == 'full' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-unit + # coverage/tmp holds raw per-process V8 dumps (>1 GB for the full + # unit suite) — exclude it; the rendered reports are the artifact. + path: | + coverage/ + !coverage/tmp + .nyc_output/ + if-no-files-found: ignore - name: Run integration tests if: matrix.scope == 'full' @@ -333,49 +354,6 @@ jobs: - name: Run security tests run: npm run test:security - coverage: - needs: changes - if: needs.changes.outputs.product_changed == 'true' - runs-on: ubuntu-latest - timeout-minutes: 15 - env: - GSD_PLUGIN_ROOT: .ci-gsd-plugin-root-disabled - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - persist-credentials: true - token: ${{ github.token }} - - name: Guard — require GitHub-hosted runner - run: node scripts/ci-guard-runner.cjs - - name: Rebase check — merge PR base branch into PR head - if: github.event_name == 'pull_request' - env: - GITHUB_TOKEN: ${{ github.token }} - run: node scripts/ci-rebase-check.cjs - - name: Set up Node.js 24 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - with: - node-version: 24 - cache: 'npm' - - name: Install dependencies - run: npm ci - - name: Dependency integrity gate - run: node scripts/check-npm-integrity.cjs - - name: Unit coverage - env: - NODE_OPTIONS: --max-old-space-size=6144 - run: npm run test:coverage:unit - - name: Upload coverage artifact - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: coverage-unit - path: | - coverage/ - .nyc_output/ - if-no-files-found: ignore - required-tests: name: Required tests needs: @@ -384,7 +362,6 @@ jobs: - test - test-inert - test-full - - coverage if: always() runs-on: ubuntu-latest timeout-minutes: 1 @@ -398,7 +375,6 @@ jobs: TEST_RESULT: ${{ needs.test.result }} INERT_RESULT: ${{ needs.test-inert.result }} FULL_TEST_RESULT: ${{ needs.test-full.result }} - COVERAGE_RESULT: ${{ needs.coverage.result }} run: | set -euo pipefail echo "code_changed=$CODE_CHANGED" @@ -408,7 +384,6 @@ jobs: echo "test=$TEST_RESULT" echo "test-inert=$INERT_RESULT" echo "test-full=$FULL_TEST_RESULT" - echo "coverage=$COVERAGE_RESULT" if [ "$CHANGES_RESULT" != "success" ]; then echo "::error::test scope detection did not pass" @@ -430,14 +405,12 @@ jobs: echo "::error::test matrix did not pass" exit 1 fi + # The coverage gates (lib 70% + scripts/ 55% floor) run inside the + # ubuntu/24 full lane of the `test` matrix, so TEST_RESULT covers them. if [ "$FULL_TEST_RESULT" != "success" ] && [ "$FULL_TEST_RESULT" != "skipped" ]; then echo "::error::full parity matrix did not pass" exit 1 fi - if [ "$COVERAGE_RESULT" != "success" ]; then - echo "::error::coverage did not pass" - exit 1 - fi else if [ "$INERT_RESULT" != "success" ]; then echo "::error::inert CI lane did not pass" diff --git a/stryker.config.mjs b/stryker.config.mjs index 5cd341deb..f21ee66b3 100644 --- a/stryker.config.mjs +++ b/stryker.config.mjs @@ -29,6 +29,15 @@ // force a full tsc rebuild per mutant — far too slow for the 30-min CI budget.) // Large/low-coverage modules are excluded (the command's test set does not // exercise them, so they would only ever produce survived mutants). +// +// KNOWN BLIND SPOT (2026-06 CI audit): this list excludes ~14.2k of ~29.8k +// lib lines (~48%), including the most central modules (state, core, +// commands, phase, verify). Mutation results therefore speak only for the +// well-tested half of the lib. Shrinking the list is deliberate tracked work: +// bring one module into scope per release by first giving it per-module +// *.unit.test.cjs / *.property.test.cjs coverage, then deleting its entry — +// never delete an entry without that coverage (it will only produce survived +// mutants and trip the break threshold). const UNMUTATED = [ '!gsd-core/bin/lib/command-aliases.cjs', '!gsd-core/bin/lib/commands.cjs', From 9db7958a70b5bbd6c5564e536af5a546c1daa2b1 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 9 Jun 2026 23:40:50 -0400 Subject: [PATCH 5/7] fix(review): single eslint home, threshold co-location, helper reuse, docs clarity Review-pass fixes: lint:ci composes npm run lint (one eslint invocation home); the scripts/ coverage floor moves to package.json (test:coverage:scripts-floor) so both thresholds live together; the ratchet test uses helpers.createTempDir; TESTING-SUITES.md clarifies what the Windows scoped lane runs and why feat-*/enh-* files are exempt from the bug-* ratchet. Co-Authored-By: Claude Fable 5 --- .github/workflows/test.yml | 12 +++++++----- docs/TESTING-SUITES.md | 19 ++++++++++++++----- package.json | 3 ++- tests/lint-regression-test-names.test.cjs | 9 +++++---- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e217bd008..146bbf679 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,9 +104,10 @@ jobs: - name: Build runtime lib (required by lint scripts) run: npm run build:lib # Single orchestrated lint entry point (npm run lint:ci) so local and CI - # always run the identical set. ESLint's --cache flag is intentionally - # NOT used here: CI never restores node_modules/.cache, so the flag was - # a no-op that only implied caching existed. + # always run the identical set. It composes `npm run lint`, keeping one + # eslint invocation home; the --cache flag inside it is a no-op in CI + # (node_modules/.cache is never restored) but still speeds local runs. + # Each sub-lint prints its own banner, so a failure identifies itself. - name: Lint — all (ESLint, skill deps, test-file count, command contract, PR checks, legacy name, regression-test names) run: npm run lint:ci @@ -203,10 +204,11 @@ jobs: # the SAME V8 coverage data left in coverage/tmp by the run above — no # extra suite execution. Audit 2026-06: scripts/ measured 65.95%; the # 55% floor prevents a collapse to zero-coverage tooling while leaving - # headroom for variance. Raise deliberately, never lower. + # headroom for variance. The threshold lives in package.json next to + # the 70% lib gate — raise deliberately, never lower. - name: Coverage floor — scripts/ tooling (≥55%) if: matrix.scope == 'full' - run: npx c8 check-coverage --lines 55 --include 'scripts/**/*.cjs' --exclude 'tests/**' --all + run: npm run test:coverage:scripts-floor - name: Upload coverage artifact if: always() && matrix.scope == 'full' diff --git a/docs/TESTING-SUITES.md b/docs/TESTING-SUITES.md index b5f31e053..6454bf961 100644 --- a/docs/TESTING-SUITES.md +++ b/docs/TESTING-SUITES.md @@ -46,6 +46,12 @@ identity ratchet (`npm run lint:regression-names`, part of `npm run lint:ci`): - **Deleting/consolidating** a grandfathered file requires pruning its allowlist entry, so the baseline only ever shrinks. +The ratchet deliberately covers only `bug-*`. Files named `feat-NNNN-*` / +`enh-NNNN-*` are *feature* test files — one (or one per suite) per feature is +the sanctioned layout (see the #443 strategy below), not a one-off regression +pattern. If `issue-*`/`perf-*` one-offs start accumulating the same way +`bug-*` did, extend the ratchet's regex and regenerate the allowlist. + ## Running suites locally ```bash @@ -96,11 +102,14 @@ The `Tests` workflow runs every PR through a scoped gate generated by tests. The default PR gate runs the broad `unit` (under the c8 coverage gate), -`integration`, and `security` suites once on Ubuntu / Node 24, scoped smoke on -Ubuntu / Node 22, and scoped Windows-sensitive tests on Windows / Node 24. -**Every changed test file always joins the Windows scoped lane** (the #494 -invariant, narrowed): a modified test is exercised on the divergent OS before -merge at per-file cost, without paying for the three full parity lanes. +`integration`, and `security` suites once on Ubuntu / Node 24, scoped tests on +Ubuntu / Node 22, and scoped tests on Windows / Node 24. "Scoped" means the +diff-selected list from the rule table — not the full suite and not a fixed +smoke set (the fixed smoke list is only the empty-selection fallback). The +Windows lane's list is the Windows-sensitive subset of the selection, plus +**every changed test file, unconditionally** (the #494 invariant, narrowed): a +modified test is exercised on the divergent OS before merge at per-file cost, +without paying for the three full parity lanes. PRs touching workflow, package, test-runner, install, release, or Windows-sensitive surfaces also run the full parity matrix on macOS and the diff --git a/package.json b/package.json index 57f6ce2ad..6838f89ae 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,7 @@ "pretest:coverage": "npm run build:lib && npm run lint:skill-deps", "lint": "eslint . --cache --cache-location node_modules/.cache/eslint/", "lint:fix": "eslint . --fix", - "lint:ci": "eslint . && npm run lint:skill-deps && node scripts/lint-test-file-count.cjs && node scripts/lint-command-contract.cjs && node scripts/lint-pr-check-project-dir.cjs && npm run lint:legacy-name && node scripts/lint-regression-test-names.cjs", + "lint:ci": "npm run lint && npm run lint:skill-deps && node scripts/lint-test-file-count.cjs && node scripts/lint-command-contract.cjs && node scripts/lint-pr-check-project-dir.cjs && npm run lint:legacy-name && node scripts/lint-regression-test-names.cjs", "lint:regression-names": "node scripts/lint-regression-test-names.cjs", "lint:descriptions": "node scripts/lint-descriptions.cjs", "lint:skill-deps": "node scripts/lint-skill-deps.cjs", @@ -111,6 +111,7 @@ "test:slow": "node scripts/run-tests.cjs --suite slow", "test:affected": "node scripts/run-affected-tests.cjs", "test:coverage": "c8 --check-coverage --lines 70 --reporter text --include 'gsd-core/bin/lib/*.cjs' --exclude 'tests/**' --all node scripts/run-tests.cjs", + "test:coverage:scripts-floor": "c8 check-coverage --lines 55 --include 'scripts/**/*.cjs' --exclude 'tests/**' --all", "test:coverage:unit": "c8 --check-coverage --lines 70 --reporter text --include 'gsd-core/bin/lib/*.cjs' --exclude 'tests/**' --all node scripts/run-tests.cjs --suite unit", "test:coverage:all": "npm run test:coverage", "test:mutation": "stryker run", diff --git a/tests/lint-regression-test-names.test.cjs b/tests/lint-regression-test-names.test.cjs index 8f30cd816..430d57d4f 100644 --- a/tests/lint-regression-test-names.test.cjs +++ b/tests/lint-regression-test-names.test.cjs @@ -9,17 +9,18 @@ const { describe, test, before, after } = require('node:test'); const assert = require('node:assert/strict'); const { spawnSync } = require('child_process'); const fs = require('fs'); -const os = require('node:os'); const path = require('path'); -const { cleanup } = require('./helpers.cjs'); +const { createTempDir, cleanup } = require('./helpers.cjs'); const ROOT = path.join(__dirname, '..'); const SCRIPT = path.join(ROOT, 'scripts', 'lint-regression-test-names.cjs'); let sandbox; +let fixtureCount = 0; + function runLint({ files, allowlist }) { - const testsDir = path.join(sandbox, `tests-${Math.random().toString(36).slice(2)}`); + const testsDir = path.join(sandbox, `tests-${fixtureCount++}`); fs.mkdirSync(testsDir, { recursive: true }); for (const f of files) fs.writeFileSync(path.join(testsDir, f), ''); const allowlistPath = path.join(testsDir, 'allowlist.json'); @@ -37,7 +38,7 @@ function runLint({ files, allowlist }) { describe('lint-regression-test-names', () => { before(() => { - sandbox = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-lint-regression-')); + sandbox = createTempDir('gsd-lint-regression-'); }); after(() => { From 8bb67840097ecb908561027f8050c14925453d94 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 9 Jun 2026 23:56:57 -0400 Subject: [PATCH 6/7] chore(ratchet): regenerate bug-* allowlist after rebase onto next (244 -> 257) 13 bug-* files landed upstream between the audit baseline and this branch's rebase; they predate the ratchet policy, so they are grandfathered. Co-Authored-By: Claude Fable 5 --- scripts/lint-regression-test-names.allowlist.json | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/lint-regression-test-names.allowlist.json b/scripts/lint-regression-test-names.allowlist.json index f99f80a62..c47663826 100644 --- a/scripts/lint-regression-test-names.allowlist.json +++ b/scripts/lint-regression-test-names.allowlist.json @@ -215,6 +215,7 @@ "bug-3784-gsd-settings-model-profile-ui-omits-adaptive.test.cjs", "bug-3805-fast-md-log-to-state-schema.test.cjs", "bug-3808-codex-adapter-text-mode-fallback.test.cjs", + "bug-3810-no-gsd-sdk-runtime-refs.test.cjs", "bug-384-agents-runtime-aware.test.cjs", "bug-397-state-preserve-executor-authored.test.cjs", "bug-410-install-defaults-test-mode-guard.test.cjs", @@ -242,5 +243,17 @@ "bug-663-redos-roadmap-phase-parsing.test.cjs", "bug-685-windowshide-spawn.test.cjs", "bug-687-agy-timeout.test.cjs", - "bug-704-codex-launcher-path-corruption.test.cjs" + "bug-704-codex-launcher-path-corruption.test.cjs", + "bug-730-milestone-phase-details-scope.test.cjs", + "bug-782-cline-skills-emission.test.cjs", + "bug-783-kilo-global-skills-base.test.cjs", + "bug-853-bg-dispatch-runtime-gating.test.cjs", + "bug-866-profile-pipeline-temp-root.test.cjs", + "bug-891-non-claude-runtime-home-fallback.test.cjs", + "bug-892-validate-checklist-roadmap-phases.test.cjs", + "bug-905-state-syncstatefrontmatter-preserve-scalars.test.cjs", + "bug-924-claude-flat-skill-layout.test.cjs", + "bug-925-context-monitor-hook-event-name.test.cjs", + "bug-936-no-nested-spawner-wrap.test.cjs", + "bug-941-managed-hooks-registry-manifest.test.cjs" ] From 533b518553d82dc46b0875938d8762a01df719e4 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 10 Jun 2026 00:15:02 -0400 Subject: [PATCH 7/7] fix(security-scan): update scanner self-exemption allowlists for renamed suite files The three shell scanners exempt their own adversarial test fixtures by exact filename; the *.security.test.cjs renames broke those entries, so the PR diff scan flagged the scanners' own test payloads. Verified locally with all three scanners in --diff origin/next mode (0 findings) and the security suite (207/207). The .sh files were missed in the original reference sweep because the rename grep filtered to .cjs/.yml/.json/.md extensions. Co-Authored-By: Claude Fable 5 --- scripts/base64-scan.sh | 2 +- scripts/prompt-injection-scan.sh | 8 ++++---- scripts/secret-scan.sh | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/base64-scan.sh b/scripts/base64-scan.sh index f7c5c7a5a..a64c44aaf 100755 --- a/scripts/base64-scan.sh +++ b/scripts/base64-scan.sh @@ -156,7 +156,7 @@ should_skip_file() { # Skip the scan scripts themselves and test files case "$file" in */base64-scan.sh) return 0 ;; - */security-scan.test.cjs) return 0 ;; + */security-scan.security.test.cjs) return 0 ;; esac # Skip scanner fixture directories — they contain deliberate injection samples case "$file" in diff --git a/scripts/prompt-injection-scan.sh b/scripts/prompt-injection-scan.sh index 78231ef12..5fc8c29fb 100755 --- a/scripts/prompt-injection-scan.sh +++ b/scripts/prompt-injection-scan.sh @@ -69,15 +69,15 @@ ALLOWLIST=( 'scripts/prompt-injection-scan.sh' 'scripts/base64-scan.sh' 'scripts/secret-scan.sh' - 'tests/security-scan.test.cjs' + 'tests/security-scan.security.test.cjs' 'tests/security.test.cjs' - 'tests/prompt-injection-scan.test.cjs' + 'tests/prompt-injection-scan.security.test.cjs' 'tests/verify.test.cjs' 'gsd-core/bin/lib/security.cjs' 'hooks/gsd-prompt-guard.js' 'hooks/gsd-read-injection-scanner.js' - 'tests/read-injection-scanner.test.cjs' - 'tests/security-prompt-injection.test.cjs' + 'tests/read-injection-scanner.security.test.cjs' + 'tests/security-prompt-injection.security.test.cjs' 'tests/fixtures/adversarial/security/' 'SECURITY.md' # These files contain intentional injection examples / security-model prose diff --git a/scripts/secret-scan.sh b/scripts/secret-scan.sh index 82d2b5ab7..9653c8bbd 100755 --- a/scripts/secret-scan.sh +++ b/scripts/secret-scan.sh @@ -218,9 +218,9 @@ should_skip_file() { # Skip the scan scripts themselves and test files case "$file" in */secret-scan.sh) return 0 ;; - */secret-scan-lint.test.cjs) return 0 ;; - */security-scan.test.cjs) return 0 ;; - */security-prompt-injection.test.cjs) return 0 ;; + */secret-scan-lint.security.test.cjs) return 0 ;; + */security-scan.security.test.cjs) return 0 ;; + */security-prompt-injection.security.test.cjs) return 0 ;; tests/fixtures/adversarial/security/*|*/tests/fixtures/adversarial/security/*) return 0 ;; esac return 1