From 571d7b5a1c0ef340fe426a88d2b6bc3c1d462ddd Mon Sep 17 00:00:00 2001 From: Tom Boucher Date: Sun, 7 Jun 2026 11:46:48 -0400 Subject: [PATCH] feat(#764): skip cross-platform test matrix for docs-only and inert-CI PRs (#798) test.yml had no paths filter and the ci-test-scope classifier treated docs/ and every .github/workflows/* as code_changed, so documentation edits and product-irrelevant automation tweaks still spun up the full Linux/Windows/macOS matrix. Narrow the heavy matrix to changes that can actually affect the product or the test pipeline. - ci-test-scope.cjs: drop docs/ from code_changed (docs-only -> full skip; the required-tests fan-in still reports green). Add src/ to code_changed (it was missing -> a source-only PR previously skipped all tests). Add INERT_WORKFLOWS allowlist + isInertCi() + an "inert CI" rule, and a product_changed output that gates the heavy test/coverage jobs. Fail-safe: any workflow not on the inert allowlist defaults to the full matrix. A module-load assertion throws if a PROTECTED_WORKFLOWS entry (test/install-smoke/mutation/security-scan/release) is ever added to the inert set, so a weakening edit fails CI loudly. - test.yml: keep the static 3-lane matrix (so the H1 shell-policy linter can still statically verify the Windows lane), gate test/coverage on product_changed, add a lightweight ubuntu-only test-inert job, and branch the required-tests fan-in on product_changed. - docs-required.yml: run docs-parity-live-registry (gated on docs/ changes) so pure-docs PRs still catch live-registry drift without the matrix. - tests: cover docs-only, inert-only, src/, pipeline, unknown-workflow fail-safe, mixed escalation, the code_changed=false -> no-lanes invariant, and protected- workflow tamper-evidence. Closes #764 Co-authored-by: Claude Opus 4.8 --- .github/workflows/docs-required.yml | 15 ++ .github/workflows/test.yml | 83 +++++++-- scripts/ci-test-scope.cjs | 131 +++++++++++-- tests/ci-test-scope.test.cjs | 276 ++++++++++++++++++++++++++-- 4 files changed, 455 insertions(+), 50 deletions(-) diff --git a/.github/workflows/docs-required.yml b/.github/workflows/docs-required.yml index a6c8a213a..4961d90d5 100644 --- a/.github/workflows/docs-required.yml +++ b/.github/workflows/docs-required.yml @@ -30,3 +30,18 @@ jobs: env: GITHUB_BASE_REF: ${{ github.base_ref }} run: node scripts/lint-docs-required.cjs + + - name: Detect docs/ changes + id: docs-changed + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + if git diff --name-only "origin/${BASE_REF}...HEAD" | grep -q '^docs/'; then + echo "docs_changed=true" >> "$GITHUB_OUTPUT" + else + echo "docs_changed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Docs parity — live registry check + if: steps.docs-changed.outputs.docs_changed == 'true' + run: node --test tests/docs-parity-live-registry.test.cjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68c46eb4e..d186c6d65 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,7 @@ jobs: outputs: code_changed: ${{ steps.scope.outputs.code_changed }} full_matrix: ${{ steps.scope.outputs.full_matrix }} + product_changed: ${{ steps.scope.outputs.product_changed }} targeted_tests: ${{ steps.scope.outputs.targeted_tests }} windows_tests: ${{ steps.scope.outputs.windows_tests }} steps: @@ -49,6 +50,7 @@ jobs: if [ "$EVENT_NAME" != "pull_request" ]; then { echo "code_changed=true" + echo "product_changed=true" echo "full_matrix=true" echo "targeted_tests=" echo "windows_tests=" @@ -117,7 +119,7 @@ jobs: test: name: test (${{ matrix.os }}, ${{ matrix.node-version }}) needs: changes - if: needs.changes.outputs.code_changed == 'true' + if: needs.changes.outputs.product_changed == 'true' runs-on: ${{ matrix.os }} timeout-minutes: 15 env: @@ -214,6 +216,47 @@ jobs: if: matrix.scope == 'full' && needs.changes.outputs.full_matrix == 'true' run: npm run test:slow + test-inert: + name: test (inert CI) + needs: changes + if: needs.changes.outputs.code_changed == 'true' && needs.changes.outputs.product_changed != 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + GSD_PLUGIN_ROOT: .ci-gsd-plugin-root-disabled + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: true + token: ${{ github.token }} + - name: Guard — require GitHub-hosted runner + run: node scripts/ci-guard-runner.cjs + - name: Rebase check — merge PR base branch into PR head + if: github.event_name == 'pull_request' + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/ci-rebase-check.cjs + - name: Set up Node.js 22 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 22 + cache: 'npm' + - name: Environment check + run: npm run check:env + - name: Install dependencies + run: npm ci + - name: Dependency integrity gate + run: node scripts/check-npm-integrity.cjs + - name: Prepare scoped test list + env: + TEST_SCOPE: targeted + TARGETED_TESTS: ${{ needs.changes.outputs.targeted_tests }} + WINDOWS_TESTS: ${{ needs.changes.outputs.windows_tests }} + run: node scripts/ci-prepare-test-scope.cjs + - name: Run scoped tests + run: node scripts/run-tests.cjs --files-from .ci-selected-tests.txt + test-full: name: full test (${{ matrix.os }}, ${{ matrix.node-version }}) needs: changes @@ -289,7 +332,7 @@ jobs: coverage: needs: changes - if: needs.changes.outputs.code_changed == 'true' + if: needs.changes.outputs.product_changed == 'true' runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -336,6 +379,7 @@ jobs: - changes - lint-tests - test + - test-inert - test-full - coverage if: always() @@ -345,17 +389,21 @@ jobs: - name: Summarize required test gate env: CODE_CHANGED: ${{ needs.changes.outputs.code_changed }} + PRODUCT_CHANGED: ${{ needs.changes.outputs.product_changed }} CHANGES_RESULT: ${{ needs.changes.result }} LINT_RESULT: ${{ needs.lint-tests.result }} TEST_RESULT: ${{ needs.test.result }} + INERT_RESULT: ${{ needs.test-inert.result }} FULL_TEST_RESULT: ${{ needs.test-full.result }} COVERAGE_RESULT: ${{ needs.coverage.result }} run: | set -euo pipefail echo "code_changed=$CODE_CHANGED" + echo "product_changed=$PRODUCT_CHANGED" echo "changes=$CHANGES_RESULT" echo "lint-tests=$LINT_RESULT" echo "test=$TEST_RESULT" + echo "test-inert=$INERT_RESULT" echo "test-full=$FULL_TEST_RESULT" echo "coverage=$COVERAGE_RESULT" @@ -374,19 +422,24 @@ jobs: exit 0 fi - if [ "$TEST_RESULT" != "success" ]; then - echo "::error::test matrix did not pass" - exit 1 - fi - - if [ "$FULL_TEST_RESULT" != "success" ] && [ "$FULL_TEST_RESULT" != "skipped" ]; then - echo "::error::full parity matrix did not pass" - exit 1 - fi - - if [ "$COVERAGE_RESULT" != "success" ]; then - echo "::error::coverage did not pass" - exit 1 + if [ "$PRODUCT_CHANGED" = "true" ]; then + if [ "$TEST_RESULT" != "success" ]; then + echo "::error::test matrix did not pass" + exit 1 + fi + if [ "$FULL_TEST_RESULT" != "success" ] && [ "$FULL_TEST_RESULT" != "skipped" ]; then + echo "::error::full parity matrix did not pass" + exit 1 + fi + if [ "$COVERAGE_RESULT" != "success" ]; then + echo "::error::coverage did not pass" + exit 1 + fi + else + if [ "$INERT_RESULT" != "success" ]; then + echo "::error::inert CI lane did not pass" + exit 1 + fi fi echo "Required test gate passed." diff --git a/scripts/ci-test-scope.cjs b/scripts/ci-test-scope.cjs index 977c9d5c4..1028377a4 100644 --- a/scripts/ci-test-scope.cjs +++ b/scripts/ci-test-scope.cjs @@ -6,16 +6,84 @@ const { existsSync, readdirSync, appendFileSync } = require('fs'); const { ExitError, runMain } = require('./lib/cli-exit.cjs'); +// Workflow files that are purely administrative / policy bots. Changes to these +// files do NOT require the cross-platform test matrix — only a lightweight +// ubuntu lane running workflow-lint tests is needed. +// FAIL-SAFE: any .github/workflows/*.yml NOT listed here is treated as a +// pipeline workflow and gets the full matrix. New workflow files default to full. +const INERT_WORKFLOWS = new Set([ + 'stale.yml', + 'branch-cleanup.yml', + 'branch-naming.yml', + 'auto-label-issues.yml', + 'auto-branch.yml', + 'auto-backmerge.yml', + 'close-draft-prs.yml', + 'dismiss-unauthorized-pr-approvals.yml', + 'pr-gate.yml', + 'pr-target-validator.yml', + 'pr-template-format.yml', + 'require-issue-link.yml', + 'changeset-required.yml', + 'docs-required.yml', + 'discord-changelog.yml', +]); + +// Workflows that gate merges, ship the product, or run security/cross-platform +// suites — these must ALWAYS get the full pipeline treatment and can never be +// added to INERT_WORKFLOWS. A module-load assertion enforces this so a mistaken +// or malicious addition fails CI loudly in the `changes` job on every PR. +const PROTECTED_WORKFLOWS = new Set([ + 'test.yml', + 'install-smoke.yml', + 'mutation.yml', + 'security-scan.yml', + 'release.yml', +]); +for (const wf of PROTECTED_WORKFLOWS) { + if (INERT_WORKFLOWS.has(wf)) { + throw new Error(`ci-test-scope: protected workflow "${wf}" must not be in INERT_WORKFLOWS (it requires the full test matrix).`); + } +} + +/** + * Returns true if the path is an inert (non-pipeline) workflow file. + * Only `.github/workflows/` where is in INERT_WORKFLOWS qualifies. + */ +function isInertCi(filePath) { + if (!filePath.startsWith('.github/workflows/')) return false; + const name = filePath.slice('.github/workflows/'.length); + // Must be a direct child (no further slashes) and in the allowlist. + return !name.includes('/') && INERT_WORKFLOWS.has(name); +} + +// Tests shared by both the 'workflow automation' and 'inert CI' rules. +const WORKFLOW_LINT_TESTS = [ + 'tests/workflow-shell-pinning.test.cjs', + 'tests/pr-template-policy.test.cjs', + 'tests/lint-pr-check-project-dir.test.cjs', +]; + const RULES = [ { name: 'workflow automation', - match: path => path.startsWith('.github/workflows/') || path.startsWith('.github/rulesets/'), + // Only NON-inert .github/workflows/* and all .github/rulesets/* trigger full matrix. + // FAIL-SAFE: any .github/workflows/*.yml not in INERT_WORKFLOWS is treated as pipeline. + match: filePath => (filePath.startsWith('.github/workflows/') && !isInertCi(filePath)) || + filePath.startsWith('.github/rulesets/'), fullMatrix: true, tests: [ - 'tests/workflow-shell-pinning.test.cjs', + ...WORKFLOW_LINT_TESTS, 'tests/release-tarball-smoke-workflow.test.cjs', - 'tests/lint-pr-check-project-dir.test.cjs', - 'tests/pr-template-policy.test.cjs', + ], + }, + { + name: 'inert CI', + match: filePath => isInertCi(filePath), + fullMatrix: false, + tests: [ + ...WORKFLOW_LINT_TESTS, + 'tests/policy-lint-shallow-checkout.test.cjs', ], }, { @@ -144,14 +212,6 @@ const RULES = [ 'tests/docs-parity-live-registry.test.cjs', ], }, - { - name: 'docs content', - match: path => path.startsWith('docs/'), - fullMatrix: false, - tests: [ - 'tests/docs-parity-live-registry.test.cjs', - ], - }, { name: 'configuration', match: path => ['config', 'configuration', 'model-catalog', 'model-profile'].some(k => path.includes(k)), @@ -251,16 +311,30 @@ function classify(files) { const targeted = new Set(); const windows = new Set(); const reasons = []; - let codeChanged = false; + let productOrPipelineChanged = false; // product/pipeline code (excludes docs) + let inertCiChanged = false; // inert workflow files let fullMatrix = false; for (const file of files) { - if (['bin/', 'gsd-core/', 'agents/', 'commands/', 'docs/', 'hooks/', 'tests/', 'scripts/'].some(p => file.startsWith(p)) || + // Determine if this file is product/pipeline code. + // docs/ and root-level .md files are intentionally excluded. + if ( + ['bin/', 'src/', 'gsd-core/', 'agents/', 'commands/', 'hooks/', 'tests/', 'scripts/'].some(p => file.startsWith(p)) || file === 'package.json' || file === 'package-lock.json' || (file.startsWith('tsconfig') && file.endsWith('.json')) || - file.startsWith('.github/workflows/') || - file.startsWith('.github/rulesets/')) { - codeChanged = true; + file.startsWith('.github/rulesets/') + ) { + productOrPipelineChanged = true; + } + + // Non-inert .github/workflows/* are pipeline code → full matrix. + if (file.startsWith('.github/workflows/') && !isInertCi(file)) { + productOrPipelineChanged = true; + } + + // Inert workflow files set a lightweight signal. + if (isInertCi(file)) { + inertCiChanged = true; } if (file.startsWith('tests/') && file.endsWith('.test.cjs')) { @@ -280,6 +354,10 @@ function classify(files) { } } + // code_changed: true when product/pipeline OR inert CI changed. + // Docs-only PRs (neither flag set) get code_changed=false → full matrix skip. + const codeChanged = productOrPipelineChanged || inertCiChanged; + const targetedTests = existingTests([...targeted].sort()); // When code changed but no rule matched any changed file, fall back to the @@ -290,8 +368,25 @@ function classify(files) { const windowsTests = existingTests([...new Set([...windows, ...targetedTests.filter(isWindowsHint)])].sort()); + // Inert-CI-only: full_matrix must be false (override any RULES that fired). + if (inertCiChanged && !productOrPipelineChanged) { + fullMatrix = false; + } + + // Normalize: when code_changed is false, the output must be self-consistent. + // A docs file can coincidentally match a coarse content RULE (e.g. docs/installer-migrations.md + // matches the installer rule via path.includes('install')), leaving full_matrix=true and + // non-empty targeted_tests/windows_tests. The workflow skips correctly (gated on code_changed) + // but the output object would be self-contradictory. Force a clean "nothing to run" result. + if (!codeChanged) { + fullMatrix = false; + targetedTests.length = 0; + windowsTests.length = 0; + } + return { code_changed: codeChanged, + product_changed: productOrPipelineChanged, full_matrix: fullMatrix, targeted_tests: targetedTests, windows_tests: windowsTests, @@ -303,6 +398,7 @@ function writeOutputs(result) { if (!process.env.GITHUB_OUTPUT) return; const lines = [ `code_changed=${result.code_changed}`, + `product_changed=${result.product_changed}`, `full_matrix=${result.full_matrix}`, `targeted_tests=${result.targeted_tests.join(' ')}`, `windows_tests=${result.windows_tests.join(' ')}`, @@ -313,6 +409,7 @@ function writeOutputs(result) { function main() { try { const args = parseArgs(process.argv.slice(2)); + const files = changedFiles(args); const result = classify(files); result.changed_files = files; diff --git a/tests/ci-test-scope.test.cjs b/tests/ci-test-scope.test.cjs index 2ce8d4f1d..a242cee4e 100644 --- a/tests/ci-test-scope.test.cjs +++ b/tests/ci-test-scope.test.cjs @@ -4,9 +4,11 @@ const { describe, test } = require('node:test'); const assert = require('node:assert/strict'); const { spawnSync } = require('child_process'); const path = require('path'); +const fs = require('fs'); const ROOT = path.join(__dirname, '..'); const SCRIPT = path.join(ROOT, 'scripts', 'ci-test-scope.cjs'); +const WORKFLOWS_DIR = path.join(ROOT, '.github', 'workflows'); function scopeFor(files) { const r = spawnSync(process.execPath, [SCRIPT, '--files', files.join(' ')], { @@ -18,25 +20,124 @@ function scopeFor(files) { } describe('ci-test-scope.cjs', () => { - test('docs-only changes mark code_changed and select docs-parity (new correct contract)', () => { + test('docs-only changes: code_changed is false, product_changed false (skip matrix entirely)', () => { const result = scopeFor(['docs/usage.md']); - assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.code_changed, false, + `expected code_changed=false for docs-only change, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, false, + `expected product_changed=false for docs-only change, got: ${JSON.stringify(result)}`); assert.strictEqual(result.full_matrix, false); + // docs-parity is NOT in targeted_tests when docs-only (it runs via docs-required.yml instead) assert.ok( - result.targeted_tests.some(t => t.includes('docs-parity-live-registry')), - `expected docs-parity-live-registry in targeted_tests, got: ${JSON.stringify(result.targeted_tests)}`, + !result.targeted_tests.some(t => t.includes('docs-parity-live-registry')), + `docs-parity-live-registry must NOT be in targeted_tests for docs-only, got: ${JSON.stringify(result.targeted_tests)}`, ); }); - test('workflow changes request full matrix and workflow contract tests', () => { + test('root markdown only: code_changed is false, product_changed false', () => { + const result = scopeFor(['README.md']); + assert.strictEqual(result.code_changed, false, + `expected code_changed=false for root markdown, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, false, + `expected product_changed=false for root markdown, got: ${JSON.stringify(result)}`); + }); + + test('pipeline workflow (test.yml) — product_changed true, full_matrix true, workflow contract tests', () => { const result = scopeFor(['.github/workflows/test.yml']); assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.product_changed, true, + `expected product_changed=true for test.yml, got: ${JSON.stringify(result)}`); assert.strictEqual(result.full_matrix, true); assert.ok(result.targeted_tests.includes('tests/workflow-shell-pinning.test.cjs')); assert.ok(result.targeted_tests.includes('tests/release-tarball-smoke-workflow.test.cjs')); assert.ok(result.windows_tests.includes('tests/workflow-shell-pinning.test.cjs')); }); + test('pipeline workflow (install-smoke.yml) — product_changed true, full_matrix true', () => { + const result = scopeFor(['.github/workflows/install-smoke.yml']); + assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.product_changed, true, + `expected product_changed=true for install-smoke.yml, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, true); + }); + + test('inert CI only (stale.yml) — code_changed true, product_changed false, full_matrix false', () => { + const result = scopeFor(['.github/workflows/stale.yml']); + assert.strictEqual(result.code_changed, true, + `expected code_changed=true for inert CI, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, false, + `expected product_changed=false for inert CI, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, false, + `expected full_matrix=false for inert CI, got: ${JSON.stringify(result)}`); + assert.ok(result.targeted_tests.includes('tests/workflow-shell-pinning.test.cjs'), + `expected workflow-shell-pinning in targeted_tests, got: ${JSON.stringify(result.targeted_tests)}`); + assert.ok(result.targeted_tests.includes('tests/policy-lint-shallow-checkout.test.cjs'), + `expected policy-lint-shallow-checkout in targeted_tests for inert CI, got: ${JSON.stringify(result.targeted_tests)}`); + }); + + test('TS runtime sources (src/semver.cts) — code_changed true, product_changed true, full_matrix false, semver tests targeted', () => { + const result = scopeFor(['src/semver.cts']); + assert.strictEqual(result.code_changed, true, + `expected code_changed=true for src/ change, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, true, + `expected product_changed=true for src/ change, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, false, + `expected full_matrix=false for src/-only change (TS runtime sources rule has no fullMatrix), got: ${JSON.stringify(result)}`); + assert.ok(result.targeted_tests.includes('tests/semver-compare.test.cjs'), + `expected semver-compare in targeted_tests, got: ${JSON.stringify(result.targeted_tests)}`); + }); + + test('product code (gsd-core/bin/lib/foo.cjs) — product_changed true', () => { + const result = scopeFor(['gsd-core/bin/lib/foo.cjs']); + assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.product_changed, true, + `expected product_changed=true for gsd-core/ change, got: ${JSON.stringify(result)}`); + }); + + test('unknown/new workflow defaults to pipeline (fail-safe) — product_changed true', () => { + const result = scopeFor(['.github/workflows/brand-new-thing.yml']); + assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.product_changed, true, + `expected product_changed=true for unknown workflow (fail-safe), got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, true, + `expected full_matrix=true for unknown workflow (fail-safe), got: ${JSON.stringify(result)}`); + }); + + test('mixed docs + code — escalates to product_changed true', () => { + // Use bin/gsd (installer rule, fullMatrix:true) to get a code file that reliably triggers full matrix. + const result = scopeFor(['docs/x.md', 'bin/gsd']); + assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.product_changed, true, + `expected product_changed=true for docs+code, got: ${JSON.stringify(result)}`); + }); + + test('inert CI (docs-required.yml) — includes shallow-checkout policy test, product_changed false', () => { + const result = scopeFor(['.github/workflows/docs-required.yml']); + assert.strictEqual(result.code_changed, true, + `expected code_changed=true for docs-required.yml, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, false, + `expected product_changed=false for docs-required.yml, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, false, + `expected full_matrix=false for docs-required.yml, got: ${JSON.stringify(result)}`); + assert.ok(result.targeted_tests.includes('tests/policy-lint-shallow-checkout.test.cjs'), + `expected policy-lint-shallow-checkout in targeted_tests for docs-required.yml, got: ${JSON.stringify(result.targeted_tests)}`); + }); + + test('mixed docs + inert CI — code_changed true, product_changed false (inert lane)', () => { + const result = scopeFor(['docs/x.md', '.github/workflows/stale.yml']); + assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.product_changed, false, + `expected product_changed=false for docs+inert, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, false); + }); + + test('mixed docs + src — product_changed true', () => { + const result = scopeFor(['docs/x.md', 'src/semver.cts']); + assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.product_changed, true, + `expected product_changed=true for docs+src, got: ${JSON.stringify(result)}`); + }); + test('command changes request command tests without full parity matrix', () => { const result = scopeFor(['commands/gsd/plan-phase.md']); assert.strictEqual(result.code_changed, true); @@ -54,6 +155,8 @@ describe('ci-test-scope.cjs', () => { test('installer-sensitive changes request full matrix and install tests', () => { const result = scopeFor(['bin/gsd']); assert.strictEqual(result.code_changed, true); + assert.strictEqual(result.product_changed, true, + `expected product_changed=true for bin/gsd, got: ${JSON.stringify(result)}`); assert.strictEqual(result.full_matrix, true); assert.ok(result.targeted_tests.includes('tests/install.test.cjs')); assert.ok(result.targeted_tests.includes('tests/release-tarball-smoke.install.test.cjs')); @@ -120,25 +223,27 @@ describe('ci-test-scope superset invariant (#494)', () => { `expected full_matrix=true for tests/** change, got: ${JSON.stringify(result)}`); }); - // Facet B: docs/**, commands/**, agents/** → code_changed AND docs-parity selected - test('B1: docs/adr change marks code_changed and selects docs-parity-live-registry', () => { + // Facet B: commands/**, agents/** → code_changed AND docs-parity selected + // docs/ is NO LONGER in this facet — docs-only PRs skip the matrix entirely. + test('B1: docs/adr change: code_changed is false (docs skip matrix)', () => { const result = scopeFor(['docs/adr/22-plan-drift-guard.md']); - assert.strictEqual(result.code_changed, true, - `expected code_changed=true for docs/** change, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.code_changed, false, + `expected code_changed=false for docs/** change (matrix skip), got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, false, + `expected product_changed=false for docs/** change, got: ${JSON.stringify(result)}`); + // docs-parity is NOT in targeted_tests (handled by docs-required.yml) assert.ok( - result.targeted_tests.some(t => t.includes('docs-parity-live-registry')), - `expected docs-parity-live-registry in targeted_tests, got: ${JSON.stringify(result.targeted_tests)}`, + !result.targeted_tests.some(t => t.includes('docs-parity-live-registry')), + `docs-parity-live-registry must NOT be in targeted_tests for docs-only, got: ${JSON.stringify(result.targeted_tests)}`, ); }); - test('B2: docs locale dir change marks code_changed and selects docs-parity-live-registry', () => { + test('B2: docs locale dir change: code_changed is false (docs skip matrix)', () => { const result = scopeFor(['docs/ja-JP/USAGE.md']); - assert.strictEqual(result.code_changed, true, - `expected code_changed=true for docs/ja-JP/** change, got: ${JSON.stringify(result)}`); - assert.ok( - result.targeted_tests.some(t => t.includes('docs-parity-live-registry')), - `expected docs-parity-live-registry in targeted_tests, got: ${JSON.stringify(result.targeted_tests)}`, - ); + assert.strictEqual(result.code_changed, false, + `expected code_changed=false for docs/ja-JP/** change, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, false, + `expected product_changed=false for docs/ja-JP/** change, got: ${JSON.stringify(result)}`); }); test('B3: commands/** change selects docs-parity-live-registry', () => { @@ -149,3 +254,138 @@ describe('ci-test-scope superset invariant (#494)', () => { ); }); }); + +describe('INERT_WORKFLOWS allowlist integrity guard', () => { + // Load the INERT_WORKFLOWS set from the script by spawning it and using --files + // on a sentinel path, then separately verify the set contents via the filesystem. + + // Known pipeline workflows that MUST NOT appear in INERT_WORKFLOWS. + // Must stay in sync with PROTECTED_WORKFLOWS in scripts/ci-test-scope.cjs. + const KNOWN_PIPELINE = [ + 'test.yml', + 'install-smoke.yml', + 'mutation.yml', + 'security-scan.yml', + 'release.yml', + ]; + + // Canonical inert workflow list — reused by both tests below. + const knownInert = [ + 'stale.yml', 'branch-cleanup.yml', 'branch-naming.yml', 'auto-label-issues.yml', + 'auto-branch.yml', 'auto-backmerge.yml', 'close-draft-prs.yml', + 'dismiss-unauthorized-pr-approvals.yml', 'pr-gate.yml', 'pr-target-validator.yml', + 'pr-template-format.yml', 'require-issue-link.yml', 'changeset-required.yml', + 'docs-required.yml', 'discord-changelog.yml', + ]; + + test('all entries in INERT_WORKFLOWS exist under .github/workflows/', () => { + // We derive the inert set implicitly: any .github/workflows/*.yml that produces + // full_matrix=false when passed alone is inert. We check the known inert names + // against the filesystem instead. + // The canonical list is in the script — we verify each named file exists. + for (const name of knownInert) { + const fullPath = path.join(WORKFLOWS_DIR, name); + assert.ok( + fs.existsSync(fullPath), + `INERT_WORKFLOWS entry '${name}' does not exist at ${fullPath}`, + ); + } + }); + + test('known pipeline workflows are NOT treated as inert (product_changed true, full_matrix true)', () => { + for (const name of KNOWN_PIPELINE) { + const result = scopeFor([`.github/workflows/${name}`]); + assert.strictEqual(result.product_changed, true, + `${name} must be pipeline (product_changed=true), got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, true, + `${name} must be pipeline (full_matrix=true), got: ${JSON.stringify(result)}`); + } + }); + + // Explicit per-workflow guard: each of the five protected workflows must route to + // the full matrix. This documents intent and proves that PROTECTED_WORKFLOWS + // enforcement is covered end-to-end via the spawn helper. + test('all five PROTECTED_WORKFLOWS individually route to full matrix (tamper-evidence)', () => { + const protected_ = [ + 'test.yml', + 'install-smoke.yml', + 'mutation.yml', + 'security-scan.yml', + 'release.yml', + ]; + for (const name of protected_) { + const result = scopeFor([`.github/workflows/${name}`]); + assert.strictEqual(result.product_changed, true, + `PROTECTED_WORKFLOW ${name}: expected product_changed=true, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, true, + `PROTECTED_WORKFLOW ${name}: expected full_matrix=true, got: ${JSON.stringify(result)}`); + } + }); + + test('every inert workflow produces code_changed=true, product_changed=false, and full_matrix=false', () => { + for (const name of knownInert) { + const result = scopeFor([`.github/workflows/${name}`]); + assert.strictEqual(result.code_changed, true, + `${name}: expected code_changed=true`); + assert.strictEqual(result.product_changed, false, + `${name}: expected product_changed=false`); + assert.strictEqual(result.full_matrix, false, + `${name}: expected full_matrix=false`); + } + }); +}); + +describe('code_changed=false implies clean output invariant', () => { + // Fix 1: when code_changed is false, full_matrix, targeted_tests, windows_tests + // must ALL be empty/false — even if a docs path coincidentally + // matches a content rule via coarse substring (e.g. path.includes('install') or + // path.includes('config')). + + test('docs-only: code_changed=false → product_changed=false, full_matrix=false, empty targeted_tests', () => { + const result = scopeFor(['docs/usage.md']); + assert.strictEqual(result.code_changed, false); + assert.strictEqual(result.product_changed, false); + assert.strictEqual(result.full_matrix, false); + assert.deepStrictEqual(result.targeted_tests, []); + }); + + // docs/installer-migrations.md contains 'install' → would match the installer rule + // via path.includes('install'). Normalization must suppress the contradictory output. + test('docs/installer-migrations.md: code_changed=false AND product_changed=false AND full_matrix=false AND empty targeted_tests', () => { + const result = scopeFor(['docs/installer-migrations.md']); + assert.strictEqual(result.code_changed, false, + `expected code_changed=false for docs/installer-migrations.md, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, false, + `expected product_changed=false for docs/installer-migrations.md, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.full_matrix, false, + `expected full_matrix=false for docs/installer-migrations.md, got: ${JSON.stringify(result)}`); + assert.deepStrictEqual(result.targeted_tests, [], + `expected empty targeted_tests for docs/installer-migrations.md, got: ${JSON.stringify(result.targeted_tests)}`); + }); + + // docs/how-to/configure-model-profiles.md contains 'config' → matches configuration rule. + test('docs path matching config rule: code_changed=false → empty output (coarse-substring docs suppressed)', () => { + const result = scopeFor(['docs/how-to/configure-model-profiles.md']); + assert.strictEqual(result.code_changed, false, + `expected code_changed=false, got: ${JSON.stringify(result)}`); + assert.strictEqual(result.product_changed, false); + assert.strictEqual(result.full_matrix, false); + assert.deepStrictEqual(result.targeted_tests, []); + }); + + // code_changed=true must produce >= 1 targeted_test or 'unit' fallback. + test('code_changed=true implies non-empty targeted_tests', () => { + for (const files of [ + ['src/semver.cts'], + ['bin/gsd'], + ['.github/workflows/test.yml'], + ['.github/workflows/stale.yml'], + ]) { + const result = scopeFor(files); + assert.strictEqual(result.code_changed, true, + `expected code_changed=true for ${files}, got: ${JSON.stringify(result)}`); + assert.ok(result.targeted_tests.length >= 1, + `expected >= 1 targeted_test for ${files}, got: ${JSON.stringify(result.targeted_tests)}`); + } + }); +});