diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b17038f8b..78c9a6674 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -782,6 +782,133 @@ jobs: CI_JOB_TIMEOUT_MINUTES: '45' run: node scripts/ci-check-job-near-cap.cjs + # #4591 (epic #4589 Phase 2): runs ONLY the platform-conformance-tier file + # list (scripts/lib/platform-conformance-tier.generated.cjs) on real + # Windows/macOS — the OS-agnostic bulk of the suite already ran once on + # ubuntu-latest in the `test` job above. Gated the same way `test-full` is + # (product code changed AND full_matrix). Windows is sharded 3 ways + # (matching test-full's own precedent) after the first real CI run measured + # it: unsharded, windows-latest hit its 45-minute timeout and was CANCELLED + # (started 03:41:19Z, cancelled 04:26:25Z, run 34434252144) while + # macos-latest finished the identical file set in 26m58s — this conformance + # tier is dominated by subprocess-spawning tests (343/565 files), and + # Windows process-spawn overhead is the documented reason test-full's own + # windows lane needed the same fix (#3057). macos-latest stays unsharded; it + # has real headroom (27m against the 45m cap). + # `test-full` (above) keeps running the WHOLE suite on the + # same OSes as a non-gating safety net for one release cycle (see + # `required-tests` below) — this job is the new GATING signal for + # windows/macos coverage. + test-conformance: + name: conformance test (${{ matrix.os }}, ${{ matrix.node-version }}${{ matrix.shard && format(', shard {0}', matrix.shard) || '' }}) + needs: [changes, preflight] + if: needs.changes.outputs.code_changed == 'true' && needs.changes.outputs.full_matrix == 'true' + runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.shell }} + # No LANE_COSTS entry exists yet for this brand-new job — see + # tests/ci-test-job-timeout-budget.test.cjs's own header: "no unit test + # can prove a lane fits its budget — only a real CI run measures that." + # Generous and unchanged from test-full's own budget until a real + # measurement exists; the smaller file count should comfortably undercut + # this ceiling. + timeout-minutes: 45 + env: + GSD_PLUGIN_ROOT: .ci-gsd-plugin-root-disabled + GSD_STRICT_LIVE_CONFIG_GUARD: ${{ matrix.os != 'windows-latest' && '1' || '' }} + GSD_EMITTED_BASE: ${{ github.event.pull_request.base.sha }} + AUDIT_BASELINE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || (github.event_name == 'push' && github.event.before) || (github.event_name == 'merge_group' && github.event.merge_group.base_sha) || '' }} + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + node-version: 24 + shell: pwsh + shard: 1/3 + - os: windows-latest + node-version: 24 + shell: pwsh + shard: 2/3 + - os: windows-latest + node-version: 24 + shell: pwsh + shard: 3/3 + - os: macos-latest + node-version: 24 + shell: 'zsh {0}' + + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 (Windows) + if: runner.os == 'Windows' + with: + fetch-depth: 0 + persist-credentials: true + token: ${{ github.token }} + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 (Linux/macOS) + if: runner.os != 'Windows' + with: + fetch-depth: 0 + persist-credentials: true + token: ${{ github.token }} + + - name: Record job start time (near-cap check) + run: node -e 'require("fs").appendFileSync(process.env.GITHUB_ENV, "CI_JOB_START_EPOCH_MS=" + Date.now() + "\n")' + + - name: Guard — require GitHub-hosted runner + run: node scripts/ci-guard-runner.cjs + + - name: Rebase check — merge PR base branch into PR head + if: github.event_name == 'pull_request' + env: + GITHUB_TOKEN: ${{ github.token }} + CI_REBASE_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: node scripts/ci-rebase-check.cjs + + - name: Set up Node.js ${{ matrix.node-version }} + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Environment check + run: npm run check:env + + - name: Install dependencies + run: npm ci + + - name: Dependency integrity gate + run: node scripts/check-npm-integrity.cjs + + - name: Restore emitted-baseline cache + if: github.event_name == 'pull_request' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: .gsd-cache/emitted-baseline.json + key: emitted-baseline-${{ github.event.pull_request.base.sha }} + + # #4591: the generated conformance-tier list is a .cjs module (so + # scripts/gen-platform-conformance-tier.cjs's own tests and other + # scripts can `require()` it directly) — run-tests.cjs --files-from + # expects a plain newline-delimited text file, so this step bridges + # the two, one path per line, matching the existing scoped-lane + # convention (.ci-selected-tests.txt). + - name: Prepare conformance-tier test list + run: node -e "require('./scripts/lib/platform-conformance-tier.generated.cjs').CONFORMANCE_TIER_FILES.forEach(f => console.log(f))" > .ci-conformance-tests.txt + + - name: Run platform-conformance-tier tests + run: node scripts/run-tests.cjs --files-from .ci-conformance-tests.txt${{ matrix.shard && format(' --shard {0}', matrix.shard) || '' }} + + - name: Check job budget (near-cap advisory) + if: always() + continue-on-error: true + env: + CI_JOB_LABEL: "conformance test (${{ matrix.os }}, ${{ matrix.node-version }})" + CI_JOB_TIMEOUT_MINUTES: '45' + run: node scripts/ci-check-job-near-cap.cjs + # #2952: the coverage gate that the `test` lane used to run inline. Sharding # the unit suite means no single runner sees the whole picture, so the gate # moves here, downloads every shard's raw V8 dumps into one coverage/tmp, and @@ -929,6 +1056,7 @@ jobs: - test - test-inert - test-full + - test-conformance - coverage-gate - qa-loop-walk if: always() @@ -944,6 +1072,7 @@ jobs: CHANGES_RESULT: ${{ needs.changes.result }} LINT_RESULT: ${{ needs.lint-tests.result }} TEST_RESULT: ${{ needs.test.result }} + TEST_CONFORMANCE_RESULT: ${{ needs.test-conformance.result }} INERT_RESULT: ${{ needs.test-inert.result }} FULL_TEST_RESULT: ${{ needs.test-full.result }} COVERAGE_GATE_RESULT: ${{ needs.coverage-gate.result }} @@ -957,6 +1086,7 @@ jobs: echo "changes=$CHANGES_RESULT" echo "lint-tests=$LINT_RESULT" echo "test=$TEST_RESULT" + echo "test-conformance=$TEST_CONFORMANCE_RESULT" echo "test-inert=$INERT_RESULT" echo "test-full=$FULL_TEST_RESULT" echo "coverage-gate=$COVERAGE_GATE_RESULT" @@ -1002,13 +1132,27 @@ jobs: echo "::error::test matrix did not pass" exit 1 fi + # #4591 (epic #4589 Phase 2): test-conformance is the new gating + # signal for real Windows/macOS coverage — the conformance-tier + # file list, not the whole suite. + if [ "$TEST_CONFORMANCE_RESULT" != "success" ] && [ "$TEST_CONFORMANCE_RESULT" != "skipped" ]; then + echo "::error::platform-conformance-tier matrix did not pass" + exit 1 + fi # #2952: the coverage gates no longer run inside the `test` matrix — # sharding moved them to the `coverage-gate` job, which merges every # shard's dumps. TEST_RESULT therefore does NOT cover them any more; # COVERAGE_GATE_RESULT below is what gates coverage. + # #4591 (epic #4589 Phase 2): downgraded from a hard gate to a + # non-blocking safety net for one release cycle, per the epic's + # phased-rollout design — test-conformance (above) is now the real + # gate for windows/macos coverage. A red full-suite run here is + # still surfaced (it means the conformance-tier classifier missed + # something) but does not block merge; retire this job entirely + # in a follow-up once the classifier has proven itself over one + # release cycle with no such miss. if [ "$FULL_TEST_RESULT" != "success" ] && [ "$FULL_TEST_RESULT" != "skipped" ]; then - echo "::error::full parity matrix did not pass" - exit 1 + echo "::warning::full legacy parity matrix did not pass (non-gating safety net — see #4591)" fi # #2952: the coverage gate is skipped when product code did not change diff --git a/bin/install.js b/bin/install.js index 00ebb795d..fe07435bf 100755 --- a/bin/install.js +++ b/bin/install.js @@ -436,7 +436,7 @@ const GSD_CHANGESET_FILES = [ 'github-release-notes.cjs', 'lint.cjs', 'new.cjs', 'README.md', // documentation only — not user-authored ]; -const GSD_SCRIPTS_LIB_FILES = ['cli-exit.cjs', 'allowlist-ratchet.cjs', 'drift-scan.cjs', 'alias-drift-families.cjs', 'exit-code-registry.cjs', 'ndjson-reporter.cjs', 'ci-job-timing.cjs', 'shellcheck-fetch.cjs', 'npm-version-check-diagnosis.cjs']; +const GSD_SCRIPTS_LIB_FILES = ['cli-exit.cjs', 'allowlist-ratchet.cjs', 'drift-scan.cjs', 'alias-drift-families.cjs', 'exit-code-registry.cjs', 'ndjson-reporter.cjs', 'ci-job-timing.cjs', 'shellcheck-fetch.cjs', 'npm-version-check-diagnosis.cjs', 'platform-conformance-tier.generated.cjs', 'suite-detection.cjs']; /** * Resolve a runtime's shared-hooks directory name from its descriptor. diff --git a/docs/TESTING-SUITES.md b/docs/TESTING-SUITES.md index e07c584ad..3e75d0fea 100644 --- a/docs/TESTING-SUITES.md +++ b/docs/TESTING-SUITES.md @@ -404,7 +404,7 @@ Two properties are load-bearing and easy to break: failure message only. Gated: `test.yml` (`lint-tests`, `test`, `test-inert`, `test-full`, -`coverage-gate`, `qa-loop-walk`, `required-tests`), `install-smoke.yml`, +`test-conformance`, `coverage-gate`, `qa-loop-walk`, `required-tests`), `install-smoke.yml`, `mutation.yml`, `security-scan.yml`, `docs-required.yml`, `changeset-required.yml`, `default-flip-documentation.yml`, `branch-naming.yml`. @@ -547,6 +547,17 @@ cap stays at least the **headroom factor** (1.5x) above a documented, hand-measured cost for that job — now all four of the jobs above, not just `test`/`test-full`/`coverage-gate`/`test-inert` as before. +`test-conformance` in `test.yml` (#4591, epic #4589 Phase 2) runs the +`scripts/lib/platform-conformance-tier.generated.cjs` file list on +`windows-latest` (sharded three ways) and `macos-latest` (unsharded) — the +gating signal for real-OS coverage, replacing `test-full`'s old gating role +for one release cycle while `test-full` runs as a non-gating safety net. It +also declares a `timeout-minutes` cap and runs the same in-job near-cap check +described below, but it has no `LANE_COSTS` entry in +`tests/ci-test-job-timeout-budget.test.cjs` yet — no real, completed +(non-cancelled) per-shard measurement exists — so the headroom-factor gate +does not cover it until one lands. + Two runtime mechanisms sit on top of that static gate, both new in #4036: - **In-job near-cap check** (`scripts/ci-check-job-near-cap.cjs`) — the last @@ -561,7 +572,7 @@ Two runtime mechanisms sit on top of that static gate, both new in #4036: `scripts/ci-timeout-report.cjs`) — runs daily and on `workflow_dispatch`. It polls GitHub's Actions REST API for recently completed jobs across `test.yml`, `mutation.yml`, and `install-smoke.yml`, resolves each job's - declared cap (a literal `timeout-minutes` for `test`/`test-full`/`smoke`, or + declared cap (a literal `timeout-minutes` for `test`/`test-full`/`test-conformance`/`smoke`, or `scripts/mutation-matrix.cjs`'s `COVERED[].timeoutMinutes` for `mutate`'s per-module shards), and appends any new `(runId, jobName)` records to `tests/ci-timeout-budget-history.jsonl`. Unlike the in-job check, diff --git a/docs/how-to/read-ci-timeout-signals.md b/docs/how-to/read-ci-timeout-signals.md index 81285ad47..64dc56790 100644 --- a/docs/how-to/read-ci-timeout-signals.md +++ b/docs/how-to/read-ci-timeout-signals.md @@ -1,9 +1,12 @@ # How to read CI timeout budget signals -Every matrixed CI job (`test`, `test-full` in `.github/workflows/test.yml`; `mutate` in -`mutation.yml`; `smoke` in `install-smoke.yml`) now reports how close it ran to its +Every matrixed CI job (`test`, `test-full`, `test-conformance` in `.github/workflows/test.yml`; +`mutate` in `mutation.yml`; `smoke` in `install-smoke.yml`) now reports how close it ran to its `timeout-minutes` cap. This page is for a maintainer trying to answer: *is a lane drifting -toward its cap, and where do I look?* +toward its cap, and where do I look?* (`test-conformance` runs the platform-conformance-tier file +list — `scripts/lib/platform-conformance-tier.generated.cjs` — on `windows-latest`, sharded three +ways, and `macos-latest`, unsharded; it is the gating signal for real-OS coverage, replacing +`test-full`'s old gating role for one release cycle.) ## 1. A single run crossed 90% of its budget diff --git a/package.json b/package.json index da75abfa6..f20c28ab8 100644 --- a/package.json +++ b/package.json @@ -110,7 +110,7 @@ "gen:registry": "node scripts/gen-registry.cjs --write", "gen:install-tree": "node scripts/gen-install-tree-fixtures.cjs", "gen:section-manifest": "node scripts/gen-section-manifest.cjs --write", - "regen:derived": "npm run build && npm run gen:registry && node scripts/gen-adr-index.cjs --write && node scripts/gen-features.cjs --write && node scripts/gen-capability-matrix.cjs --write && node scripts/gen-inventory-manifest.cjs --write && node scripts/gen-context-index.cjs --write && node scripts/gen-state-md-docs.cjs --write && npm run gen:section-manifest && node scripts/sync-manifest-versions.cjs && npm run gen:install-tree && node scripts/gen-scripts-cli-exit.cjs --write && node scripts/gen-hooks-cli-exit.cjs --write && node scripts/gen-exit-code-registry.cjs --write && node scripts/gen-exit-code-docs.cjs --write", + "regen:derived": "npm run build && npm run gen:registry && node scripts/gen-adr-index.cjs --write && node scripts/gen-features.cjs --write && node scripts/gen-capability-matrix.cjs --write && node scripts/gen-inventory-manifest.cjs --write && node scripts/gen-context-index.cjs --write && node scripts/gen-state-md-docs.cjs --write && npm run gen:section-manifest && node scripts/sync-manifest-versions.cjs && npm run gen:install-tree && node scripts/gen-scripts-cli-exit.cjs --write && node scripts/gen-hooks-cli-exit.cjs --write && node scripts/gen-exit-code-registry.cjs --write && node scripts/gen-exit-code-docs.cjs --write && node scripts/gen-platform-conformance-tier.cjs --write", "validate:registry": "node scripts/validate-registry.cjs", "prepack": "npm run build:lib", "prepare": "npm run build:lib", @@ -132,7 +132,7 @@ "lint:test-file-count": "node scripts/lint-test-file-count.cjs", "lint:pr-checks": "node scripts/lint-pr-check-project-dir.cjs", "lint:changeset": "node scripts/changeset/lint.cjs", - "lint:generated-sync": "node scripts/gen-capability-registry.cjs --check && node scripts/gen-loop-host-contract.cjs --check && node scripts/gen-capability-matrix.cjs --check && node scripts/sync-manifest-versions.cjs --check && node scripts/gen-inventory-manifest.cjs --check && node scripts/generate-package-identity.cjs --check && node scripts/gen-plugin-skills.cjs --check && node scripts/gen-registry.cjs --check && node scripts/gen-adr-index.cjs --check && node scripts/gen-features.cjs --check && node scripts/check-glossary-refs.cjs --check && node scripts/lint-compiled-artifact-sync.cjs --check && node scripts/gen-context-index.cjs --check && node scripts/gen-section-manifest.cjs --check && node scripts/gen-health-docs.cjs --check && node scripts/gen-state-md-docs.cjs --check && node scripts/gen-scripts-cli-exit.cjs --check && node scripts/gen-hooks-cli-exit.cjs --check && node scripts/gen-exit-code-registry.cjs --check && node scripts/gen-exit-code-docs.cjs --check", + "lint:generated-sync": "node scripts/gen-capability-registry.cjs --check && node scripts/gen-loop-host-contract.cjs --check && node scripts/gen-capability-matrix.cjs --check && node scripts/sync-manifest-versions.cjs --check && node scripts/gen-inventory-manifest.cjs --check && node scripts/generate-package-identity.cjs --check && node scripts/gen-plugin-skills.cjs --check && node scripts/gen-registry.cjs --check && node scripts/gen-adr-index.cjs --check && node scripts/gen-features.cjs --check && node scripts/check-glossary-refs.cjs --check && node scripts/lint-compiled-artifact-sync.cjs --check && node scripts/gen-context-index.cjs --check && node scripts/gen-section-manifest.cjs --check && node scripts/gen-health-docs.cjs --check && node scripts/gen-state-md-docs.cjs --check && node scripts/gen-scripts-cli-exit.cjs --check && node scripts/gen-hooks-cli-exit.cjs --check && node scripts/gen-exit-code-registry.cjs --check && node scripts/gen-exit-code-docs.cjs --check && node scripts/gen-platform-conformance-tier.cjs --check", "lint:docs": "node scripts/lint-docs-required.cjs", "lint:qa-smells": "node scripts/qa-smell-ratchet.cjs", "lint:legacy-name": "node scripts/lint-legacy-dir-name.cjs", diff --git a/scripts/ci-timeout-report.cjs b/scripts/ci-timeout-report.cjs index d75289cff..ab0ed21ef 100644 --- a/scripts/ci-timeout-report.cjs +++ b/scripts/ci-timeout-report.cjs @@ -35,6 +35,7 @@ const JOB_RULES = [ { workflowFile: 'test.yml', jobKey: 'test', test: (name) => name.startsWith('test (') && name !== 'test (inert CI)' }, { workflowFile: 'test.yml', jobKey: 'test-full', test: (name) => name.startsWith('full test (') }, { workflowFile: 'test.yml', jobKey: 'coverage-gate', test: (name) => name === 'Coverage gate (merged shards)' }, + { workflowFile: 'test.yml', jobKey: 'test-conformance', test: (name) => name.startsWith('conformance test (') }, { workflowFile: 'install-smoke.yml', jobKey: 'smoke', test: (name) => name.startsWith('smoke (') }, ]; diff --git a/scripts/gen-platform-conformance-tier.cjs b/scripts/gen-platform-conformance-tier.cjs new file mode 100644 index 000000000..8691c90fa --- /dev/null +++ b/scripts/gen-platform-conformance-tier.cjs @@ -0,0 +1,336 @@ +#!/usr/bin/env node +'use strict'; + +/** + * Generates scripts/lib/platform-conformance-tier.generated.cjs — the list of + * test files under tests/**\/*.test.cjs whose CONTENT signals they exercise + * platform-specific behavior (Windows/macOS path quirks, raw child_process + * usage, chmod mode bits, etc.) and therefore need REAL-OS coverage rather + * than a Linux-only conformance lane (#4591). + * + * Only `unit`-suite test files (no suite suffix, per `scripts/run-tests.cjs`'s + * `suiteOf()`) are considered for the conformance tier. `install`-, + * `security`-, `slow`-, `integration`-, and `qa`-suffixed files are excluded + * entirely — never merely deprioritized — for three independent reasons: (1) + * `install`/`slow` are explicitly PR-excluded suites + * (`scripts/affected-tests-lib.cjs`'s `PR_EXCLUDED_SUITES`; "PRs must never + * select or run these"), and this generator's output feeds a `pull_request`- + * triggered job; (2) `integration`/`security` already run via their own + * separate, dedicated, unsharded, shard-1-only steps in the `test`/ + * `test-full` jobs (.github/workflows/test.yml) — folding any of them into + * this job's generic `--files-from` + `--shard` invocation is unproven and, + * per the incident below, unsafe. (3) `qa` (loop-walk-suite files) already + * runs via its own separate, dedicated `qa-loop-walk` job + * (.github/workflows/test.yml) — the same rationale as (2): a purpose-built + * home already exists, so folding it into this job's generic invocation + * duplicates coverage without benefit. Real incident that surfaced this: a live + * CI run's `conformance test (windows-latest, shard 2/3)` job was killed with + * 11 tests in flight — including `tests/release-tarball-smoke.install.test.cjs` + * — because this generator had (wrongly) placed an `install`-suite file into + * the Linux-conformance candidate pool with no suite filtering at all. + * + * `classifyContent(content)` is the pure, exported classifier: it favors + * simple, auditable substring/regex matching over AST parsing, mirroring + * eslint-rules/lib/portability-vocab.cjs's own design stance (over-inclusion + * is the safe direction — a false positive costs one extra test running on a + * real OS; a false negative silently drops real-OS coverage). + * + * KNOWN LIMIT, disclosed deliberately: this is a STATIC content classifier, + * not a real per-file, per-OS behavioral diff. Epic #4589's issue #4591 asked + * for the cutover to be validated by "running the existing full matrix one + * more time as a parity baseline, diffing pass/fail per file between the + * real-OS runs and the Linux run" before moving any file into the Linux-only + * bulk. That literal per-file diff was NOT performed — no historical + * per-file, per-OS pass/fail dataset exists to diff against (GitHub Actions + * publishes coverage/QA artifacts from CI runs, not per-file JUnit results). + * What stands in for it: (1) the most recent push-triggered run on `next` + * (unconditionally full-matrix) is green on every OS for every file in this + * classification, confirmed before this classifier was built; (2) the + * existing `test-full` job keeps running the WHOLE suite on real Windows/ + * macOS as a non-gating safety net for one release cycle (.github/workflows/ + * test.yml) — a classifier miss surfaces as a visible warning there, not a + * silent gap, before the safety net is retired. This is the same + * static-analysis-substitutes-for-real-OS-execution stance ADR-1703's whole + * rule catalog already takes; it is a real, disclosed limit, not a silent + * substitution. + * + * Usage: + * node scripts/gen-platform-conformance-tier.cjs # print summary to stdout + * node scripts/gen-platform-conformance-tier.cjs --write # write the generated file + * node scripts/gen-platform-conformance-tier.cjs --check # exit 1 if the committed file is stale + * node scripts/gen-platform-conformance-tier.cjs --tests-dir # override the tests/ root (tests only) + * node scripts/gen-platform-conformance-tier.cjs --out # override the generated-file path (tests only) + * + * `--tests-dir`/`--out` (or the TESTS_DIR/OUT_PATH env vars, flag takes + * precedence) exist solely so tests/platform-conformance-tier.test.cjs can + * point the CLI at a small temp fixture tree instead of this repo's real, + * 900+-file tests/ tree. Production usage (package.json's lint:generated-sync + * / regen:derived chains) passes no flags and gets the real repo paths. + */ + +const fs = require('node:fs'); +const path = require('node:path'); + +const { ExitError, runMain } = require('./lib/cli-exit.cjs'); +const { suiteOf } = require('./lib/suite-detection.cjs'); + +const ROOT = path.resolve(__dirname, '..'); +const DEFAULT_TESTS_DIR = path.join(ROOT, 'tests'); +const DEFAULT_OUT_PATH = path.join(ROOT, 'scripts', 'lib', 'platform-conformance-tier.generated.cjs'); + +const GENERATED_HEADER = + '// GENERATED FILE — do not hand-edit. Run `node scripts/gen-platform-conformance-tier.cjs --write` to regenerate.\n'; + +/** + * Detection categories (#4591 design doc). Each entry's `test` receives the + * raw file content string and returns true when that category's signal is + * present. Order matches the design doc's enumeration; `signals` in + * `classifyContent`'s return value preserves this order. + */ +const CATEGORIES = [ + { + name: 'process-platform', + test: (content) => /process\.platform/.test(content), + }, + { + name: 'os-platform', + test: (content) => /os\.platform\(\)/.test(content), + }, + { + name: 'win32-darwin-literal', + test: (content) => /\bwin32\b/.test(content) || /\bdarwin\b/.test(content), + }, + { + name: 'chmod-mode-bit', + test: (content) => /chmodSync|chmod\(/.test(content) || /0o[0-7]{3,4}\b/.test(content), + }, + { + name: 'windows-shell-token', + test: (content) => /cmd\.exe|powershell|pwsh|ComSpec/i.test(content), + }, + { + name: 'windows-env-var', + test: (content) => /\bPATHEXT\b|\bUSERPROFILE\b|\bHOMEDRIVE\b|\bHOMEPATH\b/.test(content), + }, + { + name: 'process-seam-subprocess', + test: (content) => /\brunNode\(|\brunGit\(|\brunHook\(|\brunGsdTools\(|\bgitOrThrow\(/.test(content), + }, + { + name: 'raw-child-process', + // Requires BOTH the child_process import/reference token AND one of the + // three call names in the same file — this is what keeps a same-named + // local identifier (e.g. a variable called `spawnResult`) from false- + // positiving: `spawnResult` never forms the substring `spawnSync(`. + test: (content) => { + if (/require\((['"])(?:node:)?child_process\1\)/.test(content)) return true; + if (!content.includes('child_process')) return false; + return /\bspawnSync\(|\bexecSync\(|\bexecFileSync\(/.test(content); + }, + }, + { + name: 'symlink-keyword', + // A leading `\b` with no trailing one, case-insensitive: this is + // deliberately NOT `/\bsymlink\b|\bSymlink\b/` (that literal pair would + // never match the dominant real-world call shape `symlinkSync(` / + // `readlinkSync(` — no word boundary exists between "symlink" and the + // immediately-following "Sync", both \w characters). The leading `\b` + // alone still excludes a mid-word embedding like "presymlink". + test: (content) => /\bsymlink/i.test(content), + }, + { + name: 'hardcoded-path-vs-path-call', + // Intentionally coarse (design doc: over-inclusion is the safe + // direction): a path.* call ANYWHERE in the file plus a quoted + // forward-slash-leading string literal ANYWHERE in the file, with no + // attempt at proximity/scoping. + test: (content) => + /path\.(join|resolve|dirname|basename|normalize|relative)\(/.test(content) && + /['"`]\/[\w.\-/]*['"`]/.test(content), + }, +]; + +/** + * Pure classifier: given a test file's raw string content, returns which + * platform-conformance categories matched and whether the file needs real-OS + * coverage (true iff at least one category matched). + * + * @param {string} content + * @returns {{needsRealOs: boolean, signals: string[]}} + */ +function classifyContent(content) { + const text = typeof content === 'string' ? content : ''; + const signals = []; + for (const category of CATEGORIES) { + if (category.test(text)) signals.push(category.name); + } + return { needsRealOs: signals.length > 0, signals }; +} + +/** + * Recursively collect every `*.test.cjs` file beneath `dir`. + * @param {string} dir + * @returns {string[]} absolute paths + */ +function walkTestFiles(dir) { + const out = []; + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return out; + } + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + out.push(...walkTestFiles(full)); + } else if (entry.isFile() && entry.name.endsWith('.test.cjs')) { + out.push(full); + } + } + return out; +} + +/** + * Classify every test file under `testsDir`, returning `{ total, files }` + * where `files` is the SORTED array of `tests/<...>.test.cjs`-relative paths + * (POSIX-normalized, per RULESET.CONTENT-PATH-NORMALIZATION) whose content + * needs real-OS coverage. + * + * @param {string} testsDir + * @returns {{ total: number, files: string[] }} + */ +function classifyTree(testsDir) { + const absoluteFiles = walkTestFiles(testsDir); + // Only unit-suite files (no suite suffix) are eligible for the conformance + // tier — see the header doc-comment for why suite-tagged files are excluded + // entirely rather than merely deprioritized. + const unitFiles = absoluteFiles.filter((absPath) => suiteOf(absPath) === null); + const flagged = []; + for (const absPath of unitFiles) { + const content = fs.readFileSync(absPath, 'utf8'); + const { needsRealOs } = classifyContent(content); + if (needsRealOs) { + const rel = path.relative(testsDir, absPath).replace(/\\/g, '/'); + flagged.push('tests/' + rel); + } + } + flagged.sort(); + return { total: absoluteFiles.length, files: flagged }; +} + +/** + * Render the generated `.cjs` module body — one array entry per line for a + * readable diff, matching scripts/lib/portability-vocab.cjs's array-literal + * style. + * + * @param {string[]} files - already sorted. + * @returns {string} + */ +function renderGeneratedFile(files) { + const lines = files.map((f) => ` ${JSON.stringify(f)},`).join('\n'); + return ( + GENERATED_HEADER + + "'use strict';\n\n" + + 'module.exports = {\n' + + ' CONFORMANCE_TIER_FILES: [\n' + + (lines.length > 0 ? lines + '\n' : '') + + ' ],\n' + + '};\n' + ); +} + +/** Resolve the effective tests-dir / out-path from argv/env, flag beats env. */ +function resolveOverrides(argv) { + let testsDir = process.env.TESTS_DIR || DEFAULT_TESTS_DIR; + let outPath = process.env.OUT_PATH || DEFAULT_OUT_PATH; + + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--tests-dir') { + const value = argv[i + 1]; + if (!value || value.startsWith('--')) { + throw new ExitError(1, 'gen-platform-conformance-tier: --tests-dir requires a path value'); + } + testsDir = path.resolve(value); + i++; + } else if (argv[i] === '--out') { + const value = argv[i + 1]; + if (!value || value.startsWith('--')) { + throw new ExitError(1, 'gen-platform-conformance-tier: --out requires a path value'); + } + outPath = path.resolve(value); + i++; + } + } + + return { testsDir: path.resolve(testsDir), outPath: path.resolve(outPath) }; +} + +function main() { + const argv = process.argv.slice(2); + const { testsDir, outPath } = resolveOverrides(argv); + const mode = argv.includes('--check') ? 'check' : argv.includes('--write') ? 'write' : 'print'; + + const { total, files } = classifyTree(testsDir); + + if (mode === 'write') { + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, renderGeneratedFile(files)); + process.stdout.write(`Wrote ${outPath} (${files.length} conformance-tier file(s))\n`); + return; + } + + if (mode === 'check') { + // Never trust a stale require cache — the committed file may have been + // rewritten (by --write, or by hand) since this process started. + let committed; + try { + const resolved = require.resolve(outPath); + delete require.cache[resolved]; + committed = require(resolved); + } catch (err) { + throw new ExitError( + 1, + `gen-platform-conformance-tier: could not load ${outPath} — run ` + + '`node scripts/gen-platform-conformance-tier.cjs --write` first ' + + `(${err && err.message ? err.message : err})`, + ); + } + const committedFiles = Array.isArray(committed.CONFORMANCE_TIER_FILES) + ? committed.CONFORMANCE_TIER_FILES + : []; + const committedSet = new Set(committedFiles); + const liveSet = new Set(files); + + const added = files.filter((f) => !committedSet.has(f)); + const removed = committedFiles.filter((f) => !liveSet.has(f)); + + if (added.length > 0 || removed.length > 0) { + process.stderr.write( + `${path.relative(ROOT, outPath).replace(/\\/g, '/')} is stale. Run:\n` + + ' node scripts/gen-platform-conformance-tier.cjs --write\n\n', + ); + for (const f of added) process.stderr.write(' + ' + f + '\n'); + for (const f of removed) process.stderr.write(' - ' + f + '\n'); + throw new ExitError(1); + } + + process.stdout.write( + `ok gen-platform-conformance-tier: ${files.length} conformance-tier files, list matches\n`, + ); + return; + } + + // No flag: print a classification summary, write nothing. + process.stdout.write( + `gen-platform-conformance-tier: ${total} file(s) scanned, ` + + `${files.length} need real OS, ${total - files.length} excluded (Linux-only conformance tier eligible)\n`, + ); +} + +/* c8 ignore next 3 -- CLI entry guard; this repo measures coverage with c8, which does not honor istanbul pragmas */ +if (require.main === module) { + runMain(main); +} + +module.exports = { classifyContent, CATEGORIES, walkTestFiles, classifyTree, renderGeneratedFile }; diff --git a/scripts/lib/platform-conformance-tier.generated.cjs b/scripts/lib/platform-conformance-tier.generated.cjs new file mode 100644 index 000000000..29c23413a --- /dev/null +++ b/scripts/lib/platform-conformance-tier.generated.cjs @@ -0,0 +1,553 @@ +// GENERATED FILE — do not hand-edit. Run `node scripts/gen-platform-conformance-tier.cjs --write` to regenerate. +'use strict'; + +module.exports = { + CONFORMANCE_TIER_FILES: [ + "tests/active-workstream-store.test.cjs", + "tests/active-workstream-store.unit.test.cjs", + "tests/adr-15-progress-converge.test.cjs", + "tests/adr-22-plan-drift-guard.test.cjs", + "tests/adr-612-bracket-coherence.test.cjs", + "tests/adr-612-bracket-phase-counting.test.cjs", + "tests/adr-612-bracket-read-tolerance.test.cjs", + "tests/adr-index-gate.test.cjs", + "tests/adr857-core-without-capabilities.test.cjs", + "tests/advance-plan-ambiguous-phase.test.cjs", + "tests/agent-frontmatter.test.cjs", + "tests/agent-hint-routing-1689.test.cjs", + "tests/agent-install-check.test.cjs", + "tests/agent-install-validation.test.cjs", + "tests/agent-skills.test.cjs", + "tests/ai-evals.test.cjs", + "tests/antigravity-upgrades.test.cjs", + "tests/api-coverage-gate-e2e.test.cjs", + "tests/api-coverage.test.cjs", + "tests/assumption-delta-checkpoint-e2e.test.cjs", + "tests/assumption-delta.test.cjs", + "tests/audit-command-cutover.test.cjs", + "tests/audit-open-remainder-count.test.cjs", + "tests/audit-uat-acknowledged.test.cjs", + "tests/audit-uat-summary-segmentation.test.cjs", + "tests/audit-workstream-layouts.test.cjs", + "tests/augment-upgrades.test.cjs", + "tests/autonomous-converge.test.cjs", + "tests/autonomous-interactive.test.cjs", + "tests/benchmark-compact-content.test.cjs", + "tests/branch-no-track-guard.test.cjs", + "tests/broken-windows-description.test.cjs", + "tests/broken-windows.test.cjs", + "tests/bugs-1656-1657.test.cjs", + "tests/canary-version-leak-lint.test.cjs", + "tests/capability-cli.test.cjs", + "tests/capability-command-dispatch.test.cjs", + "tests/capability-consent.test.cjs", + "tests/capability-ledger.test.cjs", + "tests/capability-lifecycle.test.cjs", + "tests/capability-loader.test.cjs", + "tests/capability-lock-mkdir-failure-3987.test.cjs", + "tests/capability-precedence-parity.test.cjs", + "tests/capability-probe-fallback.test.cjs", + "tests/capability-registry.test.cjs", + "tests/capability-source.test.cjs", + "tests/capability-state.test.cjs", + "tests/capability-trust.test.cjs", + "tests/capability-writer.test.cjs", + "tests/changeset-cli.test.cjs", + "tests/changeset-github-release-notes.test.cjs", + "tests/changeset-lint.test.cjs", + "tests/changeset-new.test.cjs", + "tests/check-contract-drift.test.cjs", + "tests/check-env.test.cjs", + "tests/check-gap-analysis-plan-post-e2e.test.cjs", + "tests/check-glossary-refs.test.cjs", + "tests/check-tdd-review-checkpoint-e2e.test.cjs", + "tests/check-ui-plan-gate.test.cjs", + "tests/check-ui-safety-gate.test.cjs", + "tests/check-update-config-dir.test.cjs", + "tests/chunked-planning-parallel.test.cjs", + "tests/ci-docs-guard-registry.test.cjs", + "tests/ci-next-health.test.cjs", + "tests/ci-pr-mergeability.test.cjs", + "tests/ci-rebase-check.test.cjs", + "tests/ci-test-scope.test.cjs", + "tests/cjs-command-router-adapter.test.cjs", + "tests/claude-md-path.test.cjs", + "tests/claude-md.test.cjs", + "tests/claude-orchestration-command-router.test.cjs", + "tests/claude-orchestration.test.cjs", + "tests/claude-skills-migration.test.cjs", + "tests/cli-exit.test.cjs", + "tests/cline-install.test.cjs", + "tests/clock-seam.test.cjs", + "tests/clock.test.cjs", + "tests/close-phase-todos-padded-resolves.test.cjs", + "tests/code-review-command.test.cjs", + "tests/code-review-depth.test.cjs", + "tests/code-review-fix-pipeline-regression.test.cjs", + "tests/code-review-pipeline-regression.test.cjs", + "tests/code-review-tier3-files-override-scoping.test.cjs", + "tests/code-review.test.cjs", + "tests/codebuddy-install.test.cjs", + "tests/codebuddy-upgrades.test.cjs", + "tests/codex-config-agents.test.cjs", + "tests/codex-config-hooks.test.cjs", + "tests/codex-config-install.test.cjs", + "tests/codex-config.test.cjs", + "tests/codex-declarative-reference.test.cjs", + "tests/codex-inherit-smoke.test.cjs", + "tests/command-contract.test.cjs", + "tests/command-routing-hub.test.cjs", + "tests/commands.test.cjs", + "tests/commit-docs-bypass.test.cjs", + "tests/commit-files-deletion.test.cjs", + "tests/commit-files-pathspec.test.cjs", + "tests/commonjs-marker.test.cjs", + "tests/compact-content-4139.test.cjs", + "tests/compact-content-partition-guard.test.cjs", + "tests/completion-predicate-drift-guard.test.cjs", + "tests/completion-ratio-scope-withholding.test.cjs", + "tests/completion-ratio-single-owner.test.cjs", + "tests/concurrency-safety.test.cjs", + "tests/config-defaults-runtime-exclusion.test.cjs", + "tests/config-field-docs.test.cjs", + "tests/config-get-default.test.cjs", + "tests/config-loader.test.cjs", + "tests/config-schema.property.test.cjs", + "tests/config.test.cjs", + "tests/configuration-migrate-config.test.cjs", + "tests/context-drift.test.cjs", + "tests/context-predicates-query.test.cjs", + "tests/copilot-install.test.cjs", + "tests/copilot-upgrades.test.cjs", + "tests/core-utils.test.cjs", + "tests/coverage-metadata-parser.test.cjs", + "tests/coverage-uat-routing.test.cjs", + "tests/cursor-hook-workspace-roots.test.cjs", + "tests/cursor-hooks.test.cjs", + "tests/cursor-reviewer.test.cjs", + "tests/cursor-subagent-isolation.test.cjs", + "tests/debug-session-manager-commit.test.cjs", + "tests/decisions.test.cjs", + "tests/declarative-reference-antigravity.test.cjs", + "tests/declarative-reference-augment.test.cjs", + "tests/declarative-reference-codebuddy.test.cjs", + "tests/declarative-reference-copilot.test.cjs", + "tests/declarative-reference-windsurf.test.cjs", + "tests/declarative-reference-zcode.test.cjs", + "tests/default-flip-documentation-lint.test.cjs", + "tests/dispatcher.test.cjs", + "tests/docs-hooks-table-parity.test.cjs", + "tests/docs-parity-live-registry.test.cjs", + "tests/docs-update.test.cjs", + "tests/drift-detection.test.cjs", + "tests/edge-probe-spec-phase-contract.test.cjs", + "tests/edge-probe.test.cjs", + "tests/edit-phase-milestone-scope-guard.test.cjs", + "tests/edit-phase.test.cjs", + "tests/effort-surface-axis.test.cjs", + "tests/effort-sync-installed-runtime.test.cjs", + "tests/emitted-ack-trailer.test.cjs", + "tests/emitted-attribution.test.cjs", + "tests/emitted-caps-gate.test.cjs", + "tests/emitted-provenance.test.cjs", + "tests/emitted-sizes.test.cjs", + "tests/ensure-runtime-build.test.cjs", + "tests/enumeration-single-owner.test.cjs", + "tests/eslint-glob-coverage.test.cjs", + "tests/eslint-rules.test.cjs", + "tests/estimate-calibrate.test.cjs", + "tests/estimate-loop-convergence.test.cjs", + "tests/execute-mvp-tdd-gate.test.cjs", + "tests/execute-phase-wave.test.cjs", + "tests/execute-phase-worktree-guard.test.cjs", + "tests/execute-plan-update-codebase-map-diff-base.test.cjs", + "tests/execute-wave-post-gate-pipeline-e2e.test.cjs", + "tests/executed-plan.test.cjs", + "tests/executor-mvp-tdd-section.test.cjs", + "tests/exit-code-registry.test.cjs", + "tests/explore-command.test.cjs", + "tests/external-descriptor-confinement.test.cjs", + "tests/failing-direction.test.cjs", + "tests/fallow-runner.test.cjs", + "tests/faulty-deps.test.cjs", + "tests/feat-2296-provider-escalation.test.cjs", + "tests/feat-2483-review-claude-mds-guard.test.cjs", + "tests/feat-2646-deferred-items-audit-scanner.test.cjs", + "tests/feat-3881-yaml-parser-consequences.test.cjs", + "tests/features-index-gate.test.cjs", + "tests/fixture-builder.test.cjs", + "tests/frontmatter-cli.test.cjs", + "tests/frontmatter.test.cjs", + "tests/gap-checker.property.test.cjs", + "tests/gate-predicate-evaluator-missing.test.cjs", + "tests/gemini-runtime-removed.test.cjs", + "tests/gen-context-index.test.cjs", + "tests/gen-health-docs.test.cjs", + "tests/gen-registry.test.cjs", + "tests/gen-section-manifest.test.cjs", + "tests/gen-state-md-docs.test.cjs", + "tests/git-base-branch.test.cjs", + "tests/git-fixture.test.cjs", + "tests/golden-install-tree.test.cjs", + "tests/graphify-command-cutover.test.cjs", + "tests/graphify-graph-path.test.cjs", + "tests/graphify-visualization.test.cjs", + "tests/graphify.test.cjs", + "tests/gsd-agent-isolation-guard.test.cjs", + "tests/gsd-check-update-worker-atomic-cache.test.cjs", + "tests/gsd-check-update-worker-platform-gate.test.cjs", + "tests/gsd-mcp-server-bin.test.cjs", + "tests/gsd-quick-batch-merge-integration.test.cjs", + "tests/gsd-quick-batch-workflow.test.cjs", + "tests/gsd-secret-read-guard.test.cjs", + "tests/gsd-settings-advanced.test.cjs", + "tests/gsd-statusline.test.cjs", + "tests/gsd-tools-path-refs.test.cjs", + "tests/gsd-validate-commit-crash-policy.test.cjs", + "tests/gsd-write-guard.property.test.cjs", + "tests/gsd-write-guard.test.cjs", + "tests/gsd2-import.test.cjs", + "tests/hardcoded-paths.test.cjs", + "tests/health-diagnostic-rules/agent-install.test.cjs", + "tests/health-diagnostic-rules/config-validation.test.cjs", + "tests/health-diagnostic-rules/consistency.test.cjs", + "tests/health-diagnostic-rules/phase-structure.test.cjs", + "tests/health-diagnostic-rules/root-existence.test.cjs", + "tests/health-diagnostic-rules/state-consistency.test.cjs", + "tests/health-diagnostic-rules/worktree-health.test.cjs", + "tests/health-diagnostic.test.cjs", + "tests/health-validation.test.cjs", + "tests/helpers-cleanup.test.cjs", + "tests/helpers-process-isolation.test.cjs", + "tests/hermes-skills-migration.test.cjs", + "tests/hooks-commonjs-marker.test.cjs", + "tests/hooks-crash-policy.test.cjs", + "tests/hooks-opt-in.test.cjs", + "tests/host-integration.test.cjs", + "tests/host-runtime-detection.test.cjs", + "tests/ingest-docs.test.cjs", + "tests/init-debug.test.cjs", + "tests/init-manager.test.cjs", + "tests/init.test.cjs", + "tests/injection-blocking-config.test.cjs", + "tests/inline-plan-threshold.test.cjs", + "tests/install-fs-adapter-seam.test.cjs", + "tests/install-minimal-hooks.test.cjs", + "tests/install-nested-layout.test.cjs", + "tests/install-path-detection.test.cjs", + "tests/install-regressions.test.cjs", + "tests/install-runtime-artifacts.test.cjs", + "tests/install-scope.test.cjs", + "tests/install-write-confinement.test.cjs", + "tests/install.test.cjs", + "tests/installed-surface-resolver.test.cjs", + "tests/installer-migration-antigravity-retire-confighome-artifacts.test.cjs", + "tests/installer-migration-authoring.test.cjs", + "tests/installer-migration-config-root-marker.test.cjs", + "tests/installer-migration-pi-retire-hooks-dir.test.cjs", + "tests/installer-migration-prune-stale-pristine.test.cjs", + "tests/installer-migration-rename-gsd-core.test.cjs", + "tests/installer-migration-report.test.cjs", + "tests/installer-migrations-manifest-schema.test.cjs", + "tests/installer-migrations.test.cjs", + "tests/intel-command-cutover.test.cjs", + "tests/intel.test.cjs", + "tests/inventory-manifest-sync.test.cjs", + "tests/inventory-nested-families.test.cjs", + "tests/io.test.cjs", + "tests/isolation-sentinel.test.cjs", + "tests/kilo-upgrades.test.cjs", + "tests/kimi-agent-converter.test.cjs", + "tests/kimi-normalize-payload.property.test.cjs", + "tests/kimi-upgrades.test.cjs", + "tests/kimi-variant-disambiguation.test.cjs", + "tests/learnings.test.cjs", + "tests/legacy-cleanup-config-dir.test.cjs", + "tests/lint-allow-test-rule-refs.test.cjs", + "tests/lint-compiled-artifact-sync.test.cjs", + "tests/lint-docs-command-form.test.cjs", + "tests/lint-frontmatter-scalar-broad-grep.test.cjs", + "tests/lint-hooks-runtime-build-seam.test.cjs", + "tests/lint-legacy-dir-name.test.cjs", + "tests/lint-planning-artifact-writer-drift.test.cjs", + "tests/lint-pr-check-project-dir.test.cjs", + "tests/lint-regression-test-names.test.cjs", + "tests/lint-seam-enforcement.test.cjs", + "tests/lint-skill-deps.test.cjs", + "tests/lint-test-file-count.test.cjs", + "tests/lint-workflow-shellcheck-fetch.test.cjs", + "tests/list-seeds.test.cjs", + "tests/live-config-guard.test.cjs", + "tests/live-dom-uat.test.cjs", + "tests/lockfile-cve-audit.test.cjs", + "tests/locking-bugs-1909-1916-1925-1927.test.cjs", + "tests/loop-hooks-empty-points-e2e.test.cjs", + "tests/loop-hooks-ship-pre-e2e.test.cjs", + "tests/loop-hooks-verify-post-e2e.test.cjs", + "tests/loop-render-hooks.test.cjs", + "tests/managed-hooks.test.cjs", + "tests/manifest-version-sync.test.cjs", + "tests/markdown-table.test.cjs", + "tests/mcp-catalog.property.test.cjs", + "tests/mcp-catalog.test.cjs", + "tests/milestone-archive.test.cjs", + "tests/milestone-lock.test.cjs", + "tests/milestone-prefixed-convention.test.cjs", + "tests/milestone-window-drift-guard.test.cjs", + "tests/milestone-window-single-owner.test.cjs", + "tests/milestone.test.cjs", + "tests/model-omit-when-inherit-guard.test.cjs", + "tests/model-profiles.test.cjs", + "tests/model-resolver.test.cjs", + "tests/mutation-matrix-ratchet.test.cjs", + "tests/mutation-score-ratchet.test.cjs", + "tests/mutation-workflow-base-ref.test.cjs", + "tests/mvp-phase-integration.test.cjs", + "tests/new-milestone-clear-phases.test.cjs", + "tests/next-decimal-roadmap-scan.test.cjs", + "tests/next-up-clear-order.test.cjs", + "tests/no-bare-gsd-tools-command-position.test.cjs", + "tests/no-bare-npm-exec.rule.test.cjs", + "tests/no-dead-sdk-refs.test.cjs", + "tests/no-exact-case-env-access.rule.test.cjs", + "tests/no-hardcoded-home-gsd-tools.test.cjs", + "tests/no-hardcoded-tmp.rule.test.cjs", + "tests/no-path-literal-in-assert.rule.test.cjs", + "tests/no-pending-3212-markers.test.cjs", + "tests/no-phantom-issue-refs.test.cjs", + "tests/no-posix-mode-bit-assert.rule.test.cjs", + "tests/no-private-binary-resolution.rule.test.cjs", + "tests/no-rendered-text-length-assert.rule.test.cjs", + "tests/no-swallowed-precondition.rule.test.cjs", + "tests/no-unbounded-dirname-walk.rule.test.cjs", + "tests/no-unbounded-spawn-allowlist.test.cjs", + "tests/no-unbounded-spawn.test.cjs", + "tests/no-unguarded-nonportable-exec.rule.test.cjs", + "tests/normalize-path-in-content.rule.test.cjs", + "tests/normalize-test-command.test.cjs", + "tests/npm-audit-baseline.test.cjs", + "tests/npm-integrity-gate.test.cjs", + "tests/onboard-command.test.cjs", + "tests/opencode-command-dir-plural.test.cjs", + "tests/opencode-permissions.test.cjs", + "tests/opencode-plugin-adapter.test.cjs", + "tests/orphan-worktree-detection.test.cjs", + "tests/orphaned-hooks.test.cjs", + "tests/overlay-repo-helpers.test.cjs", + "tests/package-name-single-source.test.cjs", + "tests/packaging-shipped-scripts-require-only-shipped.test.cjs", + "tests/parallel-dependent-plans.test.cjs", + "tests/path-replacement.test.cjs", + "tests/pattern-mapper.test.cjs", + "tests/pattern.test.cjs", + "tests/pause-work-context-detection.test.cjs", + "tests/perf-317-context-monitor-fs.test.cjs", + "tests/phase-command-router.test.cjs", + "tests/phase-completion-single-owner.test.cjs", + "tests/phase-estimation.test.cjs", + "tests/phase-id-drift-guard.test.cjs", + "tests/phase-locator.test.cjs", + "tests/phase-resolution-parity.test.cjs", + "tests/phase-tdd-applicable.test.cjs", + "tests/phase.test.cjs", + "tests/phase6-capstone-conformance.test.cjs", + "tests/phases-command-router.test.cjs", + "tests/pi-config-dir-env-override.test.cjs", + "tests/pi-extension-reachability.test.cjs", + "tests/pick-flag.test.cjs", + "tests/plan-bounce.test.cjs", + "tests/plan-count-single-owner.test.cjs", + "tests/plan-phase-drift-guard.test.cjs", + "tests/plan-phase-mvp-flag.test.cjs", + "tests/plan-phase-stall-detection.test.cjs", + "tests/plan-pre-hook-e2e.test.cjs", + "tests/plan-review-convergence.test.cjs", + "tests/planning-inspect.test.cjs", + "tests/planning-inspect.unit.test.cjs", + "tests/planning-lock-mkdir-failure-1884.test.cjs", + "tests/planning-prompt-drift.test.cjs", + "tests/planning-snapshot-bypass-drift.test.cjs", + "tests/planning-snapshot.test.cjs", + "tests/planning-workspace.test.cjs", + "tests/platform-conformance-tier.test.cjs", + "tests/platform-guard.unit.test.cjs", + "tests/playwright-ui-verify.test.cjs", + "tests/plugin-manifest.test.cjs", + "tests/policy-160-route0-resume.test.cjs", + "tests/policy-shell-pinning.test.cjs", + "tests/portability-rule-disable-ban.test.cjs", + "tests/portability-vocab-drift.test.cjs", + "tests/post-planning-gaps-2493.test.cjs", + "tests/pr-branch-planning-filter.test.cjs", + "tests/precommit-alias-drift-hook.test.cjs", + "tests/precondition-element.test.cjs", + "tests/prepush-enterprise-email-hook.test.cjs", + "tests/probe-core.test.cjs", + "tests/process-seam.test.cjs", + "tests/profile-output.test.cjs", + "tests/profile-pipeline.test.cjs", + "tests/prohibition-enforcement.test.cjs", + "tests/prohibition-probe.verify-tier.test.cjs", + "tests/project-instruction-file-parity.test.cjs", + "tests/project-root.test.cjs", + "tests/prompt-budget-cli.test.cjs", + "tests/prune-orphaned-worktrees.test.cjs", + "tests/quick-batch-command-router.test.cjs", + "tests/quick-batch.test.cjs", + "tests/quick-branching.test.cjs", + "tests/quick-research.test.cjs", + "tests/quick-review-scope-tip-bound.test.cjs", + "tests/qwen-upgrades.test.cjs", + "tests/read-guard.test.cjs", + "tests/read-injection-scanner.property.test.cjs", + "tests/reapply-patches.test.cjs", + "tests/reapply-verify-hunks.test.cjs", + "tests/refactor-trigger-cli.test.cjs", + "tests/registry-reviewer-parity.test.cjs", + "tests/release-hotfix-empty-cherry-pick.test.cjs", + "tests/removed-but-needed-lint.test.cjs", + "tests/repo-invariants.test.cjs", + "tests/repo-layout.test.cjs", + "tests/representative-corpus.test.cjs", + "tests/require-fs-op-fallback.rule.test.cjs", + "tests/require-full-tmpdir-triad.rule.test.cjs", + "tests/require-issue-link-policy.test.cjs", + "tests/require-userprofile-with-home.rule.test.cjs", + "tests/research-cli.test.cjs", + "tests/research-store.test.cjs", + "tests/resolve-execution-dynamic-routing.test.cjs", + "tests/resolver-hoist-guard.test.cjs", + "tests/response-language-coverage.test.cjs", + "tests/retired-artifact-cleanup.test.cjs", + "tests/reversibility-tagging.test.cjs", + "tests/review-build-prompt-optional-sections.test.cjs", + "tests/review-default-reviewers-config.test.cjs", + "tests/review-default-reviewers-workflow.test.cjs", + "tests/review-lane-descriptor.test.cjs", + "tests/review-lane-invocation.test.cjs", + "tests/review-lane-runner.test.cjs", + "tests/review-lane-windows-spawn-resolution.test.cjs", + "tests/review-model-config.test.cjs", + "tests/review-parallel-lanes.test.cjs", + "tests/review-plan-coverage-manifest.test.cjs", + "tests/review-reviewer-instances-config.test.cjs", + "tests/reviewer-config-federation.test.cjs", + "tests/reviewer-docs-parity.test.cjs", + "tests/reviewer-lane-declarations.test.cjs", + "tests/reviewer-manifest-body.test.cjs", + "tests/reviewer-step-dispatch.test.cjs", + "tests/reviewer-trust-disclosure.test.cjs", + "tests/revision-remediation-binding.test.cjs", + "tests/roadmap-mode-field.test.cjs", + "tests/roadmap-parser.test.cjs", + "tests/roadmap-phase-fallback.test.cjs", + "tests/roadmap-upgrade.test.cjs", + "tests/roadmap.test.cjs", + "tests/roadmapper-granularity.test.cjs", + "tests/run-tests-harness.test.cjs", + "tests/run-tests-temp-root.test.cjs", + "tests/run-with-timeout.test.cjs", + "tests/runtime-artifact-install-plan.test.cjs", + "tests/runtime-artifact-layout-descriptor-drive.test.cjs", + "tests/runtime-artifact-layout-install-profiles.test.cjs", + "tests/runtime-artifact-layout-surface.test.cjs", + "tests/runtime-artifact-layout.test.cjs", + "tests/runtime-converters.test.cjs", + "tests/runtime-homes-descriptor-drive.test.cjs", + "tests/runtime-homes.property.test.cjs", + "tests/runtime-identity.test.cjs", + "tests/runtime-launcher-parity.test.cjs", + "tests/runtime-name-policy.test.cjs", + "tests/safe-resume-gate-anchoring.test.cjs", + "tests/schema-drift.test.cjs", + "tests/sdk-removal-query-family-dispatch.test.cjs", + "tests/section-manifest-init-facts.test.cjs", + "tests/section-manifest.test.cjs", + "tests/security.test.cjs", + "tests/settings-integrations.test.cjs", + "tests/settings-jsonc.test.cjs", + "tests/sh-hook-paths.test.cjs", + "tests/shadow-report.test.cjs", + "tests/shared-hooks-dir-resolution.test.cjs", + "tests/shell-command-projection-dispatch.test.cjs", + "tests/shell-command-projection-path-sep.test.cjs", + "tests/ship-notes-wedged-pr.test.cjs", + "tests/shipped-reference-cites.test.cjs", + "tests/skill-frontmatter-contract.test.cjs", + "tests/skill-manifest.test.cjs", + "tests/slash-command-namespace.test.cjs", + "tests/slug-derivation-drift-guard.test.cjs", + "tests/smart-entry.unit.test.cjs", + "tests/spec-section.test.cjs", + "tests/stale-bake-guard.test.cjs", + "tests/state-command-cutover.test.cjs", + "tests/state-contract.test.cjs", + "tests/state-document.test.cjs", + "tests/state-field-drift.test.cjs", + "tests/state-io.test.cjs", + "tests/state-prune.test.cjs", + "tests/state-rebuild-cli.test.cjs", + "tests/state-todos-render.test.cjs", + "tests/state-transition.test.cjs", + "tests/state-write-path-drift-guard.test.cjs", + "tests/state.test.cjs", + "tests/stats-mvp-display.test.cjs", + "tests/stats-phase-id-shape.test.cjs", + "tests/subagent-timeout.test.cjs", + "tests/summary-status-blocked-3345.test.cjs", + "tests/surface-md-paths.regression.test.cjs", + "tests/sync-skills-cross-runtime-refuse.test.cjs", + "tests/table-schema-drift-lint.test.cjs", + "tests/tdd-backend-wiring.test.cjs", + "tests/tdd-mode.test.cjs", + "tests/tdd-red-evidence.test.cjs", + "tests/tdd-single-statement.test.cjs", + "tests/teams-status.test.cjs", + "tests/template.test.cjs", + "tests/thinking-partner.test.cjs", + "tests/todos-done-rename-guard.test.cjs", + "tests/todos-workstream-scope.test.cjs", + "tests/tracer-bullet.test.cjs", + "tests/trae-imperative-reference.test.cjs", + "tests/tsconfig-noemit.test.cjs", + "tests/uat-predicate.test.cjs", + "tests/uat.test.cjs", + "tests/ui-spec-inventory-provenance.test.cjs", + "tests/ultraplan-phase.test.cjs", + "tests/unreachable-guard-drift.test.cjs", + "tests/unreachable-shell-guard.test.cjs", + "tests/unusable-input.test.cjs", + "tests/update-context.test.cjs", + "tests/update-custom-backup.test.cjs", + "tests/update-workflow.test.cjs", + "tests/user-artifact-staging.test.cjs", + "tests/validate-context.test.cjs", + "tests/validate-registry.test.cjs", + "tests/verification-status.test.cjs", + "tests/verify-archive-dirs-live-path.test.cjs", + "tests/verify-command-grounding.test.cjs", + "tests/verify-health.test.cjs", + "tests/verify.test.cjs", + "tests/vscode-browser-no-node-api.test.cjs", + "tests/vscode-extension-reachability.test.cjs", + "tests/windows-robustness.test.cjs", + "tests/windsurf-conversion.test.cjs", + "tests/windsurf-hooks-bridge.test.cjs", + "tests/windsurf-install.test.cjs", + "tests/workflow-compat.test.cjs", + "tests/workflow-fragments.test.cjs", + "tests/workflow-guard.test.cjs", + "tests/workflow-maintainer-skip.test.cjs", + "tests/workflow-shell-pinning.test.cjs", + "tests/workspace.test.cjs", + "tests/workstream-inventory.test.cjs", + "tests/workstream-scoped-paths.test.cjs", + "tests/workstream.test.cjs", + "tests/worktree-base-ref.test.cjs", + "tests/worktree-baseref-install.test.cjs", + "tests/worktree-cleanup.test.cjs", + "tests/worktree-safety-reap.test.cjs", + "tests/worktree-safety.test.cjs", + "tests/worktree.test.cjs", + ], +}; diff --git a/scripts/lib/suite-detection.cjs b/scripts/lib/suite-detection.cjs new file mode 100644 index 000000000..a2c342dd1 --- /dev/null +++ b/scripts/lib/suite-detection.cjs @@ -0,0 +1,32 @@ +'use strict'; + +/** + * Suite classification by filename suffix (#4591 packaging fix — extracted + * from scripts/run-tests.cjs so a SHIPPED script, e.g. + * scripts/gen-platform-conformance-tier.cjs, can depend on it: run-tests.cjs + * itself does not ship (it is a repo-development-only tool), so a shipped + * file requiring it directly is MODULE_NOT_FOUND in a published install + * (tests/packaging-shipped-scripts-require-only-shipped.test.cjs, #2858). + * run-tests.cjs re-exports suiteOf from here unchanged for full backward + * compatibility — this is a pure relocation, not a behavior change. + * + * A file with no suite suffix (plain `foo.test.cjs`) belongs to the `unit` + * suite and returns `null` here; a suite-tagged file (`foo.install.test.cjs`) + * returns its marker string. + */ + +const { basename } = require('node:path'); + +const MARKED_SUITES = ['integration', 'install', 'security', 'slow', 'qa']; + +function suiteOf(filename) { + const name = basename(filename); + if (!name.endsWith('.test.cjs')) return null; + const base = name.slice(0, -'.test.cjs'.length); + const lastDot = base.lastIndexOf('.'); + if (lastDot === -1) return null; + const marker = base.slice(lastDot + 1); + return MARKED_SUITES.includes(marker) ? marker : null; +} + +module.exports = { MARKED_SUITES, suiteOf }; diff --git a/scripts/run-tests.cjs b/scripts/run-tests.cjs index e39b32aa4..f5568c410 100644 --- a/scripts/run-tests.cjs +++ b/scripts/run-tests.cjs @@ -41,6 +41,7 @@ const { tmpdir } = require('os'); const { pathToFileURL } = require('url'); const { execFileSync } = require('child_process'); const { ExitError, runMain } = require('./lib/cli-exit.cjs'); +const { suiteOf } = require('./lib/suite-detection.cjs'); const { resolveLiveConfigRoots, resolveExtraWatchTargets, @@ -179,8 +180,6 @@ function ensureBuiltHooks(overrides = {}) { runBuild(); } } -const MARKED_SUITES = ['integration', 'install', 'security', 'slow', 'qa']; - // Recursively collect *.test.cjs files under dir, returning paths relative to dir. // Skips node_modules to avoid accidentally picking up decoy files. function walkTestFiles(dir, relBase) { @@ -794,20 +793,8 @@ function parseArgs(argv) { return { suite, files, filesFrom, shard }; } -// Return the marked suite name embedded in a filename, or null if it's unmarked. -// foo.security.test.cjs -> "security" -// foo.test.cjs -> null (unit) -// Accepts either a bare filename or a relative subdir path; classification is -// based on the basename only so subdir paths classify identically to root files. -function suiteOf(filename) { - const name = basename(filename); - if (!name.endsWith('.test.cjs')) return null; - const base = name.slice(0, -'.test.cjs'.length); - const lastDot = base.lastIndexOf('.'); - if (lastDot === -1) return null; - const marker = base.slice(lastDot + 1); - return MARKED_SUITES.includes(marker) ? marker : null; -} +// suiteOf (and its backing MARKED_SUITES) is imported from +// ./lib/suite-detection.cjs — see that module's header comment for why. function selectFiles(allFiles, suite) { if (suite === null || suite === 'all') { diff --git a/tests/ci-test-job-timeout-budget.test.cjs b/tests/ci-test-job-timeout-budget.test.cjs index 29e81d7b0..d09b200b2 100644 --- a/tests/ci-test-job-timeout-budget.test.cjs +++ b/tests/ci-test-job-timeout-budget.test.cjs @@ -253,6 +253,7 @@ test('near-cap check CI_JOB_TIMEOUT_MINUTES literals match each job\'s own timeo const staticLanes = [ { workflowFile: 'test.yml', jobKey: 'test', envLiteral: '32' }, { workflowFile: 'test.yml', jobKey: 'test-full', envLiteral: '45' }, + { workflowFile: 'test.yml', jobKey: 'test-conformance', envLiteral: '45' }, { workflowFile: 'install-smoke.yml', jobKey: 'smoke', envLiteral: '12' }, ]; @@ -294,5 +295,10 @@ test('near-cap check CI_JOB_TIMEOUT_MINUTES literals match each job\'s own timeo assert.equal(testWorkflow.jobs['coverage-gate'].name, 'Coverage gate (merged shards)', 'test.yml jobs.coverage-gate.name changed — update JOB_RULES to match'); assert.equal(coverageGateRule.test('Coverage gate (merged shards)'), true); + + const testConformanceRule = JOB_RULES.find((r) => r.workflowFile === 'test.yml' && r.jobKey === 'test-conformance'); + assert.ok(testWorkflow.jobs['test-conformance'].name.startsWith('conformance test ('), + 'test.yml jobs.test-conformance.name no longer starts with "conformance test (" — update JOB_RULES to match'); + assert.equal(testConformanceRule.test('conformance test (windows-latest, 24, shard 1/3)'), true); }); }); diff --git a/tests/fixtures/install-tree/antigravity.json b/tests/fixtures/install-tree/antigravity.json index f747670c9..6394abe9c 100644 --- a/tests/fixtures/install-tree/antigravity.json +++ b/tests/fixtures/install-tree/antigravity.json @@ -624,7 +624,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/augment.json b/tests/fixtures/install-tree/augment.json index 99c30cf15..0c93b86c4 100644 --- a/tests/fixtures/install-tree/augment.json +++ b/tests/fixtures/install-tree/augment.json @@ -695,7 +695,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-ns-context/SKILL.md", "skills/gsd-ns-context/skills/docs-update/SKILL.md", "skills/gsd-ns-context/skills/extract-learnings/SKILL.md", diff --git a/tests/fixtures/install-tree/claude-local.json b/tests/fixtures/install-tree/claude-local.json index d3b12fc4c..f013bdedd 100644 --- a/tests/fixtures/install-tree/claude-local.json +++ b/tests/fixtures/install-tree/claude-local.json @@ -559,5 +559,7 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", - "scripts/lib/shellcheck-fetch.cjs" + "scripts/lib/platform-conformance-tier.generated.cjs", + "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs" ] diff --git a/tests/fixtures/install-tree/claude.json b/tests/fixtures/install-tree/claude.json index d46fcb517..611519571 100644 --- a/tests/fixtures/install-tree/claude.json +++ b/tests/fixtures/install-tree/claude.json @@ -623,7 +623,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/cline.json b/tests/fixtures/install-tree/cline.json index ec6bcad8f..64d22d177 100644 --- a/tests/fixtures/install-tree/cline.json +++ b/tests/fixtures/install-tree/cline.json @@ -585,7 +585,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-ns-context/SKILL.md", "skills/gsd-ns-context/skills/docs-update/SKILL.md", "skills/gsd-ns-context/skills/extract-learnings/SKILL.md", diff --git a/tests/fixtures/install-tree/codebuddy.json b/tests/fixtures/install-tree/codebuddy.json index 926d035c4..7b035ffe7 100644 --- a/tests/fixtures/install-tree/codebuddy.json +++ b/tests/fixtures/install-tree/codebuddy.json @@ -695,7 +695,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/codex.json b/tests/fixtures/install-tree/codex.json index 79fe0174a..a7bb472f8 100644 --- a/tests/fixtures/install-tree/codex.json +++ b/tests/fixtures/install-tree/codex.json @@ -623,5 +623,7 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", - "scripts/lib/shellcheck-fetch.cjs" + "scripts/lib/platform-conformance-tier.generated.cjs", + "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs" ] diff --git a/tests/fixtures/install-tree/copilot.json b/tests/fixtures/install-tree/copilot.json index f28a7b9fe..4b60ade55 100644 --- a/tests/fixtures/install-tree/copilot.json +++ b/tests/fixtures/install-tree/copilot.json @@ -585,7 +585,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/cursor.json b/tests/fixtures/install-tree/cursor.json index dd7521351..2d8c8c71f 100644 --- a/tests/fixtures/install-tree/cursor.json +++ b/tests/fixtures/install-tree/cursor.json @@ -596,7 +596,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/hermes.json b/tests/fixtures/install-tree/hermes.json index a6053ff59..ed62ec76a 100644 --- a/tests/fixtures/install-tree/hermes.json +++ b/tests/fixtures/install-tree/hermes.json @@ -623,7 +623,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd/DESCRIPTION.md", "skills/gsd/gsd-ns-context/SKILL.md", "skills/gsd/gsd-ns-context/skills/docs-update/SKILL.md", diff --git a/tests/fixtures/install-tree/kilo.json b/tests/fixtures/install-tree/kilo.json index d19a48a7d..52a04da47 100644 --- a/tests/fixtures/install-tree/kilo.json +++ b/tests/fixtures/install-tree/kilo.json @@ -698,7 +698,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/kimi-code.json b/tests/fixtures/install-tree/kimi-code.json index 49924e27d..cec17e031 100644 --- a/tests/fixtures/install-tree/kimi-code.json +++ b/tests/fixtures/install-tree/kimi-code.json @@ -624,7 +624,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/kimi.json b/tests/fixtures/install-tree/kimi.json index 5c05af055..37683b3be 100644 --- a/tests/fixtures/install-tree/kimi.json +++ b/tests/fixtures/install-tree/kimi.json @@ -591,7 +591,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/opencode.json b/tests/fixtures/install-tree/opencode.json index 95dbb2a13..0d0cb6b41 100644 --- a/tests/fixtures/install-tree/opencode.json +++ b/tests/fixtures/install-tree/opencode.json @@ -698,7 +698,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-add-tests/SKILL.md", "skills/gsd-ai-integration-phase/SKILL.md", "skills/gsd-audit-fix/SKILL.md", diff --git a/tests/fixtures/install-tree/pi.json b/tests/fixtures/install-tree/pi.json index 65e02bf89..319b699ba 100644 --- a/tests/fixtures/install-tree/pi.json +++ b/tests/fixtures/install-tree/pi.json @@ -425,5 +425,7 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", - "scripts/lib/shellcheck-fetch.cjs" + "scripts/lib/platform-conformance-tier.generated.cjs", + "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs" ] diff --git a/tests/fixtures/install-tree/qwen.json b/tests/fixtures/install-tree/qwen.json index 43733896a..906b2e318 100644 --- a/tests/fixtures/install-tree/qwen.json +++ b/tests/fixtures/install-tree/qwen.json @@ -623,7 +623,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-ns-context/SKILL.md", "skills/gsd-ns-context/skills/docs-update/SKILL.md", "skills/gsd-ns-context/skills/extract-learnings/SKILL.md", diff --git a/tests/fixtures/install-tree/trae.json b/tests/fixtures/install-tree/trae.json index d326780b2..57ca1a0b5 100644 --- a/tests/fixtures/install-tree/trae.json +++ b/tests/fixtures/install-tree/trae.json @@ -583,7 +583,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-ns-context/SKILL.md", "skills/gsd-ns-context/skills/docs-update/SKILL.md", "skills/gsd-ns-context/skills/extract-learnings/SKILL.md", diff --git a/tests/fixtures/install-tree/windsurf.json b/tests/fixtures/install-tree/windsurf.json index 37a238793..6baca149a 100644 --- a/tests/fixtures/install-tree/windsurf.json +++ b/tests/fixtures/install-tree/windsurf.json @@ -518,5 +518,7 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", - "scripts/lib/shellcheck-fetch.cjs" + "scripts/lib/platform-conformance-tier.generated.cjs", + "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs" ] diff --git a/tests/fixtures/install-tree/zcode.json b/tests/fixtures/install-tree/zcode.json index 6d3a1380a..ee09be87a 100644 --- a/tests/fixtures/install-tree/zcode.json +++ b/tests/fixtures/install-tree/zcode.json @@ -655,7 +655,9 @@ "scripts/lib/exit-code-registry.cjs", "scripts/lib/ndjson-reporter.cjs", "scripts/lib/npm-version-check-diagnosis.cjs", + "scripts/lib/platform-conformance-tier.generated.cjs", "scripts/lib/shellcheck-fetch.cjs", + "scripts/lib/suite-detection.cjs", "skills/gsd-ns-context/SKILL.md", "skills/gsd-ns-context/skills/docs-update/SKILL.md", "skills/gsd-ns-context/skills/extract-learnings/SKILL.md", diff --git a/tests/platform-conformance-tier.test.cjs b/tests/platform-conformance-tier.test.cjs new file mode 100644 index 000000000..9864b6cc1 --- /dev/null +++ b/tests/platform-conformance-tier.test.cjs @@ -0,0 +1,307 @@ +'use strict'; + +/** + * Test matrix: .gsd/phase/chore-4591-conformance-tier-linux-primary/50-test-matrix.md + * + * Rows 1-15 exercise the pure, exported `classifyContent(content)` classifier + * directly on short string fixtures. Rows 16-18 exercise the + * `--check`/`--write` CLI behavior against a small temp fixture tree, driven + * through the process seam (tests/helpers/process-seam.cjs's `runNode`) per + * CONTRIBUTING.md's "spawning a subprocess: use the process seam" rule. Row + * 19 is a real-tree regression proving the committed generated file matches a + * fresh sweep of this repo's actual tests/ tree. Rows 20-21 prove the + * suite-exclusion fix (#4591 CI incident): a suite-tagged file must never + * enter the conformance-tier pool even when its content would otherwise + * qualify, and the committed generated file must contain zero such files. + */ + +const { describe, test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { createTempDir, cleanup } = require('./helpers.cjs'); +const { runNode } = require('./helpers/process-seam.cjs'); +const { classifyContent, classifyTree } = require('../scripts/gen-platform-conformance-tier.cjs'); + +const ROOT = path.resolve(__dirname, '..'); +const SCRIPT = path.join(ROOT, 'scripts', 'gen-platform-conformance-tier.cjs'); +const GENERATED_PATH = path.join(ROOT, 'scripts', 'lib', 'platform-conformance-tier.generated.cjs'); + +// ─── Rows 1-15: classifyContent, pure fixtures ──────────────────────────────── + +describe('classifyContent — happy-path signals', () => { + test('flags process.platform', () => { + const { needsRealOs, signals } = classifyContent("if (process.platform === 'win32') { doThing(); }"); + assert.equal(needsRealOs, true); + assert.ok(signals.includes('process-platform')); + }); + + test('flags os.platform()', () => { + const { needsRealOs, signals } = classifyContent("const os = require('node:os');\nconst p = os.platform();"); + assert.equal(needsRealOs, true); + assert.ok(signals.includes('os-platform')); + }); + + test('flags win32 literal', () => { + const { needsRealOs, signals } = classifyContent("const platforms = ['win32', 'linux'];"); + assert.equal(needsRealOs, true); + assert.ok(signals.includes('win32-darwin-literal')); + }); + + test('flags darwin literal', () => { + const { needsRealOs, signals } = classifyContent("const platforms = ['darwin', 'linux'];"); + assert.equal(needsRealOs, true); + assert.ok(signals.includes('win32-darwin-literal')); + }); + + test('flags chmod mode-bit octal', () => { + const { needsRealOs, signals } = classifyContent('fs.chmodSync(target, 0o755);'); + assert.equal(needsRealOs, true); + assert.ok(signals.includes('chmod-mode-bit')); + }); + + test('flags Windows-shell tokens', () => { + for (const fixture of ["spawn('cmd.exe', args)", "spawn('powershell', args)", 'const e = process.env.ComSpec;']) { + const { needsRealOs, signals } = classifyContent(fixture); + assert.equal(needsRealOs, true, fixture); + assert.ok(signals.includes('windows-shell-token'), fixture); + } + }); + + test('flags Windows env-var names', () => { + for (const fixture of [ + 'const home = process.env.USERPROFILE;', + 'const drive = process.env.HOMEDRIVE;', + 'const p = process.env.HOMEPATH;', + ]) { + const { needsRealOs, signals } = classifyContent(fixture); + assert.equal(needsRealOs, true, fixture); + assert.ok(signals.includes('windows-env-var'), fixture); + } + }); + + test('flags process-seam subprocess helpers', () => { + for (const fixture of [ + "runNode(['--check'])", + "runGit(['status'])", + "runHook(HOOK_PATH, [])", + "runGsdTools(['state', 'show'])", + ]) { + const { needsRealOs, signals } = classifyContent(fixture); + assert.equal(needsRealOs, true, fixture); + assert.ok(signals.includes('process-seam-subprocess'), fixture); + } + }); + + test('flags raw child_process usage', () => { + const fixture = "const { execFileSync } = require('child_process');\nexecFileSync('ls', []);"; + const { needsRealOs, signals } = classifyContent(fixture); + assert.equal(needsRealOs, true); + assert.ok(signals.includes('raw-child-process')); + }); + + test('flags symlink keyword', () => { + const { needsRealOs, signals } = classifyContent('fs.symlinkSync(target, link);'); + assert.equal(needsRealOs, true); + assert.ok(signals.includes('symlink-keyword')); + }); + + test('flags hardcoded path literal vs path.* call', () => { + const fixture = "const p = path.join(root, 'x');\nassert.equal(rendered, '/etc/passwd');"; + const { needsRealOs, signals } = classifyContent(fixture); + assert.equal(needsRealOs, true); + assert.ok(signals.includes('hardcoded-path-vs-path-call')); + }); +}); + +describe('classifyContent — negative / hostile inputs', () => { + test('does not flag a clean in-process unit test', () => { + const fixture = "const assert = require('node:assert/strict');\ntest('adds numbers', () => { assert.equal(1 + 1, 2); });"; + const { needsRealOs, signals } = classifyContent(fixture); + assert.equal(needsRealOs, false); + assert.deepEqual(signals, []); + }); + + test('does not flag incidental use of the word "path"', () => { + const fixture = '// This test verifies the correct path through the state machine.\nassert.ok(true);'; + const { needsRealOs } = classifyContent(fixture); + assert.equal(needsRealOs, false); + }); + + test('does not crash on empty content', () => { + assert.doesNotThrow(() => classifyContent('')); + const { needsRealOs, signals } = classifyContent(''); + assert.equal(needsRealOs, false); + assert.deepEqual(signals, []); + }); + + test('does not flag an unrelated identifier containing "spawn"', () => { + const fixture = 'const spawnResult = computeSomething();\nassert.ok(spawnResult);'; + const { needsRealOs, signals } = classifyContent(fixture); + assert.equal(needsRealOs, false); + assert.deepEqual(signals, []); + }); +}); + +// ─── Rows 16-18: CLI --check/--write against a temp fixture tree ────────────── + +/** Spawn the real generator CLI via the process seam. */ +function runGen(args) { + return runNode([SCRIPT, ...args]); +} + +describe('gen-platform-conformance-tier.cjs CLI (temp fixture tree)', () => { + test('--check passes when the generated file is fresh', () => { + const tmpDir = createTempDir('gen-platform-conformance-tier-'); + try { + const testsDir = path.join(tmpDir, 'tests'); + fs.mkdirSync(testsDir, { recursive: true }); + fs.writeFileSync(path.join(testsDir, 'flagged.test.cjs'), "if (process.platform === 'win32') {}\n"); + fs.writeFileSync(path.join(testsDir, 'clean.test.cjs'), "assert.equal(1 + 1, 2);\n"); + const outPath = path.join(tmpDir, 'platform-conformance-tier.generated.cjs'); + + const write = runGen(['--write', '--tests-dir', testsDir, '--out', outPath]); + assert.equal(write.exitCode, 0, write.stderr); + assert.ok(fs.existsSync(outPath)); + + const check = runGen(['--check', '--tests-dir', testsDir, '--out', outPath]); + assert.equal(check.exitCode, 0, check.stderr); + assert.match(check.stdout, /ok gen-platform-conformance-tier/); + } finally { + cleanup(tmpDir); + } + }); + + test('--check fails and names the drift when the list is stale', () => { + const tmpDir = createTempDir('gen-platform-conformance-tier-'); + try { + const testsDir = path.join(tmpDir, 'tests'); + fs.mkdirSync(testsDir, { recursive: true }); + fs.writeFileSync(path.join(testsDir, 'flagged.test.cjs'), "if (process.platform === 'win32') {}\n"); + fs.writeFileSync(path.join(testsDir, 'clean.test.cjs'), "assert.equal(1 + 1, 2);\n"); + const outPath = path.join(tmpDir, 'platform-conformance-tier.generated.cjs'); + + const write = runGen(['--write', '--tests-dir', testsDir, '--out', outPath]); + assert.equal(write.exitCode, 0, write.stderr); + + // Introduce drift: a brand-new signal-bearing file the committed list + // has never seen. + fs.writeFileSync(path.join(testsDir, 'new-signal.test.cjs'), 'fs.symlinkSync(target, link);\n'); + + const check = runGen(['--check', '--tests-dir', testsDir, '--out', outPath]); + assert.equal(check.exitCode, 1); + const combined = check.stdout + check.stderr; + assert.match(combined, /tests\/new-signal\.test\.cjs/, 'the drift report must name the new file'); + assert.match(combined, /\+/, 'a newly-added file is reported with a + marker'); + } finally { + cleanup(tmpDir); + } + }); + + test('--write is deterministic across repeated runs', () => { + const tmpDir = createTempDir('gen-platform-conformance-tier-'); + try { + const testsDir = path.join(tmpDir, 'tests'); + fs.mkdirSync(testsDir, { recursive: true }); + fs.writeFileSync(path.join(testsDir, 'a.test.cjs'), "process.env.PATHEXT;\n"); + fs.writeFileSync(path.join(testsDir, 'b.test.cjs'), 'fs.symlinkSync(target, link);\n'); + const out1 = path.join(tmpDir, 'out1.generated.cjs'); + const out2 = path.join(tmpDir, 'out2.generated.cjs'); + + assert.equal(runGen(['--write', '--tests-dir', testsDir, '--out', out1]).exitCode, 0); + assert.equal(runGen(['--write', '--tests-dir', testsDir, '--out', out2]).exitCode, 0); + + const content1 = fs.readFileSync(out1, 'utf8'); + const content2 = fs.readFileSync(out2, 'utf8'); + assert.equal(content1, content2, '--write must be byte-identical across independent runs over the same input'); + } finally { + cleanup(tmpDir); + } + }); +}); + +// ─── Row 20: suite-tagged files are excluded even when their content qualifies + +describe('gen-platform-conformance-tier.cjs CLI (temp fixture tree) — suite exclusion', () => { + test('a suite-suffixed file is excluded even with a qualifying content signal', () => { + const tmpDir = createTempDir('gen-platform-conformance-tier-suite-'); + try { + const testsDir = path.join(tmpDir, 'tests'); + fs.mkdirSync(testsDir, { recursive: true }); + const signal = "if (process.platform === 'win32') {}\n"; + fs.writeFileSync(path.join(testsDir, 'foo.test.cjs'), signal); + fs.writeFileSync(path.join(testsDir, 'foo.install.test.cjs'), signal); + const outPath = path.join(tmpDir, 'platform-conformance-tier.generated.cjs'); + + const write = runGen(['--write', '--tests-dir', testsDir, '--out', outPath]); + assert.equal(write.exitCode, 0, write.stderr); + + delete require.cache[require.resolve(outPath)]; + const generated = require(outPath); + + assert.deepEqual( + generated.CONFORMANCE_TIER_FILES, + ['tests/foo.test.cjs'], + 'the install-suite file must be excluded even though its content would otherwise qualify', + ); + } finally { + cleanup(tmpDir); + } + }); +}); + +// ─── Row 19: real tests/ tree, regression against the committed artifact ───── + +describe('gen-platform-conformance-tier.cjs — real repo tree (regression)', () => { + test('real tests/ tree classification matches the committed list', () => { + const realTestsDir = path.join(ROOT, 'tests'); + + let fresh; + assert.doesNotThrow(() => { + fresh = classifyTree(realTestsDir); + }, 'a full sweep of the real tests/ tree must complete without throwing'); + + // Measured 546/952 at authoring time (#4591, post suite-exclusion fix) — + // the range below is a sanity ballpark with headroom for organic + // test-suite growth in either direction, not a brittle exact-match on + // that literal. + assert.ok( + fresh.files.length >= 450 && fresh.files.length <= 700, + `expected a real, current, sanity-checked count in [450, 700], got ${fresh.files.length}`, + ); + + delete require.cache[require.resolve(GENERATED_PATH)]; + const committed = require(GENERATED_PATH); + + assert.equal( + committed.CONFORMANCE_TIER_FILES.length, + fresh.files.length, + 'the committed generated file must be fresh — run `node scripts/gen-platform-conformance-tier.cjs --write`', + ); + assert.deepEqual( + committed.CONFORMANCE_TIER_FILES.slice().sort(), + fresh.files.slice().sort(), + 'the committed list must match a fresh sweep exactly, not just in length', + ); + }); + + // ─── Row 21: no suite-tagged file ever reaches the committed conformance + // tier — the exact assertion that would have caught the CI incident before + // it ever shipped. + test('the committed conformance-tier list contains zero suite-tagged files', () => { + delete require.cache[require.resolve(GENERATED_PATH)]; + const { CONFORMANCE_TIER_FILES } = require(GENERATED_PATH); + + const suiteTaggedPattern = /\.(install|security|slow|integration|qa)\.test\.cjs$/; + const offenders = CONFORMANCE_TIER_FILES.filter((f) => suiteTaggedPattern.test(f)); + + assert.deepEqual( + offenders, + [], + 'suite-tagged files (install/security/slow/integration/qa) must never appear in the ' + + 'conformance-tier list — install/slow are PR-excluded suites and integration/security ' + + 'already run via their own dedicated steps', + ); + }); +});