feat(#4446): ban ad hoc timeout literals in tests, ship with full legacy allowlist (#4449)

Nothing enforced CONTRIBUTING.md's own stated preference ("A non-literal value
is trusted — that is the shape you should be writing") for timeout values in
tests. local/no-unbounded-spawn requires SOME bound but allows a bare literal;
no-magic-sleep-in-tests and no-elapsed-assertion cover different anti-patterns
entirely. This is exactly how PR #4428's Windows CI incident happened: two
independently-guessed 15000ms literals (one in production's check-latest-
version.cjs, one in this suite's own worker test) collided exactly and raced
two SIGKILLs against each other.

New rule local/no-adhoc-timeout-literal (eslint-rules/no-adhoc-timeout-
literal.cjs) flags a resolvable numeric timeout/timeoutMs literal; an
Identifier or MemberExpression value is trusted. No marker-comment escape —
the fix is always to extract a named constant, which is trivial.

Ships with a full legacy allowlist (126 files, 352 violations, generated
by running the rule with an empty allowlist against tests/) so it can go
live at error severity without breaking CI, mirroring how no-unbounded-spawn
itself was rolled out. Migration is tracked separately in #4445 (this PR
does not close it — only #4446, introducing the gate itself).

Documents the policy and the compliant shapes in TESTING-STANDARDS.md and
TEST-EXAMPLES.md.

Co-authored-by: sim <sim@local>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
2026-09-06 21:23:39 -04:00
committed by GitHub
parent ef30e59860
commit f15887ebb1
5 changed files with 412 additions and 0 deletions

View File

@@ -5,8 +5,10 @@ import pluginN from 'eslint-plugin-n';
import noOnlyTests from 'eslint-plugin-no-only-tests';
import { dirname } from 'path';
import { fileURLToPath } from 'url';
import { createRequire } from 'module';
const __dirname = dirname(fileURLToPath(import.meta.url));
const require = createRequire(import.meta.url);
// Local plugin with custom AST rules
import noSourceGrep from './eslint-rules/no-source-grep.cjs';
@@ -36,6 +38,9 @@ import noPrivateBinaryResolution from './eslint-rules/no-private-binary-resoluti
import requireRegisteredExit from './eslint-rules/require-registered-exit.cjs';
import noSwallowedPrecondition from './eslint-rules/no-swallowed-precondition.cjs';
import noExactCaseEnvAccess from './eslint-rules/no-exact-case-env-access.cjs';
import noAdhocTimeoutLiteral from './eslint-rules/no-adhoc-timeout-literal.cjs';
const adhocTimeoutLiteralAllowlist = require('./eslint-rules/no-adhoc-timeout-literal.allowlist.json');
const localPlugin = {
rules: {
@@ -66,6 +71,7 @@ const localPlugin = {
'require-registered-exit': requireRegisteredExit,
'no-swallowed-precondition': noSwallowedPrecondition,
'no-exact-case-env-access': noExactCaseEnvAccess,
'no-adhoc-timeout-literal': noAdhocTimeoutLiteral,
},
};
@@ -715,6 +721,11 @@ export default tseslint.config(
// exemption surface. The only sanctioned escapes are an explicit `timeout` on
// a raw spawn or the `// allow-spawn-timeout-ceiling: <reason>` marker.
'local/no-unbounded-spawn': 'error',
// Ban a bare numeric `timeout`/`timeoutMs` literal in tests (DEFECT.AD-HOC-TIMEOUT-LITERAL,
// #4428): two independently-guessed copies of the same magic number can drift apart, or
// collide exactly into a zero-margin race. Allowlist starts empty; a pre-existing violation
// gets grandfathered in here as it's found, per eslint-rules/no-adhoc-timeout-literal.allowlist.json.
'local/no-adhoc-timeout-literal': ['error', { allowlist: adhocTimeoutLiteralAllowlist }],
// Ban a consolidation-epic folded suite appearing twice in one host file (#3271).
// A second copy runs the same tests twice on every lane and drifts silently.
'local/no-duplicate-fold-marker': 'error',