a9a7a328e6
Mechanical rename produced by scripts/msd-rename.cjs: gsd/Gsd/GSD -> msd/Msd/MSD across contents and paths, upstream package/repo coordinates -> @golem15/msd-core and golem15com/msd-core. Deep links into upstream history, sibling upstream packages, the GSD-2 import feature, CHANGELOG.md and .changeset/ are kept as-is. Hand edits on top: MSD block-letter banner and logos, LICENSE copyright line, package/plugin identity, regenerated lockfile, install-tree fixtures, derived registries and benchmark baseline; migration checksum baseline re-locked (MSD keeps its own install state, so no install had applied the old sums); sort-order and regex-escaped expectations in tests adjusted.
427 lines
20 KiB
JavaScript
427 lines
20 KiB
JavaScript
'use strict';
|
|
|
|
/**
|
|
* The full test lane and the scoped Windows lane are both sharded, and the
|
|
* coverage gate that sharding displaced is still wired in —
|
|
* .github/workflows/test.yml (#2952, #3057).
|
|
*
|
|
* The `scope: full` lane was the only unsharded lane in this file. It ran the
|
|
* entire unit suite under c8 on a single runner, grew past a 15-minute cap, and
|
|
* reddened `next` (#2952). Raising the cap treated the symptom; sharding is the
|
|
* shape fix, and it is the same answer #1212 reached for the Windows lane at
|
|
* the time.
|
|
*
|
|
* The `scope: windows` lane then hit the identical cliff itself: it reached
|
|
* exactly 15m05s and was CANCELLED on PR #3094, four shas in a row. Per #869's
|
|
* stated durable follow-up, it is now sharded three ways too (#3057), leaving
|
|
* `scope: targeted` as the only lane in this job with no shard.
|
|
*
|
|
* Sharding introduces two failure modes that stay GREEN while being wrong, so
|
|
* both are pinned here:
|
|
*
|
|
* 1. An incomplete shard set. If the matrix declares shards 1/3 and 2/3 but
|
|
* never 3/3, a third of the unit suite simply stops running and every check
|
|
* still passes. The partition MATH is already covered — completeness,
|
|
* disjointness, balance and determinism are asserted against selectShard in
|
|
* run-tests-harness.test.cjs (#1212), including a fast-check property. What
|
|
* is NOT covered there, and is asserted here, is that the WORKFLOW asks for
|
|
* a complete set: same denominator everywhere, numerators exactly 1..N.
|
|
*
|
|
* 2. A dropped coverage gate. A sharded run leaves each runner with a partial
|
|
* picture — shard 2 never executes shard 1's files, so those read 0%. The
|
|
* ≥70% gate therefore cannot live on the shards; it moved to `coverage-gate`,
|
|
* which merges every shard's raw V8 dumps. If that job silently stopped
|
|
* being required, coverage enforcement would vanish without any red check.
|
|
*/
|
|
|
|
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const yaml = require('js-yaml');
|
|
const { escapeRegex } = require('../msd-core/bin/lib/pattern.cjs');
|
|
|
|
const WORKFLOWS_DIR = path.join(__dirname, '..', '.github', 'workflows');
|
|
|
|
function loadWorkflow(name) {
|
|
return yaml.load(fs.readFileSync(path.join(WORKFLOWS_DIR, name), 'utf8'));
|
|
}
|
|
|
|
/**
|
|
* Parse an `i/n` shard spec into { index, total }, or null if malformed.
|
|
* Mirrors the grammar scripts/run-tests.cjs accepts for --shard.
|
|
*/
|
|
function parseShardSpec(spec) {
|
|
const m = /^(\d+)\/(\d+)$/.exec(String(spec));
|
|
if (!m) return null;
|
|
const index = Number(m[1]);
|
|
const total = Number(m[2]);
|
|
if (!Number.isInteger(index) || !Number.isInteger(total)) return null;
|
|
if (total < 1 || index < 1 || index > total) return null;
|
|
return { index, total };
|
|
}
|
|
|
|
/** True iff `specs` is exactly one complete shard set: same N, numerators 1..N. */
|
|
function isCompleteShardSet(specs) {
|
|
if (specs.length === 0) return false;
|
|
const parsed = specs.map(parseShardSpec);
|
|
if (parsed.some((p) => p === null)) return false;
|
|
const total = parsed[0].total;
|
|
if (parsed.some((p) => p.total !== total)) return false;
|
|
if (parsed.length !== total) return false;
|
|
const seen = new Set(parsed.map((p) => p.index));
|
|
return seen.size === total && [...seen].every((i) => i >= 1 && i <= total);
|
|
}
|
|
|
|
test('the test job has no windows lane; test-conformance is the sole Windows selector (#4641)', () => {
|
|
const workflow = loadWorkflow('test.yml');
|
|
const testInclude = workflow.jobs.test.strategy.matrix.include;
|
|
const testWindowsEntries = testInclude.filter((e) => e.os === 'windows-latest');
|
|
assert.equal(
|
|
testWindowsEntries.length, 0,
|
|
`the \`test\` job's matrix.include still has ${testWindowsEntries.length} windows-latest `
|
|
+ `entr${testWindowsEntries.length === 1 ? 'y' : 'ies'} (${JSON.stringify(testWindowsEntries)}). `
|
|
+ '#4641 deletes the `test` job\'s windows lane; test-conformance is the sole Windows selector.',
|
|
);
|
|
|
|
const conformanceInclude = workflow.jobs['test-conformance'].strategy.matrix.include;
|
|
const conformanceWindowsEntries = conformanceInclude.filter((e) => e.os === 'windows-latest');
|
|
const conformanceMacosEntries = conformanceInclude.filter((e) => e.os === 'macos-latest');
|
|
|
|
// Derived, not pinned: windows-latest must be a non-empty, COMPLETE shard
|
|
// set (same denominator, numerators 1..N — see isCompleteShardSet above),
|
|
// whatever N the workflow currently declares. A bare `.length === 3` here
|
|
// would break the moment the workflow is rebalanced to a different shard
|
|
// count without a wiring regression, exactly the shape of bug this sweep
|
|
// exists to remove (the sibling macOS-tier-count magic number already did
|
|
// this once).
|
|
assert.ok(
|
|
conformanceWindowsEntries.length > 0,
|
|
`test-conformance declares no windows-latest entries: ${JSON.stringify(conformanceInclude)}`,
|
|
);
|
|
assert.ok(
|
|
isCompleteShardSet(conformanceWindowsEntries.map((e) => e.shard)),
|
|
'the windows-latest entries in test-conformance are not a complete shard set: '
|
|
+ JSON.stringify(conformanceWindowsEntries),
|
|
);
|
|
|
|
// macos-latest is, by design, a single UNSHARDED entry (see the workflow's
|
|
// "macos-latest stays unsharded; it has real headroom" comment above the
|
|
// `test-conformance` job) — unlike the windows/full shard counts, this "1"
|
|
// is not a fact about the live tree that grows with the suite, it is the
|
|
// structural claim the test exists to pin: more than one entry here would
|
|
// silently duplicate full macOS runs, and a `shard` key would mean the
|
|
// workflow started partitioning a lane the run-tests.cjs invocation below
|
|
// does not expect to be partitioned.
|
|
assert.equal(
|
|
conformanceMacosEntries.length, 1,
|
|
`expected a single unsharded macos-latest entry in test-conformance, got ${conformanceMacosEntries.length}: `
|
|
+ JSON.stringify(conformanceMacosEntries),
|
|
);
|
|
assert.equal(
|
|
conformanceMacosEntries[0] && conformanceMacosEntries[0].shard, undefined,
|
|
'the macos-latest entry in test-conformance declares a shard, contradicting the '
|
|
+ `workflow's "macos-latest stays unsharded" design: ${JSON.stringify(conformanceMacosEntries)}`,
|
|
);
|
|
});
|
|
|
|
test('the full test lane is sharded and complete (#2952)', async (t) => {
|
|
const workflow = loadWorkflow('test.yml');
|
|
const include = workflow.jobs.test.strategy.matrix.include;
|
|
const fullLanes = include.filter((e) => e.scope === 'full');
|
|
// #4641: the `test` job's `scope: windows` lane is deleted — there is no
|
|
// longer a second sharded scope in this job to pin alongside `full`.
|
|
const shardedScopes = { full: fullLanes };
|
|
|
|
for (const [scope, lanes] of Object.entries(shardedScopes)) {
|
|
await t.test(`the \`scope: ${scope}\` lane is actually sharded, not a single runner`, () => {
|
|
assert.ok(lanes.length > 0, `expected at least one \`scope: ${scope}\` matrix entry`);
|
|
assert.ok(
|
|
lanes.length > 1,
|
|
`the \`scope: ${scope}\` lane is back to a single unsharded entry. That is `
|
|
+ 'the #2952/#3057 regression: the whole suite on one runner grows past '
|
|
+ 'its cap and reddens `next`.',
|
|
);
|
|
for (const lane of lanes) {
|
|
assert.ok(
|
|
lane.shard !== undefined,
|
|
`a \`scope: ${scope}\` matrix entry declares no shard: ${JSON.stringify(lane)}`,
|
|
);
|
|
}
|
|
});
|
|
|
|
await t.test(`the \`scope: ${scope}\` lane's declared shards form one complete set`, () => {
|
|
const specs = lanes.map((e) => e.shard);
|
|
assert.ok(
|
|
isCompleteShardSet(specs),
|
|
`the \`scope: ${scope}\` lane's shards ${JSON.stringify(specs)} are not a `
|
|
+ 'complete set. Every entry must share one denominator N and the '
|
|
+ 'numerators must be exactly 1..N — a missing numerator silently stops '
|
|
+ 'running that slice of the suite while every check stays green.',
|
|
);
|
|
});
|
|
}
|
|
|
|
await t.test('no other lane is sharded', () => {
|
|
// #4641: the `test` job's `scope: windows` lane is deleted, so `full` is
|
|
// the only sharded scope left in this job's matrix.
|
|
for (const lane of include.filter((e) => e.scope !== 'full')) {
|
|
assert.equal(
|
|
lane.shard, undefined,
|
|
`lane ${JSON.stringify(lane)} declares a shard but is not \`scope: full\` — `
|
|
+ 'the targeted lane runs a selected file list, not a partition.',
|
|
);
|
|
}
|
|
});
|
|
|
|
await t.test('the scoped Windows shards are passed through to run-tests.cjs', () => {
|
|
const scopedStep = workflow.jobs.test.steps.find(
|
|
(s) => s.name === 'Run scoped tests',
|
|
);
|
|
assert.ok(scopedStep, 'no "Run scoped tests" step in the test job');
|
|
assert.match(
|
|
scopedStep.run, /matrix\.shard/,
|
|
'the "Run scoped tests" step does not reference matrix.shard, so the '
|
|
+ 'windows lane\'s three shards would each run the entire selected file '
|
|
+ 'list — N times the cost, no speedup.',
|
|
);
|
|
});
|
|
|
|
await t.test('each shard runs its own slice, not the whole suite', () => {
|
|
const unitStep = workflow.jobs.test.steps.find(
|
|
(s) => typeof s.run === 'string' && s.run.includes('test:coverage:unit:raw'),
|
|
);
|
|
assert.ok(unitStep, 'no step in the test job runs the raw unit coverage script');
|
|
assert.match(
|
|
unitStep.run, /--shard \$\{\{ matrix\.shard \}\}/,
|
|
'the unit step does not pass matrix.shard through to run-tests.cjs, so '
|
|
+ 'every shard would run the ENTIRE suite — N times the cost, no speedup.',
|
|
);
|
|
});
|
|
|
|
await t.test('the aux suites are pinned to one shard that actually exists', () => {
|
|
// Pinning to a LIVE shard is the whole assertion. A pin to a shard the
|
|
// matrix no longer declares — say the count moves to 4 and these `if:`
|
|
// conditions keep naming 1/3 — means the aux suites stop running entirely
|
|
// while every check stays green. Checking only that some shard literal is
|
|
// present would not catch that, so the literal is resolved against the
|
|
// shards the matrix actually declares.
|
|
const declared = fullLanes.map((e) => String(e.shard));
|
|
const auxScripts = ['test:integration', 'test:security', 'test:install', 'test:slow'];
|
|
const pins = new Set();
|
|
|
|
for (const script of auxScripts) {
|
|
const step = workflow.jobs.test.steps.find(
|
|
(s) => typeof s.run === 'string' && s.run.includes(script),
|
|
);
|
|
assert.ok(step, `no step runs \`npm run ${script}\``);
|
|
|
|
const pin = /matrix\.shard == '([^']+)'/.exec(String(step.if));
|
|
assert.ok(
|
|
pin,
|
|
`the \`${script}\` step is not pinned to a single shard; it would run `
|
|
+ 'once per shard and multiply its cost for no extra signal.',
|
|
);
|
|
assert.ok(
|
|
declared.includes(pin[1]),
|
|
`the \`${script}\` step is pinned to shard '${pin[1]}', which the matrix `
|
|
+ `does not declare (${JSON.stringify(declared)}). That condition can `
|
|
+ 'never be true, so this suite would silently never run.',
|
|
);
|
|
pins.add(pin[1]);
|
|
}
|
|
|
|
assert.equal(
|
|
pins.size, 1,
|
|
`the aux suites are split across shards ${JSON.stringify([...pins])}; they `
|
|
+ 'are meant to run together on exactly one.',
|
|
);
|
|
});
|
|
});
|
|
|
|
// #4070: the aux suites (integration/security/install/slow) are pinned to
|
|
// shard 1 only, but the LPT unit-test packer that balances shards 1/2/3 had
|
|
// no visibility into that fixed cost — it balanced the unit-test slice as if
|
|
// all three shards carried equal fixed cost, structurally overloading shard
|
|
// 1. RUN_TESTS_SHARD_RESERVE tells scripts/run-tests.cjs's selectShard to
|
|
// give shard 1 a virtual head start in weight equal to the aux suites'
|
|
// measured cost, so the packer routes fewer unit-test files to it. See
|
|
// tests/run-tests-harness.test.cjs's "selectShard reserved-weight partition
|
|
// (#4070)" describe block for the packer-side math; this file only pins the
|
|
// WORKFLOW wiring — that the reserve literal actually reaches the full-scope
|
|
// shards and does NOT leak onto the unrelated windows lane.
|
|
test('shard 1\'s aux-suite cost is reserved out of the unit-test packer (#4070)', async (t) => {
|
|
const workflow = loadWorkflow('test.yml');
|
|
|
|
// Reserve value derivation (see .msd/bug/fix-4070-shard1-aux-suite-budget/
|
|
// 10-diagnosis.md "Rejected fixes" for the full computation): aux-suite
|
|
// fixed cost measured at ~216-224s combined across two real runs (issue
|
|
// #4070's cited evidence, runs 33278340189 and 33285384930); shard 1's
|
|
// current unit-only real cost (~584-615s, same issue) divided by its
|
|
// current unit-only LPT weight (209.46, computed locally against
|
|
// tests/test-timings.json) gives an empirical ~2.86s-per-weight-unit
|
|
// conversion; 220s / 2.86 ≈ 77 weight units — roughly 77 average-cost test
|
|
// files' worth of virtual head start.
|
|
const EXPECTED_RESERVE = '1:77';
|
|
|
|
await t.test('the unit-test step reserves shard 1\'s aux-suite cost for the full-scope lane', () => {
|
|
const unitStep = workflow.jobs.test.steps.find(
|
|
(s) => typeof s.run === 'string' && s.run.includes('test:coverage:unit:raw'),
|
|
);
|
|
assert.ok(unitStep, 'no step in the test job runs the raw unit coverage script');
|
|
const reserveEnv = unitStep.env && unitStep.env.RUN_TESTS_SHARD_RESERVE;
|
|
assert.ok(
|
|
reserveEnv,
|
|
'the unit-test step declares no RUN_TESTS_SHARD_RESERVE env var — shard 1 would carry '
|
|
+ 'the aux-suite cost with no offsetting reduction in its unit-test share (#4070)',
|
|
);
|
|
assert.match(
|
|
String(reserveEnv), /matrix\.scope == 'full'/,
|
|
'RUN_TESTS_SHARD_RESERVE must be conditioned on `scope == \'full\'` — the windows lane '
|
|
+ 'runs no aux suites on its own shard 1 and must not be penalized for a cost it never pays',
|
|
);
|
|
assert.match(
|
|
String(reserveEnv), new RegExp(escapeRegex(EXPECTED_RESERVE)),
|
|
`RUN_TESTS_SHARD_RESERVE does not carry the documented reserve literal ${EXPECTED_RESERVE} — `
|
|
+ 'if the aux-suite cost was re-measured, update BOTH this literal and the derivation '
|
|
+ 'comment above (and the diagnosis doc) together, the same discipline LANE_COSTS uses',
|
|
);
|
|
});
|
|
|
|
await t.test('the windows lane is not given a reserve it does not need', () => {
|
|
// The reserve env is a single ternary string shared by every scope, so
|
|
// this is really the same assertion as above stated the other way: for
|
|
// any matrix entry that is NOT scope:full, the expression must resolve
|
|
// falsy. Evaluated structurally (parsing the ternary) rather than by
|
|
// simulating GitHub's expression engine.
|
|
const unitStep = workflow.jobs.test.steps.find(
|
|
(s) => typeof s.run === 'string' && s.run.includes('test:coverage:unit:raw'),
|
|
);
|
|
const reserveEnv = String((unitStep.env && unitStep.env.RUN_TESTS_SHARD_RESERVE) || '');
|
|
const ternary = /\$\{\{\s*matrix\.scope == 'full'\s*&&\s*'[^']*'\s*\|\|\s*'([^']*)'\s*\}\}/.exec(reserveEnv);
|
|
assert.ok(ternary, `RUN_TESTS_SHARD_RESERVE is not a scope-gated ternary: ${reserveEnv}`);
|
|
assert.equal(
|
|
ternary[1], '',
|
|
'the non-full-scope branch of the reserve ternary must resolve to an empty string, so '
|
|
+ 'the windows and targeted lanes see no reserve at all',
|
|
);
|
|
});
|
|
});
|
|
|
|
test('the merged coverage gate survives sharding (#2952)', async (t) => {
|
|
const workflow = loadWorkflow('test.yml');
|
|
|
|
await t.test('a dedicated coverage-gate job exists and consumes the shards', () => {
|
|
const gate = workflow.jobs['coverage-gate'];
|
|
assert.ok(gate, '.github/workflows/test.yml declares no `coverage-gate` job');
|
|
assert.ok(
|
|
(gate.needs || []).includes('test'),
|
|
'coverage-gate must depend on `test` — it merges that job\'s shard artifacts',
|
|
);
|
|
|
|
const merges = gate.steps.some(
|
|
(s) => String(s.uses || '').includes('download-artifact')
|
|
&& s.with && s.with['merge-multiple'] === true,
|
|
);
|
|
assert.ok(
|
|
merges,
|
|
'coverage-gate does not download the shard artifacts with merge-multiple. '
|
|
+ 'Without every shard merged into one coverage/tmp, the gate scores a '
|
|
+ 'partial run: files no shard in hand executed read 0%.',
|
|
);
|
|
});
|
|
|
|
await t.test('both coverage thresholds are still enforced', () => {
|
|
const runs = workflow.jobs['coverage-gate'].steps
|
|
.map((s) => s.run).filter((r) => typeof r === 'string').join('\n');
|
|
const pkg = JSON.parse(
|
|
fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8'),
|
|
);
|
|
|
|
assert.match(
|
|
runs, /test:coverage:report/,
|
|
'coverage-gate never runs the coverage report+gate script — the >=70% '
|
|
+ 'lines / >=60% branches gate on msd-core/bin/lib would be gone',
|
|
);
|
|
assert.match(
|
|
runs, /test:coverage:scripts-floor/,
|
|
'coverage-gate never enforces the >=55% scripts/ floor',
|
|
);
|
|
|
|
// The workflow calls npm scripts precisely so the thresholds are defined
|
|
// once. If a future edit inlines c8 into the YAML, the two surfaces can
|
|
// drift silently — the gate would still be green while measuring something
|
|
// other than what package.json says.
|
|
assert.match(
|
|
pkg.scripts['test:coverage:report'], /check-coverage-gate\.cjs/,
|
|
'test:coverage:report no longer runs check-coverage-gate.cjs',
|
|
);
|
|
assert.match(
|
|
pkg.scripts['test:coverage:scripts-floor'], /--lines 55/,
|
|
'test:coverage:scripts-floor no longer enforces 55%',
|
|
);
|
|
assert.match(
|
|
pkg.scripts['test:coverage:unit:raw'], /--suite unit/,
|
|
'test:coverage:unit:raw no longer runs the unit suite',
|
|
);
|
|
assert.doesNotMatch(
|
|
pkg.scripts['test:coverage:unit:raw'], /--shard/,
|
|
'the shard must come from the workflow matrix, not be baked into the script',
|
|
);
|
|
});
|
|
|
|
await t.test('required-tests fails when the coverage gate fails', () => {
|
|
const required = workflow.jobs['required-tests'];
|
|
assert.ok(
|
|
(required.needs || []).includes('coverage-gate'),
|
|
'required-tests does not depend on coverage-gate, so a red gate could not '
|
|
+ 'block a merge',
|
|
);
|
|
|
|
const summarize = required.steps[0];
|
|
assert.ok(
|
|
'COVERAGE_GATE_RESULT' in (summarize.env || {}),
|
|
'required-tests does not read needs.coverage-gate.result',
|
|
);
|
|
assert.match(
|
|
summarize.run, /coverage-gate did not pass/,
|
|
'required-tests never fails on a red coverage-gate — depending on a job '
|
|
+ 'without checking its result makes the dependency decorative',
|
|
);
|
|
});
|
|
|
|
await t.test('a skipped coverage gate is not treated as a failure', () => {
|
|
// coverage-gate is conditioned on product_changed, exactly like the test
|
|
// lane. A docs-only PR skips both; treating `skipped` as red would block
|
|
// every one of them.
|
|
assert.match(
|
|
workflow.jobs['required-tests'].steps[0].run,
|
|
/COVERAGE_GATE_RESULT" != "skipped"/,
|
|
'required-tests treats a skipped coverage-gate as a failure',
|
|
);
|
|
});
|
|
});
|
|
|
|
test('shard-spec parsing is exact at its boundaries (#2952)', () => {
|
|
assert.deepEqual(parseShardSpec('1/3'), { index: 1, total: 3 });
|
|
assert.deepEqual(parseShardSpec('3/3'), { index: 3, total: 3 });
|
|
assert.deepEqual(parseShardSpec('1/1'), { index: 1, total: 1 });
|
|
|
|
// index 0 is below the range, index total+1 above it.
|
|
assert.equal(parseShardSpec('0/3'), null);
|
|
assert.equal(parseShardSpec('4/3'), null);
|
|
assert.equal(parseShardSpec('1/0'), null);
|
|
|
|
assert.equal(parseShardSpec('1'), null);
|
|
assert.equal(parseShardSpec('a/3'), null);
|
|
assert.equal(parseShardSpec(''), null);
|
|
assert.equal(parseShardSpec(undefined), null);
|
|
|
|
// A complete set, and the three ways one stops being complete.
|
|
assert.equal(isCompleteShardSet(['1/3', '2/3', '3/3']), true);
|
|
assert.equal(isCompleteShardSet(['1/3', '2/3']), false, 'missing numerator');
|
|
assert.equal(isCompleteShardSet(['1/3', '2/3', '2/3']), false, 'duplicate numerator');
|
|
assert.equal(isCompleteShardSet(['1/3', '2/3', '3/4']), false, 'mixed denominator');
|
|
assert.equal(isCompleteShardSet([]), false, 'empty set');
|
|
});
|