msd-core/tests/ci-full-lane-sharding.test.cjs

'use strict';

/**
 * The full test lane and the scoped Windows lane are both sharded, and the
 * coverage gate that sharding displaced is still wired in —
 * .github/workflows/test.yml (#2952, #3057).
 *
 * The `scope: full` lane was the only unsharded lane in this file. It ran the
 * entire unit suite under c8 on a single runner, grew past a 15-minute cap, and
 * reddened `next` (#2952). Raising the cap treated the symptom; sharding is the
 * shape fix, and it is the same answer #1212 reached for the Windows lane at
 * the time.
 *
 * The `scope: windows` lane then hit the identical cliff itself: it reached
 * exactly 15m05s and was CANCELLED on PR #3094, four shas in a row. Per #869's
 * stated durable follow-up, it is now sharded three ways too (#3057), leaving
 * `scope: targeted` as the only lane in this job with no shard.
 *
 * Sharding introduces two failure modes that stay GREEN while being wrong, so
 * both are pinned here:
 *
 *  1. An incomplete shard set. If the matrix declares shards 1/3 and 2/3 but
 *     never 3/3, a third of the unit suite simply stops running and every check
 *     still passes. The partition MATH is already covered — completeness,
 *     disjointness, balance and determinism are asserted against selectShard in
 *     run-tests-harness.test.cjs (#1212), including a fast-check property. What
 *     is NOT covered there, and is asserted here, is that the WORKFLOW asks for
 *     a complete set: same denominator everywhere, numerators exactly 1..N.
 *
 *  2. A dropped coverage gate. A sharded run leaves each runner with a partial
 *     picture — shard 2 never executes shard 1's files, so those read 0%. The
 *     ≥70% gate therefore cannot live on the shards; it moved to `coverage-gate`,
 *     which merges every shard's raw V8 dumps. If that job silently stopped
 *     being required, coverage enforcement would vanish without any red check.
 */

const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const yaml = require('js-yaml');
const { escapeRegex } = require('../gsd-core/bin/lib/pattern.cjs');

const WORKFLOWS_DIR = path.join(__dirname, '..', '.github', 'workflows');

function loadWorkflow(name) {
  return yaml.load(fs.readFileSync(path.join(WORKFLOWS_DIR, name), 'utf8'));
}

/**
 * Parse an `i/n` shard spec into { index, total }, or null if malformed.
 * Mirrors the grammar scripts/run-tests.cjs accepts for --shard.
 */
function parseShardSpec(spec) {
  const m = /^(\d+)\/(\d+)$/.exec(String(spec));
  if (!m) return null;
  const index = Number(m[1]);
  const total = Number(m[2]);
  if (!Number.isInteger(index) || !Number.isInteger(total)) return null;
  if (total < 1 || index < 1 || index > total) return null;
  return { index, total };
}

/** True iff `specs` is exactly one complete shard set: same N, numerators 1..N. */
function isCompleteShardSet(specs) {
  if (specs.length === 0) return false;
  const parsed = specs.map(parseShardSpec);
  if (parsed.some((p) => p === null)) return false;
  const total = parsed[0].total;
  if (parsed.some((p) => p.total !== total)) return false;
  if (parsed.length !== total) return false;
  const seen = new Set(parsed.map((p) => p.index));
  return seen.size === total && [...seen].every((i) => i >= 1 && i <= total);
}

test('the full test lane is sharded and complete (#2952)', async (t) => {
  const workflow = loadWorkflow('test.yml');
  const include = workflow.jobs.test.strategy.matrix.include;
  const fullLanes = include.filter((e) => e.scope === 'full');
  const windowsLanes = include.filter((e) => e.scope === 'windows');
  // The only lane in this job with no shard is `scope: targeted` — the fast,
  // single-runner default lane. Both `full` and `windows` are sharded.
  const shardedScopes = { full: fullLanes, windows: windowsLanes };

  for (const [scope, lanes] of Object.entries(shardedScopes)) {
    await t.test(`the \`scope: ${scope}\` lane is actually sharded, not a single runner`, () => {
      assert.ok(lanes.length > 0, `expected at least one \`scope: ${scope}\` matrix entry`);
      assert.ok(
        lanes.length > 1,
        `the \`scope: ${scope}\` lane is back to a single unsharded entry. That is `
        + 'the #2952/#3057 regression: the whole suite on one runner grows past '
        + 'its cap and reddens `next`.',
      );
      for (const lane of lanes) {
        assert.ok(
          lane.shard !== undefined,
          `a \`scope: ${scope}\` matrix entry declares no shard: ${JSON.stringify(lane)}`,
        );
      }
    });

    await t.test(`the \`scope: ${scope}\` lane's declared shards form one complete set`, () => {
      const specs = lanes.map((e) => e.shard);
      assert.ok(
        isCompleteShardSet(specs),
        `the \`scope: ${scope}\` lane's shards ${JSON.stringify(specs)} are not a `
        + 'complete set. Every entry must share one denominator N and the '
        + 'numerators must be exactly 1..N — a missing numerator silently stops '
        + 'running that slice of the suite while every check stays green.',
      );
    });
  }

  await t.test('no other lane is sharded', () => {
    for (const lane of include.filter((e) => e.scope !== 'full' && e.scope !== 'windows')) {
      assert.equal(
        lane.shard, undefined,
        `lane ${JSON.stringify(lane)} declares a shard but is neither \`scope: full\` `
        + 'nor `scope: windows` — the targeted lane runs a selected file list, not '
        + 'a partition.',
      );
    }
  });

  await t.test('the scoped Windows shards are passed through to run-tests.cjs', () => {
    const scopedStep = workflow.jobs.test.steps.find(
      (s) => s.name === 'Run scoped tests',
    );
    assert.ok(scopedStep, 'no "Run scoped tests" step in the test job');
    assert.match(
      scopedStep.run, /matrix\.shard/,
      'the "Run scoped tests" step does not reference matrix.shard, so the '
      + 'windows lane\'s three shards would each run the entire selected file '
      + 'list — N times the cost, no speedup.',
    );
  });

  await t.test('each shard runs its own slice, not the whole suite', () => {
    const unitStep = workflow.jobs.test.steps.find(
      (s) => typeof s.run === 'string' && s.run.includes('test:coverage:unit:raw'),
    );
    assert.ok(unitStep, 'no step in the test job runs the raw unit coverage script');
    assert.match(
      unitStep.run, /--shard \$\{\{ matrix\.shard \}\}/,
      'the unit step does not pass matrix.shard through to run-tests.cjs, so '
      + 'every shard would run the ENTIRE suite — N times the cost, no speedup.',
    );
  });

  await t.test('the aux suites are pinned to one shard that actually exists', () => {
    // Pinning to a LIVE shard is the whole assertion. A pin to a shard the
    // matrix no longer declares — say the count moves to 4 and these `if:`
    // conditions keep naming 1/3 — means the aux suites stop running entirely
    // while every check stays green. Checking only that some shard literal is
    // present would not catch that, so the literal is resolved against the
    // shards the matrix actually declares.
    const declared = fullLanes.map((e) => String(e.shard));
    const auxScripts = ['test:integration', 'test:security', 'test:install', 'test:slow'];
    const pins = new Set();

    for (const script of auxScripts) {
      const step = workflow.jobs.test.steps.find(
        (s) => typeof s.run === 'string' && s.run.includes(script),
      );
      assert.ok(step, `no step runs \`npm run ${script}\``);

      const pin = /matrix\.shard == '([^']+)'/.exec(String(step.if));
      assert.ok(
        pin,
        `the \`${script}\` step is not pinned to a single shard; it would run `
        + 'once per shard and multiply its cost for no extra signal.',
      );
      assert.ok(
        declared.includes(pin[1]),
        `the \`${script}\` step is pinned to shard '${pin[1]}', which the matrix `
        + `does not declare (${JSON.stringify(declared)}). That condition can `
        + 'never be true, so this suite would silently never run.',
      );
      pins.add(pin[1]);
    }

    assert.equal(
      pins.size, 1,
      `the aux suites are split across shards ${JSON.stringify([...pins])}; they `
      + 'are meant to run together on exactly one.',
    );
  });
});

// #4070: the aux suites (integration/security/install/slow) are pinned to
// shard 1 only, but the LPT unit-test packer that balances shards 1/2/3 had
// no visibility into that fixed cost — it balanced the unit-test slice as if
// all three shards carried equal fixed cost, structurally overloading shard
// 1. RUN_TESTS_SHARD_RESERVE tells scripts/run-tests.cjs's selectShard to
// give shard 1 a virtual head start in weight equal to the aux suites'
// measured cost, so the packer routes fewer unit-test files to it. See
// tests/run-tests-harness.test.cjs's "selectShard reserved-weight partition
// (#4070)" describe block for the packer-side math; this file only pins the
// WORKFLOW wiring — that the reserve literal actually reaches the full-scope
// shards and does NOT leak onto the unrelated windows lane.
test('shard 1\'s aux-suite cost is reserved out of the unit-test packer (#4070)', async (t) => {
  const workflow = loadWorkflow('test.yml');

  // Reserve value derivation (see .gsd/bug/fix-4070-shard1-aux-suite-budget/
  // 10-diagnosis.md "Rejected fixes" for the full computation): aux-suite
  // fixed cost measured at ~216-224s combined across two real runs (issue
  // #4070's cited evidence, runs 33278340189 and 33285384930); shard 1's
  // current unit-only real cost (~584-615s, same issue) divided by its
  // current unit-only LPT weight (209.46, computed locally against
  // tests/test-timings.json) gives an empirical ~2.86s-per-weight-unit
  // conversion; 220s / 2.86 ≈ 77 weight units — roughly 77 average-cost test
  // files' worth of virtual head start.
  const EXPECTED_RESERVE = '1:77';

  await t.test('the unit-test step reserves shard 1\'s aux-suite cost for the full-scope lane', () => {
    const unitStep = workflow.jobs.test.steps.find(
      (s) => typeof s.run === 'string' && s.run.includes('test:coverage:unit:raw'),
    );
    assert.ok(unitStep, 'no step in the test job runs the raw unit coverage script');
    const reserveEnv = unitStep.env && unitStep.env.RUN_TESTS_SHARD_RESERVE;
    assert.ok(
      reserveEnv,
      'the unit-test step declares no RUN_TESTS_SHARD_RESERVE env var — shard 1 would carry '
      + 'the aux-suite cost with no offsetting reduction in its unit-test share (#4070)',
    );
    assert.match(
      String(reserveEnv), /matrix\.scope == 'full'/,
      'RUN_TESTS_SHARD_RESERVE must be conditioned on `scope == \'full\'` — the windows lane '
      + 'runs no aux suites on its own shard 1 and must not be penalized for a cost it never pays',
    );
    assert.match(
      String(reserveEnv), new RegExp(escapeRegex(EXPECTED_RESERVE)),
      `RUN_TESTS_SHARD_RESERVE does not carry the documented reserve literal ${EXPECTED_RESERVE} — `
      + 'if the aux-suite cost was re-measured, update BOTH this literal and the derivation '
      + 'comment above (and the diagnosis doc) together, the same discipline LANE_COSTS uses',
    );
  });

  await t.test('the windows lane is not given a reserve it does not need', () => {
    // The reserve env is a single ternary string shared by every scope, so
    // this is really the same assertion as above stated the other way: for
    // any matrix entry that is NOT scope:full, the expression must resolve
    // falsy. Evaluated structurally (parsing the ternary) rather than by
    // simulating GitHub's expression engine.
    const unitStep = workflow.jobs.test.steps.find(
      (s) => typeof s.run === 'string' && s.run.includes('test:coverage:unit:raw'),
    );
    const reserveEnv = String((unitStep.env && unitStep.env.RUN_TESTS_SHARD_RESERVE) || '');
    const ternary = /\$\{\{\s*matrix\.scope == 'full'\s*&&\s*'[^']*'\s*\|\|\s*'([^']*)'\s*\}\}/.exec(reserveEnv);
    assert.ok(ternary, `RUN_TESTS_SHARD_RESERVE is not a scope-gated ternary: ${reserveEnv}`);
    assert.equal(
      ternary[1], '',
      'the non-full-scope branch of the reserve ternary must resolve to an empty string, so '
      + 'the windows and targeted lanes see no reserve at all',
    );
  });
});

test('the merged coverage gate survives sharding (#2952)', async (t) => {
  const workflow = loadWorkflow('test.yml');

  await t.test('a dedicated coverage-gate job exists and consumes the shards', () => {
    const gate = workflow.jobs['coverage-gate'];
    assert.ok(gate, '.github/workflows/test.yml declares no `coverage-gate` job');
    assert.ok(
      (gate.needs || []).includes('test'),
      'coverage-gate must depend on `test` — it merges that job\'s shard artifacts',
    );

    const merges = gate.steps.some(
      (s) => String(s.uses || '').includes('download-artifact')
        && s.with && s.with['merge-multiple'] === true,
    );
    assert.ok(
      merges,
      'coverage-gate does not download the shard artifacts with merge-multiple. '
      + 'Without every shard merged into one coverage/tmp, the gate scores a '
      + 'partial run: files no shard in hand executed read 0%.',
    );
  });

  await t.test('both coverage thresholds are still enforced', () => {
    const runs = workflow.jobs['coverage-gate'].steps
      .map((s) => s.run).filter((r) => typeof r === 'string').join('\n');
    const pkg = JSON.parse(
      fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8'),
    );

    assert.match(
      runs, /test:coverage:report/,
      'coverage-gate never runs the coverage report+gate script — the >=70% '
      + 'lines / >=60% branches gate on gsd-core/bin/lib would be gone',
    );
    assert.match(
      runs, /test:coverage:scripts-floor/,
      'coverage-gate never enforces the >=55% scripts/ floor',
    );

    // The workflow calls npm scripts precisely so the thresholds are defined
    // once. If a future edit inlines c8 into the YAML, the two surfaces can
    // drift silently — the gate would still be green while measuring something
    // other than what package.json says.
    assert.match(
      pkg.scripts['test:coverage:report'], /check-coverage-gate\.cjs/,
      'test:coverage:report no longer runs check-coverage-gate.cjs',
    );
    assert.match(
      pkg.scripts['test:coverage:scripts-floor'], /--lines 55/,
      'test:coverage:scripts-floor no longer enforces 55%',
    );
    assert.match(
      pkg.scripts['test:coverage:unit:raw'], /--suite unit/,
      'test:coverage:unit:raw no longer runs the unit suite',
    );
    assert.doesNotMatch(
      pkg.scripts['test:coverage:unit:raw'], /--shard/,
      'the shard must come from the workflow matrix, not be baked into the script',
    );
  });

  await t.test('required-tests fails when the coverage gate fails', () => {
    const required = workflow.jobs['required-tests'];
    assert.ok(
      (required.needs || []).includes('coverage-gate'),
      'required-tests does not depend on coverage-gate, so a red gate could not '
      + 'block a merge',
    );

    const summarize = required.steps[0];
    assert.ok(
      'COVERAGE_GATE_RESULT' in (summarize.env || {}),
      'required-tests does not read needs.coverage-gate.result',
    );
    assert.match(
      summarize.run, /coverage-gate did not pass/,
      'required-tests never fails on a red coverage-gate — depending on a job '
      + 'without checking its result makes the dependency decorative',
    );
  });

  await t.test('a skipped coverage gate is not treated as a failure', () => {
    // coverage-gate is conditioned on product_changed, exactly like the test
    // lane. A docs-only PR skips both; treating `skipped` as red would block
    // every one of them.
    assert.match(
      workflow.jobs['required-tests'].steps[0].run,
      /COVERAGE_GATE_RESULT" != "skipped"/,
      'required-tests treats a skipped coverage-gate as a failure',
    );
  });
});

test('shard-spec parsing is exact at its boundaries (#2952)', () => {
  assert.deepEqual(parseShardSpec('1/3'), { index: 1, total: 3 });
  assert.deepEqual(parseShardSpec('3/3'), { index: 3, total: 3 });
  assert.deepEqual(parseShardSpec('1/1'), { index: 1, total: 1 });

  // index 0 is below the range, index total+1 above it.
  assert.equal(parseShardSpec('0/3'), null);
  assert.equal(parseShardSpec('4/3'), null);
  assert.equal(parseShardSpec('1/0'), null);

  assert.equal(parseShardSpec('1'), null);
  assert.equal(parseShardSpec('a/3'), null);
  assert.equal(parseShardSpec(''), null);
  assert.equal(parseShardSpec(undefined), null);

  // A complete set, and the three ways one stops being complete.
  assert.equal(isCompleteShardSet(['1/3', '2/3', '3/3']), true);
  assert.equal(isCompleteShardSet(['1/3', '2/3']), false, 'missing numerator');
  assert.equal(isCompleteShardSet(['1/3', '2/3', '2/3']), false, 'duplicate numerator');
  assert.equal(isCompleteShardSet(['1/3', '2/3', '3/4']), false, 'mixed denominator');
  assert.equal(isCompleteShardSet([]), false, 'empty set');
});