msd-core/tests/ci-test-job-timeout-budget.test.cjs

'use strict';

/**
 * CI job timeout budgets — .github/workflows/test.yml (#2952).
 *
 * A GitHub Actions job that exceeds its `timeout-minutes` is reported
 * `cancelled`, not `failed`. That conclusion propagates into `Required tests`
 * and reddens the branch, while reading like someone hit the cancel button —
 * which is what makes this failure mode expensive to diagnose and worth a gate.
 *
 * It has now happened three times in this repo: #1051 and #1212 on the Windows
 * full-test lane, and #2952 on the unsharded `test` lane, whose
 * `ubuntu-latest / 24` entry is the only `scope: full` matrix entry — it runs
 * the whole unit suite under c8 coverage, then the scripts/ coverage floor,
 * integration, security, install and slow, serially on one runner. Every one of
 * those was the same root cause: a budget sized to what the lane cost that
 * week, with no headroom for the suite to grow into.
 *
 * So the rule enforced here is not a fixed number per lane — it is a HEADROOM
 * FACTOR over each lane's measured cost. A lane may be slow; what it may not be
 * is budgeted to finish with seconds to spare.
 *
 * This is a budget assertion, not a duration assertion. No unit test can prove
 * a lane still FITS its budget — only a real CI run measures that. What this
 * file guarantees is that a budget cannot be quietly lowered back beneath what
 * its lane is already known to need.
 */

const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const yaml = require('js-yaml');

const WORKFLOWS_DIR = path.join(__dirname, '..', '.github', 'workflows');

function loadWorkflow(name) {
  return yaml.load(fs.readFileSync(path.join(WORKFLOWS_DIR, name), 'utf8'));
}

/**
 * Multiplier applied to a lane's measured cost to get its required budget.
 * 1.5x is enough slack for ordinary suite growth across a release cycle without
 * letting a genuinely runaway lane hide behind a large number.
 */
const HEADROOM_FACTOR = 1.5;

/**
 * Measured wall-clock cost per lane, in whole minutes rounded UP, each from a
 * named run. Raise an entry only alongside a fresh measurement — never to make
 * a red gate green. Raising a measurement raises the required budget with it,
 * which is the point: a lane that got slower must be re-budgeted, not excused.
 */
const LANE_COSTS = [
  {
    job: 'test',
    measuredMinutes: 8,
    // Sharded three ways as of #2952, so this is ONE shard's cost, not the
    // whole unit suite. Run 30677442953: shard 1/3 7m12s, 2/3 4m32s, 3/3 3m59s.
    // Shard 1 is the long pole because the unsharded aux suites ride on it.
    // Before sharding the same lane cost 15m20s and blew a 15-minute cap.
    //
    // This one `timeout-minutes` also covers the `scope: windows` matrix
    // entries — GitHub applies a single job-level budget across every matrix
    // combination, not one per entry. That lane is now sharded three ways too
    // (#3057), but no post-sharding per-shard measurement exists yet: its only
    // recorded cost is the PRE-sharding whole-suite run that hit 15m05s and was
    // CANCELLED on PR #3094. Each of its three shards should now cost roughly a
    // third of that (~5m), which is already comfortably under the 8m/12m this
    // entry requires — so no separate LANE_COSTS entry is added on a number
    // that has not actually been measured. Replace this estimate with a real
    // measured shard cost once one exists, the same discipline every other
    // entry here follows.
    evidence: 'run 30677442953 — 7m12s slowest shard',
  },
  {
    job: 'test-full',
    measuredMinutes: 27,
    // Lane moved from windows-22 to windows-latest/24 and is now sharded three
    // ways. Worst observed shard is `full test (windows-latest, 24, shard
    // 3/3)`: 26m18s on run 32614439702 (shard 2/3 23m36s, shard 1/3 19m22s),
    // and 23m17s for shard 2/3 on run 32603886007. The previous 18m59s /
    // windows-22 figure recorded here predated this cost and is stale — the
    // lane is measurably slower now, not merely relabeled.
    evidence: 'run 32614439702 — 26m18s, windows-latest/24 shard 3/3',
  },
  {
    job: 'coverage-gate',
    measuredMinutes: 2,
    // Downloads three shards' raw V8 dumps, renders one merged report and runs
    // both thresholds. Run 30677442953: 1m20s end to end, most of it npm ci.
    evidence: 'run 30677442953 — 1m20s',
  },
  {
    job: 'test-inert',
    measuredMinutes: 2,
    // Runs only the targeted-test step when no product code changed; observed
    // around a minute. Listed so its budget cannot be dropped to nothing.
    evidence: 'targeted-only lane, ~1m observed',
  },
];

function requiredBudgetMinutes(measuredMinutes, headroomFactor = HEADROOM_FACTOR) {
  return Math.ceil(measuredMinutes * headroomFactor);
}

function hasSufficientBudget(budgetMinutes, measuredMinutes, headroomFactor = HEADROOM_FACTOR) {
  return Number.isInteger(budgetMinutes)
    && budgetMinutes >= requiredBudgetMinutes(measuredMinutes, headroomFactor);
}

test('CI job timeout budgets carry headroom over measured cost (#2952)', async (t) => {
  const workflow = loadWorkflow('test.yml');

  for (const lane of LANE_COSTS) {
    await t.test(`${lane.job} is budgeted above its measured cost`, () => {
      assert.ok(
        workflow.jobs && Object.prototype.hasOwnProperty.call(workflow.jobs, lane.job),
        `.github/workflows/test.yml declares no job \`${lane.job}\`. If it was `
        + 'renamed or removed, update LANE_COSTS in this file to match — do not '
        + 'delete the entry to make this pass.',
      );

      const budget = workflow.jobs[lane.job]['timeout-minutes'];
      const required = requiredBudgetMinutes(lane.measuredMinutes);

      assert.equal(
        typeof budget, 'number',
        `.github/workflows/test.yml jobs.${lane.job} must declare timeout-minutes`,
      );
      assert.ok(
        hasSufficientBudget(budget, lane.measuredMinutes),
        `jobs.${lane.job}.timeout-minutes is ${budget}, but the lane measured `
        + `${lane.measuredMinutes}m (${lane.evidence}) and needs at least `
        + `${required} — ${HEADROOM_FACTOR}x — so suite growth does not breach `
        + 'the cap. A job that exceeds timeout-minutes is reported `cancelled` '
        + 'and reddens `Required tests`.',
      );
    });
  }

  // Boundary coverage on the predicate that decides every lane above:
  // required-1 must be rejected, required and required+1 accepted.
  await t.test('budget sufficiency is exact at the boundary', () => {
    for (const lane of LANE_COSTS) {
      const required = requiredBudgetMinutes(lane.measuredMinutes);

      assert.equal(hasSufficientBudget(required - 1, lane.measuredMinutes), false,
        `${lane.job}: a budget one minute under the requirement must be rejected`);
      assert.equal(hasSufficientBudget(required, lane.measuredMinutes), true,
        `${lane.job}: a budget exactly at the requirement must be accepted`);
      assert.equal(hasSufficientBudget(required + 1, lane.measuredMinutes), true,
        `${lane.job}: a budget over the requirement must be accepted`);
    }
  });

  await t.test('a non-integer budget is not a sufficient budget', () => {
    // `timeout-minutes: 24.5` is not something GitHub accepts; treating it as
    // sufficient would let a malformed workflow through this gate.
    assert.equal(hasSufficientBudget(24.5, 16), false);
    assert.equal(hasSufficientBudget(Number.NaN, 16), false);
    assert.equal(hasSufficientBudget(undefined, 16), false);
  });

  await t.test('requiredBudgetMinutes rounds up rather than truncating', () => {
    // 15 * 1.5 = 22.5 — truncation would hand back 22 and under-budget the lane.
    assert.equal(requiredBudgetMinutes(15, 1.5), 23);
    assert.equal(requiredBudgetMinutes(16, 1.5), 24);
    assert.equal(requiredBudgetMinutes(19, 1.5), 29);
    assert.equal(requiredBudgetMinutes(10, 1.5), 15);
  });
});