msd-core/scripts/gen-test-timings.cjs

#!/usr/bin/env node
// Regenerate the per-file test timing table used to weight chunk packing in
// scripts/run-tests.cjs (issue #2456).
//
// The chunk packer needs to know what each test file actually COSTS. Before
// #2456 it guessed from the filename (`^(?:install|codex-)` scored 12x,
// everything else 1) and was wrong in both directions — installer-migration-
// authoring.test.cjs scored 12x but runs ~0.1s, while the two heaviest files in
// the suite (run-tests-harness.test.cjs and release-tarball-smoke.install
// .test.cjs) both scored 1. This script replaces the guess with measurement.
//
// Input is one or more node:test reporter event streams as emitted by
// `msd-test` (`~/.local/state/msd-test/runs/<run-id>/test-events-<os>-node<v>
// .jsonl`). Each stream carries one `test:summary` event per test FILE, whose
// `data.duration_ms` is that file's total wall-clock and whose `data.file` is
// its absolute in-container path.
//
// Usage:
//   node scripts/gen-test-timings.cjs <events.jsonl> [<events.jsonl> ...]
//   node scripts/gen-test-timings.cjs ~/.local/state/msd-test/runs/*/test-events-*.jsonl
//   node scripts/gen-test-timings.cjs events.jsonl --out tests/test-timings.json
//
// When several streams are supplied (multiple lanes, e.g. node22 + node24), a
// file's recorded time is the MAX across them, not the mean: the packer exists
// to keep the SLOWEST lane's slowest chunk away from the per-chunk timeout, so
// the conservative bound is the right one to balance against.
//
// The table is ADVISORY and deliberately un-gated — there is no `--check` mode
// and no CI lint that fails on staleness, because timing data legitimately
// varies run to run. A file missing from the table falls back to the table's
// median weight, so a stale table degrades chunk BALANCE gracefully instead of
// failing the build. Regenerate it when the suite's cost profile has visibly
// drifted, not on a schedule.
'use strict';

const fs = require('fs');
const { basename, dirname, join } = require('path');
const { ExitError, runMain } = require('./lib/cli-exit.cjs');

const DEFAULT_OUT = join(__dirname, '..', 'tests', 'test-timings.json');
const SCHEMA_VERSION = 1;

function parseArgs(argv) {
  const inputs = [];
  let out = DEFAULT_OUT;
  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i];
    if (arg === '--out') {
      const value = argv[++i];
      if (!value) return { error: '--out requires a path' };
      out = value;
    } else if (arg.startsWith('--out=')) {
      const value = arg.slice('--out='.length);
      if (!value) return { error: '--out requires a path' };
      out = value;
    } else if (arg.startsWith('-')) {
      return { error: `unknown flag "${arg}"` };
    } else {
      inputs.push(arg);
    }
  }
  if (inputs.length === 0) {
    return { error: 'usage: gen-test-timings.cjs <reporter-events.jsonl> [...] [--out <path>]' };
  }
  return { inputs, out };
}

// Fold one reporter event stream into `acc`, keeping the MAX duration seen for
// each test file. Returns per-stream counters plus any basename collisions
// found WITHIN this stream.
//
// Keying is by BASENAME, matching how run-tests.cjs weights a selected file:
// the reporter reports absolute in-container paths (/work/tests/foo.test.cjs)
// while the harness carries paths relative to its test dir, so the basename is
// the only stable join key between the two. A basename seen in two different
// directories would make the table ambiguous, so it must fail loudly.
//
// Collision detection is scoped to a SINGLE stream deliberately. Every lane
// writes its own stream under its own container root (`/work/tests` on Linux,
// `C:/work/tests` on Windows), so comparing directories ACROSS streams reports
// every shared basename as a collision — which is the script's own documented
// usage (globbing `test-events-*.jsonl` across lanes). Within one stream the
// root is constant, so a differing directory is a real collision.
function foldStream(text, acc) {
  const dirsByBase = new Map();
  let files = 0;
  let malformed = 0;
  for (const line of text.split('\n')) {
    if (line.trim() === '') continue;
    let event;
    try {
      event = JSON.parse(line);
    } catch {
      malformed++;
      continue;
    }
    if (!event || event.type !== 'test:summary') continue;
    const data = event.data;
    if (!data || typeof data.file !== 'string') continue;
    const ms = data.duration_ms;
    if (typeof ms !== 'number' || !Number.isFinite(ms) || ms < 0) continue;
    const path = data.file.replace(/\\/g, '/');
    const base = basename(path);
    if (!dirsByBase.has(base)) dirsByBase.set(base, new Set());
    dirsByBase.get(base).add(dirname(path));
    const prev = acc.get(base);
    if (prev === undefined || ms > prev) acc.set(base, ms);
    files++;
  }
  const collisions = [...dirsByBase.entries()]
    .filter(([, dirs]) => dirs.size > 1)
    .map(([base, dirs]) => `${base} (${[...dirs].sort().join(', ')})`);
  return { files, malformed, collisions };
}

function main() {
  const parsed = parseArgs(process.argv.slice(2));
  if (parsed.error) throw new ExitError(2, `gen-test-timings: ${parsed.error}`);

  const acc = new Map();
  const allCollisions = new Set();
  const sources = [];
  for (const input of parsed.inputs) {
    let text;
    try {
      text = fs.readFileSync(input, 'utf8');
    } catch (err) {
      throw new ExitError(2, `gen-test-timings: cannot read "${input}": ${err.message}`);
    }
    const { files, malformed, collisions } = foldStream(text, acc);
    for (const c of collisions) allCollisions.add(c);
    sources.push(basename(input));
    console.error(
      `gen-test-timings: ${basename(input)} — ${files} file summaries` +
        (malformed > 0 ? `, ${malformed} unparseable lines skipped` : ''),
    );
  }

  if (acc.size === 0) {
    throw new ExitError(
      2,
      'gen-test-timings: no `test:summary` events with a file and duration_ms were found. ' +
        'Check that the input is a node:test reporter event stream (test-events-<os>-node<v>.jsonl).',
    );
  }

  // A basename that resolves to two different directories within one lane makes
  // the table ambiguous: run-tests.cjs joins on basename alone, so one file's
  // measured cost would silently be applied to the other. Fail rather than emit
  // a table that lies.
  if (allCollisions.size > 0) {
    throw new ExitError(
      2,
      `gen-test-timings: basename collision — the table cannot key on basename alone:\n  ${[...allCollisions].sort().join('\n  ')}`,
    );
  }

  // Every key must be a plain test-file basename. This is a data-integrity
  // check on a stream we do not control (the reporter emits whatever path the
  // runner saw), and it structurally excludes a computed key like `__proto__`
  // or `constructor` from being written into the table object below — the
  // `js/prototype-polluting-assignment` shape, even though the value here is
  // always a number and could not actually pollute.
  const SAFE_BASENAME_RE = /^[A-Za-z0-9._-]+\.test\.cjs$/;
  const rejected = [...acc.keys()].filter((base) => !SAFE_BASENAME_RE.test(base));
  if (rejected.length > 0) {
    throw new ExitError(
      2,
      `gen-test-timings: refusing to emit non-test-file keys: ${rejected.sort().join(', ')}`,
    );
  }

  // Sorted keys keep the checked-in diff reviewable: a regeneration shows only
  // the files whose cost actually moved, not a reshuffled object.
  const timings = Object.create(null);
  for (const base of [...acc.keys()].sort()) {
    timings[base] = Math.round(acc.get(base));
  }

  const payload = {
    schema_version: SCHEMA_VERSION,
    generated_by: 'scripts/gen-test-timings.cjs',
    unit: 'ms',
    sources: sources.sort(),
    file_count: acc.size,
    timings,
  };

  fs.writeFileSync(parsed.out, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
  const totalMs = [...acc.values()].reduce((sum, ms) => sum + ms, 0);
  console.error(
    `gen-test-timings: wrote ${parsed.out} — ${acc.size} files, ${(totalMs / 1000).toFixed(1)}s total`,
  );
  return 0;
}

if (require.main === module) {
  runMain(main);
}

module.exports = { parseArgs, foldStream };