* feat(#4036): persist CI shard/job timeout-vs-cap trending, warn at 90% Adds two new mechanisms plus an audit-coverage extension: - scripts/lib/ci-job-timing.cjs: shared elapsed-vs-cap arithmetic - scripts/ci-check-job-near-cap.cjs: in-job advisory near-cap check, wired into test/test-full/mutate/smoke as each job's last step - scripts/ci-timeout-report.cjs + .github/workflows/ci-timeout-report.yml: scheduled REST-API poll that appends new records to tests/ci-timeout-budget-history.jsonl and opens a small data-only PR - tests/ci-test-job-timeout-budget.test.cjs: extended to cover mutate (mutation.yml) and smoke (install-smoke.yml), which previously had no headroom-factor gate coverage at all Does not change any timeout-minutes value, shard composition, or shard-1 contents — those stay maintainer policy calls per the issue's own scope. * fix(#4036): address two-orthogonal-review findings - Parity tests guarding the two hand-duplicated literals this design cannot single-source through GH Actions YAML: CI_JOB_TIMEOUT_MINUTES vs each job's own timeout-minutes, and ci-timeout-report.cjs's JOB_RULES name-prefixes vs each job's actual name: template. - Thread run.event through as runEvent on every persisted record, so PR-context and push-context install-smoke timings (genuinely different matrix shape) are distinguishable in the history rather than silently conflated under one job name. - Replace the Windows near-cap start-time step's ambiguous PowerShell +/>> precedence with GitHub's documented string-interpolation form. - Move github.run_id out of direct ${{ }} shell interpolation into an env: var in the new scheduled workflow, per this repo's own expression-injection-safe convention. * test(#4036): regenerate golden install-tree fixtures for scripts/lib/ci-job-timing.cjs npm run gen:install-tree — scripts/ ships wholesale into the installed package (per ADR/known-defect precedent from #4012's own PR history: a new scripts/lib/*.cjs file needs its golden entry regenerated or every runtime's install-tree test fails). Confirmed via gsd-test: this was the sole cause of the first real verification run's 25 failures (all in tests/golden-install-tree.test.cjs, one per runtime). Top-level scripts/*.cjs files (ci-check-job-near-cap.cjs, ci-timeout-report.cjs) are not individually tracked in these fixtures — consistent with every other existing top-level scripts/*.cjs file, so no entry was expected or added for those two. * fix(#4036): register new lib file with installer, fix H1 shell policy - bin/install.js: add ci-job-timing.cjs to GSD_SCRIPTS_LIB_FILES (a hand-maintained registry, not generated — tests/install.test.cjs asserts every scripts/lib/ file is enumerated here) - test.yml: replace the two OS-conditional "Record job start time" step pairs (test + test-full jobs) with a single unconditional `node -e` step. The prior pair's Windows variant declared an explicit shell: pwsh, which scripts/workflow-policy.cjs's H1 checker statically flags against every OS a job's matrix can realize, independent of the step's own if: gate. A single Node one-liner needs no shell override at all — it's syntactically valid and behaves identically under bash, zsh, and pwsh — which is both H1 compliant and removes the last OS-specific shell syntax from this change entirely. Both defects were found by a real gsd-test run, not local gates — lint:ci and build:lib were clean throughout because neither the scripts/lib/ install-manifest parity check nor the H1 shell-policy baseline runs as part of lint:ci; both are gsd-test-only suites. * docs(#4036): how-to for reading CI timeout budget signals The phase-gate docs check correctly flagged the enablement sequence as 3 real steps (read the near-cap warning, find the accumulated trend file, pick the right maintainer lever) — a reference table can't carry a sequence. Adds docs/how-to/read-ci-timeout-signals.md, indexed from docs/README.md. * chore(#4036): backfill changeset PR number (4043) --------- Co-authored-by: sim <sim@local>
265 lines
7.5 KiB
JavaScript
265 lines
7.5 KiB
JavaScript
'use strict';
|
|
|
|
/**
|
|
* tests/ci-timeout-report.test.cjs
|
|
*
|
|
* Unit tests for scripts/ci-timeout-report.cjs's pure exports (#4036).
|
|
* main() is impure orchestration requiring a live Octokit/GitHub Actions
|
|
* context and is intentionally NOT covered here.
|
|
*/
|
|
|
|
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
|
|
const {
|
|
resolveJobTimeoutMinutes,
|
|
parseJobRecord,
|
|
buildReportLines,
|
|
dedupeAgainstHistory,
|
|
} = require('../scripts/ci-timeout-report.cjs');
|
|
|
|
test('resolveJobTimeoutMinutes', async (t) => {
|
|
await t.test('static job: resolves timeout-minutes from workflow YAML', () => {
|
|
const yamlText = [
|
|
'jobs:',
|
|
' test:',
|
|
' timeout-minutes: 15',
|
|
'',
|
|
].join('\n');
|
|
|
|
const result = resolveJobTimeoutMinutes({
|
|
jobName: 'test (ubuntu-latest, 24, shard 1/3)',
|
|
workflowFile: 'test.yml',
|
|
workflowYamlText: yamlText,
|
|
covered: null,
|
|
});
|
|
|
|
assert.equal(result, 15);
|
|
});
|
|
|
|
await t.test('mutation job: resolves override timeoutMinutes from COVERED', () => {
|
|
const result = resolveJobTimeoutMinutes({
|
|
jobName: 'Stryker (frontmatter)',
|
|
workflowFile: 'mutation.yml',
|
|
workflowYamlText: null,
|
|
covered: { frontmatter: { timeoutMinutes: 20 }, 'adr-parser': {} },
|
|
});
|
|
|
|
assert.equal(result, 20);
|
|
});
|
|
|
|
await t.test('mutation job: falls back to default 15 when no override', () => {
|
|
const result = resolveJobTimeoutMinutes({
|
|
jobName: 'Stryker (adr-parser)',
|
|
workflowFile: 'mutation.yml',
|
|
workflowYamlText: null,
|
|
covered: { frontmatter: { timeoutMinutes: 20 }, 'adr-parser': {} },
|
|
});
|
|
|
|
assert.equal(result, 15);
|
|
});
|
|
|
|
await t.test('mutation job: unknown module returns null', () => {
|
|
const result = resolveJobTimeoutMinutes({
|
|
jobName: 'Stryker (totally-unknown-module)',
|
|
workflowFile: 'mutation.yml',
|
|
workflowYamlText: null,
|
|
covered: { frontmatter: { timeoutMinutes: 20 }, 'adr-parser': {} },
|
|
});
|
|
|
|
assert.equal(result, null);
|
|
});
|
|
|
|
await t.test('test-inert resolves against the test-inert job key, not test', () => {
|
|
const yamlText = [
|
|
'jobs:',
|
|
' test:',
|
|
' timeout-minutes: 15',
|
|
' test-inert:',
|
|
' timeout-minutes: 2',
|
|
'',
|
|
].join('\n');
|
|
|
|
const result = resolveJobTimeoutMinutes({
|
|
jobName: 'test (inert CI)',
|
|
workflowFile: 'test.yml',
|
|
workflowYamlText: yamlText,
|
|
covered: null,
|
|
});
|
|
|
|
assert.equal(result, 2);
|
|
});
|
|
});
|
|
|
|
test('parseJobRecord', async (t) => {
|
|
await t.test('still-running job (completed_at null) returns null', () => {
|
|
const result = parseJobRecord({
|
|
job: {
|
|
name: 'test (ubuntu-latest, 24, shard 1/3)',
|
|
completed_at: null,
|
|
started_at: '2026-08-29T00:00:00Z',
|
|
run_id: 1,
|
|
head_sha: 'abc123',
|
|
runEvent: 'pull_request',
|
|
},
|
|
workflowFile: 'test.yml',
|
|
workflowYamlText: 'jobs:\n test:\n timeout-minutes: 15\n',
|
|
covered: null,
|
|
});
|
|
|
|
assert.equal(result, null);
|
|
});
|
|
|
|
await t.test('untracked job name returns null', () => {
|
|
for (const jobName of ['preflight', 'changes', 'lint-tests']) {
|
|
const result = parseJobRecord({
|
|
job: {
|
|
name: jobName,
|
|
completed_at: '2026-08-29T00:10:00Z',
|
|
started_at: '2026-08-29T00:00:00Z',
|
|
run_id: 1,
|
|
head_sha: 'abc123',
|
|
runEvent: 'pull_request',
|
|
},
|
|
workflowFile: 'test.yml',
|
|
workflowYamlText: 'jobs:\n test:\n timeout-minutes: 15\n',
|
|
covered: null,
|
|
});
|
|
|
|
assert.equal(result, null, `expected null for job name ${jobName}`);
|
|
}
|
|
});
|
|
|
|
await t.test('valid smoke job returns a full record', () => {
|
|
const yamlText = [
|
|
'jobs:',
|
|
' smoke:',
|
|
' timeout-minutes: 12',
|
|
'',
|
|
].join('\n');
|
|
|
|
const result = parseJobRecord({
|
|
job: {
|
|
name: 'smoke (ubuntu-latest)',
|
|
started_at: '2026-08-29T00:00:00Z',
|
|
completed_at: '2026-08-29T00:06:00Z',
|
|
run_id: 42,
|
|
head_sha: 'deadbeef',
|
|
runEvent: 'push',
|
|
},
|
|
workflowFile: 'install-smoke.yml',
|
|
workflowYamlText: yamlText,
|
|
covered: null,
|
|
});
|
|
|
|
assert.ok(result);
|
|
assert.equal(result.jobName, 'smoke (ubuntu-latest)');
|
|
assert.equal(result.workflowFile, 'install-smoke.yml');
|
|
assert.equal(result.runId, 42);
|
|
assert.equal(result.sha, 'deadbeef');
|
|
assert.equal(result.runEvent, 'push');
|
|
assert.equal(result.timeoutMinutes, 12);
|
|
assert.equal(typeof result.pct, 'number');
|
|
});
|
|
});
|
|
|
|
test('dedupeAgainstHistory', async (t) => {
|
|
await t.test('excludes only the record already present in history', () => {
|
|
const records = [
|
|
{ runId: 1, jobName: 'test (ubuntu-latest, 24, shard 1/3)', pct: 0.5 },
|
|
{ runId: 2, jobName: 'test (ubuntu-latest, 24, shard 2/3)', pct: 0.6 },
|
|
];
|
|
const historyText = `${JSON.stringify({ runId: 1, jobName: 'test (ubuntu-latest, 24, shard 1/3)' })}\n`;
|
|
|
|
const result = dedupeAgainstHistory(records, historyText);
|
|
|
|
assert.equal(result.length, 1);
|
|
assert.equal(result[0].runId, 2);
|
|
});
|
|
|
|
await t.test('same runId, different jobName: both kept when history is empty', () => {
|
|
const records = [
|
|
{ runId: 1, jobName: 'test (ubuntu-latest, 24, shard 1/3)', pct: 0.5 },
|
|
{ runId: 1, jobName: 'test (ubuntu-latest, 24, shard 2/3)', pct: 0.6 },
|
|
];
|
|
|
|
const result = dedupeAgainstHistory(records, '');
|
|
|
|
assert.equal(result.length, 2);
|
|
});
|
|
|
|
await t.test('malformed history lines are skipped, not thrown', () => {
|
|
const records = [
|
|
{ runId: 1, jobName: 'test (ubuntu-latest, 24, shard 1/3)', pct: 0.5 },
|
|
{ runId: 2, jobName: 'test (ubuntu-latest, 24, shard 2/3)', pct: 0.6 },
|
|
];
|
|
const historyText = [
|
|
JSON.stringify({ runId: 1, jobName: 'test (ubuntu-latest, 24, shard 1/3)' }),
|
|
'',
|
|
'not json{',
|
|
'',
|
|
].join('\n');
|
|
|
|
const result = dedupeAgainstHistory(records, historyText);
|
|
|
|
assert.equal(result.length, 1);
|
|
assert.equal(result[0].runId, 2);
|
|
});
|
|
});
|
|
|
|
test('buildReportLines', async (t) => {
|
|
await t.test('end-to-end: only tracked+completed jobs produce records', () => {
|
|
const workflowYamlText = [
|
|
'jobs:',
|
|
' test:',
|
|
' timeout-minutes: 15',
|
|
'',
|
|
].join('\n');
|
|
|
|
const runs = [
|
|
{
|
|
run: { id: 1, head_sha: 'sha1', event: 'pull_request' },
|
|
jobs: [
|
|
{
|
|
name: 'test (ubuntu-latest, 24, shard 1/3)',
|
|
started_at: '2026-08-29T00:00:00Z',
|
|
completed_at: '2026-08-29T00:05:00Z',
|
|
},
|
|
{
|
|
name: 'test (ubuntu-latest, 24, shard 2/3)',
|
|
started_at: '2026-08-29T00:00:00Z',
|
|
completed_at: null,
|
|
},
|
|
{
|
|
name: 'lint-tests',
|
|
started_at: '2026-08-29T00:00:00Z',
|
|
completed_at: '2026-08-29T00:01:00Z',
|
|
},
|
|
],
|
|
},
|
|
{
|
|
run: { id: 2, head_sha: 'sha2', event: 'push' },
|
|
jobs: [
|
|
{
|
|
name: 'test (ubuntu-latest, 24, shard 3/3)',
|
|
started_at: '2026-08-29T00:00:00Z',
|
|
completed_at: '2026-08-29T00:07:00Z',
|
|
},
|
|
],
|
|
},
|
|
];
|
|
|
|
const result = buildReportLines(runs, { workflowFile: 'test.yml', workflowYamlText, covered: null });
|
|
|
|
assert.equal(result.length, 2);
|
|
const names = result.map((r) => r.jobName).sort();
|
|
assert.deepEqual(names, [
|
|
'test (ubuntu-latest, 24, shard 1/3)',
|
|
'test (ubuntu-latest, 24, shard 3/3)',
|
|
]);
|
|
for (const name of names) {
|
|
assert.equal(names.filter((n) => n === name).length, 1);
|
|
}
|
|
});
|
|
});
|