* feat(#4036): persist CI shard/job timeout-vs-cap trending, warn at 90% Adds two new mechanisms plus an audit-coverage extension: - scripts/lib/ci-job-timing.cjs: shared elapsed-vs-cap arithmetic - scripts/ci-check-job-near-cap.cjs: in-job advisory near-cap check, wired into test/test-full/mutate/smoke as each job's last step - scripts/ci-timeout-report.cjs + .github/workflows/ci-timeout-report.yml: scheduled REST-API poll that appends new records to tests/ci-timeout-budget-history.jsonl and opens a small data-only PR - tests/ci-test-job-timeout-budget.test.cjs: extended to cover mutate (mutation.yml) and smoke (install-smoke.yml), which previously had no headroom-factor gate coverage at all Does not change any timeout-minutes value, shard composition, or shard-1 contents — those stay maintainer policy calls per the issue's own scope. * fix(#4036): address two-orthogonal-review findings - Parity tests guarding the two hand-duplicated literals this design cannot single-source through GH Actions YAML: CI_JOB_TIMEOUT_MINUTES vs each job's own timeout-minutes, and ci-timeout-report.cjs's JOB_RULES name-prefixes vs each job's actual name: template. - Thread run.event through as runEvent on every persisted record, so PR-context and push-context install-smoke timings (genuinely different matrix shape) are distinguishable in the history rather than silently conflated under one job name. - Replace the Windows near-cap start-time step's ambiguous PowerShell +/>> precedence with GitHub's documented string-interpolation form. - Move github.run_id out of direct ${{ }} shell interpolation into an env: var in the new scheduled workflow, per this repo's own expression-injection-safe convention. * test(#4036): regenerate golden install-tree fixtures for scripts/lib/ci-job-timing.cjs npm run gen:install-tree — scripts/ ships wholesale into the installed package (per ADR/known-defect precedent from #4012's own PR history: a new scripts/lib/*.cjs file needs its golden entry regenerated or every runtime's install-tree test fails). Confirmed via gsd-test: this was the sole cause of the first real verification run's 25 failures (all in tests/golden-install-tree.test.cjs, one per runtime). Top-level scripts/*.cjs files (ci-check-job-near-cap.cjs, ci-timeout-report.cjs) are not individually tracked in these fixtures — consistent with every other existing top-level scripts/*.cjs file, so no entry was expected or added for those two. * fix(#4036): register new lib file with installer, fix H1 shell policy - bin/install.js: add ci-job-timing.cjs to GSD_SCRIPTS_LIB_FILES (a hand-maintained registry, not generated — tests/install.test.cjs asserts every scripts/lib/ file is enumerated here) - test.yml: replace the two OS-conditional "Record job start time" step pairs (test + test-full jobs) with a single unconditional `node -e` step. The prior pair's Windows variant declared an explicit shell: pwsh, which scripts/workflow-policy.cjs's H1 checker statically flags against every OS a job's matrix can realize, independent of the step's own if: gate. A single Node one-liner needs no shell override at all — it's syntactically valid and behaves identically under bash, zsh, and pwsh — which is both H1 compliant and removes the last OS-specific shell syntax from this change entirely. Both defects were found by a real gsd-test run, not local gates — lint:ci and build:lib were clean throughout because neither the scripts/lib/ install-manifest parity check nor the H1 shell-policy baseline runs as part of lint:ci; both are gsd-test-only suites. * docs(#4036): how-to for reading CI timeout budget signals The phase-gate docs check correctly flagged the enablement sequence as 3 real steps (read the near-cap warning, find the accumulated trend file, pick the right maintainer lever) — a reference table can't carry a sequence. Adds docs/how-to/read-ci-timeout-signals.md, indexed from docs/README.md. * chore(#4036): backfill changeset PR number (4043) --------- Co-authored-by: sim <sim@local>
This commit is contained in:
Tom Boucher
49
scripts/ci-check-job-near-cap.cjs
Normal file
49
scripts/ci-check-job-near-cap.cjs
Normal file
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const { computeElapsedPct, isNearCap, formatNearCapNotice } = require('./lib/ci-job-timing.cjs');
|
||||
|
||||
function run(env = process.env, nowMs = Date.now(), appendFileSync = fs.appendFileSync) {
|
||||
const label = env.CI_JOB_LABEL;
|
||||
const startEpochMs = Number(env.CI_JOB_START_EPOCH_MS);
|
||||
const timeoutMinutes = Number(env.CI_JOB_TIMEOUT_MINUTES);
|
||||
|
||||
if (!label || !Number.isFinite(startEpochMs) || !Number.isFinite(timeoutMinutes) || timeoutMinutes <= 0) {
|
||||
console.error(
|
||||
'ci-check-job-near-cap: missing or invalid CI_JOB_LABEL / CI_JOB_START_EPOCH_MS / '
|
||||
+ 'CI_JOB_TIMEOUT_MINUTES — skipping near-cap check (advisory only, not a failure).',
|
||||
);
|
||||
return { skipped: true };
|
||||
}
|
||||
|
||||
const { elapsedMs, capMs, pct } = computeElapsedPct({
|
||||
startedAt: new Date(startEpochMs).toISOString(),
|
||||
completedAt: new Date(nowMs).toISOString(),
|
||||
timeoutMinutes,
|
||||
});
|
||||
|
||||
if (!isNearCap(pct)) {
|
||||
return { skipped: false, nearCap: false, pct };
|
||||
}
|
||||
|
||||
const { warningLine, summaryMarkdown } = formatNearCapNotice({ label, pct, elapsedMs, capMs });
|
||||
console.log(warningLine);
|
||||
|
||||
if (env.GITHUB_STEP_SUMMARY) {
|
||||
try {
|
||||
appendFileSync(env.GITHUB_STEP_SUMMARY, `${summaryMarkdown}\n`);
|
||||
} catch (err) {
|
||||
console.error(`ci-check-job-near-cap: could not write GITHUB_STEP_SUMMARY: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { skipped: false, nearCap: true, pct };
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
|
||||
if (require.main === module) {
|
||||
run();
|
||||
process.exitCode = 0;
|
||||
}
|
||||
230
scripts/ci-timeout-report.cjs
Normal file
230
scripts/ci-timeout-report.cjs
Normal file
@@ -0,0 +1,230 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* scripts/ci-timeout-report.cjs
|
||||
*
|
||||
* Scheduled CI-timeout trending report (#4036). Polls GitHub's Actions REST
|
||||
* API for recently completed jobs across test.yml, mutation.yml, and
|
||||
* install-smoke.yml, resolves each job's declared `timeout-minutes` budget,
|
||||
* computes elapsed-vs-cap via scripts/lib/ci-job-timing.cjs, and appends
|
||||
* new (never-before-seen) records to a JSONL history file. Every record also
|
||||
* carries the triggering `runEvent` (e.g. `push`/`pull_request`) so entries
|
||||
* for jobs whose matrix genuinely differs by trigger (e.g. `smoke`'s
|
||||
* push-only macOS row) can be told apart in the persisted trend — records
|
||||
* are never filtered by event, only labeled.
|
||||
*
|
||||
* Invoked from a GitHub Actions workflow via actions/github-script, e.g.:
|
||||
* const report = require(`${process.env.GITHUB_WORKSPACE}/scripts/ci-timeout-report.cjs`);
|
||||
* const result = await report.main({ github, context, core });
|
||||
*/
|
||||
|
||||
const yaml = require('js-yaml');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const {
|
||||
computeElapsedPct, isNearCap, formatNearCapNotice,
|
||||
} = require('./lib/ci-job-timing.cjs');
|
||||
|
||||
const HISTORY_PATH = path.join(__dirname, '..', 'tests', 'ci-timeout-budget-history.jsonl');
|
||||
const WORKFLOWS_DIR = path.join(__dirname, '..', '.github', 'workflows');
|
||||
|
||||
// Static job name → job-id rules, first match wins, checked in array order
|
||||
// (test-inert before test, since both job names start with "test ").
|
||||
const JOB_RULES = [
|
||||
{ workflowFile: 'test.yml', jobKey: 'test-inert', test: (name) => name === 'test (inert CI)' },
|
||||
{ workflowFile: 'test.yml', jobKey: 'test', test: (name) => name.startsWith('test (') && name !== 'test (inert CI)' },
|
||||
{ workflowFile: 'test.yml', jobKey: 'test-full', test: (name) => name.startsWith('full test (') },
|
||||
{ workflowFile: 'test.yml', jobKey: 'coverage-gate', test: (name) => name === 'Coverage gate (merged shards)' },
|
||||
{ workflowFile: 'install-smoke.yml', jobKey: 'smoke', test: (name) => name.startsWith('smoke (') },
|
||||
];
|
||||
|
||||
function resolveJobTimeoutMinutes({ jobName, workflowFile, workflowYamlText, covered }) {
|
||||
if (workflowFile === 'mutation.yml') {
|
||||
const m = jobName.match(/^Stryker \(([^)]+)\)$/);
|
||||
if (!m) return null;
|
||||
const moduleName = m[1];
|
||||
if (!covered || !Object.prototype.hasOwnProperty.call(covered, moduleName)) return null;
|
||||
return covered[moduleName].timeoutMinutes || 15;
|
||||
}
|
||||
|
||||
const rule = JOB_RULES.find((r) => r.workflowFile === workflowFile && r.test(jobName));
|
||||
if (!rule) return null;
|
||||
|
||||
const doc = yaml.load(workflowYamlText);
|
||||
const budget = doc && doc.jobs && doc.jobs[rule.jobKey] ? doc.jobs[rule.jobKey]['timeout-minutes'] : undefined;
|
||||
return typeof budget === 'number' ? budget : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{job: object, workflowFile: string, workflowYamlText: ?string, covered: ?object}} args
|
||||
* `job.runEvent` is the triggering event (e.g. `push`/`pull_request`) — carried through to
|
||||
* the returned record so entries whose matrix genuinely differs by trigger (e.g. `smoke`'s
|
||||
* push-only macOS row) can be distinguished in the persisted history.
|
||||
*/
|
||||
function parseJobRecord({ job, workflowFile, workflowYamlText, covered }) {
|
||||
if (!job.completed_at) return null;
|
||||
|
||||
const timeoutMinutes = resolveJobTimeoutMinutes({ jobName: job.name, workflowFile, workflowYamlText, covered });
|
||||
if (timeoutMinutes == null) return null;
|
||||
|
||||
const { elapsedMs, pct } = computeElapsedPct({
|
||||
startedAt: job.started_at, completedAt: job.completed_at, timeoutMinutes,
|
||||
});
|
||||
|
||||
return {
|
||||
runId: job.run_id,
|
||||
jobName: job.name,
|
||||
workflowFile,
|
||||
sha: job.head_sha,
|
||||
runEvent: job.runEvent,
|
||||
completedAt: job.completed_at,
|
||||
elapsedMs,
|
||||
timeoutMinutes,
|
||||
pct,
|
||||
};
|
||||
}
|
||||
|
||||
function buildReportLines(runs, { workflowFile, workflowYamlText, covered }) {
|
||||
const records = [];
|
||||
for (const { run, jobs } of runs) {
|
||||
for (const job of jobs) {
|
||||
const rec = parseJobRecord({
|
||||
job: {
|
||||
...job, run_id: run.id, head_sha: run.head_sha, runEvent: run.event,
|
||||
},
|
||||
workflowFile,
|
||||
workflowYamlText,
|
||||
covered,
|
||||
});
|
||||
if (rec) records.push(rec);
|
||||
}
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
function dedupeAgainstHistory(newRecords, historyText) {
|
||||
const seen = new Set();
|
||||
for (const line of String(historyText || '').split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const rec = JSON.parse(line);
|
||||
seen.add(`${rec.runId}::${rec.jobName}`);
|
||||
} catch {
|
||||
// Malformed history line — skip it rather than crash the whole report.
|
||||
}
|
||||
}
|
||||
return newRecords.filter((r) => !seen.has(`${r.runId}::${r.jobName}`));
|
||||
}
|
||||
|
||||
function formatHistoryLine(record) {
|
||||
return `${JSON.stringify(record)}\n`;
|
||||
}
|
||||
|
||||
const WORKFLOW_FILES = ['test.yml', 'mutation.yml', 'install-smoke.yml'];
|
||||
const MAX_RUNS_PER_WORKFLOW = 15;
|
||||
|
||||
/**
|
||||
* Orchestration entry point — impure, invoked from actions/github-script.
|
||||
*
|
||||
* @param {{github: object, context: object, core: object, historyPath?: string, fs?: object}} args
|
||||
* @returns {Promise<{added: number, nearCap: number}>}
|
||||
*/
|
||||
async function main({
|
||||
github, context, core, historyPath = HISTORY_PATH, fs: fsImpl = fs,
|
||||
}) {
|
||||
const { owner, repo } = context.repo;
|
||||
const mutationMatrix = require('./mutation-matrix.cjs');
|
||||
|
||||
const allNewRecords = [];
|
||||
|
||||
for (const workflowFile of WORKFLOW_FILES) {
|
||||
const covered = workflowFile === 'mutation.yml' ? mutationMatrix.COVERED : null;
|
||||
const workflowYamlText = workflowFile === 'mutation.yml'
|
||||
? null
|
||||
: fsImpl.readFileSync(path.join(WORKFLOWS_DIR, workflowFile), 'utf8');
|
||||
|
||||
let runsList;
|
||||
try {
|
||||
runsList = await github.paginate(github.rest.actions.listWorkflowRuns, {
|
||||
owner,
|
||||
repo,
|
||||
workflow_id: workflowFile,
|
||||
status: 'completed',
|
||||
per_page: 30,
|
||||
});
|
||||
} catch (err) {
|
||||
core.warning(`ci-timeout-report: failed to list runs for ${workflowFile}: ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const runs = runsList.slice(0, MAX_RUNS_PER_WORKFLOW);
|
||||
const runsWithJobs = [];
|
||||
|
||||
for (const run of runs) {
|
||||
try {
|
||||
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||
owner,
|
||||
repo,
|
||||
run_id: run.id,
|
||||
per_page: 50,
|
||||
});
|
||||
runsWithJobs.push({ run, jobs });
|
||||
} catch (err) {
|
||||
core.warning(`ci-timeout-report: failed to list jobs for ${workflowFile} run ${run.id}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const records = buildReportLines(runsWithJobs, { workflowFile, workflowYamlText, covered });
|
||||
allNewRecords.push(...records);
|
||||
}
|
||||
|
||||
let historyText = '';
|
||||
try {
|
||||
historyText = fsImpl.readFileSync(historyPath, 'utf8');
|
||||
} catch {
|
||||
// First run — history file does not exist yet, treat as empty.
|
||||
historyText = '';
|
||||
}
|
||||
|
||||
const deduped = dedupeAgainstHistory(allNewRecords, historyText);
|
||||
|
||||
if (deduped.length > 0) {
|
||||
const newLines = deduped.map(formatHistoryLine).join('');
|
||||
fsImpl.appendFileSync(historyPath, newLines);
|
||||
}
|
||||
// First-run bootstrap when there is nothing new to append is handled by
|
||||
// `git add` picking up whatever the history file already contains.
|
||||
|
||||
let nearCapCount = 0;
|
||||
for (const record of deduped) {
|
||||
if (!isNearCap(record.pct)) continue;
|
||||
nearCapCount += 1;
|
||||
|
||||
const notice = formatNearCapNotice({
|
||||
label: `${record.jobName} (run ${record.runId})`,
|
||||
pct: record.pct,
|
||||
elapsedMs: record.elapsedMs,
|
||||
capMs: record.timeoutMinutes * 60000,
|
||||
});
|
||||
|
||||
core.warning(notice.warningLine.replace(/^::warning title=CI budget::/, ''));
|
||||
|
||||
if (core.summary) {
|
||||
core.summary.addRaw(`${notice.summaryMarkdown}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
return { added: deduped.length, nearCap: nearCapCount };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
HISTORY_PATH,
|
||||
WORKFLOWS_DIR,
|
||||
JOB_RULES,
|
||||
resolveJobTimeoutMinutes,
|
||||
parseJobRecord,
|
||||
buildReportLines,
|
||||
dedupeAgainstHistory,
|
||||
formatHistoryLine,
|
||||
main,
|
||||
};
|
||||
72
scripts/lib/ci-job-timing.cjs
Normal file
72
scripts/lib/ci-job-timing.cjs
Normal file
@@ -0,0 +1,72 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* scripts/lib/ci-job-timing.cjs
|
||||
*
|
||||
* Pure budget-percentage arithmetic shared by the in-job near-cap check
|
||||
* (scripts/ci-check-job-near-cap.cjs) and the scheduled trending report
|
||||
* (scripts/ci-timeout-report.cjs). See #4036.
|
||||
*/
|
||||
|
||||
/** A job at or above this fraction of its `timeout-minutes` cap is "near-cap". */
|
||||
const THRESHOLD_PCT = 0.9;
|
||||
|
||||
/**
|
||||
* @param {{startedAt: string, completedAt: string, timeoutMinutes: number}} args
|
||||
* @returns {{elapsedMs: number, capMs: number, pct: number}}
|
||||
*/
|
||||
function computeElapsedPct({ startedAt, completedAt, timeoutMinutes }) {
|
||||
if (!Number.isFinite(timeoutMinutes) || timeoutMinutes <= 0) {
|
||||
throw new Error(`timeoutMinutes must be a positive finite number, got ${timeoutMinutes}`);
|
||||
}
|
||||
|
||||
const startMs = new Date(startedAt).getTime();
|
||||
const endMs = new Date(completedAt).getTime();
|
||||
|
||||
if (!Number.isFinite(startMs)) {
|
||||
throw new Error(`startedAt is not a valid timestamp: ${startedAt}`);
|
||||
}
|
||||
if (!Number.isFinite(endMs)) {
|
||||
throw new Error(`completedAt is not a valid timestamp: ${completedAt}`);
|
||||
}
|
||||
|
||||
const elapsedMs = endMs - startMs;
|
||||
if (elapsedMs < 0) {
|
||||
throw new Error(`completedAt (${completedAt}) is before startedAt (${startedAt})`);
|
||||
}
|
||||
|
||||
const capMs = timeoutMinutes * 60000;
|
||||
return { elapsedMs, capMs, pct: elapsedMs / capMs };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number} pct
|
||||
* @param {number} [threshold]
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isNearCap(pct, threshold = THRESHOLD_PCT) {
|
||||
return pct >= threshold;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{label: string, pct: number, elapsedMs: number, capMs: number}} args
|
||||
* @returns {{warningLine: string, summaryMarkdown: string}}
|
||||
*/
|
||||
function formatNearCapNotice({ label, pct, elapsedMs, capMs }) {
|
||||
const pctStr = `${Math.round(pct * 100)}%`;
|
||||
const elapsedMin = (elapsedMs / 60000).toFixed(1);
|
||||
const capMin = (capMs / 60000).toFixed(1);
|
||||
const detail = `${label} at ${pctStr} of its ${capMin}m cap (${elapsedMin}m elapsed)`;
|
||||
|
||||
return {
|
||||
warningLine: `::warning title=CI budget::${detail}`,
|
||||
summaryMarkdown: `- **${label}** — ${pctStr} of ${capMin}m cap (${elapsedMin}m elapsed)`,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
THRESHOLD_PCT,
|
||||
computeElapsedPct,
|
||||
isNearCap,
|
||||
formatNearCapNotice,
|
||||
};
|
||||
Reference in New Issue
Block a user