'use strict'; /** * tests/mutation-score-ratchet.test.cjs * * Regression net for scripts/check-mutation-score-ratchet.cjs (#3881 * follow-up, mutation-matrix piece 3: "the floor must ratchet up, not sit * where it is forever"). Drives the pure `evaluateRatchet` against boundary * inputs, then proves the CLI end-to-end against a synthetic Stryker `json` * reporter fixture: FAILS when the achieved score clears the floor by more * than the documented slack, PASSES when raised (or when within slack). */ const { test, describe } = require('node:test'); const assert = require('node:assert/strict'); const path = require('node:path'); const fs = require('node:fs'); const os = require('node:os'); const { RATCHET_SLACK, evaluateRatchet, extractAchievedScore, } = require('../scripts/check-mutation-score-ratchet.cjs'); const { runNode } = require('./helpers/process-seam.cjs'); const { createTempDir, cleanup } = require('./helpers.cjs'); const { PROBE_TIMEOUT_MS } = require('./helpers/timeouts.cjs'); const SCRIPT = path.resolve(__dirname, '..', 'scripts', 'check-mutation-score-ratchet.cjs'); const REPO_ROOT = path.resolve(__dirname, '..'); describe('evaluateRatchet: pure boundary behaviour', () => { test(`achieved exactly floor + ${RATCHET_SLACK} does NOT ratchet (boundary, inclusive)`, () => { const { shouldRatchet } = evaluateRatchet(65 + RATCHET_SLACK, 65); assert.equal(shouldRatchet, false); }); test(`achieved floor + ${RATCHET_SLACK} + 0.01 DOES ratchet (just past the boundary)`, () => { const { shouldRatchet, suggestedFloor } = evaluateRatchet(65 + RATCHET_SLACK + 0.01, 65); assert.equal(shouldRatchet, true); assert.equal(suggestedFloor, Math.floor(65 + RATCHET_SLACK + 0.01) - 1); }); test('achieved below floor does NOT ratchet (a floor breach is Stryker\'s own MUTATION_BREAK problem, not this script\'s)', () => { const { shouldRatchet } = evaluateRatchet(40, 65); assert.equal(shouldRatchet, false); }); test('achieved equal to floor does NOT ratchet', () => { const { shouldRatchet } = evaluateRatchet(65, 65); assert.equal(shouldRatchet, false); }); test('suggestedFloor follows floor(achieved) - 1 exactly (this file\'s own documented convention)', () => { const { suggestedFloor } = evaluateRatchet(92.7, 60); assert.equal(suggestedFloor, 91); }); }); describe('extractAchievedScore: Stryker json-reporter document', () => { test('computes the same score Stryker itself would report for a mixed killed/survived set', () => { const report = { schemaVersion: '1.0', thresholds: { high: 80, low: 60 }, files: { 'foo.js': { language: 'javascript', source: 'x', mutants: [ { id: '1', mutatorName: 'a', status: 'Killed', location: { start: { line: 1, column: 1 }, end: { line: 1, column: 2 } } }, { id: '2', mutatorName: 'a', status: 'Killed', location: { start: { line: 1, column: 1 }, end: { line: 1, column: 2 } } }, { id: '3', mutatorName: 'a', status: 'Killed', location: { start: { line: 1, column: 1 }, end: { line: 1, column: 2 } } }, { id: '4', mutatorName: 'a', status: 'Survived', location: { start: { line: 1, column: 1 }, end: { line: 1, column: 2 } } }, ], }, }, }; assert.equal(extractAchievedScore(report), 75); }); test('throws when the report has no scoreable mutants (empty files map — a wiring bug, never a 0% score)', () => { assert.throws(() => extractAchievedScore({ schemaVersion: '1.0', thresholds: {}, files: {} }), /no scoreable mutants/); }); // #3915: coverageAnalysis flips 'off' -> 'perTest' for the tap-runner swap, which is // what makes NoCoverage a status Stryker can now actually emit for these shards (the // command runner's 'off' analysis never produced it). extractAchievedScore must already // treat NoCoverage exactly like Survived in the denominator, or every shard's floor // silently becomes easier to clear the moment mutants start reporting NoCoverage instead // of Survived. function buildCountReport({ Killed = 0, Survived = 0, NoCoverage = 0 } = {}) { const mutants = []; let id = 0; const push = (status, n) => { for (let i = 0; i < n; i += 1) { mutants.push({ id: String(id++), mutatorName: 'a', status, location: { start: { line: 1, column: 1 }, end: { line: 1, column: 2 } }, }); } }; push('Killed', Killed); push('Survived', Survived); push('NoCoverage', NoCoverage); return { schemaVersion: '1.0', thresholds: { high: 80, low: 60 }, files: { 'foo.js': { language: 'javascript', source: 'x', mutants } }, }; } test('8 Killed + 2 Survived → 80', () => { assert.strictEqual(extractAchievedScore(buildCountReport({ Killed: 8, Survived: 2 })), 80); }); // NoCoverage counts in the mutation-score denominator exactly as Survived does, which is // what makes the #3915 coverageAnalysis 'off'->'perTest' switch score-neutral; Stryker's // own break threshold reads this same field (mutation-test-report-helper.js). test('8 Killed + 2 NoCoverage → 80, NOT 100', () => { assert.strictEqual(extractAchievedScore(buildCountReport({ Killed: 8, NoCoverage: 2 })), 80); }); // Non-vacuity guard (Goodhart guard): proves the previous test is not a tautology by // showing mutationScore and mutationScoreBasedOnCoveredCode genuinely diverge on the same // report. This fails the moment someone "improves" extractAchievedScore to read the // better-sounding covered-code field, which would make every floor trivially satisfiable. test('non-vacuity: mutationScoreBasedOnCoveredCode (100) diverges from extractAchievedScore (80) on the same report', () => { const { calculateMutationTestMetrics } = require('mutation-testing-metrics'); const report = buildCountReport({ Killed: 8, NoCoverage: 2 }); const metrics = calculateMutationTestMetrics(report); assert.strictEqual(metrics.systemUnderTestMetrics.metrics.mutationScoreBasedOnCoveredCode, 100); assert.strictEqual(extractAchievedScore(report), 80); }); test('0 Killed + 10 NoCoverage → 0 (a wholly-uncovered shard scores zero, not 100)', () => { assert.strictEqual(extractAchievedScore(buildCountReport({ NoCoverage: 10 })), 0); }); }); // ── CLI end-to-end: fail on a planted over-floor score, pass when the floor is raised ─── // SYNTHETIC_FLOOR is a fixture value this test file owns outright — never a real module's // minScore from scripts/mutation-matrix.cjs. That real config ratchets UP as modules improve // (exactly what this script's own CLI enforces), so a row that hardcodes a real floor breaks // every time the ratchet does its job (see config-schema: 52 -> 74 by commit 973321541, which // broke this file's previous PLANTED/RESTORED rows). Building a synthetic `--matrix` fixture // with a floor this test controls makes the rows indifferent to any real module's floor moving. const SYNTHETIC_MODULE = 'synthetic-ratchet-fixture'; const SYNTHETIC_FLOOR = 65; function withMatrixFixture(floor, fn) { const dir = createTempDir('mutation-score-ratchet-matrix-'); const matrixPath = path.join(dir, 'fixture-mutation-matrix.cjs'); fs.writeFileSync( matrixPath, `'use strict';\nmodule.exports = { COVERED: { '${SYNTHETIC_MODULE}': { minScore: ${floor} } } };\n`, 'utf8' ); try { return fn(matrixPath); } finally { cleanup(dir); } } function withReportFixture(mutationScore, fn) { // Build a Stryker json-reporter-shaped document whose achieved score is EXACTLY // `mutationScore` via N killed + (100 - N) survived out of 100 mutants — avoids floating // point ambiguity in the fixture itself. const killed = Math.round(mutationScore); const mutants = []; for (let i = 0; i < 100; i++) { mutants.push({ id: String(i), mutatorName: 'a', status: i < killed ? 'Killed' : 'Survived', location: { start: { line: 1, column: 1 }, end: { line: 1, column: 2 } }, }); } const report = { schemaVersion: '1.0', thresholds: { high: 80, low: 60 }, files: { 'foo.js': { language: 'javascript', source: 'x', mutants } }, }; const dir = createTempDir('mutation-score-ratchet-test-'); const reportPath = path.join(dir, 'mutation.json'); fs.writeFileSync(reportPath, JSON.stringify(report), 'utf8'); try { return fn(reportPath); } finally { cleanup(dir); } } describe('check-mutation-score-ratchet CLI: end-to-end fail-then-pass', () => { test(`PLANTED: achieved score far above a synthetic module's floor (${SYNTHETIC_FLOOR}) FAILS the ratchet`, () => { withMatrixFixture(SYNTHETIC_FLOOR, (matrixPath) => { withReportFixture(SYNTHETIC_FLOOR + RATCHET_SLACK + 10, (reportPath) => { const result = runNode( [SCRIPT, '--module', SYNTHETIC_MODULE, '--report', reportPath, '--matrix', matrixPath], { cwd: REPO_ROOT, timeoutMs: PROBE_TIMEOUT_MS } ); assert.notEqual(result.exitCode, 0, `expected nonzero exit; stderr: ${result.stderr}`); assert.match(result.stderr, /raise the floor/); assert.match(result.stderr, new RegExp(SYNTHETIC_MODULE)); }); }); }); test('RESTORED: an achieved score within slack of the same synthetic floor PASSES', () => { withMatrixFixture(SYNTHETIC_FLOOR, (matrixPath) => { withReportFixture(SYNTHETIC_FLOOR + 1, (reportPath) => { const result = runNode( [SCRIPT, '--module', SYNTHETIC_MODULE, '--report', reportPath, '--matrix', matrixPath], { cwd: REPO_ROOT, timeoutMs: PROBE_TIMEOUT_MS } ); assert.equal(result.exitCode, 0, `expected exit 0; stderr: ${result.stderr}`); assert.match(result.stdout, /ok mutation-score-ratchet/); }); }); }); test('an unknown --module exits nonzero with a clear message', () => { withMatrixFixture(SYNTHETIC_FLOOR, (matrixPath) => { withReportFixture(90, (reportPath) => { const result = runNode( [SCRIPT, '--module', 'does-not-exist', '--report', reportPath, '--matrix', matrixPath], { cwd: REPO_ROOT, timeoutMs: PROBE_TIMEOUT_MS } ); assert.notEqual(result.exitCode, 0); assert.match(result.stderr, /unknown module/); }); }); }); test('a missing report file exits nonzero with a clear message', () => { withMatrixFixture(SYNTHETIC_FLOOR, (matrixPath) => { const result = runNode( [SCRIPT, '--module', SYNTHETIC_MODULE, '--report', path.join(os.tmpdir(), 'does-not-exist-mutation.json'), '--matrix', matrixPath], { cwd: REPO_ROOT, timeoutMs: PROBE_TIMEOUT_MS } ); assert.notEqual(result.exitCode, 0); assert.match(result.stderr, /report not found/); }); }); });