cec7e704d6
`expandRunsOn` enumerated a multi-axis `strategy.matrix` one key at a time,
producing partial realization contexts. A true `os × shell` matrix therefore
left `${{ matrix.shell }}` unresolvable against any `{ os: ... }`-only context,
firing spurious `UNRESOLVABLE_MATRIX` violations and leaving shell-pinning
coverage incomplete on Cartesian jobs.
Enumerate the full GitHub Actions cross-product of all base-list matrix keys
(every `matrix.<k>` array, excluding the `include`/`exclude` control keys) via
a named `cartesianProduct` helper. Each realization's context now carries a
value for every matrix key, so `${{ matrix.<key> }}` resolves per realization.
Single-axis matrices keep byte-for-byte identical output; only multi-axis
matrices change shape. The `include` and `exclude` blocks are unchanged
(full tuple-aware exclude is a documented out-of-scope follow-up).
Tests: updated the `os × shell` test to assert post-fix behavior (4 step
realizations, 2 WRONG_SHELL_FOR_OS, 0 UNRESOLVABLE_MATRIX); added a compliant
`os × node-version` cross-product test; added a fast-check property test that
the realization count equals the product of axis lengths and that no axis key
is dropped from any context.
Closes #435
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
739 lines
27 KiB
JavaScript
739 lines
27 KiB
JavaScript
'use strict';
|
||
|
||
const { test, describe } = require('node:test');
|
||
const assert = require('node:assert/strict');
|
||
const path = require('path');
|
||
|
||
const {
|
||
VIOLATION,
|
||
inspectWorkflow,
|
||
runPolicyLint,
|
||
} = require('../scripts/workflow-policy.cjs');
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 1 — Baseline: repo's current workflow files must yield ZERO violations
|
||
// (RED on origin/next; GREEN after YAML fixes are committed)
|
||
// ---------------------------------------------------------------------------
|
||
describe('baseline: repo workflows comply with H1 shell policy', () => {
|
||
test('runPolicyLint on .github/workflows produces zero violations', () => {
|
||
const workflowsDir = path.resolve(__dirname, '..', '.github', 'workflows');
|
||
const result = runPolicyLint({ workflowsDir });
|
||
|
||
if (result.violations.length > 0) {
|
||
const top10 = result.violations.slice(0, 10);
|
||
const msg = top10.map(v =>
|
||
` ${path.basename(v.filePath)}:${v.evidence.line} [${v.jobId}/${v.stepName}] runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`
|
||
).join('\n');
|
||
assert.fail(
|
||
`Expected 0 violations but found ${result.violations.length}. ` +
|
||
`Top violations (mechanism: each step on a macos-* or windows-* runner must use native shell):\n${msg}`
|
||
);
|
||
}
|
||
|
||
assert.strictEqual(
|
||
result.violations.length,
|
||
0,
|
||
'All workflow steps must comply with H1 shell policy'
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 2 — Positive synthetic: compliant workflow yields zero violations
|
||
// Three separate jobs, one per OS, each using H1-compliant shell configuration:
|
||
// ubuntu: no shell pin (runner default bash = policy bash)
|
||
// macos: job-level defaults.run.shell: zsh
|
||
// windows: no shell pin (runner default pwsh = policy pwsh)
|
||
// ---------------------------------------------------------------------------
|
||
describe('synthetic: fully-compliant per-OS jobs workflow', () => {
|
||
const COMPLIANT_YAML = `
|
||
name: Compliant Workflow
|
||
jobs:
|
||
linux-job:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: Run tests on ubuntu
|
||
run: npm test
|
||
macos-job:
|
||
runs-on: macos-latest
|
||
defaults:
|
||
run:
|
||
shell: zsh
|
||
steps:
|
||
- name: Run tests on macOS
|
||
run: npm test
|
||
windows-job:
|
||
runs-on: windows-latest
|
||
steps:
|
||
- name: Run tests on windows
|
||
run: npm test
|
||
`;
|
||
|
||
test('compliant per-OS workflow (ubuntu no pin, macos job-defaults zsh, windows no pin) produces zero violations', () => {
|
||
const result = inspectWorkflow(COMPLIANT_YAML, { filePath: '<synthetic-compliant>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
0,
|
||
'Compliant per-OS workflow must have zero violations. Got: ' +
|
||
violations.map(v => `${v.runner}/${v.violation}`).join(', ')
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 3 — Counter-test: macOS missing explicit zsh → MACOS_MISSING_EXPLICIT_ZSH
|
||
// (mechanism: macos-latest default is bash, not zsh; H1 requires explicit shell: zsh)
|
||
// ---------------------------------------------------------------------------
|
||
describe('counter-test: macOS step without explicit shell: zsh', () => {
|
||
const MACOS_NO_SHELL_YAML = `
|
||
name: macOS No Shell
|
||
jobs:
|
||
build:
|
||
runs-on: macos-latest
|
||
steps:
|
||
- name: Run tests
|
||
run: npm test
|
||
`;
|
||
|
||
test('macos-latest step with no shell pin produces exactly one MACOS_MISSING_EXPLICIT_ZSH violation', () => {
|
||
const result = inspectWorkflow(MACOS_NO_SHELL_YAML, { filePath: '<synthetic-macos-no-shell>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
1,
|
||
`Expected exactly 1 violation (MACOS_MISSING_EXPLICIT_ZSH on macos-latest) but got ${violations.length}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].violation,
|
||
VIOLATION.MACOS_MISSING_EXPLICIT_ZSH,
|
||
`Expected violation type MACOS_MISSING_EXPLICIT_ZSH but got ${violations[0].violation}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].runner,
|
||
'macos-latest',
|
||
`Expected violation runner to be macos-latest but got ${violations[0].runner}`
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 4 — Counter-test: wrong shell for OS → WRONG_SHELL_FOR_OS
|
||
// (mechanism: windows-2025 default is pwsh; specifying shell: bash is a policy violation)
|
||
// ---------------------------------------------------------------------------
|
||
describe('counter-test: windows step with explicit shell: bash', () => {
|
||
const WINDOWS_BASH_YAML = `
|
||
name: Windows Bash
|
||
jobs:
|
||
build:
|
||
runs-on: windows-2025
|
||
steps:
|
||
- name: Run tests with wrong shell
|
||
shell: bash
|
||
run: npm test
|
||
`;
|
||
|
||
test('windows-2025 step with shell: bash produces exactly one WRONG_SHELL_FOR_OS violation', () => {
|
||
const result = inspectWorkflow(WINDOWS_BASH_YAML, { filePath: '<synthetic-windows-bash>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
1,
|
||
`Expected exactly 1 violation (WRONG_SHELL_FOR_OS on windows-2025) but got ${violations.length}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].violation,
|
||
VIOLATION.WRONG_SHELL_FOR_OS,
|
||
`Expected violation type WRONG_SHELL_FOR_OS but got ${violations[0].violation}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].runner,
|
||
'windows-2025',
|
||
`Expected violation runner to be windows-2025 but got ${violations[0].runner}`
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 5 — Counter-test: matrix expansion with shell: bash on every step
|
||
// (mechanism: ubuntu realizations are compliant since bash IS policy for ubuntu;
|
||
// macos → WRONG_SHELL_FOR_OS (explicit wrong pin); windows → WRONG_SHELL_FOR_OS)
|
||
// Expected: 2 violations total (1 macos + 1 windows), zero for ubuntu
|
||
// Note: MACOS_MISSING_EXPLICIT_ZSH fires only when NO shell is set at any level;
|
||
// here shell: bash is explicit, so WRONG_SHELL_FOR_OS is the correct subtype.
|
||
// ---------------------------------------------------------------------------
|
||
describe('counter-test: three-OS matrix with shell: bash on every step', () => {
|
||
const ALL_BASH_MATRIX_YAML = `
|
||
name: All Bash Matrix
|
||
jobs:
|
||
build:
|
||
runs-on: \${{ matrix.os }}
|
||
strategy:
|
||
matrix:
|
||
os: [ubuntu-latest, macos-latest, windows-2025]
|
||
steps:
|
||
- name: Run tests
|
||
shell: bash
|
||
run: npm test
|
||
`;
|
||
|
||
test('three-OS matrix with shell: bash produces exactly 2 WRONG_SHELL_FOR_OS violations (macos + windows), zero for ubuntu', () => {
|
||
const result = inspectWorkflow(ALL_BASH_MATRIX_YAML, { filePath: '<synthetic-all-bash-matrix>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
2,
|
||
`Expected exactly 2 violations (macos-latest + windows-2025) but got ${violations.length}: ` +
|
||
violations.map(v => `${v.runner}/${v.violation}`).join(', ')
|
||
);
|
||
|
||
const macosViolation = violations.find(v => v.runner === 'macos-latest');
|
||
assert.ok(
|
||
macosViolation,
|
||
'Expected a violation for macos-latest realization'
|
||
);
|
||
// When an explicit shell: bash is set on the step, the violation is WRONG_SHELL_FOR_OS
|
||
// (the explicit pin is wrong for the OS). MACOS_MISSING_EXPLICIT_ZSH only fires when
|
||
// there is NO shell set at any level and the runner default (bash) is inherited silently.
|
||
assert.strictEqual(
|
||
macosViolation.violation,
|
||
VIOLATION.WRONG_SHELL_FOR_OS,
|
||
`macos-latest with explicit shell: bash should be WRONG_SHELL_FOR_OS (explicit wrong pin) but got ${macosViolation?.violation}`
|
||
);
|
||
|
||
const windowsViolation = violations.find(v => v.runner === 'windows-2025');
|
||
assert.ok(
|
||
windowsViolation,
|
||
'Expected a violation for windows-2025 realization'
|
||
);
|
||
assert.strictEqual(
|
||
windowsViolation.violation,
|
||
VIOLATION.WRONG_SHELL_FOR_OS,
|
||
`windows-2025 violation should be WRONG_SHELL_FOR_OS but got ${windowsViolation?.violation}`
|
||
);
|
||
|
||
const ubuntuViolations = violations.filter(v => v.runner === 'ubuntu-latest');
|
||
assert.strictEqual(
|
||
ubuntuViolations.length,
|
||
0,
|
||
`ubuntu-latest should produce zero violations (bash is both runner default and policy) but got ${ubuntuViolations.length}`
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 6 — Counter-test: unknown runner → UNKNOWN_RUNNER
|
||
// (mechanism: self-hosted is not in POLICY, so runner cannot be validated)
|
||
// ---------------------------------------------------------------------------
|
||
describe('counter-test: self-hosted runner produces UNKNOWN_RUNNER violation', () => {
|
||
const SELF_HOSTED_YAML = `
|
||
name: Self-Hosted
|
||
jobs:
|
||
build:
|
||
runs-on: self-hosted
|
||
steps:
|
||
- name: Run build
|
||
run: npm build
|
||
`;
|
||
|
||
test('self-hosted runner step produces exactly one UNKNOWN_RUNNER violation', () => {
|
||
const result = inspectWorkflow(SELF_HOSTED_YAML, { filePath: '<synthetic-self-hosted>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
1,
|
||
`Expected exactly 1 UNKNOWN_RUNNER violation but got ${violations.length}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].violation,
|
||
VIOLATION.UNKNOWN_RUNNER,
|
||
`Expected violation type UNKNOWN_RUNNER but got ${violations[0].violation}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].runner,
|
||
'self-hosted',
|
||
`Expected violation runner to be self-hosted but got ${violations[0].runner}`
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 7a — Positive: matrix.include with shell key + defaults.run.shell: ${{ matrix.shell }}
|
||
// (mechanism: each realization carries its own shell value; the linter resolves
|
||
// the matrix expression against the realization context before checking policy)
|
||
// ---------------------------------------------------------------------------
|
||
describe('matrix.shell: ${{ matrix.shell }} resolves per realization — zero violations', () => {
|
||
const MATRIX_SHELL_YAML = `
|
||
name: Matrix Shell Positive
|
||
jobs:
|
||
build:
|
||
runs-on: \${{ matrix.os }}
|
||
defaults:
|
||
run:
|
||
shell: \${{ matrix.shell }}
|
||
strategy:
|
||
matrix:
|
||
include:
|
||
- os: macos-latest
|
||
shell: zsh
|
||
- os: windows-2025
|
||
shell: pwsh
|
||
steps:
|
||
- name: Run tests
|
||
run: npm test
|
||
`;
|
||
|
||
test('matrix.include with shell:zsh for macOS + shell:pwsh for Windows + defaults.run.shell: ${{ matrix.shell }} yields zero violations', () => {
|
||
const result = inspectWorkflow(MATRIX_SHELL_YAML, { filePath: '<synthetic-matrix-shell-positive>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
0,
|
||
'matrix.shell resolved per realization must produce zero violations. Got: ' +
|
||
violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ')
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 7b — Counter-test: matrix.include row with wrong shell value
|
||
// (mechanism: if a row's shell value doesn't match its OS policy, WRONG_SHELL_FOR_OS fires)
|
||
// ---------------------------------------------------------------------------
|
||
describe('matrix.shell: ${{ matrix.shell }} with wrong value per row — WRONG_SHELL_FOR_OS', () => {
|
||
const MATRIX_SHELL_WRONG_YAML = `
|
||
name: Matrix Shell Wrong Row
|
||
jobs:
|
||
build:
|
||
runs-on: \${{ matrix.os }}
|
||
defaults:
|
||
run:
|
||
shell: \${{ matrix.shell }}
|
||
strategy:
|
||
matrix:
|
||
include:
|
||
- os: macos-latest
|
||
shell: bash
|
||
- os: windows-2025
|
||
shell: pwsh
|
||
steps:
|
||
- name: Run tests
|
||
run: npm test
|
||
`;
|
||
|
||
test('matrix.include row with shell:bash for macOS produces WRONG_SHELL_FOR_OS (bash is wrong for macOS)', () => {
|
||
const result = inspectWorkflow(MATRIX_SHELL_WRONG_YAML, { filePath: '<synthetic-matrix-shell-wrong>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
// macOS realization: shell resolves to bash → WRONG_SHELL_FOR_OS
|
||
// Windows realization: shell resolves to pwsh → compliant
|
||
assert.strictEqual(
|
||
violations.length,
|
||
1,
|
||
`Expected exactly 1 violation (macos-latest bash→WRONG_SHELL_FOR_OS) but got ${violations.length}: ` +
|
||
violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ')
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].runner,
|
||
'macos-latest',
|
||
`Expected violation for macos-latest but got ${violations[0].runner}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].violation,
|
||
VIOLATION.WRONG_SHELL_FOR_OS,
|
||
`Expected WRONG_SHELL_FOR_OS but got ${violations[0].violation}`
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 7c — Counter-test: matrix.include row missing the shell key while
|
||
// defaults.run.shell: ${{ matrix.shell }} references it → UNRESOLVABLE_MATRIX
|
||
// ---------------------------------------------------------------------------
|
||
describe('matrix.shell expression references missing key — UNRESOLVABLE_MATRIX', () => {
|
||
const MATRIX_SHELL_MISSING_KEY_YAML = `
|
||
name: Matrix Shell Missing Key
|
||
jobs:
|
||
build:
|
||
runs-on: \${{ matrix.os }}
|
||
defaults:
|
||
run:
|
||
shell: \${{ matrix.shell }}
|
||
strategy:
|
||
matrix:
|
||
include:
|
||
- os: ubuntu-latest
|
||
node-version: 24
|
||
steps:
|
||
- name: Run tests
|
||
run: npm test
|
||
`;
|
||
|
||
test('matrix.include row without shell key while defaults.run.shell: ${{ matrix.shell }} → UNRESOLVABLE_MATRIX', () => {
|
||
const result = inspectWorkflow(MATRIX_SHELL_MISSING_KEY_YAML, { filePath: '<synthetic-matrix-shell-missing-key>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
1,
|
||
`Expected exactly 1 UNRESOLVABLE_MATRIX violation but got ${violations.length}: ` +
|
||
violations.map(v => `runner=${v.runner} type=${v.violation}`).join(', ')
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].violation,
|
||
VIOLATION.UNRESOLVABLE_MATRIX,
|
||
`Expected UNRESOLVABLE_MATRIX but got ${violations[0].violation}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].runner,
|
||
'ubuntu-latest',
|
||
`Expected runner ubuntu-latest but got ${violations[0].runner}`
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 7 — Counter-test: workflow-level defaults.run.shell: zsh satisfies macOS H1
|
||
// (mechanism: resolution order puts workflow defaults above runner default;
|
||
// zsh at workflow level means macOS steps inherit it without step-level pin)
|
||
// ---------------------------------------------------------------------------
|
||
describe('counter-test: workflow-level defaults.run.shell: zsh satisfies macos-* H1', () => {
|
||
const WORKFLOW_DEFAULTS_ZSH_YAML = `
|
||
name: Workflow Defaults ZSH
|
||
defaults:
|
||
run:
|
||
shell: zsh
|
||
jobs:
|
||
build:
|
||
runs-on: macos-latest
|
||
steps:
|
||
- name: Run tests on macOS
|
||
run: npm test
|
||
`;
|
||
|
||
test('workflow-level shell: zsh + macos-latest + no step-level shell produces zero violations', () => {
|
||
const result = inspectWorkflow(WORKFLOW_DEFAULTS_ZSH_YAML, { filePath: '<synthetic-workflow-defaults-zsh>' });
|
||
|
||
assert.strictEqual(
|
||
result.workflowDefaultsShell,
|
||
'zsh',
|
||
`Expected workflowDefaultsShell to be zsh but got ${result.workflowDefaultsShell}`
|
||
);
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
0,
|
||
`Workflow-level shell: zsh must satisfy H1 for macos-latest steps (resolution-order rule). Got ${violations.length} violations: ` +
|
||
violations.map(v => `${v.violation}`).join(', ')
|
||
);
|
||
});
|
||
|
||
test('effective shell for macOS step is zsh when inherited from workflow defaults', () => {
|
||
const result = inspectWorkflow(WORKFLOW_DEFAULTS_ZSH_YAML, { filePath: '<synthetic-workflow-defaults-zsh>' });
|
||
|
||
const step = result.jobs[0]?.steps[0];
|
||
assert.ok(step, 'Expected at least one step');
|
||
|
||
assert.strictEqual(
|
||
step.effectiveShell,
|
||
'zsh',
|
||
`Expected effectiveShell to be zsh (inherited from workflow defaults) but got ${step.effectiveShell}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
step.stepShell,
|
||
null,
|
||
`Expected stepShell to be null (no step-level pin) but got ${step.stepShell}`
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 8a — Cartesian matrix os × shell — full cross-product expansion resolves
|
||
// matrix.shell per realization
|
||
// (mechanism: matrix.os: [macos-latest, macos-latest] with matrix.shell: [zsh, bash]
|
||
// and runs-on: ${{ matrix.os }}, step shell: ${{ matrix.shell }}.
|
||
// GitHub Actions realizes a 2×2 grid (4 jobs). After the Cartesian-product fix,
|
||
// expandRunsOn must produce 4 realizations, each carrying BOTH os AND shell in
|
||
// context so that ${{ matrix.shell }} resolves per realization.
|
||
//
|
||
// Expected post-fix behavior:
|
||
// - 4 total step-results (2 os × 2 shell)
|
||
// - exactly 2 WRONG_SHELL_FOR_OS violations — the two bash cells on macos-latest
|
||
// - ZERO UNRESOLVABLE_MATRIX violations (matrix.shell is now fully resolved)
|
||
// ---------------------------------------------------------------------------
|
||
describe('Cartesian matrix os × shell — full cross-product expansion resolves matrix.shell per realization', () => {
|
||
const CARTESIAN_MATRIX_YAML = `
|
||
name: Cartesian Matrix
|
||
jobs:
|
||
build:
|
||
runs-on: \${{ matrix.os }}
|
||
strategy:
|
||
matrix:
|
||
os: [macos-latest, macos-latest]
|
||
shell: [zsh, bash]
|
||
steps:
|
||
- name: Run tests
|
||
shell: \${{ matrix.shell }}
|
||
run: echo hi
|
||
`;
|
||
|
||
test('2×2 Cartesian product yields 4 step-results, 2 WRONG_SHELL_FOR_OS violations, 0 UNRESOLVABLE_MATRIX', () => {
|
||
const result = inspectWorkflow(CARTESIAN_MATRIX_YAML, { filePath: '<synthetic-cartesian-matrix>' });
|
||
|
||
const allSteps = result.jobs.flatMap(j => j.steps);
|
||
const violations = allSteps.filter(s => s.violation !== null);
|
||
const unresolvable = violations.filter(s => s.violation === VIOLATION.UNRESOLVABLE_MATRIX);
|
||
const wrongShell = violations.filter(s => s.violation === VIOLATION.WRONG_SHELL_FOR_OS);
|
||
|
||
// 4 step-results: 2 os values × 2 shell values = 4 realizations, each with 1 step
|
||
assert.strictEqual(
|
||
allSteps.length,
|
||
4,
|
||
`Expected 4 step-results (2×2 Cartesian product) but got ${allSteps.length}. All steps: ` +
|
||
allSteps.map(s => `runner=${s.runner} shell=${s.effectiveShell} violation=${s.violation}`).join(', ')
|
||
);
|
||
|
||
// Core regression proof: ZERO UNRESOLVABLE_MATRIX (matrix.shell now resolves)
|
||
assert.strictEqual(
|
||
unresolvable.length,
|
||
0,
|
||
`Expected 0 UNRESOLVABLE_MATRIX violations but got ${unresolvable.length}: ` +
|
||
unresolvable.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ')
|
||
);
|
||
|
||
// Exactly 2 WRONG_SHELL_FOR_OS: the two bash cells on macos-latest
|
||
assert.strictEqual(
|
||
wrongShell.length,
|
||
2,
|
||
`Expected exactly 2 WRONG_SHELL_FOR_OS violations (bash on macos-latest) but got ${wrongShell.length}: ` +
|
||
violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ')
|
||
);
|
||
|
||
for (const v of wrongShell) {
|
||
assert.strictEqual(
|
||
v.runner,
|
||
'macos-latest',
|
||
`Expected violation runner to be macos-latest but got ${v.runner}`
|
||
);
|
||
assert.strictEqual(
|
||
v.effectiveShell,
|
||
'bash',
|
||
`Expected effectiveShell to be bash (the violating cell) but got ${v.effectiveShell}`
|
||
);
|
||
}
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 8b — Cartesian matrix os × node-version — compliant cross-product carries
|
||
// extra axis without violations
|
||
// (mechanism: matrix.os: [ubuntu-latest, ubuntu-latest] with
|
||
// matrix.node-version: [22, 24], step shell: bash (literal).
|
||
// The cross-product yields 4 realizations. ubuntu-latest + bash = compliant.
|
||
// Pre-fix: only os is expanded → 2 step-results. Post-fix: 4 step-results.
|
||
// This is the fail-first signal for the Cartesian expansion fix.
|
||
// ---------------------------------------------------------------------------
|
||
describe('Cartesian matrix os × node-version — compliant cross-product yields 4 step-results with 0 violations', () => {
|
||
const CARTESIAN_COMPLIANT_YAML = `
|
||
name: Cartesian Compliant
|
||
jobs:
|
||
build:
|
||
runs-on: \${{ matrix.os }}
|
||
strategy:
|
||
matrix:
|
||
os: [ubuntu-latest, ubuntu-latest]
|
||
node-version: [22, 24]
|
||
steps:
|
||
- name: Run tests
|
||
shell: bash
|
||
run: npm test
|
||
`;
|
||
|
||
test('2×2 Cartesian product yields 4 step-results (fail-first signal pre-fix: 2) and 0 violations', () => {
|
||
const result = inspectWorkflow(CARTESIAN_COMPLIANT_YAML, { filePath: '<synthetic-cartesian-compliant>' });
|
||
|
||
const allSteps = result.jobs.flatMap(j => j.steps);
|
||
const violations = allSteps.filter(s => s.violation !== null);
|
||
|
||
// Pre-fix: only os is expanded → 2 step-results; post-fix: 4
|
||
assert.strictEqual(
|
||
allSteps.length,
|
||
4,
|
||
`Expected 4 step-results (2 os × 2 node-version Cartesian product) but got ${allSteps.length}. ` +
|
||
`If you see 2, the Cartesian expansion fix is not yet applied. All steps: ` +
|
||
allSteps.map(s => `runner=${s.runner} shell=${s.effectiveShell} violation=${s.violation}`).join(', ')
|
||
);
|
||
|
||
// ubuntu-latest + literal bash = compliant; no violations expected
|
||
assert.strictEqual(
|
||
violations.length,
|
||
0,
|
||
`Expected 0 violations (ubuntu-latest + bash is compliant) but got ${violations.length}: ` +
|
||
violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ')
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 8c — Property-based: for any 1–3 base-list axes with 1–3 values each,
|
||
// step count === Cartesian product of axis lengths
|
||
// (fast-check is a devDependency: fast-check ^4.8.0)
|
||
// ---------------------------------------------------------------------------
|
||
const fc = require('fast-check');
|
||
|
||
describe('property-based: step count equals product of all axis lengths for arbitrary base-list matrices', () => {
|
||
test('step count === product(axisLengths) for 1–3 axes with 1–3 values each', () => {
|
||
// Axis keys named k0, k1, k2; values restricted to [A-Za-z0-9-] to keep YAML well-formed.
|
||
// runs-on references ${{ matrix.k0 }}; the single step's shell references the
|
||
// LAST matrix axis key (${{ matrix.kLast }}) so that a dropped axis key would
|
||
// surface as UNRESOLVABLE_MATRIX rather than silently resolving.
|
||
const axisValueArb = fc.stringMatching(/^[A-Za-z][A-Za-z0-9-]{0,7}$/);
|
||
const axisArb = fc.array(axisValueArb, { minLength: 1, maxLength: 3 });
|
||
const matrixArb = fc.array(axisArb, { minLength: 1, maxLength: 3 });
|
||
|
||
fc.assert(
|
||
fc.property(matrixArb, (axes) => {
|
||
// Build YAML matrix block
|
||
const keys = axes.map((_, i) => `k${i}`);
|
||
const lastKey = keys[keys.length - 1];
|
||
const matrixLines = keys.map((k, i) => ` ${k}: [${axes[i].join(', ')}]`);
|
||
|
||
const yaml = [
|
||
'name: PropertyTest',
|
||
'jobs:',
|
||
' build:',
|
||
' runs-on: ${{ matrix.k0 }}',
|
||
' strategy:',
|
||
' matrix:',
|
||
...matrixLines,
|
||
' steps:',
|
||
' - name: Run tests',
|
||
` shell: \${{ matrix.${lastKey} }}`,
|
||
' run: echo hi',
|
||
].join('\n');
|
||
|
||
const result = inspectWorkflow(yaml, { filePath: '<property-test>' });
|
||
const allSteps = result.jobs.flatMap(j => j.steps);
|
||
const actualSteps = allSteps.length;
|
||
const expectedSteps = axes.reduce((acc, axis) => acc * axis.length, 1);
|
||
|
||
// Every realization must carry the last axis key in context so that
|
||
// ${{ matrix.kLast }} resolves. If the key is absent from any
|
||
// realization's context, effectiveShell fires UNRESOLVABLE_MATRIX.
|
||
const noUnresolvable = allSteps.every(
|
||
s => s.violation !== VIOLATION.UNRESOLVABLE_MATRIX,
|
||
);
|
||
|
||
return actualSteps === expectedSteps && noUnresolvable;
|
||
}),
|
||
{ seed: 42, numRuns: 100 }
|
||
);
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Test 8 — Counter-test: two macos-latest matrix.include rows where
|
||
// row 1 has shell: zsh (compliant) and row 2 has shell: bash (violation).
|
||
// Guards against the dedup bug where runner-label-only deduplication would
|
||
// collapse both rows into one, hiding the second row's policy violation.
|
||
// Expected: EXACTLY ONE WRONG_SHELL_FOR_OS violation (on the second row).
|
||
// ---------------------------------------------------------------------------
|
||
describe('counter-test: two macos-latest rows — dedup must not hide second row violation', () => {
|
||
const TWO_MACOS_ROWS_YAML = `
|
||
name: Two macOS Rows
|
||
jobs:
|
||
build:
|
||
runs-on: \${{ matrix.os }}
|
||
strategy:
|
||
matrix:
|
||
include:
|
||
- os: macos-latest
|
||
node-version: 22
|
||
shell: zsh
|
||
- os: macos-latest
|
||
node-version: 24
|
||
shell: bash
|
||
steps:
|
||
- name: Run tests
|
||
shell: \${{ matrix.shell }}
|
||
run: npm test
|
||
`;
|
||
|
||
test('two macos-latest matrix.include rows (zsh + bash) produce exactly one WRONG_SHELL_FOR_OS violation on the second row', () => {
|
||
const result = inspectWorkflow(TWO_MACOS_ROWS_YAML, { filePath: '<synthetic-two-macos-rows>' });
|
||
|
||
const violations = result.jobs
|
||
.flatMap(j => j.steps)
|
||
.filter(s => s.violation !== null);
|
||
|
||
assert.strictEqual(
|
||
violations.length,
|
||
1,
|
||
`Expected exactly 1 violation (second macos-latest row shell:bash → WRONG_SHELL_FOR_OS) but got ${violations.length}: ` +
|
||
violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ')
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].violation,
|
||
VIOLATION.WRONG_SHELL_FOR_OS,
|
||
`Expected WRONG_SHELL_FOR_OS but got ${violations[0].violation}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].runner,
|
||
'macos-latest',
|
||
`Expected violation runner to be macos-latest but got ${violations[0].runner}`
|
||
);
|
||
|
||
assert.strictEqual(
|
||
violations[0].effectiveShell,
|
||
'bash',
|
||
`Expected effectiveShell to be bash (the violating row) but got ${violations[0].effectiveShell}`
|
||
);
|
||
});
|
||
});
|