'use strict'; const { test, describe } = require('node:test'); const assert = require('node:assert/strict'); const path = require('path'); const { VIOLATION, inspectWorkflow, runPolicyLint, } = require('../scripts/workflow-policy.cjs'); // --------------------------------------------------------------------------- // Test 1 — Baseline: repo's current workflow files must yield ZERO violations // (RED on origin/next; GREEN after YAML fixes are committed) // --------------------------------------------------------------------------- describe('baseline: repo workflows comply with H1 shell policy', () => { test('runPolicyLint on .github/workflows produces zero violations', () => { const workflowsDir = path.resolve(__dirname, '..', '.github', 'workflows'); const result = runPolicyLint({ workflowsDir }); if (result.violations.length > 0) { const top10 = result.violations.slice(0, 10); const msg = top10.map(v => ` ${path.basename(v.filePath)}:${v.evidence.line} [${v.jobId}/${v.stepName}] runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}` ).join('\n'); assert.fail( `Expected 0 violations but found ${result.violations.length}. ` + `Top violations (mechanism: each step on a macos-* or windows-* runner must use native shell):\n${msg}` ); } assert.strictEqual( result.violations.length, 0, 'All workflow steps must comply with H1 shell policy' ); }); }); // --------------------------------------------------------------------------- // Test 2 — Positive synthetic: compliant workflow yields zero violations // Three separate jobs, one per OS, each using H1-compliant shell configuration: // ubuntu: no shell pin (runner default bash = policy bash) // macos: job-level defaults.run.shell: zsh // windows: no shell pin (runner default pwsh = policy pwsh) // --------------------------------------------------------------------------- describe('synthetic: fully-compliant per-OS jobs workflow', () => { const COMPLIANT_YAML = ` name: Compliant Workflow jobs: linux-job: runs-on: ubuntu-latest steps: - name: Run tests on ubuntu run: npm test macos-job: runs-on: macos-latest defaults: run: shell: zsh steps: - name: Run tests on macOS run: npm test windows-job: runs-on: windows-latest steps: - name: Run tests on windows run: npm test `; test('compliant per-OS workflow (ubuntu no pin, macos job-defaults zsh, windows no pin) produces zero violations', () => { const result = inspectWorkflow(COMPLIANT_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 0, 'Compliant per-OS workflow must have zero violations. Got: ' + violations.map(v => `${v.runner}/${v.violation}`).join(', ') ); }); }); // --------------------------------------------------------------------------- // Test 3 — Counter-test: macOS missing explicit zsh → MACOS_MISSING_EXPLICIT_ZSH // (mechanism: macos-latest default is bash, not zsh; H1 requires explicit shell: zsh) // --------------------------------------------------------------------------- describe('counter-test: macOS step without explicit shell: zsh', () => { const MACOS_NO_SHELL_YAML = ` name: macOS No Shell jobs: build: runs-on: macos-latest steps: - name: Run tests run: npm test `; test('macos-latest step with no shell pin produces exactly one MACOS_MISSING_EXPLICIT_ZSH violation', () => { const result = inspectWorkflow(MACOS_NO_SHELL_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 1, `Expected exactly 1 violation (MACOS_MISSING_EXPLICIT_ZSH on macos-latest) but got ${violations.length}` ); assert.strictEqual( violations[0].violation, VIOLATION.MACOS_MISSING_EXPLICIT_ZSH, `Expected violation type MACOS_MISSING_EXPLICIT_ZSH but got ${violations[0].violation}` ); assert.strictEqual( violations[0].runner, 'macos-latest', `Expected violation runner to be macos-latest but got ${violations[0].runner}` ); }); }); // --------------------------------------------------------------------------- // Test 4 — Counter-test: wrong shell for OS → WRONG_SHELL_FOR_OS // (mechanism: windows-2025 default is pwsh; specifying shell: bash is a policy violation) // --------------------------------------------------------------------------- describe('counter-test: windows step with explicit shell: bash', () => { const WINDOWS_BASH_YAML = ` name: Windows Bash jobs: build: runs-on: windows-2025 steps: - name: Run tests with wrong shell shell: bash run: npm test `; test('windows-2025 step with shell: bash produces exactly one WRONG_SHELL_FOR_OS violation', () => { const result = inspectWorkflow(WINDOWS_BASH_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 1, `Expected exactly 1 violation (WRONG_SHELL_FOR_OS on windows-2025) but got ${violations.length}` ); assert.strictEqual( violations[0].violation, VIOLATION.WRONG_SHELL_FOR_OS, `Expected violation type WRONG_SHELL_FOR_OS but got ${violations[0].violation}` ); assert.strictEqual( violations[0].runner, 'windows-2025', `Expected violation runner to be windows-2025 but got ${violations[0].runner}` ); }); }); // --------------------------------------------------------------------------- // Test 5 — Counter-test: matrix expansion with shell: bash on every step // (mechanism: ubuntu realizations are compliant since bash IS policy for ubuntu; // macos → WRONG_SHELL_FOR_OS (explicit wrong pin); windows → WRONG_SHELL_FOR_OS) // Expected: 2 violations total (1 macos + 1 windows), zero for ubuntu // Note: MACOS_MISSING_EXPLICIT_ZSH fires only when NO shell is set at any level; // here shell: bash is explicit, so WRONG_SHELL_FOR_OS is the correct subtype. // --------------------------------------------------------------------------- describe('counter-test: three-OS matrix with shell: bash on every step', () => { const ALL_BASH_MATRIX_YAML = ` name: All Bash Matrix jobs: build: runs-on: \${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, macos-latest, windows-2025] steps: - name: Run tests shell: bash run: npm test `; test('three-OS matrix with shell: bash produces exactly 2 WRONG_SHELL_FOR_OS violations (macos + windows), zero for ubuntu', () => { const result = inspectWorkflow(ALL_BASH_MATRIX_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 2, `Expected exactly 2 violations (macos-latest + windows-2025) but got ${violations.length}: ` + violations.map(v => `${v.runner}/${v.violation}`).join(', ') ); const macosViolation = violations.find(v => v.runner === 'macos-latest'); assert.ok( macosViolation, 'Expected a violation for macos-latest realization' ); // When an explicit shell: bash is set on the step, the violation is WRONG_SHELL_FOR_OS // (the explicit pin is wrong for the OS). MACOS_MISSING_EXPLICIT_ZSH only fires when // there is NO shell set at any level and the runner default (bash) is inherited silently. assert.strictEqual( macosViolation.violation, VIOLATION.WRONG_SHELL_FOR_OS, `macos-latest with explicit shell: bash should be WRONG_SHELL_FOR_OS (explicit wrong pin) but got ${macosViolation?.violation}` ); const windowsViolation = violations.find(v => v.runner === 'windows-2025'); assert.ok( windowsViolation, 'Expected a violation for windows-2025 realization' ); assert.strictEqual( windowsViolation.violation, VIOLATION.WRONG_SHELL_FOR_OS, `windows-2025 violation should be WRONG_SHELL_FOR_OS but got ${windowsViolation?.violation}` ); const ubuntuViolations = violations.filter(v => v.runner === 'ubuntu-latest'); assert.strictEqual( ubuntuViolations.length, 0, `ubuntu-latest should produce zero violations (bash is both runner default and policy) but got ${ubuntuViolations.length}` ); }); }); // --------------------------------------------------------------------------- // Test 6 — Counter-test: unknown runner → UNKNOWN_RUNNER // (mechanism: self-hosted is not in POLICY, so runner cannot be validated) // --------------------------------------------------------------------------- describe('counter-test: self-hosted runner produces UNKNOWN_RUNNER violation', () => { const SELF_HOSTED_YAML = ` name: Self-Hosted jobs: build: runs-on: self-hosted steps: - name: Run build run: npm build `; test('self-hosted runner step produces exactly one UNKNOWN_RUNNER violation', () => { const result = inspectWorkflow(SELF_HOSTED_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 1, `Expected exactly 1 UNKNOWN_RUNNER violation but got ${violations.length}` ); assert.strictEqual( violations[0].violation, VIOLATION.UNKNOWN_RUNNER, `Expected violation type UNKNOWN_RUNNER but got ${violations[0].violation}` ); assert.strictEqual( violations[0].runner, 'self-hosted', `Expected violation runner to be self-hosted but got ${violations[0].runner}` ); }); }); // --------------------------------------------------------------------------- // Test 7a — Positive: matrix.include with shell key + defaults.run.shell: ${{ matrix.shell }} // (mechanism: each realization carries its own shell value; the linter resolves // the matrix expression against the realization context before checking policy) // --------------------------------------------------------------------------- describe('matrix.shell: ${{ matrix.shell }} resolves per realization — zero violations', () => { const MATRIX_SHELL_YAML = ` name: Matrix Shell Positive jobs: build: runs-on: \${{ matrix.os }} defaults: run: shell: \${{ matrix.shell }} strategy: matrix: include: - os: macos-latest shell: zsh - os: windows-2025 shell: pwsh steps: - name: Run tests run: npm test `; test('matrix.include with shell:zsh for macOS + shell:pwsh for Windows + defaults.run.shell: ${{ matrix.shell }} yields zero violations', () => { const result = inspectWorkflow(MATRIX_SHELL_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 0, 'matrix.shell resolved per realization must produce zero violations. Got: ' + violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ') ); }); }); // --------------------------------------------------------------------------- // Test 7b — Counter-test: matrix.include row with wrong shell value // (mechanism: if a row's shell value doesn't match its OS policy, WRONG_SHELL_FOR_OS fires) // --------------------------------------------------------------------------- describe('matrix.shell: ${{ matrix.shell }} with wrong value per row — WRONG_SHELL_FOR_OS', () => { const MATRIX_SHELL_WRONG_YAML = ` name: Matrix Shell Wrong Row jobs: build: runs-on: \${{ matrix.os }} defaults: run: shell: \${{ matrix.shell }} strategy: matrix: include: - os: macos-latest shell: bash - os: windows-2025 shell: pwsh steps: - name: Run tests run: npm test `; test('matrix.include row with shell:bash for macOS produces WRONG_SHELL_FOR_OS (bash is wrong for macOS)', () => { const result = inspectWorkflow(MATRIX_SHELL_WRONG_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); // macOS realization: shell resolves to bash → WRONG_SHELL_FOR_OS // Windows realization: shell resolves to pwsh → compliant assert.strictEqual( violations.length, 1, `Expected exactly 1 violation (macos-latest bash→WRONG_SHELL_FOR_OS) but got ${violations.length}: ` + violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ') ); assert.strictEqual( violations[0].runner, 'macos-latest', `Expected violation for macos-latest but got ${violations[0].runner}` ); assert.strictEqual( violations[0].violation, VIOLATION.WRONG_SHELL_FOR_OS, `Expected WRONG_SHELL_FOR_OS but got ${violations[0].violation}` ); }); }); // --------------------------------------------------------------------------- // Test 7c — Counter-test: matrix.include row missing the shell key while // defaults.run.shell: ${{ matrix.shell }} references it → UNRESOLVABLE_MATRIX // --------------------------------------------------------------------------- describe('matrix.shell expression references missing key — UNRESOLVABLE_MATRIX', () => { const MATRIX_SHELL_MISSING_KEY_YAML = ` name: Matrix Shell Missing Key jobs: build: runs-on: \${{ matrix.os }} defaults: run: shell: \${{ matrix.shell }} strategy: matrix: include: - os: ubuntu-latest node-version: 24 steps: - name: Run tests run: npm test `; test('matrix.include row without shell key while defaults.run.shell: ${{ matrix.shell }} → UNRESOLVABLE_MATRIX', () => { const result = inspectWorkflow(MATRIX_SHELL_MISSING_KEY_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 1, `Expected exactly 1 UNRESOLVABLE_MATRIX violation but got ${violations.length}: ` + violations.map(v => `runner=${v.runner} type=${v.violation}`).join(', ') ); assert.strictEqual( violations[0].violation, VIOLATION.UNRESOLVABLE_MATRIX, `Expected UNRESOLVABLE_MATRIX but got ${violations[0].violation}` ); assert.strictEqual( violations[0].runner, 'ubuntu-latest', `Expected runner ubuntu-latest but got ${violations[0].runner}` ); }); }); // --------------------------------------------------------------------------- // Test 7 — Counter-test: workflow-level defaults.run.shell: zsh satisfies macOS H1 // (mechanism: resolution order puts workflow defaults above runner default; // zsh at workflow level means macOS steps inherit it without step-level pin) // --------------------------------------------------------------------------- describe('counter-test: workflow-level defaults.run.shell: zsh satisfies macos-* H1', () => { const WORKFLOW_DEFAULTS_ZSH_YAML = ` name: Workflow Defaults ZSH defaults: run: shell: zsh jobs: build: runs-on: macos-latest steps: - name: Run tests on macOS run: npm test `; test('workflow-level shell: zsh + macos-latest + no step-level shell produces zero violations', () => { const result = inspectWorkflow(WORKFLOW_DEFAULTS_ZSH_YAML, { filePath: '' }); assert.strictEqual( result.workflowDefaultsShell, 'zsh', `Expected workflowDefaultsShell to be zsh but got ${result.workflowDefaultsShell}` ); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 0, `Workflow-level shell: zsh must satisfy H1 for macos-latest steps (resolution-order rule). Got ${violations.length} violations: ` + violations.map(v => `${v.violation}`).join(', ') ); }); test('effective shell for macOS step is zsh when inherited from workflow defaults', () => { const result = inspectWorkflow(WORKFLOW_DEFAULTS_ZSH_YAML, { filePath: '' }); const step = result.jobs[0]?.steps[0]; assert.ok(step, 'Expected at least one step'); assert.strictEqual( step.effectiveShell, 'zsh', `Expected effectiveShell to be zsh (inherited from workflow defaults) but got ${step.effectiveShell}` ); assert.strictEqual( step.stepShell, null, `Expected stepShell to be null (no step-level pin) but got ${step.stepShell}` ); }); }); // --------------------------------------------------------------------------- // Test 8a — Cartesian matrix os × shell — full cross-product expansion resolves // matrix.shell per realization // (mechanism: matrix.os: [macos-latest, macos-latest] with matrix.shell: [zsh, bash] // and runs-on: ${{ matrix.os }}, step shell: ${{ matrix.shell }}. // GitHub Actions realizes a 2×2 grid (4 jobs). After the Cartesian-product fix, // expandRunsOn must produce 4 realizations, each carrying BOTH os AND shell in // context so that ${{ matrix.shell }} resolves per realization. // // Expected post-fix behavior: // - 4 total step-results (2 os × 2 shell) // - exactly 2 WRONG_SHELL_FOR_OS violations — the two bash cells on macos-latest // - ZERO UNRESOLVABLE_MATRIX violations (matrix.shell is now fully resolved) // --------------------------------------------------------------------------- describe('Cartesian matrix os × shell — full cross-product expansion resolves matrix.shell per realization', () => { const CARTESIAN_MATRIX_YAML = ` name: Cartesian Matrix jobs: build: runs-on: \${{ matrix.os }} strategy: matrix: os: [macos-latest, macos-latest] shell: [zsh, bash] steps: - name: Run tests shell: \${{ matrix.shell }} run: echo hi `; test('2×2 Cartesian product yields 4 step-results, 2 WRONG_SHELL_FOR_OS violations, 0 UNRESOLVABLE_MATRIX', () => { const result = inspectWorkflow(CARTESIAN_MATRIX_YAML, { filePath: '' }); const allSteps = result.jobs.flatMap(j => j.steps); const violations = allSteps.filter(s => s.violation !== null); const unresolvable = violations.filter(s => s.violation === VIOLATION.UNRESOLVABLE_MATRIX); const wrongShell = violations.filter(s => s.violation === VIOLATION.WRONG_SHELL_FOR_OS); // 4 step-results: 2 os values × 2 shell values = 4 realizations, each with 1 step assert.strictEqual( allSteps.length, 4, `Expected 4 step-results (2×2 Cartesian product) but got ${allSteps.length}. All steps: ` + allSteps.map(s => `runner=${s.runner} shell=${s.effectiveShell} violation=${s.violation}`).join(', ') ); // Core regression proof: ZERO UNRESOLVABLE_MATRIX (matrix.shell now resolves) assert.strictEqual( unresolvable.length, 0, `Expected 0 UNRESOLVABLE_MATRIX violations but got ${unresolvable.length}: ` + unresolvable.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ') ); // Exactly 2 WRONG_SHELL_FOR_OS: the two bash cells on macos-latest assert.strictEqual( wrongShell.length, 2, `Expected exactly 2 WRONG_SHELL_FOR_OS violations (bash on macos-latest) but got ${wrongShell.length}: ` + violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ') ); for (const v of wrongShell) { assert.strictEqual( v.runner, 'macos-latest', `Expected violation runner to be macos-latest but got ${v.runner}` ); assert.strictEqual( v.effectiveShell, 'bash', `Expected effectiveShell to be bash (the violating cell) but got ${v.effectiveShell}` ); } }); }); // --------------------------------------------------------------------------- // Test 8b — Cartesian matrix os × node-version — compliant cross-product carries // extra axis without violations // (mechanism: matrix.os: [ubuntu-latest, ubuntu-latest] with // matrix.node-version: [22, 24], step shell: bash (literal). // The cross-product yields 4 realizations. ubuntu-latest + bash = compliant. // Pre-fix: only os is expanded → 2 step-results. Post-fix: 4 step-results. // This is the fail-first signal for the Cartesian expansion fix. // --------------------------------------------------------------------------- describe('Cartesian matrix os × node-version — compliant cross-product yields 4 step-results with 0 violations', () => { const CARTESIAN_COMPLIANT_YAML = ` name: Cartesian Compliant jobs: build: runs-on: \${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, ubuntu-latest] node-version: [22, 24] steps: - name: Run tests shell: bash run: npm test `; test('2×2 Cartesian product yields 4 step-results (fail-first signal pre-fix: 2) and 0 violations', () => { const result = inspectWorkflow(CARTESIAN_COMPLIANT_YAML, { filePath: '' }); const allSteps = result.jobs.flatMap(j => j.steps); const violations = allSteps.filter(s => s.violation !== null); // Pre-fix: only os is expanded → 2 step-results; post-fix: 4 assert.strictEqual( allSteps.length, 4, `Expected 4 step-results (2 os × 2 node-version Cartesian product) but got ${allSteps.length}. ` + `If you see 2, the Cartesian expansion fix is not yet applied. All steps: ` + allSteps.map(s => `runner=${s.runner} shell=${s.effectiveShell} violation=${s.violation}`).join(', ') ); // ubuntu-latest + literal bash = compliant; no violations expected assert.strictEqual( violations.length, 0, `Expected 0 violations (ubuntu-latest + bash is compliant) but got ${violations.length}: ` + violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ') ); }); }); // --------------------------------------------------------------------------- // Test 8c — Property-based: for any 1–3 base-list axes with 1–3 values each, // step count === Cartesian product of axis lengths // (fast-check is a devDependency: fast-check ^4.8.0) // --------------------------------------------------------------------------- const fc = require('fast-check'); describe('property-based: step count equals product of all axis lengths for arbitrary base-list matrices', () => { test('step count === product(axisLengths) for 1–3 axes with 1–3 values each', () => { // Axis keys named k0, k1, k2; values restricted to [A-Za-z0-9-] to keep YAML well-formed. // runs-on references ${{ matrix.k0 }}; the single step's shell references the // LAST matrix axis key (${{ matrix.kLast }}) so that a dropped axis key would // surface as UNRESOLVABLE_MATRIX rather than silently resolving. const axisValueArb = fc.stringMatching(/^[A-Za-z][A-Za-z0-9-]{0,7}$/); const axisArb = fc.array(axisValueArb, { minLength: 1, maxLength: 3 }); const matrixArb = fc.array(axisArb, { minLength: 1, maxLength: 3 }); fc.assert( fc.property(matrixArb, (axes) => { // Build YAML matrix block const keys = axes.map((_, i) => `k${i}`); const lastKey = keys[keys.length - 1]; const matrixLines = keys.map((k, i) => ` ${k}: [${axes[i].join(', ')}]`); const yaml = [ 'name: PropertyTest', 'jobs:', ' build:', ' runs-on: ${{ matrix.k0 }}', ' strategy:', ' matrix:', ...matrixLines, ' steps:', ' - name: Run tests', ` shell: \${{ matrix.${lastKey} }}`, ' run: echo hi', ].join('\n'); const result = inspectWorkflow(yaml, { filePath: '' }); const allSteps = result.jobs.flatMap(j => j.steps); const actualSteps = allSteps.length; const expectedSteps = axes.reduce((acc, axis) => acc * axis.length, 1); // Every realization must carry the last axis key in context so that // ${{ matrix.kLast }} resolves. If the key is absent from any // realization's context, effectiveShell fires UNRESOLVABLE_MATRIX. const noUnresolvable = allSteps.every( s => s.violation !== VIOLATION.UNRESOLVABLE_MATRIX, ); return actualSteps === expectedSteps && noUnresolvable; }), { seed: 42, numRuns: 100 } ); }); }); // --------------------------------------------------------------------------- // Test 8 — Counter-test: two macos-latest matrix.include rows where // row 1 has shell: zsh (compliant) and row 2 has shell: bash (violation). // Guards against the dedup bug where runner-label-only deduplication would // collapse both rows into one, hiding the second row's policy violation. // Expected: EXACTLY ONE WRONG_SHELL_FOR_OS violation (on the second row). // --------------------------------------------------------------------------- describe('counter-test: two macos-latest rows — dedup must not hide second row violation', () => { const TWO_MACOS_ROWS_YAML = ` name: Two macOS Rows jobs: build: runs-on: \${{ matrix.os }} strategy: matrix: include: - os: macos-latest node-version: 22 shell: zsh - os: macos-latest node-version: 24 shell: bash steps: - name: Run tests shell: \${{ matrix.shell }} run: npm test `; test('two macos-latest matrix.include rows (zsh + bash) produce exactly one WRONG_SHELL_FOR_OS violation on the second row', () => { const result = inspectWorkflow(TWO_MACOS_ROWS_YAML, { filePath: '' }); const violations = result.jobs .flatMap(j => j.steps) .filter(s => s.violation !== null); assert.strictEqual( violations.length, 1, `Expected exactly 1 violation (second macos-latest row shell:bash → WRONG_SHELL_FOR_OS) but got ${violations.length}: ` + violations.map(v => `runner=${v.runner} shell=${v.effectiveShell} type=${v.violation}`).join(', ') ); assert.strictEqual( violations[0].violation, VIOLATION.WRONG_SHELL_FOR_OS, `Expected WRONG_SHELL_FOR_OS but got ${violations[0].violation}` ); assert.strictEqual( violations[0].runner, 'macos-latest', `Expected violation runner to be macos-latest but got ${violations[0].runner}` ); assert.strictEqual( violations[0].effectiveShell, 'bash', `Expected effectiveShell to be bash (the violating row) but got ${violations[0].effectiveShell}` ); }); });