Files
msd-core/tests/npm-audit-baseline.test.cjs
Tom Boucher 114dfcb739 fix(#4196): npm-audit gate blocks only NEW advisories, not pre-existing ones (#4214)
* fix(#4196): npm-audit gate blocks only NEW advisories, not pre-existing ones

The #3588 gate failed on ANY advisory in the production tree, regardless
of whether the PR/push actually introduced it. Because npm's advisory
database updates continuously and independently of repo state, a commit
could pass this gate at merge time and fail it minutes later on the
identical tree -- proven on PR #4188/dce40eeb6, which passed on all 3
OSes at 15:57-16:27 and failed the same assertion at 16:19-16:30 on the
unchanged commit, purely because GHSA-jqff-g426-hqxp was disclosed for
fast-uri in the interim.

scripts/npm-audit-baseline.cjs diffs the head tree's vulnerable-package
set against a resolved baseline (the PR's target branch, or the prior
commit on a direct push) and blocks only newly-introduced advisories.
When no baseline can be resolved, falls back to the original
zero-tolerance behavior -- fail-closed, never silently weaker.

* fix(#4196): pin the npm-audit baseline instead of using a drift-prone ref

Two orthogonal reviews found the same class of bug this repo already
fixed once for a different gate (see GSD_EMITTED_BASE's own incident
comment in test.yml): origin/<branch> is live under fetch-depth: 0 and
can advance mid-run, so resolveBaselineRef()'s fallback to
origin/${GITHUB_BASE_REF} could silently disagree with the tree
ci-rebase-check.cjs actually merged. Wire AUDIT_BASELINE_REF from the
workflow to github.event.pull_request.base.sha / github.event.before,
the same pinned values GSD_EMITTED_BASE already relies on.

Also: HEAD~1 assumed exactly one commit per push, which this repo's
allow_rebase_merge:true setting can violate (a rebase-merged PR lands
as several discrete commits in one push) -- github.event.before is
git's own record of the correct pre-push state, not an assumed offset.
HEAD~1 remains as a documented last-resort fallback for out-of-band
invocations (e.g. gsd-test) that don't set any of the above, alongside
a new local-branch fallback for gsd-test's local `next` (not
origin/next) sandbox shape.

---------

Co-authored-by: sim <sim@local>
2026-09-02 22:38:37 -04:00

297 lines
12 KiB
JavaScript

'use strict';
/**
* Unit tests for scripts/npm-audit-baseline.cjs (#4196).
*
* Covers the pure diff/verdict functions directly, plus the git-object-level
* extraction and env-driven ref resolution using real throwaway git fixture
* repos (no network, no real npm registry round-trip -- runPackageLockAudit
* is only exercised here for its filesystem-only skip conditions).
*/
const { test, describe, beforeEach, afterEach } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const { execFileSync } = require('node:child_process');
const {
AUDIT_DIFF_REASON,
diffNewVulnerablePackages,
evaluateAuditDiff,
runPackageLockAudit,
extractBaselineTree,
resolveBaselineRef,
NULL_SHA,
} = require('../scripts/npm-audit-baseline.cjs');
const { createTempDir, cleanup } = require('./helpers.cjs');
const GIT_TIMEOUT_MS = 30_000;
function git(args, cwd) {
return execFileSync('git', args, {
cwd,
encoding: 'utf-8',
stdio: ['ignore', 'pipe', 'pipe'],
timeout: GIT_TIMEOUT_MS,
}).trim();
}
/**
* Builds a throwaway git repo with one commit containing package.json +
* package-lock.json (and optionally under a subdir), returning
* { dir, commitSha }.
*/
function makeCommittedFixtureRepo(t, { subdir = '', pkgContent = '{"name":"fixture"}', lockContent = '{"lockfileVersion":3}' } = {}) {
const dir = createTempDir('gsd-audit-baseline-fixture-');
t.after(() => cleanup(dir));
git(['init', '-q'], dir);
git(['config', 'user.email', 'test@example.com'], dir);
git(['config', 'user.name', 'Test'], dir);
const targetDir = subdir ? path.join(dir, subdir) : dir;
fs.mkdirSync(targetDir, { recursive: true });
fs.writeFileSync(path.join(targetDir, 'package.json'), pkgContent);
fs.writeFileSync(path.join(targetDir, 'package-lock.json'), lockContent);
git(['add', '-A'], dir);
git(['commit', '-q', '-m', 'fixture commit'], dir);
const commitSha = git(['rev-parse', 'HEAD'], dir);
return { dir, commitSha };
}
// ─── diffNewVulnerablePackages ────────────────────────────────────────────
describe('diffNewVulnerablePackages', () => {
test('empty baseline + empty head -> []', () => {
assert.deepStrictEqual(diffNewVulnerablePackages({}, {}), []);
});
test('baseline and head share the same packages -> [] (nothing new)', () => {
const baseline = { a: {}, b: {} };
const head = { a: {}, b: {} };
assert.deepStrictEqual(diffNewVulnerablePackages(baseline, head), []);
});
test('head adds a package not in baseline -> only the addition', () => {
const baseline = { a: {} };
const head = { a: {}, b: {} };
assert.deepStrictEqual(diffNewVulnerablePackages(baseline, head), ['b']);
});
test('empty baseline, head has one package -> that package', () => {
assert.deepStrictEqual(diffNewVulnerablePackages({}, { a: {} }), ['a']);
});
test('packages removed from head (present in baseline only) are not "new"', () => {
const baseline = { a: {}, b: {}, c: {} };
const head = { a: {} };
assert.deepStrictEqual(diffNewVulnerablePackages(baseline, head), []);
});
test('baseline and head both undefined -> [] (must not throw)', () => {
assert.deepStrictEqual(diffNewVulnerablePackages(undefined, undefined), []);
});
});
// ─── evaluateAuditDiff ─────────────────────────────────────────────────────
describe('evaluateAuditDiff', () => {
test('no new packages -> ok:true with OK_NO_NEW_VULNERABILITIES and preExisting list', () => {
const baselineVulnerabilities = { a: {} };
const headVulnerabilities = { a: {} };
const result = evaluateAuditDiff({ baselineVulnerabilities, headVulnerabilities });
assert.deepStrictEqual(result, {
ok: true,
reason: AUDIT_DIFF_REASON.OK_NO_NEW_VULNERABILITIES,
preExisting: ['a'],
});
});
test('one new package -> ok:false with FAIL_NEW_VULNERABLE_PACKAGE and exact newlyIntroduced', () => {
const baselineVulnerabilities = { a: {} };
const headVulnerabilities = { a: {}, b: {} };
const result = evaluateAuditDiff({ baselineVulnerabilities, headVulnerabilities });
assert.strictEqual(result.ok, false);
assert.strictEqual(result.reason, AUDIT_DIFF_REASON.FAIL_NEW_VULNERABLE_PACKAGE);
assert.deepStrictEqual(result.newlyIntroduced, ['b']);
});
test('multiple new packages -> all listed', () => {
const baselineVulnerabilities = {};
const headVulnerabilities = { a: {}, b: {}, c: {} };
const result = evaluateAuditDiff({ baselineVulnerabilities, headVulnerabilities });
assert.strictEqual(result.ok, false);
assert.deepStrictEqual(result.newlyIntroduced.sort(), ['a', 'b', 'c']);
});
});
// ─── resolveBaselineRef ─────────────────────────────────────────────────────
describe('resolveBaselineRef', () => {
const envKeys = ['AUDIT_BASELINE_REF', 'GITHUB_BASE_REF', 'GITHUB_EVENT_NAME'];
let originalEnv;
beforeEach(() => {
originalEnv = Object.fromEntries(envKeys.map((key) => [key, process.env[key]]));
for (const key of envKeys) delete process.env[key];
});
afterEach(() => {
for (const key of envKeys) {
if (originalEnv[key] === undefined) delete process.env[key];
else process.env[key] = originalEnv[key];
}
});
test('AUDIT_BASELINE_REF set -> returned verbatim, highest priority even with others set', (t) => {
const { dir } = makeCommittedFixtureRepo(t);
process.env.AUDIT_BASELINE_REF = 'some/explicit-ref';
process.env.GITHUB_BASE_REF = 'next';
process.env.GITHUB_EVENT_NAME = 'push';
assert.strictEqual(resolveBaselineRef(dir), 'some/explicit-ref');
});
test('AUDIT_BASELINE_REF unset, GITHUB_BASE_REF=next -> origin/next', (t) => {
const { dir } = makeCommittedFixtureRepo(t);
process.env.GITHUB_BASE_REF = 'next';
assert.strictEqual(resolveBaselineRef(dir), 'origin/next');
});
test('neither set, push event, HEAD~1 resolves -> returns that parent sha', (t) => {
const dir = createTempDir('gsd-audit-baseline-fixture-');
t.after(() => cleanup(dir));
git(['init', '-q'], dir);
git(['config', 'user.email', 'test@example.com'], dir);
git(['config', 'user.name', 'Test'], dir);
fs.writeFileSync(path.join(dir, 'a.txt'), 'first');
git(['add', '-A'], dir);
git(['commit', '-q', '-m', 'first commit'], dir);
fs.writeFileSync(path.join(dir, 'a.txt'), 'second');
git(['add', '-A'], dir);
git(['commit', '-q', '-m', 'second commit'], dir);
// Compute expected parent sha independently, not via resolveBaselineRef.
const expectedParentSha = git(['rev-parse', 'HEAD~1'], dir);
process.env.GITHUB_EVENT_NAME = 'push';
assert.strictEqual(resolveBaselineRef(dir), expectedParentSha);
});
test('NULL_SHA is the documented all-zeros 40-char sentinel', () => {
assert.strictEqual(NULL_SHA, '0'.repeat(40));
assert.strictEqual(NULL_SHA.length, 40);
});
test('push event but only one commit (HEAD~1 does not exist) falls through without choking', (t) => {
const dir = createTempDir('gsd-audit-baseline-fixture-');
t.after(() => cleanup(dir));
git(['init', '-q'], dir);
git(['config', 'user.email', 'test@example.com'], dir);
git(['config', 'user.name', 'Test'], dir);
fs.writeFileSync(path.join(dir, 'a.txt'), 'only');
git(['add', '-A'], dir);
git(['commit', '-q', '-m', 'only commit'], dir);
process.env.GITHUB_EVENT_NAME = 'push';
// No origin/next remote-tracking ref exists in this throwaway repo, so
// this must fall all the way through to ''.
assert.strictEqual(resolveBaselineRef(dir), '');
});
test('no origin/next, but a plain local branch named next exists -> returns "next"', (t) => {
const dir = createTempDir('gsd-audit-baseline-fixture-');
t.after(() => cleanup(dir));
git(['init', '-q'], dir);
git(['config', 'user.email', 'test@example.com'], dir);
git(['config', 'user.name', 'Test'], dir);
fs.writeFileSync(path.join(dir, 'a.txt'), 'first');
git(['add', '-A'], dir);
git(['commit', '-q', '-m', 'first commit'], dir);
// rename the default branch to "next" so it's a plain LOCAL branch, not
// a remote-tracking origin/next ref -- mirrors gsd-test's sandbox shape.
git(['branch', '-M', 'next'], dir);
process.env.GITHUB_EVENT_NAME = 'push';
assert.strictEqual(resolveBaselineRef(dir), 'next');
});
test('nothing resolves at all (no env vars, not a git repo) -> returns ""', (t) => {
const dir = createTempDir('gsd-audit-baseline-nongit-');
t.after(() => cleanup(dir));
assert.strictEqual(resolveBaselineRef(dir), '');
});
});
// ─── extractBaselineTree ─────────────────────────────────────────────────────
describe('extractBaselineTree', () => {
test('extracts package.json + package-lock.json at root from a real commit', (t) => {
const pkgContent = JSON.stringify({ name: 'root-fixture' });
const lockContent = JSON.stringify({ lockfileVersion: 3, name: 'root-fixture' });
const { dir, commitSha } = makeCommittedFixtureRepo(t, { pkgContent, lockContent });
const extracted = extractBaselineTree(commitSha, dir);
assert.notStrictEqual(extracted, null);
t.after(() => cleanup(extracted));
assert.strictEqual(fs.readFileSync(path.join(extracted, 'package.json'), 'utf-8'), pkgContent);
assert.strictEqual(fs.readFileSync(path.join(extracted, 'package-lock.json'), 'utf-8'), lockContent);
});
test('extracts package.json + package-lock.json from a subdir', (t) => {
const pkgContent = JSON.stringify({ name: 'sdk-fixture' });
const lockContent = JSON.stringify({ lockfileVersion: 3, name: 'sdk-fixture' });
const { dir, commitSha } = makeCommittedFixtureRepo(t, { subdir: 'sdk', pkgContent, lockContent });
const extracted = extractBaselineTree(commitSha, dir, 'sdk');
assert.notStrictEqual(extracted, null);
t.after(() => cleanup(extracted));
assert.strictEqual(fs.readFileSync(path.join(extracted, 'package.json'), 'utf-8'), pkgContent);
assert.strictEqual(fs.readFileSync(path.join(extracted, 'package-lock.json'), 'utf-8'), lockContent);
});
test('a ref that does not exist -> null', (t) => {
const { dir } = makeCommittedFixtureRepo(t);
const bogusSha = 'f'.repeat(40);
assert.strictEqual(extractBaselineTree(bogusSha, dir), null);
});
test('ref exists but package-lock.json was never committed at that ref -> null', (t) => {
const dir = createTempDir('gsd-audit-baseline-nolock-');
t.after(() => cleanup(dir));
git(['init', '-q'], dir);
git(['config', 'user.email', 'test@example.com'], dir);
git(['config', 'user.name', 'Test'], dir);
fs.writeFileSync(path.join(dir, 'package.json'), '{"name":"nolock"}');
git(['add', '-A'], dir);
git(['commit', '-q', '-m', 'no lockfile'], dir);
const sha = git(['rev-parse', 'HEAD'], dir);
assert.strictEqual(extractBaselineTree(sha, dir), null);
});
});
// ─── runPackageLockAudit (filesystem-only skip conditions, no registry) ────
describe('runPackageLockAudit', () => {
test('missing package.json -> null', () => {
const dir = createTempDir('gsd-audit-baseline-empty-');
try {
assert.strictEqual(runPackageLockAudit(dir), null);
} finally {
cleanup(dir);
}
});
test('package.json present but no package-lock.json -> null', () => {
const dir = createTempDir('gsd-audit-baseline-nolock2-');
try {
fs.writeFileSync(path.join(dir, 'package.json'), '{"name":"nolock2"}');
assert.strictEqual(runPackageLockAudit(dir), null);
} finally {
cleanup(dir);
}
});
});