* fix(#4196): npm-audit gate blocks only NEW advisories, not pre-existing ones The #3588 gate failed on ANY advisory in the production tree, regardless of whether the PR/push actually introduced it. Because npm's advisory database updates continuously and independently of repo state, a commit could pass this gate at merge time and fail it minutes later on the identical tree -- proven on PR #4188/dce40eeb6, which passed on all 3 OSes at 15:57-16:27 and failed the same assertion at 16:19-16:30 on the unchanged commit, purely because GHSA-jqff-g426-hqxp was disclosed for fast-uri in the interim. scripts/npm-audit-baseline.cjs diffs the head tree's vulnerable-package set against a resolved baseline (the PR's target branch, or the prior commit on a direct push) and blocks only newly-introduced advisories. When no baseline can be resolved, falls back to the original zero-tolerance behavior -- fail-closed, never silently weaker. * fix(#4196): pin the npm-audit baseline instead of using a drift-prone ref Two orthogonal reviews found the same class of bug this repo already fixed once for a different gate (see GSD_EMITTED_BASE's own incident comment in test.yml): origin/<branch> is live under fetch-depth: 0 and can advance mid-run, so resolveBaselineRef()'s fallback to origin/${GITHUB_BASE_REF} could silently disagree with the tree ci-rebase-check.cjs actually merged. Wire AUDIT_BASELINE_REF from the workflow to github.event.pull_request.base.sha / github.event.before, the same pinned values GSD_EMITTED_BASE already relies on. Also: HEAD~1 assumed exactly one commit per push, which this repo's allow_rebase_merge:true setting can violate (a rebase-merged PR lands as several discrete commits in one push) -- github.event.before is git's own record of the correct pre-push state, not an assumed offset. HEAD~1 remains as a documented last-resort fallback for out-of-band invocations (e.g. gsd-test) that don't set any of the above, alongside a new local-branch fallback for gsd-test's local `next` (not origin/next) sandbox shape. --------- Co-authored-by: sim <sim@local>
This commit is contained in:
@@ -179,14 +179,18 @@ describe('#114: npm integrity gate — --help output', () => {
|
||||
* high or moderate npm-audit advisories.
|
||||
*
|
||||
* Strategy: run `npm audit --omit=dev --json` against both the root
|
||||
* workspace and the embedded SDK package and assert that the metadata
|
||||
* vulnerability counts are zero across info/low/moderate/high/critical.
|
||||
* workspace and the embedded SDK package, then diff the resulting
|
||||
* vulnerable-package set against a baseline tree (see #4196 and
|
||||
* scripts/npm-audit-baseline.cjs) so the gate only fails on advisories
|
||||
* this PR/push actually introduces — not on pre-existing advisories in
|
||||
* an untouched transitive dependency. When no baseline can be resolved,
|
||||
* falls back to the original zero-tolerance check across
|
||||
* info/low/moderate/high/critical.
|
||||
*
|
||||
* The test is intentionally strict — any advisory of any severity (other
|
||||
* than 'low' if the maintainer accepts it; that branch is left explicit
|
||||
* here) blocks CI. If a future advisory lands without an upstream patch,
|
||||
* either bump the patched transitive (preferred), or annotate the
|
||||
* acceptance below with a justification AND a link to the upstream tracker.
|
||||
* If a future advisory lands without an upstream patch on a package this
|
||||
* PR touches, either bump the patched transitive (preferred), or annotate
|
||||
* the acceptance below with a justification AND a link to the upstream
|
||||
* tracker.
|
||||
*
|
||||
* Skips automatically when `node_modules/` is absent (a fresh checkout
|
||||
* before `npm install`) so the test does not falsely report on developer
|
||||
@@ -198,6 +202,13 @@ const assert = require('node:assert/strict');
|
||||
const path = require('node:path');
|
||||
const fs = require('node:fs');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const {
|
||||
evaluateAuditDiff,
|
||||
runPackageLockAudit,
|
||||
extractBaselineTree,
|
||||
resolveBaselineRef,
|
||||
} = require('../scripts/npm-audit-baseline.cjs');
|
||||
const { cleanup } = require('./helpers.cjs');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const SDK = path.join(ROOT, 'sdk');
|
||||
@@ -249,37 +260,55 @@ function auditProductionVulns(cwd) {
|
||||
// (npm changed its output format, audit aborted before metadata, etc.) —
|
||||
// throw so the test fails loudly instead of skipping silently.
|
||||
if (parsed && parsed.metadata && parsed.metadata.vulnerabilities) {
|
||||
return parsed.metadata.vulnerabilities;
|
||||
return parsed;
|
||||
}
|
||||
throw new Error(`Unexpected npm audit JSON shape in ${cwd}: missing metadata.vulnerabilities`);
|
||||
}
|
||||
|
||||
describe('#3588: npm audit --omit=dev reports zero advisories', () => {
|
||||
test('root workspace production tree has no advisories', { timeout: TEST_TIMEOUT_MS }, (t) => {
|
||||
const vulns = auditProductionVulns(ROOT);
|
||||
if (vulns === null) {
|
||||
t.skip('auditable npm package not present or node_modules/ missing');
|
||||
describe('#3588: npm audit --omit=dev introduces no NEW advisories vs baseline (#4196)', () => {
|
||||
// #4196: a pre-existing advisory in an untouched transitive dependency
|
||||
// must not block this PR/push -- only an advisory THIS change actually
|
||||
// introduces should fail the gate. When no baseline can be resolved
|
||||
// (e.g. a bare local run with no git history), fall back to the
|
||||
// original #3588 zero-tolerance behavior rather than silently skipping.
|
||||
function checkTreeAgainstBaseline(t, cwd, subdir, skipMessage) {
|
||||
const audit = auditProductionVulns(cwd);
|
||||
if (audit === null) {
|
||||
t.skip(skipMessage);
|
||||
return;
|
||||
}
|
||||
assert.strictEqual(vulns.critical, 0, `expected 0 critical; got ${vulns.critical}`);
|
||||
assert.strictEqual(vulns.high, 0, `expected 0 high; got ${vulns.high}`);
|
||||
assert.strictEqual(vulns.moderate, 0, `expected 0 moderate; got ${vulns.moderate}`);
|
||||
// Low advisories are not explicitly forbidden by the #3588 acceptance
|
||||
// criterion but the issue listed only high/moderate as actual findings —
|
||||
// tighten if any future low advisory is introduced.
|
||||
assert.strictEqual(vulns.low, 0, `expected 0 low; got ${vulns.low}`);
|
||||
const baselineRef = resolveBaselineRef(ROOT);
|
||||
const baselineDir = baselineRef ? extractBaselineTree(baselineRef, ROOT, subdir) : null;
|
||||
if (baselineDir === null) {
|
||||
const vulns = audit.metadata.vulnerabilities;
|
||||
assert.strictEqual(vulns.critical, 0, `no baseline available; falling back to zero-tolerance -- expected 0 critical; got ${vulns.critical}`);
|
||||
assert.strictEqual(vulns.high, 0, `no baseline available; falling back to zero-tolerance -- expected 0 high; got ${vulns.high}`);
|
||||
assert.strictEqual(vulns.moderate, 0, `no baseline available; falling back to zero-tolerance -- expected 0 moderate; got ${vulns.moderate}`);
|
||||
assert.strictEqual(vulns.low, 0, `no baseline available; falling back to zero-tolerance -- expected 0 low; got ${vulns.low}`);
|
||||
return;
|
||||
}
|
||||
t.after(() => cleanup(baselineDir));
|
||||
const baselineAudit = runPackageLockAudit(baselineDir);
|
||||
const baselineVulns = (baselineAudit && baselineAudit.vulnerabilities) || {};
|
||||
const result = evaluateAuditDiff({
|
||||
baselineVulnerabilities: baselineVulns,
|
||||
headVulnerabilities: audit.vulnerabilities || {},
|
||||
});
|
||||
assert.strictEqual(
|
||||
result.ok,
|
||||
true,
|
||||
result.ok
|
||||
? ''
|
||||
: `new advisory introduced vs baseline (${baselineRef}): ${result.newlyIntroduced.join(', ')}. Pre-existing advisories are tracked separately (see #4196) and do not block this change.`,
|
||||
);
|
||||
}
|
||||
|
||||
test('root workspace production tree introduces no new advisories', { timeout: TEST_TIMEOUT_MS }, (t) => {
|
||||
checkTreeAgainstBaseline(t, ROOT, '', 'auditable npm package not present or node_modules/ missing');
|
||||
});
|
||||
|
||||
test('sdk/ production tree has no advisories', { timeout: TEST_TIMEOUT_MS }, (t) => {
|
||||
const vulns = auditProductionVulns(SDK);
|
||||
if (vulns === null) {
|
||||
t.skip('sdk/ is not an auditable npm package or sdk/node_modules/ is missing');
|
||||
return;
|
||||
}
|
||||
assert.strictEqual(vulns.critical, 0, `expected 0 critical; got ${vulns.critical}`);
|
||||
assert.strictEqual(vulns.high, 0, `expected 0 high; got ${vulns.high}`);
|
||||
assert.strictEqual(vulns.moderate, 0, `expected 0 moderate; got ${vulns.moderate}`);
|
||||
assert.strictEqual(vulns.low, 0, `expected 0 low; got ${vulns.low}`);
|
||||
test('sdk/ production tree introduces no new advisories', { timeout: TEST_TIMEOUT_MS }, (t) => {
|
||||
checkTreeAgainstBaseline(t, SDK, 'sdk', 'sdk/ is not an auditable npm package or sdk/node_modules/ is missing');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user