* test(#2070): fail-first tests for adaptive model_profile and models tier validation Encodes the three acceptance criteria from #2070 plus the boundary cases the resolver silently ignores today (non-string values, empty string, mistyped phase-type key), and pins VALID_TIERS to a catalog-derived set. Red phase: these fail against current src/ by design. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SLufH5sDuqA1AiEGu45cuA * fix(#2070): accept adaptive model_profile in validate health; warn on invalid models tiers (W022) W004 sourced its profile list from a hand-maintained literal that predated the adaptive profile, so `"model_profile": "adaptive"` was false-flagged. It now reads VALID_PROFILES, which model-catalog.cts derives from model-catalog.json. models.<phase_type> was validated nowhere: the resolver's tier gate silently drops unknown values, so a typo like `"planning": "opuss"` was an undiagnosable no-op. A new W022 flags unknown phase-type keys and invalid tier values (including non-string values, which the same gate also drops). VALID_TIERS moves from a function-local literal in model-resolver.cts to a catalog-derived export, so health and the resolver cannot disagree by construction rather than by parity test. Object.values(adaptiveTierMap) is ['opus','sonnet','haiku'] plus 'inherit' — identical to the previous literal, so resolution behavior is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SLufH5sDuqA1AiEGu45cuA * docs(#2070): changeset for validate health adaptive profile + W022 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SLufH5sDuqA1AiEGu45cuA * fix(#2070): close review findings — malformed models, tier-list duplication, changeset gate Review of the initial fix surfaced three real defects, folded in per the no-defer rule: 1. verify.cts: the W022 guard skipped a top-level `models` that is present but not a plain object (`[]`, `"opus"`, `5`, `true`). The resolver ignores those identically, so they were the same undiagnosable no-op #2070 targets — just one level up. They now warn; absent/null/{} stay silent. 2. config-loader.cts: RUNTIME_OVERRIDE_TIERS was a second hardcoded copy of the tier vocabulary this change had just de-hardcoded elsewhere. It now derives from the catalog via ADAPTIVE_TIER_VALUES (no 'inherit' — runtime overrides resolve to a concrete tier). Byte-equivalent to the old literal. 3. scripts/changeset/lint.cjs: USER_FACING_PREFIXES omitted `src/`. Post-ADR-457 the product source is src/*.cts compiled to a gitignored gsd-core/bin/lib, so the `gsd-core/` prefix is dead coverage for library code and a src/-only PR could merge with no release note — including this one. Adding `src/` closes the gate; tests/ stays non-user-facing. Also corrects a false docstring in the VALID_TIERS test: value-equality cannot detect a re-hardcoded literal, so the test no longer claims it does. Two review findings were rejected with evidence rather than actioned: - W021 double-allocation is governed by ADR-612 ("W021 renumber -> void ... kept, message-disambiguated"), not a defect. - Global-defaults validation would be a false-positive generator: config-loader reads ~/.gsd/defaults.json only on the "no .planning/" branch, and health early-returns E001 without .planning/, so those values provably never affect resolution in any context health can run. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SLufH5sDuqA1AiEGu45cuA * test(#2070): regenerate install goldens for the changeset-lint change scripts/ ships as an installed artifact, so scripts/changeset/lint.cjs's content hash is pinned in all 18 runtime golden fixtures. Adding 'src/' to USER_FACING_PREFIXES changed that hash and tripped every golden parity check. Regenerated via `npm run gen:golden`; the only delta is the lint.cjs hash. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SLufH5sDuqA1AiEGu45cuA * docs(#2070): backfill PR number 2336 into changeset Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SLufH5sDuqA1AiEGu45cuA --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
300 lines
11 KiB
JavaScript
300 lines
11 KiB
JavaScript
'use strict';
|
|
process.env.GSD_TEST_MODE = '1';
|
|
|
|
const { test, describe } = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const path = require('node:path');
|
|
const fs = require('node:fs');
|
|
const os = require('node:os');
|
|
const cp = require('node:child_process');
|
|
|
|
const { evaluateLint, LINT_REASON } = require(path.join(__dirname, '..', 'scripts', 'changeset', 'lint.cjs'));
|
|
|
|
const ROOT = path.join(__dirname, '..');
|
|
const LINT_SCRIPT = path.join(ROOT, 'scripts', 'changeset', 'lint.cjs');
|
|
const { cleanup } = require('./helpers.cjs');
|
|
|
|
/**
|
|
* Build a minimal temp git repo shaped like a PR branch:
|
|
* origin/main = base commit (README.md)
|
|
* pr = PR branch with caller-supplied files committed on top
|
|
*
|
|
* @param {string} tmpDir - pre-created temp directory (mkdtempSync result)
|
|
* @param {Array<{file: string, content: string}>} prFiles - files to create on the PR branch
|
|
* @param {Array<{file: string, content: string}>} [baseFiles] - extra files to create on base commit
|
|
* @returns {string} path to the temp repo (same as tmpDir)
|
|
*/
|
|
function buildTempRepo(tmpDir, prFiles, baseFiles = []) {
|
|
const git = (...args) => cp.execFileSync('git', args, { cwd: tmpDir, encoding: 'utf8' });
|
|
|
|
git('init', '-q', '-b', 'main');
|
|
git('config', 'user.email', 'test@example.com');
|
|
git('config', 'user.name', 'Test');
|
|
|
|
// Base commit: README + any caller-supplied base files
|
|
fs.writeFileSync(path.join(tmpDir, 'README.md'), '# test\n');
|
|
for (const { file, content } of baseFiles) {
|
|
const abs = path.join(tmpDir, file);
|
|
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
fs.writeFileSync(abs, content);
|
|
}
|
|
git('add', '-A');
|
|
git('commit', '-q', '-m', 'base');
|
|
|
|
// Fake origin/main so `git diff origin/main...HEAD` works without a real remote
|
|
git('update-ref', 'refs/remotes/origin/main', 'HEAD');
|
|
|
|
// PR branch
|
|
git('checkout', '-q', '-b', 'pr');
|
|
|
|
// Create or delete the PR files
|
|
for (const { file, content } of prFiles) {
|
|
const abs = path.join(tmpDir, file);
|
|
if (content === null) {
|
|
// null content = delete the file (unlinkSync, not rmSync, to stay within
|
|
// the no-raw-rmsync-in-tests rule — we are removing a single file, not a tree)
|
|
try { fs.unlinkSync(abs); } catch { /* already absent — ok */ }
|
|
} else {
|
|
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
fs.writeFileSync(abs, content);
|
|
}
|
|
}
|
|
git('add', '-A');
|
|
git('commit', '-q', '-m', 'pr changes');
|
|
|
|
return tmpDir;
|
|
}
|
|
|
|
/**
|
|
* Invoke lint.cjs --json in the given repo directory and return the parsed report.
|
|
* @param {string} repoDir
|
|
* @returns {{ status: number, report: object }}
|
|
*/
|
|
function runLint(repoDir) {
|
|
const result = cp.spawnSync(
|
|
process.execPath,
|
|
[LINT_SCRIPT, '--json'],
|
|
{
|
|
cwd: repoDir,
|
|
env: { ...process.env, GITHUB_BASE_REF: 'main', GITHUB_EVENT_PATH: '' },
|
|
encoding: 'utf8',
|
|
},
|
|
);
|
|
let report = {};
|
|
try { report = JSON.parse(result.stdout); } catch { /* leave as empty object */ }
|
|
return { status: result.status, report };
|
|
}
|
|
|
|
// evaluateLint is a pure function over file lists + label list — no fs, no git.
|
|
// Tests assert on the structured verdict: { ok: bool, reason: LINT_REASON.X }.
|
|
|
|
describe('changeset lint: pure verdict (#2975)', () => {
|
|
test('LINT_REASON enum exposes the documented codes', () => {
|
|
assert.deepEqual(
|
|
Object.keys(LINT_REASON).sort(),
|
|
['OK_FRAGMENT_PRESENT', 'OK_NO_USER_FACING_CHANGES', 'OK_OPT_OUT_LABEL', 'FAIL_MISSING_FRAGMENT', 'FAIL_INVALID_FRAGMENT'].sort(),
|
|
);
|
|
});
|
|
|
|
test('OK_FRAGMENT_PRESENT when the diff includes a new .changeset/*.md', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['bin/install.js', '.changeset/silly-bears-dance.md'],
|
|
labels: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: true, reason: LINT_REASON.OK_FRAGMENT_PRESENT });
|
|
});
|
|
|
|
test('FAIL_MISSING_FRAGMENT when user-facing files change without a fragment', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['bin/install.js', 'tests/foo.test.cjs'],
|
|
labels: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: false, reason: LINT_REASON.FAIL_MISSING_FRAGMENT });
|
|
});
|
|
|
|
test('OK_OPT_OUT_LABEL when no-changelog label present, even with user-facing changes', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['bin/install.js'],
|
|
labels: ['no-changelog'],
|
|
});
|
|
assert.deepEqual(verdict, { ok: true, reason: LINT_REASON.OK_OPT_OUT_LABEL });
|
|
});
|
|
|
|
test('OK_NO_USER_FACING_CHANGES when only test/ci/doc files change', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['tests/foo.test.cjs', '.github/workflows/x.yml', 'docs/x.md'],
|
|
labels: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: true, reason: LINT_REASON.OK_NO_USER_FACING_CHANGES });
|
|
});
|
|
|
|
test('FAIL_MISSING_FRAGMENT when CHANGELOG.md is edited directly (closes the workflow bypass)', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['CHANGELOG.md'],
|
|
labels: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: false, reason: LINT_REASON.FAIL_MISSING_FRAGMENT });
|
|
});
|
|
|
|
test('a fragment alone (no source change) is OK_FRAGMENT_PRESENT — fragment-only PR is allowed', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['.changeset/silly-bears-dance.md'],
|
|
labels: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: true, reason: LINT_REASON.OK_FRAGMENT_PRESENT });
|
|
});
|
|
|
|
test('FAIL_INVALID_FRAGMENT when fragmentFailures is non-empty', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['.changeset/bad.md'],
|
|
labels: [],
|
|
fragmentFailures: [{ file: '.changeset/bad.md', reason: 'invalid_pr', detail: '0' }],
|
|
});
|
|
assert.equal(verdict.ok, false);
|
|
assert.equal(verdict.reason, LINT_REASON.FAIL_INVALID_FRAGMENT);
|
|
assert.deepEqual(verdict.failures, [{ file: '.changeset/bad.md', reason: 'invalid_pr', detail: '0' }]);
|
|
});
|
|
|
|
test('OK_FRAGMENT_PRESENT when fragment is present and fragmentFailures is empty (regression guard)', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['bin/install.js', '.changeset/good.md'],
|
|
labels: [],
|
|
fragmentFailures: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: true, reason: LINT_REASON.OK_FRAGMENT_PRESENT });
|
|
});
|
|
|
|
test('FAIL_INVALID_FRAGMENT beats no-changelog opt-out label', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['.changeset/bad.md'],
|
|
labels: ['no-changelog'],
|
|
fragmentFailures: [{ file: '.changeset/bad.md', reason: 'invalid_pr', detail: '0' }],
|
|
});
|
|
assert.equal(verdict.ok, false);
|
|
assert.equal(verdict.reason, LINT_REASON.FAIL_INVALID_FRAGMENT);
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// src/ is user-facing (#2070 fix 2): post-ADR-457 the .cts sources under
|
|
// src/ are the product source of truth — gsd-core/bin/lib/*.cjs is a
|
|
// gitignored build artifact, so a src/-only PR previously had zero
|
|
// USER_FACING_PREFIXES coverage and could merge with no release note.
|
|
// ---------------------------------------------------------------------------
|
|
describe('changeset lint: src/ is user-facing (#2070)', () => {
|
|
test('FAIL_MISSING_FRAGMENT when only a src/*.cts file changes without a fragment', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['src/verify.cts'],
|
|
labels: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: false, reason: LINT_REASON.FAIL_MISSING_FRAGMENT });
|
|
});
|
|
|
|
test('OK_FRAGMENT_PRESENT when a src/*.cts change is accompanied by a fragment', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['src/verify.cts', '.changeset/silly-bears-dance.md'],
|
|
labels: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: true, reason: LINT_REASON.OK_FRAGMENT_PRESENT });
|
|
});
|
|
|
|
test('OK_NO_USER_FACING_CHANGES when only a tests/*.test.cjs file changes (tests are not user-facing)', () => {
|
|
const verdict = evaluateLint({
|
|
changedFiles: ['tests/foo.test.cjs'],
|
|
labels: [],
|
|
});
|
|
assert.deepEqual(verdict, { ok: true, reason: LINT_REASON.OK_NO_USER_FACING_CHANGES });
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// End-to-end integration: real main() wiring via temp git repo (#1006)
|
|
// ---------------------------------------------------------------------------
|
|
describe('changeset lint: main() end-to-end wiring (#1006)', () => {
|
|
// Each test allocates its own tmpDir so cases run independently.
|
|
|
|
test('malformed fragment (pr: 0) fails the gate end-to-end', (t) => {
|
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-lint-e2e-'));
|
|
t.after(() => cleanup(tmpDir));
|
|
|
|
buildTempRepo(tmpDir, [
|
|
// User-facing source change that requires a fragment
|
|
{ file: 'bin/thing.js', content: '// placeholder\n' },
|
|
// Malformed fragment: pr: 0 is rejected by parseFragment (pr must be > 0)
|
|
{
|
|
file: '.changeset/bad.md',
|
|
content: '---\ntype: Fixed\npr: 0\n---\n**Bad** placeholder. (#1)\n',
|
|
},
|
|
]);
|
|
|
|
const { status, report } = runLint(tmpDir);
|
|
|
|
assert.equal(status, 1, `expected exit 1, got ${status}`);
|
|
assert.equal(
|
|
report.reason,
|
|
LINT_REASON.FAIL_INVALID_FRAGMENT,
|
|
`expected FAIL_INVALID_FRAGMENT, got ${report.reason}`,
|
|
);
|
|
assert.ok(Array.isArray(report.failures), 'failures must be an array');
|
|
const badEntry = report.failures.find((f) => f.file.endsWith('.changeset/bad.md'));
|
|
assert.ok(badEntry, 'failures must contain the bad fragment file');
|
|
assert.equal(badEntry.reason, 'invalid_pr', `expected reason invalid_pr, got ${badEntry.reason}`);
|
|
});
|
|
|
|
test('valid fragment (pr: 1) passes the gate end-to-end', (t) => {
|
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-lint-e2e-'));
|
|
t.after(() => cleanup(tmpDir));
|
|
|
|
buildTempRepo(tmpDir, [
|
|
{ file: 'bin/thing.js', content: '// placeholder\n' },
|
|
{
|
|
file: '.changeset/good.md',
|
|
content: '---\ntype: Fixed\npr: 1\n---\n**Good** fix. (#1)\n',
|
|
},
|
|
]);
|
|
|
|
const { status, report } = runLint(tmpDir);
|
|
|
|
assert.equal(status, 0, `expected exit 0, got ${status}`);
|
|
assert.equal(
|
|
report.reason,
|
|
LINT_REASON.OK_FRAGMENT_PRESENT,
|
|
`expected OK_FRAGMENT_PRESENT, got ${report.reason}`,
|
|
);
|
|
});
|
|
|
|
test('deleted fragment is skipped and does not produce fail_invalid_fragment', (t) => {
|
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gsd-lint-e2e-'));
|
|
t.after(() => cleanup(tmpDir));
|
|
|
|
// Base commit includes a valid fragment (pr: 5) that the PR will delete.
|
|
buildTempRepo(
|
|
tmpDir,
|
|
[
|
|
// PR deletes the old fragment (null = delete)
|
|
{ file: '.changeset/old.md', content: null },
|
|
// PR adds a new valid fragment
|
|
{
|
|
file: '.changeset/new.md',
|
|
content: '---\ntype: Fixed\npr: 6\n---\n**New** fix. (#6)\n',
|
|
},
|
|
],
|
|
// base files: the old fragment exists before the PR
|
|
[{ file: '.changeset/old.md', content: '---\ntype: Fixed\npr: 5\n---\n**Old** fix. (#5)\n' }],
|
|
);
|
|
|
|
const { status, report } = runLint(tmpDir);
|
|
|
|
assert.equal(status, 0, `expected exit 0, got ${status}`);
|
|
assert.equal(
|
|
report.reason,
|
|
LINT_REASON.OK_FRAGMENT_PRESENT,
|
|
`expected OK_FRAGMENT_PRESENT, got ${report.reason}`,
|
|
);
|
|
// The deleted fragment must NOT appear in failures
|
|
const failures = report.failures ?? [];
|
|
const deletedEntry = failures.find((f) => f.file.endsWith('.changeset/old.md'));
|
|
assert.ok(!deletedEntry, `deleted fragment must not appear in failures, got: ${JSON.stringify(failures)}`);
|
|
});
|
|
});
|