* feat(#1579): deterministic gsd-tools query eval.score verb Split C of #1573 (pure code, lowest risk). Adds an eval.score query verb (coverage*0.6 + infra*0.4; bands 80/60/40) mirroring the verify.* chain; gsd-eval-auditor consumes it instead of doing weighted arithmetic in-prompt. Non-breaking — additive only. arXiv: 2601.15130 (Plausibility Trap/DPDM), 2507.10281 (Table Agent), 2508.15754 (TIR). * fix(#1579): address review — domain guard, property test, glossary, SKIP_ROOT, inventory/baseline - C3 input-domain: reject out-of-domain eval.score (require 0<=covered<=total; was emitting overall_score>100 / negatives) - C1 property test: add tests/eval.property.test.cjs (fast-check) — determinism, band monotonicity, [0,100] bounds, never-throws - C2 glossary: CONTEXT.md "Eval Scoring Module" entry (source-of-truth path + interface) - C4: add `eval` to SKIP_ROOT_RESOLUTION (pure arithmetic; no .planning/ access) - inventory: register generated eval.cjs/eval-command-router.cjs (INVENTORY-MANIFEST.json + INVENTORY.md rows) - size: regen agent-size baseline for gsd-eval-auditor (reused gsd_run shim + eval.score step) - eslint: ignore generated eval*.cjs (ADR-457 bin/lib migration coverage) * fix(#1579): register eval family in alias-drift gates Add EVAL_COMMAND_ALIASES/EVAL_SUBCOMMANDS to scripts/check-alias-drift.cjs families and to familyArrayKeys in the manifest-coverage test, so the eval family lands under the same drift guard as every sibling family (state/verify/init/phase/phases/validate/roadmap). Addresses trek-e review. check:alias-drift ok; feat-3251 coverage 9/9; eval suites 10/10. * docs(#1579): use half-open verdict band ranges in CLI-TOOLS overall_score is fractional and thresholds are >=80/>=60/>=40, so a score in [79,80) is correctly NEEDS WORK despite the old '60-79' label. Relabel bands as 60-<80 / 40-<60 / 0-<40 to match the code. Addresses trek-e nit. * fix(#1579): validate eval.score CLI inputs Reject unknown infra tokens and fractional counts, and pin the 80-point verdict boundary including rounding-before-banding behavior.
120 lines
3.7 KiB
JavaScript
120 lines
3.7 KiB
JavaScript
'use strict';
|
|
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
|
|
const { ExitError, runMain } = require('./lib/cli-exit.cjs');
|
|
|
|
const ROOT = path.resolve(__dirname, '..');
|
|
const aliasesPath = path.join(ROOT, 'gsd-core', 'bin', 'lib', 'command-aliases.cjs');
|
|
|
|
function fail(message) {
|
|
process.stderr.write(`${message}\n`);
|
|
throw new ExitError(1);
|
|
}
|
|
|
|
function ensureArray(value, name) {
|
|
if (!Array.isArray(value)) {
|
|
fail(`check:alias-drift: expected ${name} to be an array`);
|
|
}
|
|
}
|
|
|
|
function assertNoDuplicates(values, label) {
|
|
const seen = new Set();
|
|
for (const value of values) {
|
|
if (seen.has(value)) {
|
|
fail(`check:alias-drift: duplicate ${label} value "${value}"`);
|
|
}
|
|
seen.add(value);
|
|
}
|
|
}
|
|
|
|
function main() {
|
|
if (!fs.existsSync(aliasesPath)) {
|
|
fail(`check:alias-drift: missing ${path.relative(ROOT, aliasesPath)}`);
|
|
}
|
|
|
|
const aliases = require(aliasesPath);
|
|
|
|
const families = [
|
|
{
|
|
commandAliases: 'STATE_COMMAND_ALIASES',
|
|
subcommands: 'STATE_SUBCOMMANDS',
|
|
routerPath: path.join(ROOT, 'gsd-core', 'bin', 'lib', 'state-command-router.cjs'),
|
|
},
|
|
{
|
|
commandAliases: 'VERIFY_COMMAND_ALIASES',
|
|
subcommands: 'VERIFY_SUBCOMMANDS',
|
|
routerPath: path.join(ROOT, 'gsd-core', 'bin', 'lib', 'verify-command-router.cjs'),
|
|
},
|
|
{
|
|
commandAliases: 'INIT_COMMAND_ALIASES',
|
|
subcommands: 'INIT_SUBCOMMANDS',
|
|
routerPath: path.join(ROOT, 'gsd-core', 'bin', 'lib', 'init-command-router.cjs'),
|
|
},
|
|
{
|
|
commandAliases: 'PHASE_COMMAND_ALIASES',
|
|
subcommands: 'PHASE_SUBCOMMANDS',
|
|
routerPath: path.join(ROOT, 'gsd-core', 'bin', 'lib', 'phase-command-router.cjs'),
|
|
},
|
|
{
|
|
commandAliases: 'PHASES_COMMAND_ALIASES',
|
|
subcommands: 'PHASES_SUBCOMMANDS',
|
|
routerPath: path.join(ROOT, 'gsd-core', 'bin', 'lib', 'phases-command-router.cjs'),
|
|
},
|
|
{
|
|
commandAliases: 'VALIDATE_COMMAND_ALIASES',
|
|
subcommands: 'VALIDATE_SUBCOMMANDS',
|
|
routerPath: path.join(ROOT, 'gsd-core', 'bin', 'lib', 'validate-command-router.cjs'),
|
|
},
|
|
{
|
|
commandAliases: 'ROADMAP_COMMAND_ALIASES',
|
|
subcommands: 'ROADMAP_SUBCOMMANDS',
|
|
routerPath: path.join(ROOT, 'gsd-core', 'bin', 'lib', 'roadmap-command-router.cjs'),
|
|
},
|
|
{
|
|
commandAliases: 'EVAL_COMMAND_ALIASES',
|
|
subcommands: 'EVAL_SUBCOMMANDS',
|
|
routerPath: path.join(ROOT, 'gsd-core', 'bin', 'lib', 'eval-command-router.cjs'),
|
|
},
|
|
];
|
|
|
|
for (const family of families) {
|
|
const commandAliases = aliases[family.commandAliases];
|
|
const subcommands = aliases[family.subcommands];
|
|
|
|
ensureArray(commandAliases, family.commandAliases);
|
|
ensureArray(subcommands, family.subcommands);
|
|
|
|
const derivedSubcommands = commandAliases.map((entry) => entry && entry.subcommand);
|
|
assertNoDuplicates(derivedSubcommands, `${family.commandAliases}.subcommand`);
|
|
|
|
if (derivedSubcommands.length !== subcommands.length) {
|
|
fail(
|
|
`check:alias-drift: ${family.subcommands} length ${subcommands.length} does not match ` +
|
|
`${family.commandAliases} length ${derivedSubcommands.length}`,
|
|
);
|
|
}
|
|
|
|
for (let i = 0; i < derivedSubcommands.length; i++) {
|
|
if (derivedSubcommands[i] !== subcommands[i]) {
|
|
fail(
|
|
`check:alias-drift: ${family.subcommands}[${i}] = "${subcommands[i]}" ` +
|
|
`does not match ${family.commandAliases}[${i}].subcommand = "${derivedSubcommands[i]}"`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const routerSource = fs.readFileSync(family.routerPath, 'utf8');
|
|
if (!routerSource.includes(family.subcommands)) {
|
|
fail(
|
|
`check:alias-drift: ${path.relative(ROOT, family.routerPath)} does not reference ${family.subcommands}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
process.stdout.write('check:alias-drift ok\n');
|
|
}
|
|
|
|
runMain(main);
|