Files
msd-core/tests/policy-138-nyquist-config-default.test.cjs
Tom Boucher b55a2e7655 fix(#2117): distinguish not-yet-validated phase from validated failure in audit-milestone
audit-milestone's Nyquist scan classified a phase from `nyquist_compliant`
alone, so a phase seeded by plan-phase but never run through validate-phase
read PARTIAL — identical to a phase that validated and genuinely failed. The
template's `status` field could discriminate the two, but no workflow ever
promoted it off `draft`, so it was dead.

Make `status` live and read it:
- validate-phase.md §6: set `status: validated` in both the create (State B)
  and update (State A) VALIDATION.md paths.
- audit-milestone.md §5.5: parse `status`; add a distinct NOT-VALIDATED bucket
  keyed on `status: draft`, gate COMPLIANT/PARTIAL on `status: validated`, and
  report `not_validated_phases` in the audit YAML.
- VALIDATION.md template: document the draft → validated lifecycle.

Tests & generated artifacts:
- Regression test folded into policy-138 (owning workflow-contract file);
  fail-first verified vs origin/next (0 matches pre-fix).
- Regenerate golden-install-parity fixtures cleanly: adds the previously-missed
  qwen.json and removes a contaminated `settings.local.json` entry that had
  leaked into claude-local.json (the harness excludes hook-config files).
- Correct a stale validate-phase.md workflow-size-baseline entry.

Closes #2117

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-11 22:47:51 -04:00

127 lines
5.4 KiB
JavaScript

// allow-test-rule: runtime-contract-is-the-product — asserts GSD workflow/template markdown prose, the executable contract (#138, #2117)
'use strict';
// Policy regression test for issue #138:
// Nyquist activation must be absent-safe. ADR-857 phase 6 moved that defaulting
// into `loop render-hooks verify:post`, so workflows must consume the capability
// hook instead of calling config-get directly.
const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const WORKFLOWS_DIR = path.join(__dirname, '..', 'gsd-core', 'workflows');
function readWorkflow(name) {
return fs.readFileSync(path.join(WORKFLOWS_DIR, name), 'utf8');
}
function assertNyquistCapabilityGate(name) {
const content = readWorkflow(name);
assert.ok(
content.includes('loop render-hooks verify:post'),
`${name} must resolve Nyquist activation through verify:post capability hooks`
);
assert.ok(
content.includes('ref.skill == "validate-phase"'),
`${name} must identify the validate-phase capability hook`
);
assert.ok(
!content.includes('config-get workflow.nyquist_validation'),
`${name} must not read workflow.nyquist_validation directly after capability cutover`
);
}
function findNyquistConfigLine(filePath) {
const content = fs.readFileSync(filePath, 'utf8');
const lines = content.split(/\r?\n/);
for (let i = 0; i < lines.length; i++) {
if (lines[i].includes('config-get workflow.nyquist_validation')) {
return { lineNumber: i + 1, line: lines[i] };
}
}
return null;
}
test('validate-phase.md: Nyquist activation uses verify:post capability hook', () => {
assertNyquistCapabilityGate('validate-phase.md');
});
test('audit-milestone.md: Nyquist activation uses verify:post capability hook', () => {
assertNyquistCapabilityGate('audit-milestone.md');
});
test('legacy Nyquist config helper still detects unsafe direct reads', () => {
const tmp = path.join(os.tmpdir(), `policy-138-synthetic-${process.pid}.md`);
try {
fs.writeFileSync(tmp, 'NYQUIST_CFG=$(gsd_run query config-get workflow.nyquist_validation --raw)\n');
const result = findNyquistConfigLine(tmp);
assert.ok(result, 'synthetic direct config read should be detected');
assert.ok(!result.line.includes('--default'), 'synthetic unsafe read intentionally lacks --default');
} finally {
try {
fs.unlinkSync(tmp);
} catch {
// Best-effort cleanup for synthetic file.
}
}
});
// ─────────────────────────────────────────────────────────────────────────────
// Issue #2117: audit-milestone could not distinguish a not-yet-validated phase
// from a validated-but-failing one — both read Nyquist PARTIAL. The fix makes the
// dead `status` field live (validate-phase §6 promotes draft → validated) and has
// audit-milestone §5.5 bucket `status: draft` as a distinct NOT-VALIDATED state.
// These assertions fail-first if either half of that two-workflow contract is
// reverted, silently re-collapsing "not validated" and "validation failed".
// ─────────────────────────────────────────────────────────────────────────────
test('#2117 validate-phase.md promotes status: draft → validated when it reconciles VALIDATION.md', () => {
const content = readWorkflow('validate-phase.md');
// Both the create (State B) and update (State A) paths in §6 must set the
// terminal marker, otherwise `status` stays `draft` for the life of the file
// and audit-milestone cannot tell an unvalidated phase from a failing one.
const occurrences = content.match(/status: validated/g) || [];
assert.ok(
occurrences.length >= 2,
'validate-phase.md must set `status: validated` in both the create (State B) and update (State A) VALIDATION.md paths',
);
});
test('#2117 audit-milestone.md buckets status: draft as NOT-VALIDATED, never PARTIAL', () => {
const content = readWorkflow('audit-milestone.md');
assert.ok(
content.includes('`status`'),
'audit-milestone.md must parse the `status` frontmatter field to detect not-yet-validated phases',
);
assert.ok(
content.includes('| NOT-VALIDATED | `status: draft`'),
'audit-milestone.md must define a NOT-VALIDATED bucket keyed on `status: draft`',
);
assert.ok(
content.includes('| COMPLIANT | `status: validated`'),
'COMPLIANT must require `status: validated` so a draft file can never be scored compliant',
);
assert.ok(
content.includes('| PARTIAL | `status: validated`'),
'PARTIAL must require `status: validated`; a `status: draft` file is NOT-VALIDATED, not PARTIAL',
);
assert.ok(
content.includes('not_validated_phases'),
'audit-milestone.md must report not_validated_phases in the nyquist audit YAML aggregate',
);
});
test('#2117 VALIDATION.md template seeds status: draft (the pre-validation state)', () => {
const template = fs.readFileSync(
path.join(__dirname, '..', 'gsd-core', 'templates', 'VALIDATION.md'),
'utf8',
);
assert.ok(
/^status: draft$/m.test(template),
'VALIDATION.md template must seed `status: draft`; validate-phase promotes it to validated',
);
});