* test(#1464): add behavioral manifest-validation test for capability tutorial docs Extracts JSON capability manifests from tutorial/reference docs and validates them through the real validateCapability — closing the test gap that let issue #1464's broken tutorial manifest (step missing ref) pass undetected. Adds fix-1464-docs-manifest-validation.test.cjs with: - Suite 1: build/install/reference docs' complete manifests pass validateCapability - Suite 2: adversarial fixtures prove the original #1464 bug shapes are caught (step without ref → "steps[0].ref must be an object…"; id/folderId mismatch) - Suite 3: extractManifests helper unit tests (complete vs. partial block filtering) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * test(#1496): allowlist docs module for 3-file test cluster docs-parity-live-registry, docs-update, and the new fix-1464-docs-manifest-validation sit on the same docs production module; add the docs entry to lint-test-file-count.allowlist.json so the novel-offender CI gate passes. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
@@ -175,6 +175,14 @@
|
||||
"external-job-waiting.test.cjs"
|
||||
],
|
||||
"issue": "#1165"
|
||||
},
|
||||
"docs": {
|
||||
"files": [
|
||||
"docs-parity-live-registry.test.cjs",
|
||||
"docs-update.test.cjs",
|
||||
"fix-1464-docs-manifest-validation.test.cjs"
|
||||
],
|
||||
"issue": "1496"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
269
tests/fix-1464-docs-manifest-validation.test.cjs
Normal file
269
tests/fix-1464-docs-manifest-validation.test.cjs
Normal file
@@ -0,0 +1,269 @@
|
||||
// allow-test-rule: source-text-is-the-product see #1464
|
||||
// Tutorial docs are the product surface users follow. Reading JSON code blocks
|
||||
// from them and validating through validateCapability is behavioral, not
|
||||
// source-grep — it proves the manifests work, not just that they "mention" a term.
|
||||
|
||||
'use strict';
|
||||
|
||||
const { describe, test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const { validateCapability } = require('../scripts/gen-capability-registry.cjs');
|
||||
|
||||
const ROOT = path.join(__dirname, '..');
|
||||
|
||||
// ─── Extractor ───────────────────────────────────────────────────────────────
|
||||
|
||||
// Required top-level fields that distinguish a complete capability manifest
|
||||
// from a partial output snippet (list-entry, install-result, etc.).
|
||||
// Partial output snippets have id+role but lack steps/contributions/gates/config.
|
||||
const MANIFEST_REQUIRED_KEYS = new Set([
|
||||
'id', 'role', 'title', 'description', 'tier',
|
||||
'requires', 'runtimeCompat', 'skills', 'agents',
|
||||
'config', 'steps', 'contributions', 'gates',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Extract JSON code blocks from markdown that are complete capability manifests.
|
||||
* A complete manifest has ALL keys in MANIFEST_REQUIRED_KEYS.
|
||||
* Partial output snippets (list-entries, install-results) have only id+role and are skipped.
|
||||
*/
|
||||
function extractManifests(mdContent) {
|
||||
const manifests = [];
|
||||
const fenceRe = /```json\s*\n([\s\S]*?)```/g;
|
||||
let match;
|
||||
while ((match = fenceRe.exec(mdContent)) !== null) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(match[1]);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||||
const keys = new Set(Object.keys(parsed));
|
||||
if ([...MANIFEST_REQUIRED_KEYS].every((k) => keys.has(k))) {
|
||||
manifests.push(parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
return manifests;
|
||||
}
|
||||
|
||||
// ─── Suite 1: tutorial manifests validate ────────────────────────────────────
|
||||
|
||||
describe('docs tutorial manifests pass validateCapability (#1464 regression)', () => {
|
||||
test('build-your-first-capability.md: every manifest passes', () => {
|
||||
const content = fs.readFileSync(
|
||||
path.join(ROOT, 'docs', 'tutorials', 'build-your-first-capability.md'),
|
||||
'utf8',
|
||||
);
|
||||
const manifests = extractManifests(content);
|
||||
assert.ok(
|
||||
manifests.length > 0,
|
||||
'expected at least one capability manifest in build tutorial',
|
||||
);
|
||||
for (const cap of manifests) {
|
||||
const errors = validateCapability(cap, cap.id);
|
||||
assert.deepStrictEqual(
|
||||
errors,
|
||||
[],
|
||||
`build tutorial manifest id="${cap.id}" failed validateCapability:\n ${errors.join('\n ')}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('install-your-first-capability.md: every manifest passes', () => {
|
||||
const content = fs.readFileSync(
|
||||
path.join(ROOT, 'docs', 'tutorials', 'install-your-first-capability.md'),
|
||||
'utf8',
|
||||
);
|
||||
const manifests = extractManifests(content);
|
||||
assert.ok(
|
||||
manifests.length > 0,
|
||||
'expected at least one capability manifest in install tutorial',
|
||||
);
|
||||
for (const cap of manifests) {
|
||||
const errors = validateCapability(cap, cap.id);
|
||||
assert.deepStrictEqual(
|
||||
errors,
|
||||
[],
|
||||
`install tutorial manifest id="${cap.id}" failed validateCapability:\n ${errors.join('\n ')}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('capability-manifest.md reference example passes', () => {
|
||||
const content = fs.readFileSync(
|
||||
path.join(ROOT, 'docs', 'reference', 'capability-manifest.md'),
|
||||
'utf8',
|
||||
);
|
||||
const manifests = extractManifests(content);
|
||||
assert.ok(
|
||||
manifests.length > 0,
|
||||
'expected at least one capability manifest in reference doc',
|
||||
);
|
||||
for (const cap of manifests) {
|
||||
const errors = validateCapability(cap, cap.id);
|
||||
assert.deepStrictEqual(
|
||||
errors,
|
||||
[],
|
||||
`reference manifest id="${cap.id}" failed validateCapability:\n ${errors.join('\n ')}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Suite 2: adversarial — #1464 failure modes caught ───────────────────────
|
||||
//
|
||||
// These are the EXACT failure shapes from issue #1464.
|
||||
// They must fail validateCapability — proving this test would have caught the bug.
|
||||
|
||||
describe('validateCapability catches original #1464 bug shapes', () => {
|
||||
// #1464 high-1: step missing ref → validateStep rejects it
|
||||
test('step without ref fails (the original broken tutorial step)', () => {
|
||||
const cap = {
|
||||
id: 'hello-note',
|
||||
role: 'feature',
|
||||
version: '0.1.0',
|
||||
title: 'Hello Note',
|
||||
description: 'Test fixture for #1464 regression.',
|
||||
tier: 'standard',
|
||||
requires: [],
|
||||
engines: { gsd: '>=1.6.0' },
|
||||
runtimeCompat: { supported: ['*'], unsupported: [] },
|
||||
skills: [],
|
||||
agents: [],
|
||||
config: {},
|
||||
steps: [
|
||||
{
|
||||
// Missing ref — this was the #1464 high-1 bug in the original tutorial
|
||||
point: 'plan:pre',
|
||||
produces: ['HELLO.md'],
|
||||
consumes: [],
|
||||
onError: 'skip',
|
||||
},
|
||||
],
|
||||
contributions: [],
|
||||
gates: [],
|
||||
};
|
||||
const errors = validateCapability(cap, 'hello-note');
|
||||
assert.ok(errors.length > 0, 'expected validation errors for step without ref');
|
||||
assert.ok(
|
||||
errors.some((e) => /ref/.test(e)),
|
||||
`expected an error mentioning "ref"; got: ${errors.join('; ')}`,
|
||||
);
|
||||
});
|
||||
|
||||
// #1464 shape: id must match folder name (folderId contract)
|
||||
test('id not matching folderId fails', () => {
|
||||
const cap = {
|
||||
id: 'hello-note',
|
||||
role: 'feature',
|
||||
version: '0.1.0',
|
||||
title: 'Hello Note',
|
||||
description: 'Test fixture for id/folderId mismatch.',
|
||||
tier: 'standard',
|
||||
requires: [],
|
||||
runtimeCompat: { supported: ['*'], unsupported: [] },
|
||||
skills: [],
|
||||
agents: [],
|
||||
config: {},
|
||||
steps: [],
|
||||
contributions: [],
|
||||
gates: [],
|
||||
};
|
||||
const errors = validateCapability(cap, 'wrong-folder');
|
||||
assert.ok(errors.length > 0, 'expected id/folderId mismatch to fail validation');
|
||||
assert.ok(
|
||||
errors.some((e) => /folder/.test(e) || /equal/.test(e) || /id/.test(e)),
|
||||
`expected error about id/folderId mismatch; got: ${errors.join('; ')}`,
|
||||
);
|
||||
});
|
||||
|
||||
// Corrected shape: contribution with fragment + into (the PR #1495 fix)
|
||||
test('contribution with fragment.path + into passes (the PR #1495 fix shape)', () => {
|
||||
const cap = {
|
||||
id: 'hello-note',
|
||||
role: 'feature',
|
||||
version: '0.1.0',
|
||||
title: 'Hello Note',
|
||||
description: 'Injects a greeting note at plan:pre and produces HELLO.md.',
|
||||
tier: 'standard',
|
||||
requires: [],
|
||||
runtimeCompat: { supported: ['*'], unsupported: [] },
|
||||
skills: [],
|
||||
agents: [],
|
||||
config: {},
|
||||
steps: [],
|
||||
contributions: [
|
||||
{
|
||||
point: 'plan:pre',
|
||||
into: 'planner',
|
||||
fragment: { path: 'fragments/plan-pre.md' },
|
||||
produces: ['HELLO.md'],
|
||||
consumes: [],
|
||||
onError: 'skip',
|
||||
},
|
||||
],
|
||||
gates: [],
|
||||
};
|
||||
const errors = validateCapability(cap, 'hello-note');
|
||||
assert.deepStrictEqual(
|
||||
errors,
|
||||
[],
|
||||
`corrected contribution manifest has unexpected errors: ${errors.join('; ')}`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Suite 3: extractManifests helper ────────────────────────────────────────
|
||||
|
||||
describe('extractManifests helper unit tests', () => {
|
||||
test('returns empty array for plain text with no JSON fences', () => {
|
||||
assert.deepStrictEqual(extractManifests('No code blocks here.'), []);
|
||||
});
|
||||
|
||||
test('skips JSON blocks without all required manifest keys', () => {
|
||||
// Partial list-entry block — only has id, role, version but not steps/contributions/etc.
|
||||
const md = '```json\n{"id":"x","role":"feature","version":"1.0.0"}\n```';
|
||||
assert.deepStrictEqual(extractManifests(md), []);
|
||||
});
|
||||
|
||||
function makeCompleteManifest(overrides) {
|
||||
return {
|
||||
id: 'test-cap', role: 'feature', title: 'T', description: 'D',
|
||||
tier: 'standard', requires: [], runtimeCompat: { supported: ['*'], unsupported: [] },
|
||||
skills: [], agents: [], config: {}, steps: [], contributions: [], gates: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('extracts a complete manifest (all required keys present)', () => {
|
||||
const cap = makeCompleteManifest({ id: 'x' });
|
||||
const md = '```json\n' + JSON.stringify(cap, null, 2) + '\n```';
|
||||
const result = extractManifests(md);
|
||||
assert.strictEqual(result.length, 1);
|
||||
assert.strictEqual(result[0].id, 'x');
|
||||
});
|
||||
|
||||
test('skips malformed JSON blocks silently', () => {
|
||||
const complete = makeCompleteManifest({ id: 'y' });
|
||||
const md = '```json\n{bad json here\n```\n```json\n' + JSON.stringify(complete) + '\n```';
|
||||
const result = extractManifests(md);
|
||||
assert.strictEqual(result.length, 1);
|
||||
assert.strictEqual(result[0].id, 'y');
|
||||
});
|
||||
|
||||
test('extracts multiple complete manifests from one doc', () => {
|
||||
const a = makeCompleteManifest({ id: 'cap-a' });
|
||||
const b = makeCompleteManifest({ id: 'cap-b' });
|
||||
const md = [
|
||||
'```json\n' + JSON.stringify(a) + '\n```',
|
||||
'```json\n' + JSON.stringify(b) + '\n```',
|
||||
].join('\n');
|
||||
const result = extractManifests(md);
|
||||
assert.strictEqual(result.length, 2);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user