// allow-test-rule: source-text-is-the-product see #1464 // Tutorial docs are the product surface users follow. Reading JSON code blocks // from them and validating through validateCapability is behavioral, not // source-grep — it proves the manifests work, not just that they "mention" a term. 'use strict'; const { describe, test } = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const { validateCapability } = require('../scripts/gen-capability-registry.cjs'); const ROOT = path.join(__dirname, '..'); // ─── Extractor ─────────────────────────────────────────────────────────────── // Required top-level fields that distinguish a complete capability manifest // from a partial output snippet (list-entry, install-result, etc.). // Partial output snippets have id+role but lack steps/contributions/gates/config. const MANIFEST_REQUIRED_KEYS = new Set([ 'id', 'role', 'title', 'description', 'tier', 'requires', 'runtimeCompat', 'skills', 'agents', 'config', 'steps', 'contributions', 'gates', ]); /** * Extract JSON code blocks from markdown that are complete capability manifests. * A complete manifest has ALL keys in MANIFEST_REQUIRED_KEYS. * Partial output snippets (list-entries, install-results) have only id+role and are skipped. */ function extractManifests(mdContent) { const manifests = []; const fenceRe = /```json\s*\r?\n([\s\S]*?)```/g; let match; while ((match = fenceRe.exec(mdContent)) !== null) { let parsed; try { parsed = JSON.parse(match[1]); } catch { continue; } if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { const keys = new Set(Object.keys(parsed)); if ([...MANIFEST_REQUIRED_KEYS].every((k) => keys.has(k))) { manifests.push(parsed); } } } return manifests; } // ─── Suite 1: tutorial manifests validate ──────────────────────────────────── describe('docs tutorial manifests pass validateCapability (#1464 regression)', () => { test('build-your-first-capability.md: every manifest passes', () => { const content = fs.readFileSync( path.join(ROOT, 'docs', 'tutorials', 'build-your-first-capability.md'), 'utf8', ); const manifests = extractManifests(content); assert.ok( manifests.length > 0, 'expected at least one capability manifest in build tutorial', ); for (const cap of manifests) { const errors = validateCapability(cap, cap.id); assert.deepStrictEqual( errors, [], `build tutorial manifest id="${cap.id}" failed validateCapability:\n ${errors.join('\n ')}`, ); } }); test('install-your-first-capability.md: every manifest passes', () => { const content = fs.readFileSync( path.join(ROOT, 'docs', 'tutorials', 'install-your-first-capability.md'), 'utf8', ); const manifests = extractManifests(content); assert.ok( manifests.length > 0, 'expected at least one capability manifest in install tutorial', ); for (const cap of manifests) { const errors = validateCapability(cap, cap.id); assert.deepStrictEqual( errors, [], `install tutorial manifest id="${cap.id}" failed validateCapability:\n ${errors.join('\n ')}`, ); } }); test('capability-manifest.md reference example passes', () => { const content = fs.readFileSync( path.join(ROOT, 'docs', 'reference', 'capability-manifest.md'), 'utf8', ); const manifests = extractManifests(content); assert.ok( manifests.length > 0, 'expected at least one capability manifest in reference doc', ); for (const cap of manifests) { const errors = validateCapability(cap, cap.id); assert.deepStrictEqual( errors, [], `reference manifest id="${cap.id}" failed validateCapability:\n ${errors.join('\n ')}`, ); } }); }); // ─── Suite 2: adversarial — #1464 failure modes caught ─────────────────────── // // These are the EXACT failure shapes from issue #1464. // They must fail validateCapability — proving this test would have caught the bug. describe('validateCapability catches original #1464 bug shapes', () => { // #1464 high-1: step missing ref → validateStep rejects it test('step without ref fails (the original broken tutorial step)', () => { const cap = { id: 'hello-note', role: 'feature', version: '0.1.0', title: 'Hello Note', description: 'Test fixture for #1464 regression.', tier: 'standard', requires: [], engines: { gsd: '>=1.6.0' }, runtimeCompat: { supported: ['*'], unsupported: [] }, skills: [], agents: [], config: {}, steps: [ { // Missing ref — this was the #1464 high-1 bug in the original tutorial point: 'plan:pre', produces: ['HELLO.md'], consumes: [], onError: 'skip', }, ], contributions: [], gates: [], }; const errors = validateCapability(cap, 'hello-note'); assert.ok(errors.length > 0, 'expected validation errors for step without ref'); assert.ok( errors.some((e) => /ref/.test(e)), `expected an error mentioning "ref"; got: ${errors.join('; ')}`, ); }); // #1464 shape: id must match folder name (folderId contract) test('id not matching folderId fails', () => { const cap = { id: 'hello-note', role: 'feature', version: '0.1.0', title: 'Hello Note', description: 'Test fixture for id/folderId mismatch.', tier: 'standard', requires: [], runtimeCompat: { supported: ['*'], unsupported: [] }, skills: [], agents: [], config: {}, steps: [], contributions: [], gates: [], }; const errors = validateCapability(cap, 'wrong-folder'); assert.ok(errors.length > 0, 'expected id/folderId mismatch to fail validation'); assert.ok( errors.some((e) => /folder/.test(e) || /equal/.test(e) || /id/.test(e)), `expected error about id/folderId mismatch; got: ${errors.join('; ')}`, ); }); // Corrected shape: contribution with fragment + into (the PR #1495 fix) test('contribution with fragment.path + into passes (the PR #1495 fix shape)', () => { const cap = { id: 'hello-note', role: 'feature', version: '0.1.0', title: 'Hello Note', description: 'Injects a greeting note at plan:pre and produces HELLO.md.', tier: 'standard', requires: [], runtimeCompat: { supported: ['*'], unsupported: [] }, skills: [], agents: [], config: {}, steps: [], contributions: [ { point: 'plan:pre', into: 'planner', fragment: { path: 'fragments/plan-pre.md' }, produces: ['HELLO.md'], consumes: [], onError: 'skip', }, ], gates: [], }; const errors = validateCapability(cap, 'hello-note'); assert.deepStrictEqual( errors, [], `corrected contribution manifest has unexpected errors: ${errors.join('; ')}`, ); }); }); // ─── Suite 3: extractManifests helper ──────────────────────────────────────── describe('extractManifests helper unit tests', () => { test('returns empty array for plain text with no JSON fences', () => { assert.deepStrictEqual(extractManifests('No code blocks here.'), []); }); test('skips JSON blocks without all required manifest keys', () => { // Partial list-entry block — only has id, role, version but not steps/contributions/etc. const md = '```json\n{"id":"x","role":"feature","version":"1.0.0"}\n```'; assert.deepStrictEqual(extractManifests(md), []); }); function makeCompleteManifest(overrides) { return { id: 'test-cap', role: 'feature', title: 'T', description: 'D', tier: 'standard', requires: [], runtimeCompat: { supported: ['*'], unsupported: [] }, skills: [], agents: [], config: {}, steps: [], contributions: [], gates: [], ...overrides, }; } test('extracts a complete manifest (all required keys present)', () => { const cap = makeCompleteManifest({ id: 'x' }); const md = '```json\n' + JSON.stringify(cap, null, 2) + '\n```'; const result = extractManifests(md); assert.strictEqual(result.length, 1); assert.strictEqual(result[0].id, 'x'); }); test('skips malformed JSON blocks silently', () => { const complete = makeCompleteManifest({ id: 'y' }); const md = '```json\n{bad json here\n```\n```json\n' + JSON.stringify(complete) + '\n```'; const result = extractManifests(md); assert.strictEqual(result.length, 1); assert.strictEqual(result[0].id, 'y'); }); test('extracts multiple complete manifests from one doc', () => { const a = makeCompleteManifest({ id: 'cap-a' }); const b = makeCompleteManifest({ id: 'cap-b' }); const md = [ '```json\n' + JSON.stringify(a) + '\n```', '```json\n' + JSON.stringify(b) + '\n```', ].join('\n'); const result = extractManifests(md); assert.strictEqual(result.length, 2); }); });