From 8f013983e56cef24015c2fc252b47cbac0f9aa46 Mon Sep 17 00:00:00 2001 From: Tom Boucher Date: Thu, 3 Sep 2026 06:59:30 -0400 Subject: [PATCH] fix(#3751): provision the claude CLI in CI and cover agents/ in the plugin-validate fixture (#4229) * test(#3751): the validation fixture must cover agents/ and CI must provision the CLI * fix(#3751): cover agents/ in the plugin-validate fixture and provision the claude CLI in CI * fix(#3751): wire the strict live-config guard into the plugin-validate job * fix(#3751): job-level strict-guard env, where the guard derivation reads it * chore(#3751): changeset for the CI-provisioned plugin-validate gate * chore(#3751): backfill changeset pr number --------- Co-authored-by: sim --- .changeset/sharp-jaguars-run.md | 5 ++++ .github/workflows/test.yml | 27 +++++++++++++++++ tests/plugin-manifest.test.cjs | 53 +++++++++++++++++++++++++++++---- 3 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 .changeset/sharp-jaguars-run.md diff --git a/.changeset/sharp-jaguars-run.md b/.changeset/sharp-jaguars-run.md new file mode 100644 index 000000000..bf7f33638 --- /dev/null +++ b/.changeset/sharp-jaguars-run.md @@ -0,0 +1,5 @@ +--- +type: Fixed +pr: 4229 +--- +**`claude plugin validate --strict` now runs in CI and covers `agents/`** — a dedicated test.yml job provisions the claude CLI so the C2 tier is a real gate, and the validation fixture includes the agents/ tree the CLI validates by convention. (#3751) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3b729f35..6e082b253 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -127,6 +127,33 @@ jobs: - name: Lint — all (ESLint, skill deps, test-file count, command contract, PR checks, legacy name, regression-test names, resolution-provenance) run: npm run lint:ci + # #3751 (decision 1): provision the claude CLI so plugin-manifest's C2 tier + # (`claude plugin validate --strict`) is a real CI gate instead of a + # local-only check. Dedicated job — the matrix lanes stay CLI-free. + plugin-validate: + needs: preflight + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + # #2665 round 4: every job that runs the suite wires the strict guard + # (job-level env, where the live-config-guard derivation reads it). + GSD_STRICT_LIVE_CONFIG_GUARD: '1' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + token: ${{ github.token }} + - name: Set up Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + - name: Install dev dependencies + run: npm ci --ignore-scripts + - name: Install claude CLI (C2 gate dependency, #3751) + run: npm install -g @anthropic-ai/claude-code + - name: Run plugin-manifest suite (C1+C2+C3) + run: node scripts/run-tests.cjs --files plugin-manifest.test.cjs + test: name: test (${{ matrix.os }}, ${{ matrix.node-version }}${{ matrix.shard && format(', shard {0}', matrix.shard) || '' }}) needs: [changes, preflight] diff --git a/tests/plugin-manifest.test.cjs b/tests/plugin-manifest.test.cjs index 00368785d..82a4a52c6 100644 --- a/tests/plugin-manifest.test.cjs +++ b/tests/plugin-manifest.test.cjs @@ -421,12 +421,13 @@ describe('C: plugin.json schema validation', () => { // symlinked. An earlier revision also copied agents/, on the (correct) // observation that the CLI auto-validates it and a frontmatter-less // agents/*.md exits 1. Dropped in review: it is a NEW gate the issue does not - // ask for, on the largest of the trees, and because C2 never runs in CI it - // would be red only on contributor machines with `claude` installed — the - // same worst-of-both-states #3613 exists to remove. agents/ coverage is worth - // having and is tracked as #3751, where "should CI provision the CLI" — the - // decision it actually turns on — can be answered for it. - const COMPONENT_DIRS = ['commands', 'hooks', 'skills']; + // ask for, on the largest of the trees. RESOLVED by #3751 (2026-09-02, + // options 1+3): CI provisions the claude CLI, agents/ is in the fixture, and + // the asymmetry #3613 existed to remove is gone — C2 runs in CI. + // #3751 (decision 1+3, 2026-09-02): agents/ is included — the CLI validates + // it by convention (measured on 2.1.239), and CI now provisions the claude + // CLI in a dedicated test.yml job, so C2 is a real gate over this tree. + const COMPONENT_DIRS = ['commands', 'hooks', 'skills', 'agents']; /** * One entry that must survive the copy into each component tree, so C3 catches @@ -439,6 +440,9 @@ describe('C: plugin.json schema validation', () => { commands: 'gsd', hooks: 'hooks.json', skills: 'gsd-add-tests', + // #3751: agents/ is undeclared in plugin.json (CLI-convention pickup), so + // the expected entry is a shipped agent file, not a manifest-declared path. + agents: 'gsd-executor.md', }; /** @@ -533,6 +537,43 @@ describe('C: plugin.json schema validation', () => { } ); + // ── #3751: agents/ coverage (maintainer decision 2026-09-02: options 1+3) ──── + // + // `claude plugin validate` auto-validates agents/ by CLI CONVENTION (the + // manifest does not declare it), measured live on CLI 2.1.239 in the issue. + // Decision: CI provisions the claude CLI (a dedicated test.yml job), so C2 is + // a real gate, and the fixture + C3 cover the tree everywhere else. + + test('C3+#3751: the validation fixture covers agents/, the tree the CLI validates by convention', () => { + const pluginRoot = buildValidationPluginRoot(); + try { + const agentsDir = path.join(pluginRoot, 'agents'); + const stat = fs.lstatSync(agentsDir); + assert.ok(stat.isDirectory(), 'agents/ must be a real directory in the C2 validation fixture'); + assert.equal(stat.isSymbolicLink(), false, 'agents/ must be copied, not symlinked'); + const entries = fs.readdirSync(agentsDir); + assert.ok(entries.length > 0, 'agents/ is EMPTY in the C2 validation fixture'); + assert.ok( + entries.some((e) => /^gsd-.*\.md$/.test(e)), + 'agents/ must carry the shipped gsd-*.md files, not a stub' + ); + } finally { + cleanup(pluginRoot); + } + }); + + test('#3751: CI provisions the claude CLI so C2 is a real gate, not a local-only tier', () => { + const workflow = fs.readFileSync(path.join(ROOT, '.github', 'workflows', 'test.yml'), 'utf8'); + assert.ok( + /@anthropic-ai\/claude-code/.test(workflow), + 'test.yml must install the claude CLI (npm i -g @anthropic-ai/claude-code) in a job' + ); + assert.ok( + /plugin-manifest\.test\.cjs/.test(workflow), + 'the provisioning job must run tests/plugin-manifest.test.cjs (the C2 gate)' + ); + }); + // ── C3: Unconditional fixture-construction guard ───────────────────────────── // // #3613 regression. C2 above is the test that actually shells out to the CLI,