msd-core/tests/fix-1627-asvs-level-scaling.test.cjs

// allow-test-rule: source-text-is-the-product #1627
// Agent .md / reference .md files — their text IS what the runtime loads.
// Testing text content tests the deployed contract.
// Per CONTRIBUTING.md exception matrix.

/**
 * Fix #1627 — ASVS level scaling
 *
 * Asserts that `workflow.security_asvs_level` now scales both planner
 * threat-disposition rigor and auditor verification depth rather than
 * being display-only.
 */

'use strict';

const { describe, test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');

const ROOT = path.join(__dirname, '..');
const AGENTS_DIR = path.join(ROOT, 'agents');
const REFS_DIR = path.join(ROOT, 'gsd-core', 'references');
const MANIFEST_PATH = path.join(ROOT, 'docs', 'INVENTORY-MANIFEST.json');

describe('SECURE: ASVS level scaling (#1627)', () => {
  // ── 1. New reference file ────────────────────────────────────────────────

  describe('security-asvs-levels.md reference', () => {
    const refPath = path.join(REFS_DIR, 'security-asvs-levels.md');

    test('file exists', () => {
      assert.ok(fs.existsSync(refPath), 'gsd-core/references/security-asvs-levels.md must exist');
    });

    test('defines all three levels', () => {
      const content = fs.readFileSync(refPath, 'utf-8');
      assert.ok(content.includes('L1'), 'must define L1');
      assert.ok(content.includes('L2'), 'must define L2');
      assert.ok(content.includes('L3'), 'must define L3');
    });

    test('L1 describes opportunistic scope and planner disposition', () => {
      const content = fs.readFileSync(refPath, 'utf-8');
      assert.ok(
        content.toLowerCase().includes('opportunistic'),
        'L1 must be described as opportunistic'
      );
      assert.ok(
        content.includes('mitigate') && content.includes('accept'),
        'must describe mitigate/accept dispositions'
      );
    });

    test('L2 requires explicit rationale for accepted threats', () => {
      const content = fs.readFileSync(refPath, 'utf-8');
      // L2 must require documented rationale for accepted risks
      assert.ok(
        content.includes('rationale') || content.includes('documented'),
        'L2 must require documented rationale for accepted threats'
      );
    });

    test('L3 describes deep/comprehensive verification', () => {
      const content = fs.readFileSync(refPath, 'utf-8');
      const lower = content.toLowerCase();
      assert.ok(
        lower.includes('deep') || lower.includes('comprehensive') || lower.includes('exhaustive'),
        'L3 must describe deep/comprehensive verification'
      );
    });

    test('mentions that higher levels are supersets of lower', () => {
      const content = fs.readFileSync(refPath, 'utf-8');
      const lower = content.toLowerCase();
      assert.ok(
        lower.includes('superset') || lower.includes('higher level') || lower.includes('includes all'),
        'must note that higher levels are supersets of lower'
      );
    });

    test('describes distinct auditor verification depth for each level', () => {
      const content = fs.readFileSync(refPath, 'utf-8');
      // All three audit depth keywords should appear
      assert.ok(content.includes('grep') || content.includes('PRESENT'), 'L1 audit depth must mention grep/presence check');
      assert.ok(content.includes('boundary') || content.includes('addresses'), 'L2 audit depth must mention boundary/addresses');
      assert.ok(content.includes('end-to-end') || content.includes('bypass'), 'L3 audit depth must mention end-to-end or bypass check');
    });
  });

  // ── 2. gsd-planner.md — no hardcoded L1 in disposition ──────────────────

  describe('gsd-planner.md security disposition', () => {
    const plannerPath = path.join(AGENTS_DIR, 'gsd-planner.md');

    test('planner security instruction does not hardcode "ASVS L1"', () => {
      const content = fs.readFileSync(plannerPath, 'utf-8');
      // The old bug: "mitigate if ASVS L1 requires it" — must be gone
      assert.ok(
        !content.includes('ASVS L1 requires it'),
        'planner must not hardcode "ASVS L1 requires it"; it must reference the configured level'
      );
    });

    test('planner references the configured OWASP ASVS level', () => {
      const content = fs.readFileSync(plannerPath, 'utf-8');
      assert.ok(
        content.includes('OWASP ASVS level') || content.includes('configured OWASP'),
        'planner must reference the configured OWASP ASVS level'
      );
    });

    test('planner @-references security-asvs-levels.md', () => {
      const content = fs.readFileSync(plannerPath, 'utf-8');
      assert.ok(
        content.includes('security-asvs-levels.md'),
        'planner must @-reference security-asvs-levels.md'
      );
    });

    test('planner is under the 49152-char cap', () => {
      const content = fs.readFileSync(plannerPath, 'utf-8').replace(/\r\n/g, '\n').replace(/\r/g, '\n');
      assert.ok(
        content.length < 49152,
        `gsd-planner.md must be < 49152 chars (LF-normalized); got ${content.length}`
      );
    });
  });

  // ── 3. gsd-security-auditor.md — scaled verification depth ──────────────

  describe('gsd-security-auditor.md verification depth', () => {
    const auditorPath = path.join(AGENTS_DIR, 'gsd-security-auditor.md');

    test('auditor scales verification depth by asvs_level', () => {
      const content = fs.readFileSync(auditorPath, 'utf-8');
      assert.ok(
        content.includes('asvs_level') || content.includes('ASVS level'),
        'auditor must reference asvs_level to scale verification'
      );
    });

    test('auditor describes L1/L2/L3 depth differences', () => {
      const content = fs.readFileSync(auditorPath, 'utf-8');
      // All three levels must appear in context of depth scaling
      assert.ok(content.includes('L1'), 'auditor must mention L1 depth');
      assert.ok(content.includes('L2'), 'auditor must mention L2 depth');
      assert.ok(content.includes('L3'), 'auditor must mention L3 depth');
    });

    test('auditor @-references security-asvs-levels.md', () => {
      const content = fs.readFileSync(auditorPath, 'utf-8');
      assert.ok(
        content.includes('security-asvs-levels.md'),
        'auditor must @-reference security-asvs-levels.md'
      );
    });

    test('auditor still echoes ASVS Level in structured output', () => {
      const content = fs.readFileSync(auditorPath, 'utf-8');
      assert.ok(
        content.includes('ASVS Level:') && content.includes('{1/2/3}'),
        'auditor must still emit ASVS Level in SECURED/OPEN_THREATS output'
      );
    });
  });

  // ── 4. secure-phase.md — ASVS-aware short-circuit ──────────────────────

  describe('secure-phase.md short-circuit conditioned on asvs_level', () => {
    const wfPath = path.join(ROOT, 'gsd-core', 'workflows', 'secure-phase.md');

    test('short-circuit to Step 6 is gated on asvs_level == 1', () => {
      const content = fs.readFileSync(wfPath, 'utf-8');
      // The condition must reference asvs_level so that L2/L3 don't skip the auditor
      assert.ok(
        content.includes('asvs_level == 1'),
        'secure-phase.md must gate the skip-to-Step-6 short-circuit on asvs_level == 1'
      );
    });

    test('auditor runs at L2/L3 even when threats_open is 0 (asvs_level >= 2 branch present)', () => {
      const content = fs.readFileSync(wfPath, 'utf-8');
      // The >= 2 branch must explicitly say the auditor is spawned for L2/L3 deep verification
      assert.ok(
        content.includes('asvs_level >= 2'),
        'secure-phase.md must include asvs_level >= 2 branch that does NOT skip the auditor'
      );
      // The >= 2 branch must make clear the auditor is spawned (not skipped)
      assert.ok(
        content.includes('L2/L3 deep verification') || content.includes('L2 boundary') || content.includes('L3 end-to-end'),
        'secure-phase.md asvs_level >= 2 branch must reference L2/L3 deep verification'
      );
    });
  });

  // ── 5. security-asvs-levels.md — L1 medium-severity gap closed ──────────

  describe('security-asvs-levels.md L1 medium-severity is specified', () => {
    const refPath = path.join(REFS_DIR, 'security-asvs-levels.md');

    test('L1 explicitly handles medium-severity threats (no gap)', () => {
      const content = fs.readFileSync(refPath, 'utf-8');
      // L1 section must say something about medium-severity
      assert.ok(
        content.includes('medium-severity') || content.includes('medium severity'),
        'L1 must explicitly specify disposition for medium-severity threats (no ambiguity gap)'
      );
    });

    test('L1 medium-severity disposition is conditional (trust-boundary-aware)', () => {
      const content = fs.readFileSync(refPath, 'utf-8');
      // L1 must distinguish between medium on primary trust boundary vs not
      assert.ok(
        content.includes('trust boundary') || content.includes('primary trust'),
        'L1 medium-severity rule must reference trust boundary to disambiguate disposition'
      );
    });
  });

  // ── 6. Inventory manifest ─────────────────────────────────────────────────

  describe('inventory manifest', () => {
    test('security-asvs-levels.md is registered in INVENTORY-MANIFEST.json', () => {
      const manifest = JSON.parse(fs.readFileSync(MANIFEST_PATH, 'utf-8'));
      const refs = (manifest.families || {}).references || [];
      assert.ok(
        refs.includes('security-asvs-levels.md'),
        'security-asvs-levels.md must appear in families.references of INVENTORY-MANIFEST.json'
      );
    });
  });
});