msd-core/tests/platform-conformance-tier.test.cjs

'use strict';

/**
 * Test matrix: .gsd/phase/chore-4591-conformance-tier-linux-primary/50-test-matrix.md
 *
 * Rows 1-15 exercise the pure, exported `classifyContent(content)` classifier
 * directly on short string fixtures. Rows 16-18 exercise the
 * `--check`/`--write` CLI behavior against a small temp fixture tree, driven
 * through the process seam (tests/helpers/process-seam.cjs's `runNode`) per
 * CONTRIBUTING.md's "spawning a subprocess: use the process seam" rule. Row
 * 19 is a real-tree regression proving the committed generated file matches a
 * fresh sweep of this repo's actual tests/ tree. Rows 20-21 prove the
 * suite-exclusion fix (#4591 CI incident): a suite-tagged file must never
 * enter the conformance-tier pool even when its content would otherwise
 * qualify, and the committed generated file must contain zero such files.
 */

const { describe, test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');

const { createTempDir, cleanup } = require('./helpers.cjs');
const { runNode } = require('./helpers/process-seam.cjs');
const {
  classifyContent,
  classifyTree,
  NOISY_FOR_SOURCE_REACHABILITY,
  classifyMacosContent,
  classifyMacosTree,
} = require('../scripts/gen-platform-conformance-tier.cjs');

const ROOT = path.resolve(__dirname, '..');
const SCRIPT = path.join(ROOT, 'scripts', 'gen-platform-conformance-tier.cjs');
const GENERATED_PATH = path.join(ROOT, 'scripts', 'lib', 'platform-conformance-tier.generated.cjs');
const MACOS_GENERATED_PATH = path.join(ROOT, 'scripts', 'lib', 'macos-conformance-tier.generated.cjs');

// ─── Rows 1-15: classifyContent, pure fixtures ────────────────────────────────

describe('classifyContent — happy-path signals', () => {
  test('flags process.platform', () => {
    const { needsRealOs, signals } = classifyContent("if (process.platform === 'win32') { doThing(); }");
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('process-platform'));
  });

  test('flags os.platform()', () => {
    const { needsRealOs, signals } = classifyContent("const os = require('node:os');\nconst p = os.platform();");
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('os-platform'));
  });

  test('flags win32 literal', () => {
    const { needsRealOs, signals } = classifyContent("const platforms = ['win32', 'linux'];");
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('win32-darwin-literal'));
  });

  test('flags darwin literal', () => {
    const { needsRealOs, signals } = classifyContent("const platforms = ['darwin', 'linux'];");
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('win32-darwin-literal'));
  });

  test('flags chmod mode-bit octal', () => {
    const { needsRealOs, signals } = classifyContent('fs.chmodSync(target, 0o755);');
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('chmod-mode-bit'));
  });

  test('flags Windows-shell tokens', () => {
    for (const fixture of ["spawn('cmd.exe', args)", "spawn('powershell', args)", 'const e = process.env.ComSpec;']) {
      const { needsRealOs, signals } = classifyContent(fixture);
      assert.equal(needsRealOs, true, fixture);
      assert.ok(signals.includes('windows-shell-token'), fixture);
    }
  });

  test('flags Windows env-var names', () => {
    for (const fixture of [
      'const home = process.env.USERPROFILE;',
      'const drive = process.env.HOMEDRIVE;',
      'const p = process.env.HOMEPATH;',
    ]) {
      const { needsRealOs, signals } = classifyContent(fixture);
      assert.equal(needsRealOs, true, fixture);
      assert.ok(signals.includes('windows-env-var'), fixture);
    }
  });

  test('flags process-seam subprocess helpers', () => {
    for (const fixture of [
      "runNode(['--check'])",
      "runGit(['status'])",
      "runHook(HOOK_PATH, [])",
      "runGsdTools(['state', 'show'])",
    ]) {
      const { needsRealOs, signals } = classifyContent(fixture);
      assert.equal(needsRealOs, true, fixture);
      assert.ok(signals.includes('process-seam-subprocess'), fixture);
    }
  });

  test('flags raw child_process usage', () => {
    const fixture = "const { execFileSync } = require('child_process');\nexecFileSync('ls', []);";
    const { needsRealOs, signals } = classifyContent(fixture);
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('raw-child-process'));
  });

  test('flags symlink keyword', () => {
    const { needsRealOs, signals } = classifyContent('fs.symlinkSync(target, link);');
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('symlink-keyword'));
  });

  test('flags hardcoded path literal vs path.* call', () => {
    const fixture = "const p = path.join(root, 'x');\nassert.equal(rendered, '/etc/passwd');";
    const { needsRealOs, signals } = classifyContent(fixture);
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('hardcoded-path-vs-path-call'));
  });
});

describe('classifyContent — negative / hostile inputs', () => {
  test('does not flag a clean in-process unit test', () => {
    const fixture = "const assert = require('node:assert/strict');\ntest('adds numbers', () => { assert.equal(1 + 1, 2); });";
    const { needsRealOs, signals } = classifyContent(fixture);
    assert.equal(needsRealOs, false);
    assert.deepEqual(signals, []);
  });

  test('does not flag incidental use of the word "path"', () => {
    const fixture = '// This test verifies the correct path through the state machine.\nassert.ok(true);';
    const { needsRealOs } = classifyContent(fixture);
    assert.equal(needsRealOs, false);
  });

  test('does not crash on empty content', () => {
    assert.doesNotThrow(() => classifyContent(''));
    const { needsRealOs, signals } = classifyContent('');
    assert.equal(needsRealOs, false);
    assert.deepEqual(signals, []);
  });

  test('does not flag an unrelated identifier containing "spawn"', () => {
    const fixture = 'const spawnResult = computeSomething();\nassert.ok(spawnResult);';
    const { needsRealOs, signals } = classifyContent(fixture);
    assert.equal(needsRealOs, false);
    assert.deepEqual(signals, []);
  });
});

// ─── Row 15 (#4592): NOISY_FOR_SOURCE_REACHABILITY drift guard ────────────────

describe('NOISY_FOR_SOURCE_REACHABILITY (#4592)', () => {
  test('NOISY_FOR_SOURCE_REACHABILITY exports exactly the two noisy categories', () => {
    assert.ok(NOISY_FOR_SOURCE_REACHABILITY instanceof Set,
      `expected a Set, got: ${typeof NOISY_FOR_SOURCE_REACHABILITY}`);
    assert.deepEqual(
      [...NOISY_FOR_SOURCE_REACHABILITY].sort(),
      ['hardcoded-path-vs-path-call', 'symlink-keyword'].sort(),
      `expected exactly the two named noisy categories, got: ${JSON.stringify([...NOISY_FOR_SOURCE_REACHABILITY])}`,
    );
  });
});

// ─── Rows 16-18: CLI --check/--write against a temp fixture tree ──────────────

/** Spawn the real generator CLI via the process seam. */
function runGen(args) {
  return runNode([SCRIPT, ...args]);
}

describe('gen-platform-conformance-tier.cjs CLI (temp fixture tree)', () => {
  test('--check passes when the generated file is fresh', () => {
    const tmpDir = createTempDir('gen-platform-conformance-tier-');
    try {
      const testsDir = path.join(tmpDir, 'tests');
      fs.mkdirSync(testsDir, { recursive: true });
      fs.writeFileSync(path.join(testsDir, 'flagged.test.cjs'), "if (process.platform === 'win32') {}\n");
      fs.writeFileSync(path.join(testsDir, 'clean.test.cjs'), "assert.equal(1 + 1, 2);\n");
      const outPath = path.join(tmpDir, 'platform-conformance-tier.generated.cjs');

      const write = runGen(['--write', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(write.exitCode, 0, write.stderr);
      assert.ok(fs.existsSync(outPath));

      const check = runGen(['--check', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(check.exitCode, 0, check.stderr);
      assert.match(check.stdout, /ok gen-platform-conformance-tier/);
    } finally {
      cleanup(tmpDir);
    }
  });

  test('--check fails and names the drift when the list is stale', () => {
    const tmpDir = createTempDir('gen-platform-conformance-tier-');
    try {
      const testsDir = path.join(tmpDir, 'tests');
      fs.mkdirSync(testsDir, { recursive: true });
      fs.writeFileSync(path.join(testsDir, 'flagged.test.cjs'), "if (process.platform === 'win32') {}\n");
      fs.writeFileSync(path.join(testsDir, 'clean.test.cjs'), "assert.equal(1 + 1, 2);\n");
      const outPath = path.join(tmpDir, 'platform-conformance-tier.generated.cjs');

      const write = runGen(['--write', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(write.exitCode, 0, write.stderr);

      // Introduce drift: a brand-new signal-bearing file the committed list
      // has never seen.
      fs.writeFileSync(path.join(testsDir, 'new-signal.test.cjs'), 'fs.symlinkSync(target, link);\n');

      const check = runGen(['--check', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(check.exitCode, 1);
      const combined = check.stdout + check.stderr;
      assert.match(combined, /tests\/new-signal\.test\.cjs/, 'the drift report must name the new file');
      assert.match(combined, /\+/, 'a newly-added file is reported with a + marker');
    } finally {
      cleanup(tmpDir);
    }
  });

  test('--write is deterministic across repeated runs', () => {
    const tmpDir = createTempDir('gen-platform-conformance-tier-');
    try {
      const testsDir = path.join(tmpDir, 'tests');
      fs.mkdirSync(testsDir, { recursive: true });
      fs.writeFileSync(path.join(testsDir, 'a.test.cjs'), "process.env.PATHEXT;\n");
      fs.writeFileSync(path.join(testsDir, 'b.test.cjs'), 'fs.symlinkSync(target, link);\n');
      const out1 = path.join(tmpDir, 'out1.generated.cjs');
      const out2 = path.join(tmpDir, 'out2.generated.cjs');

      assert.equal(runGen(['--write', '--tests-dir', testsDir, '--out', out1]).exitCode, 0);
      assert.equal(runGen(['--write', '--tests-dir', testsDir, '--out', out2]).exitCode, 0);

      const content1 = fs.readFileSync(out1, 'utf8');
      const content2 = fs.readFileSync(out2, 'utf8');
      assert.equal(content1, content2, '--write must be byte-identical across independent runs over the same input');
    } finally {
      cleanup(tmpDir);
    }
  });
});

// ─── Row 20: suite-tagged files are excluded even when their content qualifies

describe('gen-platform-conformance-tier.cjs CLI (temp fixture tree) — suite exclusion', () => {
  test('a suite-suffixed file is excluded even with a qualifying content signal', () => {
    const tmpDir = createTempDir('gen-platform-conformance-tier-suite-');
    try {
      const testsDir = path.join(tmpDir, 'tests');
      fs.mkdirSync(testsDir, { recursive: true });
      const signal = "if (process.platform === 'win32') {}\n";
      fs.writeFileSync(path.join(testsDir, 'foo.test.cjs'), signal);
      fs.writeFileSync(path.join(testsDir, 'foo.install.test.cjs'), signal);
      const outPath = path.join(tmpDir, 'platform-conformance-tier.generated.cjs');

      const write = runGen(['--write', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(write.exitCode, 0, write.stderr);

      delete require.cache[require.resolve(outPath)];
      const generated = require(outPath);

      assert.deepEqual(
        generated.CONFORMANCE_TIER_FILES,
        ['tests/foo.test.cjs'],
        'the install-suite file must be excluded even though its content would otherwise qualify',
      );
    } finally {
      cleanup(tmpDir);
    }
  });
});

// ─── Row 19: real tests/ tree, regression against the committed artifact ─────

describe('gen-platform-conformance-tier.cjs — real repo tree (regression)', () => {
  test('real tests/ tree classification matches the committed list', () => {
    const realTestsDir = path.join(ROOT, 'tests');

    let fresh;
    assert.doesNotThrow(() => {
      fresh = classifyTree(realTestsDir);
    }, 'a full sweep of the real tests/ tree must complete without throwing');

    // Measured 546/952 at authoring time (#4591, post suite-exclusion fix) —
    // the range below is a sanity ballpark with headroom for organic
    // test-suite growth in either direction, not a brittle exact-match on
    // that literal.
    assert.ok(
      fresh.files.length >= 450 && fresh.files.length <= 700,
      `expected a real, current, sanity-checked count in [450, 700], got ${fresh.files.length}`,
    );

    delete require.cache[require.resolve(GENERATED_PATH)];
    const committed = require(GENERATED_PATH);

    assert.equal(
      committed.CONFORMANCE_TIER_FILES.length,
      fresh.files.length,
      'the committed generated file must be fresh — run `node scripts/gen-platform-conformance-tier.cjs --write`',
    );
    assert.deepEqual(
      committed.CONFORMANCE_TIER_FILES.slice().sort(),
      fresh.files.slice().sort(),
      'the committed list must match a fresh sweep exactly, not just in length',
    );
  });

  // ─── Row 21: no suite-tagged file ever reaches the committed conformance
  // tier — the exact assertion that would have caught the CI incident before
  // it ever shipped.
  test('the committed conformance-tier list contains zero suite-tagged files', () => {
    delete require.cache[require.resolve(GENERATED_PATH)];
    const { CONFORMANCE_TIER_FILES } = require(GENERATED_PATH);

    const suiteTaggedPattern = /\.(install|security|slow|integration|qa)\.test\.cjs$/;
    const offenders = CONFORMANCE_TIER_FILES.filter((f) => suiteTaggedPattern.test(f));

    assert.deepEqual(
      offenders,
      [],
      'suite-tagged files (install/security/slow/integration/qa) must never appear in the ' +
        'conformance-tier list — install/slow are PR-excluded suites and integration/security ' +
        'already run via their own dedicated steps',
    );
  });
});

// ─── #4593: macOS-specific classifier (classifyMacosContent / MACOS_CATEGORIES)

describe('classifyMacosContent — happy-path signals', () => {
  test('flags darwin literal', () => {
    const { needsRealOs, signals } = classifyMacosContent("const platforms = ['darwin', 'linux'];");
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('darwin-literal'));
  });

  test('flags zsh dispatch', () => {
    const { needsRealOs, signals } = classifyMacosContent("shell: 'zsh {0}'");
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('zsh-dispatch'));
  });

  test('flags case-sensitivity phrasing', () => {
    for (const fixture of [
      '// verify the case-insensitive lookup',
      '// verify the case-sensitive lookup',
      '// verify case insensitivity',
    ]) {
      const { needsRealOs, signals } = classifyMacosContent(fixture);
      assert.equal(needsRealOs, true, fixture);
      assert.ok(signals.includes('case-sensitivity'), fixture);
    }
  });

  test('flags chmod mode-bit octal', () => {
    const { needsRealOs, signals } = classifyMacosContent('fs.chmodSync(target, 0o755);');
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('chmod-mode-bit'));
  });

  test('flags symlink keyword', () => {
    const { needsRealOs, signals } = classifyMacosContent('fs.symlinkSync(target, link);');
    assert.equal(needsRealOs, true);
    assert.ok(signals.includes('symlink-keyword'));
  });
});

describe('classifyMacosContent — negative case', () => {
  test('does not flag a clean file with none of the 5 macOS signals', () => {
    const fixture =
      "const assert = require('node:assert/strict');\n" +
      "if (process.platform === 'win32') { doThing(); }\n" +
      "test('adds numbers', () => { assert.equal(1 + 1, 2); });";
    const { needsRealOs, signals } = classifyMacosContent(fixture);
    assert.equal(needsRealOs, false);
    assert.deepEqual(signals, []);
  });

  test('does not crash on empty content', () => {
    assert.doesNotThrow(() => classifyMacosContent(''));
    const { needsRealOs, signals } = classifyMacosContent('');
    assert.equal(needsRealOs, false);
    assert.deepEqual(signals, []);
  });
});

// ─── #4593: CLI --target macos ─────────────────────────────────────────────

describe('gen-platform-conformance-tier.cjs CLI --target macos (temp fixture tree)', () => {
  test('--target macos --check passes when the generated file is fresh', () => {
    const tmpDir = createTempDir('gen-platform-conformance-tier-macos-');
    try {
      const testsDir = path.join(tmpDir, 'tests');
      fs.mkdirSync(testsDir, { recursive: true });
      fs.writeFileSync(path.join(testsDir, 'flagged.test.cjs'), 'fs.symlinkSync(target, link);\n');
      fs.writeFileSync(path.join(testsDir, 'clean.test.cjs'), "assert.equal(1 + 1, 2);\n");
      const outPath = path.join(tmpDir, 'macos-conformance-tier.generated.cjs');

      const write = runGen(['--target', 'macos', '--write', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(write.exitCode, 0, write.stderr);
      assert.ok(fs.existsSync(outPath));

      delete require.cache[require.resolve(outPath)];
      const generated = require(outPath);
      assert.deepEqual(generated.MACOS_CONFORMANCE_TIER_FILES, ['tests/flagged.test.cjs']);

      const check = runGen(['--target', 'macos', '--check', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(check.exitCode, 0, check.stderr);
      assert.match(check.stdout, /ok gen-platform-conformance-tier --target macos/);
    } finally {
      cleanup(tmpDir);
    }
  });

  test('--target macos --check fails and names the drift when the list is stale', () => {
    const tmpDir = createTempDir('gen-platform-conformance-tier-macos-');
    try {
      const testsDir = path.join(tmpDir, 'tests');
      fs.mkdirSync(testsDir, { recursive: true });
      fs.writeFileSync(path.join(testsDir, 'flagged.test.cjs'), 'fs.symlinkSync(target, link);\n');
      const outPath = path.join(tmpDir, 'macos-conformance-tier.generated.cjs');

      const write = runGen(['--target', 'macos', '--write', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(write.exitCode, 0, write.stderr);

      fs.writeFileSync(path.join(testsDir, 'new-signal.test.cjs'), "const platforms = ['darwin'];\n");

      const check = runGen(['--target', 'macos', '--check', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(check.exitCode, 1);
      const combined = check.stdout + check.stderr;
      assert.match(combined, /tests\/new-signal\.test\.cjs/, 'the drift report must name the new file');
    } finally {
      cleanup(tmpDir);
    }
  });

  test('a suite-suffixed file is excluded even with a qualifying macOS content signal', () => {
    const tmpDir = createTempDir('gen-platform-conformance-tier-macos-suite-');
    try {
      const testsDir = path.join(tmpDir, 'tests');
      fs.mkdirSync(testsDir, { recursive: true });
      const signal = 'fs.symlinkSync(target, link);\n';
      fs.writeFileSync(path.join(testsDir, 'foo.test.cjs'), signal);
      fs.writeFileSync(path.join(testsDir, 'foo.install.test.cjs'), signal);
      const outPath = path.join(tmpDir, 'macos-conformance-tier.generated.cjs');

      const write = runGen(['--target', 'macos', '--write', '--tests-dir', testsDir, '--out', outPath]);
      assert.equal(write.exitCode, 0, write.stderr);

      delete require.cache[require.resolve(outPath)];
      const generated = require(outPath);

      assert.deepEqual(
        generated.MACOS_CONFORMANCE_TIER_FILES,
        ['tests/foo.test.cjs'],
        'the install-suite file must be excluded even though its content would otherwise qualify',
      );
    } finally {
      cleanup(tmpDir);
    }
  });
});

// ─── #4593: real tests/ tree, regression against the committed macOS artifact

describe('gen-platform-conformance-tier.cjs — real repo tree, macOS target (regression)', () => {
  test('real tests/ tree macOS classification matches the committed list', () => {
    const realTestsDir = path.join(ROOT, 'tests');

    let fresh;
    assert.doesNotThrow(() => {
      fresh = classifyMacosTree(realTestsDir);
    }, 'a full sweep of the real tests/ tree must complete without throwing');

    // Measured 196/930 at authoring time (#4593 design doc). Sanity ballpark
    // with headroom for organic test-suite growth, not a brittle exact match.
    assert.ok(
      fresh.files.length >= 100 && fresh.files.length <= 350,
      `expected a real, current, sanity-checked count in [100, 350], got ${fresh.files.length}`,
    );

    delete require.cache[require.resolve(MACOS_GENERATED_PATH)];
    const committed = require(MACOS_GENERATED_PATH);

    assert.equal(
      committed.MACOS_CONFORMANCE_TIER_FILES.length,
      fresh.files.length,
      'the committed macOS generated file must be fresh — run ' +
        '`node scripts/gen-platform-conformance-tier.cjs --target macos --write`',
    );
    assert.deepEqual(
      committed.MACOS_CONFORMANCE_TIER_FILES.slice().sort(),
      fresh.files.slice().sort(),
      'the committed macOS list must match a fresh sweep exactly, not just in length',
    );
  });

  test('the committed macOS conformance-tier list contains zero suite-tagged files', () => {
    delete require.cache[require.resolve(MACOS_GENERATED_PATH)];
    const { MACOS_CONFORMANCE_TIER_FILES } = require(MACOS_GENERATED_PATH);

    const suiteTaggedPattern = /\.(install|security|slow|integration|qa)\.test\.cjs$/;
    const offenders = MACOS_CONFORMANCE_TIER_FILES.filter((f) => suiteTaggedPattern.test(f));

    assert.deepEqual(
      offenders,
      [],
      'suite-tagged files must never appear in the macOS conformance-tier list either',
    );
  });
});