msd-core/tests/helpers/overlay-repo.cjs

'use strict';

/**
 * overlay-repo.cjs — shared "overlay repo" builder for install-spawning test
 * suites (extracted from tests/workflow-fragments-emission.install.test.cjs,
 * issue #2933, so a second divergent copy is never written — see
 * CONTEXT.md's Generative Fix Divergence anti-pattern).
 *
 * ── The overlay technique ────────────────────────────────────────────────
 *
 * A test that needs a spawned `bin/install.js` to read a DIFFERENT
 * `msd-core/workflows/execute-phase.md` (or any other repo file) than this
 * checkout's real one, without paying to copy the ~400 MB repository (mostly
 * node_modules) for every run, calls `buildOverlayRepo` with a map of
 * POSIX-relative-path -> replacement content. `buildOverlayRepo` mirrors the
 * repo tree with real directories (so `copyWithPathReplacement`'s own
 * `entry.isDirectory()` / `entry.isFile()` Dirent checks — which do NOT
 * follow symlinks — see the correct type) and HARD-LINKS every unmodified
 * leaf file (not symlinks: a symlinked leaf file also fails an `isFile()`
 * Dirent check elsewhere in the installer, verified empirically — "Failed
 * to install agents: directory is empty" against a symlink-leaf overlay).
 * Only `node_modules` and `.git` are symlinked at the top level (install.js
 * never walks into either), which is what keeps the overlay build fast.
 * Every overlay-spawned installer should run with `--preserve-symlinks
 * --preserve-symlinks-main` as a defensive belt: with an all-hardlink leaf
 * layout this checkout does not currently NEED symlink-preservation for
 * correctness, but the flag is free insurance against a future install.js
 * change that resolves a node_modules package by real path.
 *
 * `buildOverlayRepo` can only REPLACE the content of a real leaf file that
 * already exists somewhere under `REPO_ROOT` — it cannot graft in a net-new
 * path (a `fileOverrides` key naming a path with no existing file/directory
 * ancestor in the real tree is silently never created, since `place()` only
 * walks `fs.readdirSync` of the REAL source directory).
 *
 * ── `opts.mode`: 'link' (default) vs 'copy' ─────────────────────────────
 *
 * `'link'` (the default, and every pre-existing caller's behavior) hard-links
 * every unmodified leaf — cheap, but a `--write` generator run inside the
 * overlay does an in-place `writeFileSync` through that hard link, i.e. the
 * SAME INODE as this real checkout's own tracked file, silently corrupting
 * it. `'copy'` mode instead COPIES every unmodified leaf (`fs.copyFileSync`,
 * a real independent inode), so a real `--write` generator — or a full `npm
 * run regen:derived` chain — can safely run to completion inside the overlay
 * without ever touching `REPO_ROOT`. `node_modules` and `.git` are still
 * symlinked at the top level in BOTH modes (unchanged from `'link'` mode):
 * `install.js`/`npm`/`tsc` never write into either through the overlay path,
 * only read/resolve through them, and symlinking is what keeps even
 * `'copy'` mode affordable (`node_modules` alone dwarfs the rest of the
 * tree).
 */

const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');

const REPO_ROOT = path.join(__dirname, '..', '..');

const OVERLAY_SKIP_TOP = new Set(['node_modules', '.git']);

/**
 * True when `err` reports that a path was not there.
 *
 * @param {unknown} err
 * @returns {boolean}
 */
function isMissingPath(err) {
  return Boolean(err) && typeof err === 'object' && err.code === 'ENOENT';
}

/**
 * Run `attempt` against a source path that another process may be replacing
 * underneath the walk, and report whether the leaf was actually placed.
 *
 * `buildOverlayRepo` enumerates names with `readdirSync` and then acts on them,
 * which is a TOCTOU window. It is not theoretical: `hooks/dist` is regenerated
 * by an ATOMIC REPLACE (`scripts/build-hooks.js` unlinks and renames), so any
 * concurrently running test that rebuilds hooks makes a just-listed name vanish
 * mid-walk. That took down three runs on three different branches with a bare
 * `ENOENT ... link '/work/hooks/dist/...'`.
 *
 * On ENOENT the source is re-examined ONCE rather than slept on: an atomic
 * rename is a single syscall, so by the time the failure surfaces the successor
 * is either already in place (retry succeeds) or the path is genuinely gone from
 * the tree (nothing to mirror, so the leaf is skipped). No sleep, no spin — a
 * timing-based wait here would be the flake this is fixing, not a fix for it.
 *
 * The retry itself can also lose the race — a second atomic replace landing in
 * the same window makes the retry throw ENOENT too (e.g. two concurrent
 * `build:hooks` runs). That is still just "the path is vanishing": the same
 * conclusion the single-vanish case reaches, so it is likewise treated as
 * skipped rather than left to escape as a bare uncaught ENOENT (#3108). There
 * is still only ONE retry — a second retry would turn this into the sleep/spin
 * loop the comment above already rejects.
 *
 * Returns false only when the path left the source tree entirely (on the
 * first attempt OR the retry); the overlay mirrors the tree, and a file that
 * is no longer in it is not part of the snapshot. Every other error — from
 * EITHER attempt, non-ENOENT — propagates untouched; that invariant must hold
 * for any future widening of this tolerance.
 *
 * When no `destPath` is supplied, an ENOENT is tolerated as a vanished
 * source: on the FIRST attempt's ENOENT, `srcPath` is re-checked with
 * `fs.existsSync` to decide whether the retry is even worth attempting (gone
 * already -> skip, no retry); if the retry's own attempt ALSO throws ENOENT,
 * that is tolerated UNCONDITIONALLY — by the time a second atomic replace has
 * landed in the same window there is nothing left to meaningfully re-check,
 * and this is deliberately optimistic rather than throwing on the race this
 * function exists to tolerate.
 *
 * Two discriminators that look like they should tell a vanished-source ENOENT
 * apart from a dest-side one (a missing DEST parent directory, e.g. a Windows
 * MAX_PATH failure or a concurrently-removed dest subtree) both fail, and
 * must not be reached for again here:
 *   - `fs.existsSync(srcPath)` re-checked at catch time: in the genuine
 *     double-vanish race the source is being atomically REPLACED (e.g.
 *     `hooks/dist`'s unlink+rename), so it can be present again by the time
 *     the ENOENT is handled even though the ENOENT was genuinely
 *     source-side. Gating the RETRY's ENOENT on it throws on exactly the
 *     race this function exists to tolerate (#3108 regression).
 *   - `err.path`: empirically, Node's `fs.linkSync` reports the SOURCE path
 *     in `err.path` for BOTH a missing source and a missing dest parent
 *     directory — it does not distinguish them either.
 *
 * The only discriminator that actually works is the DEST PARENT DIRECTORY,
 * because `buildOverlayRepo` builds its own dest tree (`fs.mkdirSync(destDir,
 * {recursive:true})` before every walk, into a private `mkdtempSync` root no
 * other process touches) — so a missing dest parent is always a bug, never
 * the atomic-replace race. Callers that know the dest path (`linkOrCopyFile`,
 * the `copy`-mode branch in `place()`) pass it as `destPath`; when supplied,
 * it REPLACES the source-existence check entirely (on both the first attempt
 * and the retry): an ENOENT is tolerated as a vanished source only if the
 * dest parent is confirmed present, and rethrown untouched if the dest
 * parent is missing. Callers with no dest to check keep the source-only
 * logic above, unchanged.
 *
 * @param {string} srcPath
 * @param {() => void} attempt
 * @param {string} [destPath] - when supplied, an ENOENT (on either attempt)
 *   is tolerated as a vanished source only if
 *   `fs.existsSync(path.dirname(destPath))`; a missing dest parent rethrows
 *   instead (see discriminator discussion above).
 * @returns {boolean} whether the leaf was placed
 */
function placeVanishableLeaf(srcPath, attempt, destPath) {
  function destParentPresent() {
    return fs.existsSync(path.dirname(destPath));
  }
  try {
    attempt();
    return true;
  } catch (err) {
    if (!isMissingPath(err)) throw err;
    if (destPath !== undefined) {
      if (!destParentPresent()) throw err;
    } else if (!fs.existsSync(srcPath)) {
      return false;
    }
    try {
      attempt();
      return true;
    } catch (retryErr) {
      if (!isMissingPath(retryErr)) throw retryErr;
      if (destPath !== undefined && !destParentPresent()) throw retryErr;
      return false;
    }
  }
}

/** Hard-link a file, falling back to a real copy only if the two paths sit on
 *  different filesystems/devices (EXDEV) or linking is denied (EPERM) — both
 *  cross-platform-legitimate, unlike a symlink's Dirent type-detection gap.
 *  Returns whether the leaf was placed; false means the source vanished
 *  mid-walk (see `placeVanishableLeaf`). */
function linkOrCopyFile(src, dest) {
  return placeVanishableLeaf(
    src,
    () => {
      try {
        fs.linkSync(src, dest);
      } catch (err) {
        if (err.code === 'EXDEV' || err.code === 'EPERM') {
          fs.copyFileSync(src, dest);
        } else {
          throw err;
        }
      }
    },
    dest,
  );
}

/**
 * Build a throwaway mirror of REPO_ROOT with real directories throughout and
 * every unmodified leaf file hard-linked (or copied — see `opts.mode`
 * above), except the paths named in `fileOverrides`
 * (POSIX-relative-path -> content string), which are written as real files.
 * Returns the mirror's absolute path; caller must
 * `fs.rmSync(..., {recursive:true, force:true})` it away.
 *
 * @param {{[relPath: string]: string}} fileOverrides
 * @param {{mode?: 'link'|'copy', warn?: (msg: string) => void}} [opts] -
 *   `mode` defaults to `'link'` so every pre-existing caller is unchanged.
 *   Pass `{mode: 'copy'}` when the overlay must survive a real `--write`
 *   generator run (see the module doc above) — every leaf file becomes a
 *   real independent inode, so no write inside the overlay can ever reach
 *   `REPO_ROOT`. `warn` defaults to `console.warn` (byte-identical to every
 *   existing caller) and exists so a test can inject a spy to assert on the
 *   skipped-leaf warning without capturing real console output.
 */
function buildOverlayRepo(fileOverrides, opts = {}) {
  const mode = opts.mode || 'link';
  const warn = opts.warn || console.warn;
  // #3900: test-only root override (mirrors cold-runtime-lib-fixture's
  // repoRoot) — the #3900 regression tests build the overlay from a tiny
  // synthetic root instead of walking the whole repo; every existing caller
  // uses the default REPO_ROOT and is unaffected.
  const root = opts.root || REPO_ROOT;
  const tmpRepo = fs.mkdtempSync(path.join(os.tmpdir(), 'msd-2930-overlay-'));
  const entries = Object.entries(fileOverrides).map(([relPath, content]) => ({
    parts: relPath.split('/'),
    content,
  }));
  const skipped = [];

  function place(srcDir, destDir, pending, isTop) {
    fs.mkdirSync(destDir, { recursive: true });
    const grouped = new Map();
    for (const e of pending) {
      const [head, ...rest] = e.parts;
      if (!grouped.has(head)) grouped.set(head, []);
      grouped.get(head).push({ parts: rest, content: e.content });
    }
    for (const de of fs.readdirSync(srcDir, { withFileTypes: true })) {
      if (isTop && OVERLAY_SKIP_TOP.has(de.name)) {
        fs.symlinkSync(path.join(srcDir, de.name), path.join(destDir, de.name));
        continue;
      }
      const srcPath = path.join(srcDir, de.name);
      const destPath = path.join(destDir, de.name);
      const overridden = grouped.get(de.name);
      const leaf = overridden && overridden.find((s) => s.parts.length === 0);
      if (leaf) {
        fs.writeFileSync(destPath, leaf.content);
        continue;
      }
      // fs.statSync follows symlinks (unlike Dirent.isDirectory()), so a
      // symlinked source directory is still recursed as a REAL directory in
      // the overlay — the property copyWithPathReplacement itself needs.
      //
      // The stat sits in the same TOCTOU window as the copy/link below: the
      // name came from readdirSync, and an atomic replace elsewhere in the tree
      // can retire it before we get here.
      let srcStat;
      try {
        srcStat = fs.statSync(srcPath);
      } catch (err) {
        if (isMissingPath(err)) continue;
        throw err;
      }
      if (srcStat.isDirectory()) {
        place(srcPath, destPath, overridden || [], false);
      } else if (!srcStat.isFile()) {
        // #3900: sockets, FIFOs, device nodes are not repository content.
        // Classifying them as files made copyFileSync throw ENXIO on a
        // socket (a FIFO would block until a writer appears) — 32 local
        // test failures from a daemon's working-tree socket, none about the
        // code under test. Skip: the overlay mirrors files, not devices.
        continue;
      } else if (mode === 'copy') {
        // Real independent inode — a write through this path in the overlay
        // can never alias back to REPO_ROOT's own tracked file (see
        // opts.mode doc above).
        const placed = placeVanishableLeaf(
          srcPath,
          () => fs.copyFileSync(srcPath, destPath),
          destPath,
        );
        if (!placed) skipped.push(srcPath);
      } else {
        const placed = linkOrCopyFile(srcPath, destPath);
        if (!placed) skipped.push(srcPath);
      }
    }
  }

  place(root, tmpRepo, entries, true);

  if (skipped.length > 0) {
    // Not thrown: a source that left the tree mid-walk is genuinely not part of
    // the snapshot, and failing here would reintroduce the crash this tolerance
    // exists to remove. But it must not be SILENT either — a dropped leaf can
    // surface later as a confusing "file missing" in an unrelated assertion, or
    // as nothing at all for a test that never touches it.
    warn(
      `buildOverlayRepo: ${skipped.length} source file(s) vanished mid-walk and were ` +
      `omitted from the overlay (likely a concurrent atomic replace, e.g. hooks/dist — ` +
      `run \`npm run build:hooks\` to regenerate it):\n  ` +
      skipped.join('\n  '),
    );
  }

  return tmpRepo;
}

module.exports = { buildOverlayRepo, linkOrCopyFile, placeVanishableLeaf, isMissingPath, REPO_ROOT, OVERLAY_SKIP_TOP };