golem15/msd-core
4
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(#1217): bound acquireStateLock retries on recoverable errno (no busy-spin) (#1236)

Browse Source 
* fix(#1217): bound acquireStateLock retries on recoverable errno (no busy-spin)

The recoverable-errno branch (`ACQUIRE_LOCK_RETRY_ERRNOS`) previously called
`continue` directly, skipping both `clock.sleep()` and the 30 000 ms budget
check. A permanently-failing ENOENT (e.g. parent dir removed) would spin at
100% CPU forever with the event loop fully blocked, making the OS-level
`timeout` the only escape.

Fix: extract a `checkBudgetAndSleep(context)` helper and call it from BOTH
the recoverable-errno path and the EEXIST contention path so every retry is
bounded and backed off identically.

Regression tests added to tests/clock-seam.test.cjs:
- persistent ENOENT throws budget-exceeded error (clock must advance via sleep)
- transient ENOENT (2 retries then success) acquires lock normally

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* chore(#1217): backfill changeset PR number (#1236)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tom Boucher
2026-06-14 15:10:59 -04:00
committed by GitHub
parent cafb874c4a
commit e97b5d6b7b
3 changed files with 499 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/state.cts
View File

@@ -1440,6 +1440,22 @@ function acquireStateLock(statePath: string, clock?: StateLockClock): string {
const maxWaitMs = 30000;
const startedAt = clock.now();
// Shared helper: check the time budget then back off with jitter before the
// next retry. Both the EEXIST contention path and the recoverable-errno path
// must go through this so neither can busy-spin (#1217).
const checkBudgetAndSleep = (context: string) => {
if (clock.now() - startedAt >= maxWaitMs) {
const e = new Error(
'acquireStateLock: ' + lockPath + ' ' + context + ' for ' +
(clock.now() - startedAt) + 'ms (exceeded ' + maxWaitMs + 'ms budget)'
);
(e as unknown as Record<string, unknown>).lockBudgetExceeded = true;
throw e;
}
const jitter = Math.floor(Math.random() * 50);
clock.sleep(retryDelay + jitter);
};
while (true) {
try {
const fd = fs.openSync(lockPath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY);
@@ -1450,9 +1466,13 @@ function acquireStateLock(statePath: string, clock?: StateLockClock): string {
return lockPath;
} catch (err) {
// Transient filesystem errors (Docker overlay-fs, NFS, OS signals, AV scanners)
// are recoverable — retry the acquisition loop rather than propagating.
// are recoverable — retry with the same budget + backoff as the EEXIST path so
// a permanently-failing errno cannot busy-spin at 100% CPU (#1217).
// See ACQUIRE_LOCK_RETRY_ERRNOS for the full list and rationale.
if (ACQUIRE_LOCK_RETRY_ERRNOS.has((err as NodeJS.ErrnoException).code as string)) { continue; }
if (ACQUIRE_LOCK_RETRY_ERRNOS.has((err as NodeJS.ErrnoException).code as string)) {
checkBudgetAndSleep((err as NodeJS.ErrnoException).code + ' persisted');
continue;
}
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; // propagate — silent bypass causes lost updates
// Only unlink a lock we did not place when it has crossed the staleness
// threshold (crashed holder). Nuking a fresh lock held by a slow-but-live
@@ -1460,18 +1480,33 @@ function acquireStateLock(statePath: string, clock?: StateLockClock): string {
try {
const stat = fs.statSync(lockPath);
if ((clock).now() - stat.mtimeMs > staleThresholdMs) {
try { fs.unlinkSync(lockPath); } catch { /* already gone */ }
let removed = false;
try { fs.unlinkSync(lockPath); removed = true; } catch { /* swallow: bounded below */ }
if (removed) {
// Successful steal — retry immediately to grab the just-freed lock.
// Must NOT call checkBudgetAndSleep here: a throw-after-delete would
// corrupt the filesystem state, and the budget is already bounded on
// the next iteration's EEXIST or open attempt (#1217 regression fix).
continue;
}
// Persistent unlinkSync failure — apply budget + backoff so it cannot
// busy-spin (#1217).
checkBudgetAndSleep('stale lock removal failed');
continue;
}
} catch { continue; /* released between EEXIST and stat */ }
if ((clock).now() - startedAt >= maxWaitMs) {
throw new Error(
'acquireStateLock: ' + lockPath + ' held by live process for ' +
((clock).now() - startedAt) + 'ms (exceeded ' + maxWaitMs + 'ms budget)'
);
} catch (err) {
// Re-throw a budget-exceeded error from the unlinkSync failure path above
// unchanged — its message already names the real cause ("stale lock removal
// failed") and double-wrapping it would replace that with the misleading
// "statSync failed after EEXIST" context string (#1217 diagnostic fix).
if ((err as Record<string, unknown>)?.lockBudgetExceeded) throw err;
// statSync failed — lock was likely released between our EEXIST and this
// stat call. Apply budget + backoff so a persistent statSync failure
// cannot busy-spin (#1217).
checkBudgetAndSleep('statSync failed after EEXIST');
continue;
}
const jitter = Math.floor(Math.random() * 50);
(clock).sleep(retryDelay + jitter);
checkBudgetAndSleep('held by live process');
}
}
}