'use strict'; // Producer-half of the async external-job contract (#1164 / #1105). // The CORE consumer half (external_job_waiting) is covered by // external-job-waiting.test.cjs. These tests assert the scheduler-adapter // Capability's pure module: SLURM state mapping, manifest build/validate, // sbatch/squeue/sacct parsers, and the fail-closed manifest writer that // mirrors the consumer's duplicate-execution guard // (docs/reference/planning-artifacts.md). const { test } = require('node:test'); const assert = require('node:assert'); const path = require('node:path'); const fc = require('fast-check'); const m = require('../gsd-core/bin/lib/external-job.cjs'); const { MANIFEST_VERSION, MANIFEST_STATUS, NON_TERMINAL_STATUSES, TERMINAL_FAILURE_STATUSES, mapSlurmState, buildManifest, validateManifest, parseSbatchParsable, parseSqueueLine, parseSacctRow, writeManifest, manifestPath, } = m; // ─── Closed status enum (Hyrum's Law: stability contract) ───────────────────── test('MANIFEST_STATUS is the closed scheduler-agnostic enum from the contract', () => { assert.deepStrictEqual([...MANIFEST_STATUS].sort(), [ 'cancelled', 'completed-unverified', 'failed', 'running', 'submitted', 'timeout', ]); assert.strictEqual(MANIFEST_VERSION, '1.0'); }); test('NON_TERMINAL / TERMINAL_FAILURE partition the enum without overlap', () => { for (const s of MANIFEST_STATUS) { const inNon = NON_TERMINAL_STATUSES.includes(s); const inTerm = TERMINAL_FAILURE_STATUSES.includes(s); // completed-unverified is neither non-terminal nor a failure — its own bucket. if (s === 'completed-unverified') { assert.ok(!inNon && !inTerm, 'completed-unverified is its own bucket'); } else { assert.ok(inNon !== inTerm, `${s} must sit in exactly one partition`); } } }); // ─── SLURM state mapping ────────────────────────────────────────────────────── test('mapSlurmState maps every documented SLURM state to the closed enum', () => { const cases = { PENDING: 'submitted', CONFIGURING: 'submitted', RUNNING: 'running', COMPLETED: 'completed-unverified', COMPLETING: 'running', FAILED: 'failed', CANCELLED: 'cancelled', TIMEOUT: 'timeout', OUT_OF_MEMORY: 'failed', BOOT_FAIL: 'failed', NODE_FAIL: 'failed', PREEMPTED: 'failed', }; for (const [slurm, expected] of Object.entries(cases)) { assert.strictEqual(mapSlurmState(slurm), expected, `${slurm} -> ${expected}`); } }); test('mapSlurmState is case-insensitive and trims whitespace', () => { assert.strictEqual(mapSlurmState('running'), 'running'); assert.strictEqual(mapSlurmState(' PENDING '), 'submitted'); assert.strictEqual(mapSlurmState('Cancelled'), 'cancelled'); }); test('mapSlurmState returns null for unknown states (no guessing)', () => { // Boundary: unknown must not collapse to a terminal failure silently. assert.strictEqual(mapSlurmState('NO_SUCH_STATE'), null); assert.strictEqual(mapSlurmState(''), null); assert.strictEqual(mapSlurmState('COMPLETED2'), null); }); // ─── Manifest build ─────────────────────────────────────────────────────────── function baseInput() { return { plan_id: '3.1', phase: '3', job_id: '12345', backend: 'slurm', submit_command: 'sbatch --parsable ./run.sh', status: 'submitted', expected_artifacts: ['Artifacts/jobs/12345/result.h5'], verification_command: 'python -m verify.py 12345', resume_command: '/gsd:execute-phase 3', }; } test('buildManifest stamps version and submitted_at via the clock seam', () => { const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const out = buildManifest(baseInput(), { clock }); assert.strictEqual(out.version, '1.0'); assert.strictEqual(out.submitted_at, '2020-06-15T12:00:00.000Z'); assert.strictEqual(out.terminal_details, null, 'non-terminal -> null terminal_details'); assert.strictEqual(out.plan_id, '3.1'); }); test('buildManifest rejects missing required fields', () => { for (const key of ['plan_id', 'phase', 'job_id', 'backend', 'submit_command', 'status', 'expected_artifacts', 'verification_command', 'resume_command']) { const bad = baseInput(); delete bad[key]; assert.throws(() => buildManifest(bad), { message: new RegExp(key) }, `missing ${key} must throw`); } }); test('buildManifest rejects an out-of-enum status', () => { const bad = baseInput(); bad.status = 'done'; assert.throws(() => buildManifest(bad), /status/i); }); test('buildManifest sets terminal_details when status is a terminal failure', () => { const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; for (const status of TERMINAL_FAILURE_STATUSES) { const input = { ...baseInput(), status, terminal_details: { reason: 'oom', exit_code: 137 } }; const out = buildManifest(input, { clock }); assert.deepStrictEqual(out.terminal_details, { reason: 'oom', exit_code: 137 }, `${status} carries terminal_details`); } }); // ─── validateManifest (producer-side mirror of the trust boundary) ──────────── test('validateManifest accepts a well-formed manifest', () => { const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const res = validateManifest(buildManifest(baseInput(), { clock })); assert.strictEqual(res.ok, true); }); test('validateManifest rejects bad version, status, and missing plan_id', () => { const good = buildManifest(baseInput(), { clock: { nowIso: () => '2020-06-15T12:00:00.000Z' } }); const badVersion = { ...good, version: '9.9' }; assert.strictEqual(validateManifest(badVersion).ok, false); const badStatus = { ...good, status: 'finished' }; assert.strictEqual(validateManifest(badStatus).ok, false); const noPlan = { ...good }; delete noPlan.plan_id; assert.strictEqual(validateManifest(noPlan).ok, false); }); // ─── Parsers ────────────────────────────────────────────────────────────────── test('parseSbatchParsable parses a bare number and a number;cluster form', () => { assert.deepStrictEqual(parseSbatchParsable('12345'), { ok: true, job_id: '12345' }); assert.deepStrictEqual(parseSbatchParsable('12345;mycluster\n'), { ok: true, job_id: '12345' }); }); test('parseSbatchParsable fails closed on empty or non-numeric output', () => { assert.strictEqual(parseSbatchParsable('').ok, false); assert.strictEqual(parseSbatchParsable('Submitted batch job 12345').ok, false, 'non-parsable prose rejected'); assert.strictEqual(parseSbatchParsable('abc;cluster').ok, false); }); test('parseSqueueLine parses " " and returns null for malformed', () => { assert.deepStrictEqual(parseSqueueLine('12345 RUNNING'), { job_id: '12345', state: 'RUNNING' }); assert.strictEqual(parseSqueueLine('header'), null); assert.strictEqual(parseSqueueLine(''), null); }); test('parseSacctRow parses [jobid, state] columns', () => { assert.deepStrictEqual(parseSacctRow(['12345', 'COMPLETED']), { job_id: '12345', state: 'COMPLETED' }); assert.strictEqual(parseSacctRow(['x']), null); assert.strictEqual(parseSacctRow([], ), null); }); // ─── writeManifest (fail-closed duplicate guard + fs injection) ──────────────── function memFs(files = {}, opts = {}) { const store = new Map(Object.entries(files)); const failReads = new Map(Object.entries(opts.failReads || {})); return { mkdirSync: () => undefined, readdirSync: (d) => { const set = store.get(d); return Array.isArray(set) ? set : []; }, readFileSync: (p) => { if (failReads.has(p)) { throw new Error(failReads.get(p)); } if (!store.has(p)) { const e = new Error('enoent'); e.code = 'ENOENT'; throw e; } return store.get(p); }, writeFileSync: (p, c) => { store.set(p, c); }, existsSync: (p) => store.has(p), }; } test('manifestPath projects to .planning/async-jobs/.json', () => { assert.strictEqual( manifestPath('.planning', '12345'), path.join('.planning', 'async-jobs', '12345.json'), ); }); test('writeManifest writes a new manifest and returns its path', () => { const fs = memFs({ [path.join('.planning', 'async-jobs')]: [] }); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, true); assert.ok(res.path.endsWith(path.join('async-jobs', '12345.json'))); const written = JSON.parse(fs.readFileSync(res.path)); assert.strictEqual(written.plan_id, '3.1'); }); test('writeManifest allows updating the SAME job_id (status progression)', () => { const dir = path.join('.planning', 'async-jobs'); const existingPath = path.join(dir, '12345.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const submitted = buildManifest(baseInput(), { clock }); const existing = { ...submitted }; const fs = memFs({ [dir]: ['12345.json'], [existingPath]: JSON.stringify(existing) }); const running = buildManifest({ ...baseInput(), status: 'running' }, { clock }); const res = writeManifest(running, '.planning', { fs, clock }); assert.strictEqual(res.ok, true, 'same job_id progression must be allowed'); }); test('writeManifest FAILS CLOSED when a different non-terminal job exists for the same plan_id', () => { // Duplicate-execution guard: a second dispatch for the same plan would // duplicate the external job. Mirror of planning-artifacts.md fail-closed. const dir = path.join('.planning', 'async-jobs'); const otherPath = path.join(dir, '99999.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const other = buildManifest({ ...baseInput(), job_id: '99999' }, { clock }); const fs = memFs({ [dir]: ['99999.json'], [otherPath]: JSON.stringify(other) }); const second = buildManifest({ ...baseInput(), job_id: '12345' }, { clock }); const res = writeManifest(second, '.planning', { fs, clock }); assert.strictEqual(res.ok, false); assert.strictEqual(res.kind, 'duplicate_plan_id'); }); test('writeManifest allows a NEW job once the prior plan_id job is terminal', () => { const dir = path.join('.planning', 'async-jobs'); const otherPath = path.join(dir, '99999.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const dead = buildManifest({ ...baseInput(), job_id: '99999', status: 'failed', terminal_details: { code: 1 } }, { clock }); const fs = memFs({ [dir]: ['99999.json'], [otherPath]: JSON.stringify(dead) }); const next = buildManifest({ ...baseInput(), job_id: '12345' }, { clock }); const res = writeManifest(next, '.planning', { fs, clock }); assert.strictEqual(res.ok, true, 'terminal prior job must not block a new dispatch'); }); test('writeManifest fails closed on a malformed existing manifest', () => { const dir = path.join('.planning', 'async-jobs'); const brokenPath = path.join(dir, '12345.json'); const fs = memFs({ [dir]: ['12345.json'], [brokenPath]: '{not json' }); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, false); assert.strictEqual(res.kind, 'malformed_existing'); }); // ─── writeManifest — negative-space: an incomplete duplicate scan (#3057) ───── // A corrupt/unreadable SIBLING manifest must not be silently skipped: the // duplicate scan cannot be trusted to have inspected it, so the write must // refuse rather than risk dispatching a duplicate external job. test('an unreadable sibling manifest refuses the write', () => { // A1 const dir = path.join('.planning', 'async-jobs'); const siblingPath = path.join(dir, '99999.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const fs = memFs( { [dir]: ['99999.json'] }, { failReads: { [siblingPath]: 'eio: input/output error' } }, ); const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, false); assert.strictEqual(res.kind, 'scan_incomplete'); assert.strictEqual(res.offendingPath, siblingPath, 'offendingPath must name the unreadable file'); }); test('an unparseable sibling manifest refuses the write', () => { // A2 const dir = path.join('.planning', 'async-jobs'); const siblingPath = path.join(dir, '99999.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const fs = memFs({ [dir]: ['99999.json'], [siblingPath]: '{not json' }); const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, false); assert.strictEqual(res.kind, 'scan_incomplete'); assert.strictEqual(res.offendingPath, siblingPath, 'offendingPath must name the unparseable file'); }); test('target and sibling failures stay distinct kinds', () => { // A3 const dir = path.join('.planning', 'async-jobs'); const targetPath = path.join(dir, '12345.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const fs = memFs({ [dir]: ['12345.json'], [targetPath]: '{not json' }); const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, false); assert.strictEqual(res.kind, 'malformed_existing', 'the TARGET failure must stay malformed_existing, not scan_incomplete'); }); // A4 (paired): prove the sibling really holds a duplicate (control), then // prove that making that SAME content unreadable still refuses (regression). test('control: a readable sibling with a non-terminal duplicate plan_id is duplicate_plan_id', () => { // A4a — control. Same construction as the "FAILS CLOSED on duplicate // plan_id" test above: valid JSON, same plan_id, different job_id, // non-terminal status. This establishes the sibling content genuinely // triggers the duplicate guard when it CAN be read. const dir = path.join('.planning', 'async-jobs'); const siblingPath = path.join(dir, '99999.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const duplicate = buildManifest({ ...baseInput(), job_id: '99999' }, { clock }); const fs = memFs({ [dir]: ['99999.json'], [siblingPath]: JSON.stringify(duplicate) }); const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, false); assert.strictEqual(res.kind, 'duplicate_plan_id', 'the sibling content must be a real duplicate trigger when readable'); }); test('a corrupt sibling can no longer hide a duplicate dispatch', () => { // A4b — the bug, as a test. Same sibling PATH as the control above, which // that test proves can hold a genuine non-terminal duplicate for this // plan_id. Here the read itself faults (opts.failReads), so no content is // reachable at all — and that is the point: the guard now refuses on ANY // unreadable sibling precisely because it cannot know whether that file // was the duplicate. The control supplies the "it could have been"; this // supplies the "and we no longer gamble on it". // // Before the fix, an unreadable sibling was `continue`d past, the // duplicate was never seen, and this returned ok:true — dispatching a // duplicate external job. const dir = path.join('.planning', 'async-jobs'); const siblingPath = path.join(dir, '99999.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const fs = memFs( { [dir]: ['99999.json'] }, { failReads: { [siblingPath]: 'eio: input/output error' } }, ); const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, false, 'a corrupt sibling must refuse, not silently permit a possible duplicate dispatch'); assert.strictEqual(res.kind, 'scan_incomplete'); }); test('happy path unaffected', () => { // A5 const dir = path.join('.planning', 'async-jobs'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const fs = memFs({ [dir]: [] }); const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, true); }); test('a terminal prior job is still allowed', () => { // A6 const dir = path.join('.planning', 'async-jobs'); const otherPath = path.join(dir, '99999.json'); const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const dead = buildManifest({ ...baseInput(), job_id: '99999', status: 'failed', terminal_details: { code: 1 } }, { clock }); const fs = memFs({ [dir]: ['99999.json'], [otherPath]: JSON.stringify(dead) }); const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, true, 'the duplicate guard only protects against re-dispatching live work'); }); test('a directory read failure is io_error, not scan_incomplete', () => { // A7 const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const fs = memFs({}, { failReads: {} }); fs.readdirSync = () => { throw new Error('eio: input/output error'); }; const manifest = buildManifest(baseInput(), { clock }); const res = writeManifest(manifest, '.planning', { fs, clock }); assert.strictEqual(res.ok, false); assert.strictEqual(res.kind, 'io_error'); }); // ─── Property-based (CLAUDE.md: parsers/contracts need a fast-check test) ───── test('property: mapSlurmState is total and idempotent over the known alphabet', () => { fc.assert( fc.property(fc.constantFrom( 'PENDING', 'CONFIGURING', 'RUNNING', 'COMPLETING', 'COMPLETED', 'FAILED', 'CANCELLED', 'TIMEOUT', 'OUT_OF_MEMORY', 'BOOT_FAIL', 'NODE_FAIL', 'PREEMPTED', ), (state) => { const a = mapSlurmState(state); const b = mapSlurmState(state); return a !== null && a === b && MANIFEST_STATUS.includes(a); }), { numRuns: 200 }, ); }); test('property: buildManifest -> validateManifest round-trips for valid generated input', () => { fc.assert( fc.property( fc.record({ plan_id: fc.stringMatching(/^[0-9]+\.[0-9]+$/), phase: fc.stringMatching(/^[0-9]+$/), job_id: fc.stringMatching(/^[0-9]{1,8}$/), backend: fc.constantFrom('slurm'), submit_command: fc.constantFrom('sbatch --parsable ./run.sh'), status: fc.constantFrom(...MANIFEST_STATUS), expected_artifacts: fc.array(fc.constantFrom('Artifacts/jobs/x/out.h5'), { minLength: 1 }), verification_command: fc.constantFrom('python -m verify.py'), resume_command: fc.constantFrom('/gsd:execute-phase 3'), terminal_details: fc.oneof(fc.constant(null), fc.record({ code: fc.integer() })), }), (input) => { const clock = { nowIso: () => '2020-06-15T12:00:00.000Z' }; const td = input.status === 'completed-unverified' ? null : input.terminal_details; const built = buildManifest({ ...input, terminal_details: td }, { clock }); return validateManifest(built).ok === true; }, ), { numRuns: 100 }, ); });