Adds the shared adversarial-input harness described in TEST-EXAMPLES.md
§"CLI Negative Matrix" and applies it across two layers:
1. tests/helpers/cli-negative.cjs — runCli() wraps spawnSync of
get-shit-done/bin/gsd-tools.cjs, prepends --json-errors by default,
and returns a typed IR { status, ok, reason, message,
hasStackTrace, ... } so adversarial-case tests assert on
reason codes — never on stderr prose.
2. tests/feat-3593-cli-negative-config.test.cjs — full 12-category
matrix for the config command family (the highest-risk read/write
surface): missing/empty/whitespace args, duplicate --cwd,
unknown subcommand, value-looks-like-a-flag, corrupt config.json,
50KB key, Unicode/emoji keys and values, and 9 distinct shell-
metacharacter payloads asserted as NOT-executed via per-test
sentinel-file probes.
3. tests/feat-3593-cli-negative-universal.test.cjs — narrower
cross-family sweep (phase, roadmap, state, config, workstream,
init, validate). Pins the three universal invariants every
family must satisfy: bare invocation does not crash with a V8
stack trace, unknown subcommand emits a typed reason, and shell
payloads as argv values are not executed.
4. tests/feat-3593-cli-negative-harness.test.cjs — meta-test that
pins the harness IR contract so a future regression in the
parser (stack-trace detection, JSON shape extraction, hostile
stderr handling) surfaces before it cascades through every
matrix file.
Bug fix surfaced by the new tests:
get-shit-done/bin/lib/config.cjs cmdConfigSet — invoking
`config-set <key>` with no value silently returned exit 0 and
emitted { updated: true } even though the value parameter was
undefined. JSON.stringify dropped the key during the write or
persisted a corrupt entry. Now rejected with typed ERROR_REASON.USAGE
before any write. Matching guard added to SDK configSet for parity.
Harness coverage delivered: 58 new tests (9 meta + 26 config + 23
universal sweep). Pre-existing config suites (101 tests) all pass.
lint-no-source-grep clean.
Refs #3593
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
134 lines
5.7 KiB
JavaScript
134 lines
5.7 KiB
JavaScript
/**
|
|
* CLI negative-matrix harness (#3593).
|
|
*
|
|
* Wraps spawnSync of get-shit-done/bin/gsd-tools.cjs so test files can
|
|
* assert on structured outputs (exit code, typed reason, stack-trace
|
|
* absence) without each test re-implementing the JSON-errors parsing
|
|
* dance. Hostile values are passed as argv elements — never composed
|
|
* into a shell string — so shell metacharacters in test inputs (;, &&,
|
|
* $(), backticks, quotes, newlines, null bytes) reach the CLI as
|
|
* opaque data, not as shell syntax.
|
|
*
|
|
* Designed against CONTRIBUTING.md §"Testing Standards" and
|
|
* TEST-EXAMPLES.md §"CLI Negative Matrix":
|
|
*
|
|
* - spawnSync (no shell) per the "no shell strings" rule.
|
|
* - Returns a typed IR { status, ok, reason, message, ... } so tests
|
|
* assert on `.reason === REASON_CODE`, never on prose.
|
|
* - Detects stack-trace leakage in stderr; tests can fail when the
|
|
* CLI surfaces an unexpected exception under hostile input.
|
|
* - --json-errors is on by default; pass jsonErrors:false to verify
|
|
* human-formatted stderr paths.
|
|
*
|
|
* The harness does NOT decide what the test asserts — it just shapes
|
|
* the data so the assertion is mechanical and prose-free.
|
|
*/
|
|
|
|
'use strict';
|
|
|
|
const path = require('node:path');
|
|
const { spawnSync } = require('node:child_process');
|
|
|
|
const TOOLS_PATH = path.resolve(__dirname, '..', '..', 'get-shit-done', 'bin', 'gsd-tools.cjs');
|
|
|
|
/**
|
|
* Run gsd-tools with the given argv against a project directory.
|
|
*
|
|
* @param {string[]} argv - argument vector passed to gsd-tools.cjs. Each
|
|
* element reaches the child as a single argv element regardless of
|
|
* shell metacharacters in its value.
|
|
* @param {object} options
|
|
* @param {string} options.cwd - working directory for the child (REQUIRED).
|
|
* @param {object} [options.env] - env vars merged on top of process.env.
|
|
* @param {boolean} [options.jsonErrors=true] - prepend --json-errors so
|
|
* the CLI emits a structured `{ ok, reason, message }` payload to
|
|
* stderr. Set false to exercise the human-formatted path.
|
|
* @param {number} [options.timeoutMs=10000] - kill child if it runs longer.
|
|
* @returns {object} typed IR — see field docs below.
|
|
*/
|
|
function runCli(argv, options) {
|
|
if (!options || typeof options.cwd !== 'string' || options.cwd.length === 0) {
|
|
throw new TypeError('runCli: options.cwd (string) is required');
|
|
}
|
|
if (!Array.isArray(argv)) {
|
|
throw new TypeError('runCli: argv must be an array (no shell strings)');
|
|
}
|
|
const jsonErrors = options.jsonErrors !== false;
|
|
const finalArgs = jsonErrors ? ['--json-errors', ...argv] : argv.slice();
|
|
const env = { ...process.env, ...(options.env || {}) };
|
|
const spawnResult = spawnSync(process.execPath, [TOOLS_PATH, ...finalArgs], {
|
|
cwd: options.cwd,
|
|
encoding: 'utf-8',
|
|
timeout: typeof options.timeoutMs === 'number' ? options.timeoutMs : 10000,
|
|
env,
|
|
});
|
|
return parseSpawnResult(spawnResult, { jsonErrorsRequested: jsonErrors });
|
|
}
|
|
|
|
/**
|
|
* Shape a raw spawnSync result into the harness IR.
|
|
*
|
|
* Returned fields:
|
|
* - status {number|null} child exit code (null if killed by signal)
|
|
* - signal {string|null} terminating signal name when killed
|
|
* - stdout {string} captured stdout
|
|
* - stderr {string} captured stderr
|
|
* - ok {boolean|null} parsed from JSON-errors payload; null when
|
|
* stderr is not JSON-shaped
|
|
* - reason {string|null} ERROR_REASON code from JSON-errors payload
|
|
* - message {string|null} human message from JSON-errors payload
|
|
* - hasStackTrace {boolean} stderr contains a `\n at ` frame line.
|
|
* Tests use this to assert the CLI didn't
|
|
* leak an unexpected exception.
|
|
* - jsonErrorsRequested {boolean} true when --json-errors was added by
|
|
* the harness; tests use this to decide
|
|
* whether the missing-reason case is a bug
|
|
* vs. expected (jsonErrors:false path).
|
|
* - spawnError {Error|null} spawnSync's `error` field (e.g. ENOENT
|
|
* on process.execPath, timeout signal).
|
|
*/
|
|
function parseSpawnResult(spawnResult, meta) {
|
|
const stderr = typeof spawnResult.stderr === 'string' ? spawnResult.stderr : '';
|
|
const stdout = typeof spawnResult.stdout === 'string' ? spawnResult.stdout : '';
|
|
// Stack-trace leak detection: V8 prints " at Function (path:line:col)"
|
|
// frames into stderr when an uncaught throw escapes. Any such line means
|
|
// the CLI hit a code path that wasn't wrapped in error()/typed reason.
|
|
// \n is required so we don't match the word "at" appearing in prose.
|
|
const hasStackTrace = /\n\s{2,}at\s+/.test(stderr);
|
|
|
|
let ok = null;
|
|
let reason = null;
|
|
let message = null;
|
|
// JSON-errors mode emits a single-line JSON document to stderr followed by
|
|
// a newline. Parse defensively — any non-JSON stderr leaves the fields null
|
|
// and lets the test surface the discrepancy.
|
|
const trimmed = stderr.trim();
|
|
if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
|
|
try {
|
|
const parsed = JSON.parse(trimmed);
|
|
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
if (typeof parsed.ok === 'boolean') ok = parsed.ok;
|
|
if (typeof parsed.reason === 'string') reason = parsed.reason;
|
|
if (typeof parsed.message === 'string') message = parsed.message;
|
|
}
|
|
} catch {
|
|
// Not JSON after all — leave parsed fields null.
|
|
}
|
|
}
|
|
|
|
return {
|
|
status: spawnResult.status,
|
|
signal: spawnResult.signal,
|
|
stdout,
|
|
stderr,
|
|
ok,
|
|
reason,
|
|
message,
|
|
hasStackTrace,
|
|
jsonErrorsRequested: meta && meta.jsonErrorsRequested === true,
|
|
spawnError: spawnResult.error || null,
|
|
};
|
|
}
|
|
|
|
module.exports = { runCli, parseSpawnResult, TOOLS_PATH };
|