From f2b6704fc4b7191963f7e68d464e168cfa6ab830 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 14:26:56 +0000 Subject: [PATCH] fix: Run mutation tests in a sandbox and activate mutants with Vitest 5 (no-changelog) The mutation tool now runs Stryker in a sandbox mirror and never changes the working tree. A vitest-compat runner fixes the test name filter so that mutants run their tests under Vitest 5. The --test-command flag runs a package-specific test command with the Stryker command runner. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018gK4jZbBjL8toMAnrfJ4mh --- packages/@n8n/scheduler/stryker.config.mjs | 4 +- packages/workflow/stryker.config.mjs | 4 +- scripts/mutation-health/README.md | 169 +- scripts/mutation-health/main.test.mjs | 315 ++++ scripts/mutation-health/mutate.mjs | 1360 ++++------------- scripts/mutation-health/mutate.test.mjs | 1008 +++--------- scripts/mutation-health/plan-diff.test.mjs | 268 ++++ scripts/mutation-health/plan.mjs | 224 +++ scripts/mutation-health/plan.test.mjs | 207 +++ scripts/mutation-health/properties.test.mjs | 324 ++++ scripts/mutation-health/run-job.test.mjs | 440 ++++++ scripts/mutation-health/run-outcome.test.mjs | 155 ++ scripts/mutation-health/run-stryker.test.mjs | 182 +++ scripts/mutation-health/samples.mjs | 46 + scripts/mutation-health/sandbox-mirror.mjs | 107 ++ .../mutation-health/sandbox-mirror.test.mjs | 171 +++ .../mutation-health/stryker-process.test.mjs | 279 ++++ scripts/mutation-health/stryker.default.mjs | 8 +- scripts/mutation-health/stryker.mjs | 410 +++++ scripts/mutation-health/stryker.test.mjs | 458 ++++++ .../mutation-health/summary-report.test.mjs | 106 ++ scripts/mutation-health/summary.mjs | 327 ++++ scripts/mutation-health/summary.test.mjs | 353 +++++ scripts/mutation-health/targets.mjs | 159 ++ scripts/mutation-health/targets.test.mjs | 377 +++++ scripts/mutation-health/test-doubles.mjs | 140 ++ .../mutation-health/vitest-compat-runner.mjs | 15 + scripts/mutation-health/vitest-compat.mjs | 117 ++ .../mutation-health/vitest-compat.test.mjs | 409 +++++ scripts/mutation-health/working-tree.test.mjs | 179 +++ 30 files changed, 6401 insertions(+), 1920 deletions(-) create mode 100644 scripts/mutation-health/main.test.mjs create mode 100644 scripts/mutation-health/plan-diff.test.mjs create mode 100644 scripts/mutation-health/plan.mjs create mode 100644 scripts/mutation-health/plan.test.mjs create mode 100644 scripts/mutation-health/properties.test.mjs create mode 100644 scripts/mutation-health/run-job.test.mjs create mode 100644 scripts/mutation-health/run-outcome.test.mjs create mode 100644 scripts/mutation-health/run-stryker.test.mjs create mode 100644 scripts/mutation-health/samples.mjs create mode 100644 scripts/mutation-health/sandbox-mirror.mjs create mode 100644 scripts/mutation-health/sandbox-mirror.test.mjs create mode 100644 scripts/mutation-health/stryker-process.test.mjs create mode 100644 scripts/mutation-health/stryker.mjs create mode 100644 scripts/mutation-health/stryker.test.mjs create mode 100644 scripts/mutation-health/summary-report.test.mjs create mode 100644 scripts/mutation-health/summary.mjs create mode 100644 scripts/mutation-health/summary.test.mjs create mode 100644 scripts/mutation-health/targets.mjs create mode 100644 scripts/mutation-health/targets.test.mjs create mode 100644 scripts/mutation-health/test-doubles.mjs create mode 100644 scripts/mutation-health/vitest-compat-runner.mjs create mode 100644 scripts/mutation-health/vitest-compat.mjs create mode 100644 scripts/mutation-health/vitest-compat.test.mjs create mode 100644 scripts/mutation-health/working-tree.test.mjs diff --git a/packages/@n8n/scheduler/stryker.config.mjs b/packages/@n8n/scheduler/stryker.config.mjs index a18a3dbf47a..f83e2f0a22b 100644 --- a/packages/@n8n/scheduler/stryker.config.mjs +++ b/packages/@n8n/scheduler/stryker.config.mjs @@ -17,8 +17,8 @@ export default { incrementalFile: 'reports/mutation/stryker-incremental.json', // Skip module-level (static) mutants: each forces a full re-instrument. ignoreStatic: true, - // Default empty: the `mutate` npm script always passes --mutate . - // Direct invocation with no --mutate will fail fast (allowEmpty: false). + // Empty: mutate.mjs sets `mutate` to the run's targets in the run config + // that it gives Stryker (reports/mutation/stryker.run.json). mutate: [], jsonReporter: { fileName: 'reports/mutation/raw.json' }, // Cap low: a string/utils mutant running 15s+ is a spinning loop-condition mutant; fail it fast. diff --git a/packages/workflow/stryker.config.mjs b/packages/workflow/stryker.config.mjs index d23fd03b304..32dab64274b 100644 --- a/packages/workflow/stryker.config.mjs +++ b/packages/workflow/stryker.config.mjs @@ -17,8 +17,8 @@ export default { incrementalFile: 'reports/mutation/stryker-incremental.json', // Skip module-level (static) mutants — each forces a full re-instrument. ignoreStatic: true, - // Default empty — the `mutate` npm script always passes --mutate . - // Direct invocation with no --mutate will fail fast (allowEmpty: false). + // Empty: mutate.mjs sets `mutate` to the run's targets in the run config + // that it gives Stryker (reports/mutation/stryker.run.json). mutate: [], jsonReporter: { fileName: 'reports/mutation/raw.json' }, // Cap low: a string/utils mutant running 15s+ is a spinning loop-condition mutant; fail it fast. diff --git a/scripts/mutation-health/README.md b/scripts/mutation-health/README.md index f5168097244..9a8f4eed78f 100644 --- a/scripts/mutation-health/README.md +++ b/scripts/mutation-health/README.md @@ -54,8 +54,13 @@ That divergence is exactly why this project exists. | File | Role | | --- | --- | -| `mutate.mjs` | The whole engine. Runs Stryker over a package and emits an actionable summary. Exposed as `pnpm mutate`. | -| `mutate.test.mjs` | Unit tests for its pure helpers (`node --test 'scripts/mutation-health/*.test.mjs'`). | +| `mutate.mjs` | The entry point, exposed as `pnpm mutate`. Reads the command line, plans the runs, runs Stryker and prints the summary. | +| `plan.mjs`, `targets.mjs` | What a run mutates: targets, diff ranges, test files and which packages can be scored. | +| `stryker.mjs` | How a run starts: the run config, the Stryker process and signal handling. | +| `sandbox-mirror.mjs` | Where the sandbox goes. See [Sandbox runs](#sandbox-runs). | +| `summary.mjs` | Scoring, the gate and `summary.json`. | +| `vitest-compat.mjs`, `vitest-compat-runner.mjs` | The `vitest-compat` Stryker test runner. See [Vitest 5](#vitest-5). | +| `*.test.mjs`, `test-doubles.mjs`, `samples.mjs` | Unit tests (`node --test scripts/mutation-health/*.test.mjs`). They start no Stryker run. CI runs them in the "Workflow scripts" job, which installs only `.github/scripts`, so a test must not need a root dependency such as Stryker or fast-check. Inject a stand-in, or skip the test when the dependency is missing. `properties.test.mjs` checks the pure helpers with seeded generated inputs from `samples.mjs`. `working-tree.test.mjs` checks the main promise of the tool: an edit made while a run is in flight stays, whatever way the run ends. | | `stryker.default.mjs` | Shared Stryker config for any vitest package. A package that needs special handling ships its own `stryker.config.mjs`, which `mutate.mjs` prefers. | | `stryker.cli.mjs` | The default plus `vitest.related: false`, used for `packages/cli` targets. See [Scoping the tests](#scoping-the-tests-with---test-files). | @@ -63,6 +68,10 @@ Outputs land in `/reports/mutation/` (gitignored): - `raw.json` — the full Stryker Mutation Testing Elements report (600 KB+; don't read it directly). - `summary.json` — the compact actionable summary: every survivor's location, mutator, replacement, and covering tests. **This is the file to read.** +- `stryker.run.json` — the exact config the tool gave Stryker. Read it when a run does not do what + you expect. It can name the sandbox mirror (see [Sandbox runs](#sandbox-runs)), which is gone + after the run. To repeat a run, run `pnpm mutate` again with the same arguments, not + `stryker run` on this file. ## Usage @@ -86,27 +95,45 @@ pnpm mutate src/cron.ts --package-dir packages/workflow # One file, scoped to the tests that must kill its mutants. pnpm mutate packages/cli/src/credentials/external-secrets.utils.ts:32-68 \ --test-files packages/cli/src/credentials/__tests__/external-secrets.utils.test.ts + +# A package whose `test` script does not run vitest: name the test command. +pnpm mutate packages/quality/testing/playwright/coverage-options.ts \ + --test-files coverage-options.test.ts --test-command 'pnpm exec vitest run' ``` Exit codes: `0` gate passed · `1` gate failed (summary.json still written — this is the -iterate signal) · `2` usage error · `3` Stryker could not run. A toolchain failure is +iterate signal) · `2` usage or config error · `3` Stryker could not run, or its test runner ran +no test for a mutant that tests cover · `130` / `143` the run was cancelled. A toolchain failure is **never** `1`, so a broken checkout can't be mistaken for a score of zero. ### Why `--diff` is fast -Two things do the work: +Three things do the work: 1. **Patch scoping.** Stryker's mutation-range syntax (`file.ts:13-16`) means only the mutants inside your changed lines are generated. You're scored on the lines you touched, not on inherited debt. -2. **One dry run per package.** Targets are comma-joined into a single `--mutate` argument. - Repeated `--mutate` flags silently *overwrite* each other in Stryker's CLI, so comma-joining - is the only way to batch — and it means a package pays for its dry run once, not once per file. +2. **One dry run per package.** All the targets of a package go into one Stryker run, so a + package pays for its dry run once, not once per file. +3. **Per-test coverage.** The default runner records which tests cover each mutant during the + dry run. Each mutant then runs only those tests, in a Vitest process that stays alive + between mutants. -On top of that, Stryker's vitest runner only loads the tests *related* to the mutated files, so -cost tracks the related suite rather than package size. Measured end-to-end, whole-file: -`@n8n/decorators` 1s · `@n8n/scheduler` 3s · `packages/workflow` 13s · `nodes-base` 26s · -`packages/cli` 88s. Line-scoping cuts these further. +On top of that, Stryker's vitest runner loads only the tests *related* to the mutated files, so +cost tracks the related suite rather than package size. + +Measured with the sandbox runs of this tool (Vitest 5.0.1, 4 workers): + +| Target | Mutants | Default runner | Command runner | +| --- | --- | --- | --- | +| `@n8n/instance-ai` `utils/model-config-id.ts` | 38 | 5–11 s | 16 s | +| `@n8n/scheduler` `core/clock-skew.ts` | 15 | 5 s | — | +| `@n8n/instance-ai` `automation/schedule-phrase.ts` | 350 | 54 s | 11 min 45 s ¹ | +| `packages/cli` `credentials/external-secrets.utils.ts:32-68` | 11 | 26 s | — | + +¹ Measured with 2 workers. Both runners found the same 8 survivors. + +A `packages/cli` run spends most of its time on the sandbox copy and the cli test setup. ### Scoping the tests with `--test-files` @@ -115,13 +142,16 @@ walks the import graph of the mutated file and runs every test file that reaches right default for most packages, and it is what keeps cost tracking the related suite rather than package size. -`--test-files` replaces that discovery with an explicit list. The value goes to Stryker's -[`testFiles`](https://stryker-mutator.io/docs/stryker-js/configuration/#testfiles-string) config -field, so only those files run — which also answers a sharper question: can this module's *own* -unit tests kill its mutants, without help from the rest of the suite? +`--test-files` replaces that discovery with an explicit list. With the default runner the list +goes to Stryker's [`testFiles`](https://stryker-mutator.io/docs/stryker-js/configuration/#testfiles-string) +config field, so only those files run. With `--test-command` the files go onto the end of the +command, because the command runner does not accept `testFiles`. The list also answers a +sharper question: can this module's *own* unit tests kill its mutants, without help from the +rest of the suite? The flag repeats and it also takes a comma-separated list. Paths may be repo-relative (what you -type) or package-relative (what Stryker matches); `mutate.mjs` converts them. Globs work. +type) or package-relative (what Stryker matches); `mutate.mjs` converts them. Globs work with the +default runner. A test command gets each path as one quoted argument, so a glob is not expanded. ```bash pnpm mutate packages/@n8n/crdt/src/utils.ts --test-files packages/@n8n/crdt/src/__tests__/utils.test.ts @@ -145,38 +175,100 @@ Mutating packages/cli needs --test-files. ``` Those runs also get `stryker.cli.mjs` instead of the shared default — same settings, with -`vitest.related` turned off, because the explicit list already decides the scope. With it, the -example above runs its 36-test file and finishes in seconds. +`vitest.related` turned off, because the explicit list already decides the scope. In diff mode, the changed CLI test files provide the same explicit list. If a CLI patch changes source without changing a test, the command fails before Stryker starts. Add or update a covering test, or run a named target with the existing covering tests passed through `--test-files`. -### In-place mutation +### Test runners -Runs use Stryker's `--inPlace`. Its default sandbox copy breaks on any package whose vitest -config resolves a workspace dependency through a path alias — the alias doesn't survive the -copy, and `packages/cli` dies on `ERR_LOAD_URL … .stryker-tmp/@n8n/backend-test-utils`. +| Runner | When | How it runs a mutant | Coverage in the summary | +| --- | --- | --- | --- | +| `vitest-compat` | By default, for every config that uses the `vitest` runner | Runs only the tests that cover the mutant, in a Vitest process that stays alive | Yes | +| `command` | With `--test-command ` | Runs ` ` in a new shell for each mutant | No (`cov n/a`) | -Stryker restores your files on a clean exit and on `Ctrl-C`, but not after a crash, a timeout or -a `SIGTERM` — and its preprocessing reaches past the mutate targets, so a target-only snapshot -left mutated files behind. +Use `--test-command` only for a package whose `test` script does not run vitest, such as the +playwright package (`test:unit` runs its unit tests). The command runner has no per-test +coverage, so: -`mutate.mjs` therefore snapshots the **whole working tree** before the run: +- every mutant runs every named test file, and pays the start-up time of the command; +- `NoCoverage` never appears: a mutant that no test reaches is `Survived`, which also fails the gate; +- `coveringTests` in `summary.json` is empty; +- a command that finds no test file fails the dry run. The tool records it as a score-0 red + result, as it does when the vitest runner finds no test. -- the exact bytes of every tracked file that is already dirty, plus every target. In `--diff` - mode those files hold *uncommitted work*, so `git checkout --` is not a safe undo. -- the dirty set itself. Anything dirty *after* the run that was clean before it was changed by - Stryker, and git holds its pre-run state, so `git checkout --` is the right undo there. +Name the test files with `--test-files`. Without them the command runs the package's whole +suite for each mutant. `--test-command` needs a single target, so it does not combine with +`--diff`. -One cleanup routine restores that snapshot and deletes every `stryker-setup-*.js` the vitest -runner left under the repo root. It is idempotent and registered on every exit path: the usual -one, an uncaught exception, `SIGINT` and `SIGTERM`. A cancelled run exits `130` (`143` for -`SIGTERM`) with a clean `git status`. +## Vitest 5 -One caveat: files you edit in another terminal *while a run is in flight* look like Stryker's -work and are reverted. Don't edit the repo during a run. +`@stryker-mutator/vitest-runner` 10.0.0 is tested against Vitest 4.1. The repo uses Vitest 5. +With the plain runner every mutant survived, and Stryker printed `Ran 0.00 tests per mutant`. +The cause is the test filter. For each mutant the runner runs only the tests that cover it, and it +selects them with a `testNamePattern` regex built from names that join the suite and the test +with a space (`suite test`). Vitest 4 matched the pattern against names in that form. Vitest 5 +matches it against `fullTestName`, which joins them with ` > ` (`suite > test`). The pattern then +misses every test inside a `describe`, so no test runs. `coverageAnalysis: 'off'` and `'all'` do +not help: the runner records per-test coverage in each dry run, and Stryker then sends the same +per-test filter. + +`vitest-compat-runner.mjs` is a small Stryker plugin that wraps the runner. Before Vitest starts, +it changes each space in the pattern to "space or ` > `". It makes one more change: a test file +that throws on import fails the dry run with that file's error. The plain runner drops such a +file without an error, and the run then reads as "no covering tests". + +`mutate.mjs` gives the `vitest-compat` runner to every config that uses `vitest`, including +the package-local configs of `packages/workflow` and `@n8n/scheduler`. Remove the name fix when +the runner supports Vitest 5. + +The tool also checks each report for this failure, so that a later runner or Vitest change +cannot bring it back without a sign. Stryker records the tests that cover each mutant +(`coveredBy`) and the number of tests it ran for the mutant (`testsCompleted`). A mutant that tests +cover but that survived with no test run means that the runner did not run its covering tests. +The run then exits `3`, lists the first such mutants and writes no `summary.json`. A covering test +that skips itself at run time has the same effect, because Stryker does not count a skipped test. + +## Sandbox runs + +Every run uses Stryker's sandbox: Stryker copies the package and mutates the copy, never your +files. Nothing is restored after a run, and you can keep working in the repo while a run is in +flight. + +The default sandbox sits two levels below the package (`/.stryker-tmp/sandbox-*/`). A +config path that leaves the package then points to the wrong place. Examples are the cli aliases +`../@n8n/telemetry/src` and `../@n8n/mcp-apps/src/server`, and the `@nodes-testing` alias of +`nodes-base` and `nodes-langchain`. Stryker corrects such paths in `tsconfig.json`, but not in a +vite or vitest config. + +`mutate.mjs` therefore puts the sandbox in a **mirror** of the repo under `.stryker-tmp/` at the +repo root (gitignored). The mirror has real directories along the package's path, with a symbolic +link to each real entry beside that path. The sandbox has the same depth as the package, so `..` +from the sandbox reaches the same files as `..` from the package. Stryker's tsconfig correction is +turned off for these runs, because the paths are already right. + +After each run Stryker removes its sandbox and the tool removes the mirror. The removal unlinks +each link and never follows one, so it never deletes a real file. The tool also removes the mirror +before it exits on `SIGINT`, `SIGTERM` or a crash. On a signal, Stryker stops and does not remove +its sandbox, so the mirror removal removes it. On a crash while Stryker runs, the tool tells +Stryker to stop and waits up to 10 seconds until Stryker is gone. Only then does it remove the +mirror, so it does not remove files that Stryker still uses. If Stryker is still alive after the +wait, the tool tells it to stop again, which makes Stryker exit at once. Only a stop that the tool +cannot handle, such as `SIGKILL`, leaves a `.stryker-tmp/mirror-*` directory behind. It holds only +directories and links, and you can delete it. + +The tool refuses these configs with exit `2`: + +- A config that turns on Stryker's in place mode. +- A config that uses the TypeScript checker (`checkers: ['typescript']`). The checker + type-checks with the file that `tsconfigFile` names, and the tool sets that option to a file + that does not exist to turn the tsconfig correction off. + +Results that Stryker keeps for `incremental` runs go to a file for each runner (for example +`stryker-incremental.vitest-compat.json`). The runners name tests differently, so their results do +not mix. This also drops the results that the plain runner recorded under Vitest 5. ## Which packages can be scored @@ -185,6 +277,9 @@ of them, including `nodes-base`, `nodes-langchain`, `cli` and `db`. `--diff` der per file and prints a one-line reason for anything it skips; there is no curated list to maintain. +A package whose `test` script does not run vitest, such as the playwright package, can be scored +as a named target with [`--test-command`](#test-runners). + Not scored: - `@n8n/expression-runtime` — Stryker's dry run SIGABRTs on the isolated-vm engine ([DEVP-257](https://linear.app/n8n/issue/DEVP-257)). @@ -201,7 +296,7 @@ A run passes only when **both**: 1. Mutation score meets `STRYKER_THRESHOLD` (default `80`), **and** 2. Zero `Survived` / `NoCoverage` mutants remain — every unkilled mutant must be explicitly justified as `Ignored` via a `// Stryker disable next-line : ` comment in the source. -Stryker excludes `Ignored` mutants from both numerator and denominator of the score (see `scoreFromCounts` in `mutate.mjs`), so marking a genuine equivalent as ignored is **not** padding — it's the documented mechanism for "this mutant is equivalent / not behaviour-bearing, here's why". The score becomes a coarse floor; the real gate is "no unjustified survivors". This stops agents from padding the suite with trivial tests to clear `80%` while leaving real behaviour gaps unasserted. See [DEVP-442](https://linear.app/n8n/issue/DEVP-442) for the motivation. +Stryker excludes `Ignored` mutants from both numerator and denominator of the score (see `scoreFromCounts` in `summary.mjs`), so marking a genuine equivalent as ignored is **not** padding — it's the documented mechanism for "this mutant is equivalent / not behaviour-bearing, here's why". The score becomes a coarse floor; the real gate is "no unjustified survivors". This stops agents from padding the suite with trivial tests to clear `80%` while leaving real behaviour gaps unasserted. See [DEVP-442](https://linear.app/n8n/issue/DEVP-442) for the motivation. `summary.json` surfaces every `Ignored` mutant alongside its disable-comment reason so reviewers can spot-check the justifications — those become the high-signal review artifact rather than N padding tests. diff --git a/scripts/mutation-health/main.test.mjs b/scripts/mutation-health/main.test.mjs new file mode 100644 index 00000000000..fd04e9a6412 --- /dev/null +++ b/scripts/mutation-health/main.test.mjs @@ -0,0 +1,315 @@ +import { afterEach, beforeEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { USAGE, main, reportResults } from './mutate.mjs'; +import { buildSummary } from './summary.mjs'; +import { MutateError } from './targets.mjs'; +import { + SAMPLE_REPO, + fakeProcess, + fakeSpawn, + resolveFakeStrykerBin, + sink, + writeTree, +} from './test-doubles.mjs'; + +const SOURCE = 'export const a = 1 + 2;\n'; + +// A Stryker report with one mutant for each given status. `extra` goes on each mutant. +function report(statuses, extra = {}) { + return { + files: { + 'src/a.ts': { + source: SOURCE, + mutants: statuses.map((status, index) => ({ + id: String(index), + mutatorName: 'ArithmeticOperator', + status, + location: { start: { line: 1, column: 17 }, end: { line: 1, column: 22 } }, + replacement: '1 - 2', + ...extra, + })), + }, + }, + }; +} + +// `pkgRoot` sits in a temp dir. A test that plans from the repo root seeds +// SAMPLE_REPO in that dir and passes it as `repoRoot`. +let pkgRoot; +const tempRepo = () => path.dirname(pkgRoot); + +beforeEach(() => { + pkgRoot = path.join(mkdtempSync(path.join(tmpdir(), 'mutate-main-')), 'pkg'); + mkdirSync(path.join(pkgRoot, 'src'), { recursive: true }); + writeFileSync( + path.join(pkgRoot, 'package.json'), + '{ "name": "pkg", "scripts": { "test": "vitest run" } }', + ); + writeFileSync(path.join(pkgRoot, 'src/a.ts'), SOURCE); +}); + +afterEach(() => { + rmSync(path.dirname(pkgRoot), { recursive: true, force: true }); +}); + +// Start the whole command on the temp package with a fake Stryker that writes +// `raw` as its report (no report when `raw` is null) and exits with `exitCode`. +function startMain({ + raw, + exitCode = 0, + argv = ['src/a.ts', '--package-dir', pkgRoot], + git, + repoRoot, +} = {}) { + const doubles = fakeSpawn({ + onSpawn: (child, { options }) => + setImmediate(() => { + if (raw) { + writeFileSync(path.join(options.cwd, 'reports/mutation/raw.json'), JSON.stringify(raw)); + } + child.finish(exitCode); + }), + }); + const p = fakeProcess(); + const stderr = sink(); + const stdout = sink(); + const io = { + git, + repoRoot, + resolveStrykerBin: resolveFakeStrykerBin, + spawn: doubles.spawn, + stdout, + stderr, + proc: p.proc, + exit: p.exit, + write: p.write, + }; + return { promise: main(argv, io), stderr, stdout, calls: doubles.calls }; +} + +// Run the whole command to its exit code (see startMain). +async function runMain(options) { + const run = startMain(options); + const code = await run.promise; + return { code, stderr: run.stderr.text(), stdout: run.stdout.text(), calls: run.calls }; +} + +describe('main', () => { + it('exits 0 and prints PASS when every mutant is killed or justified', async () => { + const run = await runMain({ raw: report(['Killed', 'Killed', 'Ignored']) }); + assert.equal(run.code, 0); + assert.match(run.stderr, /\n=== Mutation summary ===\n/); + assert.match(run.stderr, /\n {2}summary: .*pkg\/reports\/mutation\/summary\.json\n/); + assert.match( + run.stderr, + /\nGate: PASS {2}• {2}threshold: \d+(\.\d+)?% {2}• {2}unjustified survivors: 0 {2}• {2}ignored \(justified\): 1\n$/, + ); + }); + + it('exits 1 and prints FAIL when a mutant survives', async () => { + const run = await runMain({ + raw: report(['Killed', 'Killed', 'Killed', 'Killed', 'Survived']), + }); + assert.equal(run.code, 1); + assert.match(run.stderr, /\nGate: FAIL {2}• .*unjustified survivors: 1 /); + }); + + // The mutants a partial run never tested can be survivors. + it('exits 1 and says so when Stryker wrote a report and then failed', async () => { + const run = await runMain({ raw: report(['Killed']), exitCode: 1 }); + assert.equal(run.code, 1); + assert.match(run.stderr, /\nGate: FAIL \(partial\) {2}• /); + }); + + // The playwright package has only a `test:unit` script. + it('scores a package without a vitest test script through its test command', async () => { + writeFileSync(path.join(pkgRoot, 'package.json'), '{ "name": "pkg", "scripts": {} }'); + const run = await runMain({ + raw: report(['Killed']), + argv: ['src/a.ts', '--package-dir', pkgRoot, '--test-command', 'pnpm exec vitest run'], + }); + assert.equal(run.code, 0); + assert.match(run.stderr, /runner: command, /); + assert.equal(run.calls.length, 1); + }); + + it('prints the usage on --help and starts nothing', async () => { + const run = await runMain({ argv: ['--help'] }); + assert.equal(run.code, 0); + assert.equal(run.stdout, `${USAGE}\n`); + assert.deepEqual(run.calls, []); + }); + + it('throws a usage error with the usage text for a bad command line', async () => { + await assert.rejects( + runMain({ argv: [] }), + (error) => + error instanceof MutateError && + error.exitCode === 2 && + error.showUsage && + error.message === 'Missing mutate target.', + ); + }); + + it('refuses a cli target without test files before Stryker starts', async () => { + writeTree(tempRepo(), SAMPLE_REPO); + const target = 'packages/cli/src/credentials/external-secrets.utils.ts'; + await assert.rejects( + runMain({ argv: [target], repoRoot: tempRepo() }), + (error) => + error.exitCode === 2 && /^Mutating packages\/cli needs --test-files\./.test(error.message), + ); + }); +}); + +// A broken toolchain must never read as a score of zero. +describe('main toolchain errors', () => { + it('exits 3 when Stryker wrote no report', async () => { + const run = await runMain({ raw: null, exitCode: 1 }); + assert.equal(run.code, 3); + assert.match( + run.stderr, + /\nGate: ERROR — at least one Stryker run produced no valid report\.\n$/, + ); + }); + + // The plain vitest runner under Vitest 5 ran no test for any mutant, and + // every mutant survived. That must read as a broken toolchain, not a score. + it('exits 3 and writes no summary when covered mutants survived with no test run', async () => { + const notRun = { coveredBy: ['t1'], testsCompleted: 0 }; + const run = await runMain({ raw: report(['Survived', 'Survived'], notRun) }); + assert.equal(run.code, 3); + assert.match( + run.stderr, + /\n✗ .*pkg: 2 mutant\(s\) survived with no test run, but tests cover them:\n {3}- src\/a\.ts:1:17\n/, + ); + assert.match( + run.stderr, + /\nGate: ERROR — at least one Stryker run produced no valid report\.\n$/, + ); + assert.equal(existsSync(path.join(pkgRoot, 'reports/mutation/summary.json')), false); + }); +}); + +describe('main --diff', () => { + // A git stand-in for a branch that added lines 2-4 to each file in `names`. + const gitChanged = (names) => (args) => { + if (args[0] === 'merge-base') return { status: 0, stdout: 'abc123\n', stderr: '' }; + if (args.includes('-U0')) return { status: 0, stdout: '@@ -1,0 +2,3 @@\n+x\n', stderr: '' }; + return { status: 0, stdout: `${names.join('\n')}\n`, stderr: '' }; + }; + + it('exits 0 and says so when the branch changed no mutable source', async () => { + const run = await runMain({ argv: ['--diff'], git: gitChanged(['README.md']) }); + assert.equal(run.code, 0); + assert.equal(run.stderr, '\nNothing mutable changed vs origin/master.\n'); + assert.deepEqual(run.calls, []); + }); + + it('names each changed file that it skips, and starts no run', async () => { + writeTree(tempRepo(), SAMPLE_REPO); + const blocked = 'packages/@n8n/expression-runtime/src/index.ts'; + const run = await runMain({ + argv: ['--diff', '--base', 'upstream/master'], + git: gitChanged([blocked]), + repoRoot: tempRepo(), + }); + assert.equal(run.code, 0); + assert.match(run.stderr, /^ {2}skipped packages\/@n8n\/expression-runtime\/src\/index\.ts — /); + assert.match(run.stderr, /\nNothing mutable changed vs upstream\/master\.\n$/); + assert.deepEqual(run.calls, []); + }); + + // --test-files does not work with --diff, so the message must not ask for it. + // Every scope check runs before the first run: the instance-ai job, planned + // first, must not start either. + it('refuses a changed cli source without a changed cli test before any run starts', async () => { + writeTree(tempRepo(), SAMPLE_REPO); + const sources = [ + 'packages/@n8n/instance-ai/src/utils/model-config-id.ts', + 'packages/cli/src/credentials/external-secrets.utils.ts', + ]; + const run = startMain({ + raw: report(['Killed']), + argv: ['--diff'], + git: gitChanged(sources), + repoRoot: tempRepo(), + }); + await assert.rejects( + run.promise, + (error) => + error instanceof MutateError && + error.exitCode === 2 && + /^Mutating packages\/cli with --diff needs at least one changed test file\.\n/.test( + error.message, + ), + ); + assert.deepEqual(run.calls, []); + assert.equal(existsSync(path.join(tempRepo(), '.stryker-tmp')), false); + }); + + // The plan, the run and the sandbox mirror all use the repo that main gets. + it('runs the changed lines of each package of the repo it is given', async () => { + writeTree(tempRepo(), SAMPLE_REPO); + const source = 'packages/@n8n/instance-ai/src/utils/model-config-id.ts'; + const run = await runMain({ + raw: report(['Killed']), + argv: ['--diff'], + git: gitChanged([source]), + repoRoot: tempRepo(), + }); + assert.equal(run.code, 0); + assert.equal(run.calls.length, 1); + const pkgDir = path.join(tempRepo(), 'packages/@n8n/instance-ai'); + assert.equal(run.calls[0].options.cwd, pkgDir); + const config = JSON.parse( + readFileSync(path.join(pkgDir, 'reports/mutation/stryker.run.json'), 'utf8'), + ); + assert.deepEqual(config.mutate, [`${path.join('src', 'utils', 'model-config-id.ts')}:2-4`]); + const mirrorParent = path.join(tempRepo(), '.stryker-tmp'); + assert.match(path.relative(mirrorParent, config.tempDirName), /^mirror-[^/]+\/packages\/@n8n$/); + assert.equal(existsSync(mirrorParent), false); + assert.match( + run.stderr, + /\n {2}summary: packages\/@n8n\/instance-ai\/reports\/mutation\/summary\.json\n/, + ); + }); +}); + +describe('reportResults', () => { + const passed = { + packageDir: 'packages/a', + summaryPath: 'packages/a/reports/mutation/summary.json', + summary: buildSummary(report(['Killed']), { + threshold: 80, + target: 'src/a.ts', + generatedAt: '2026-01-01T00:00:00.000Z', + testRunner: 'vitest-compat', + }), + }; + + function gate(results) { + const lines = []; + const code = reportResults(results, (line) => lines.push(line)); + return { code, text: lines.join('\n') }; + } + + it('prints each run and passes when every run passed', () => { + const { code, text } = gate([passed]); + assert.equal(code, 0); + assert.match(text, /\n {2}summary: packages\/a\/reports\/mutation\/summary\.json\n/); + assert.match(text, /\nGate: PASS /); + }); + + // One run without a report makes the whole result a broken toolchain. + it('exits 3 when any run produced no valid report, even when another run passed', () => { + const { code, text } = gate([{ packageDir: 'packages/b', failed: true }, passed]); + assert.equal(code, 3); + assert.match(text, /summary: packages\/a\/reports\/mutation\/summary\.json/); + assert.match(text, /\nGate: ERROR — at least one Stryker run produced no valid report\.$/); + }); +}); diff --git a/scripts/mutation-health/mutate.mjs b/scripts/mutation-health/mutate.mjs index 08eae5b81ec..665be037055 100644 --- a/scripts/mutation-health/mutate.mjs +++ b/scripts/mutation-health/mutate.mjs @@ -4,7 +4,7 @@ * script works for any package. Run it as `pnpm mutate` from the repo root. * * pnpm mutate [:-] [--package-dir ] [--config ] - * [--test-files [,…]] + * [--test-files [,…]] [--test-command ] * pnpm mutate --diff [--base ] [--config ] * * Use `--diff` before you merge. It reads the changed line ranges from @@ -13,30 +13,36 @@ * per package. This makes the gate apply to the patch: it scores the lines you * changed, not the debt you inherited. * - * Use `--test-files` to name the test files that must kill the mutants. The - * value goes to Stryker's `testFiles` config field, thus Stryker runs those - * files instead of the whole related-test graph. The flag repeats and it also - * takes a comma-separated list. Named `packages/cli` targets require it; - * `--diff` uses the CLI test files changed in the patch. + * Use `--test-files` to name the test files that must kill the mutants. Stryker + * then runs those files instead of the whole related-test graph. The flag + * repeats and it also takes a comma-separated list. Named `packages/cli` + * targets require it; `--diff` uses the CLI test files changed in the patch. + * + * Use `--test-command` for a package whose `test` script does not run vitest. + * Stryker's command runner then runs the command, plus the test files, once + * per mutant. See stryker.mjs for the two runners and their cost. + * + * Every run uses Stryker's sandbox: Stryker mutates a copy of the package + * under `.stryker-tmp/`, never the working tree, so nothing is restored after + * a run and other work in the tree is safe while a run is in flight. The + * sandbox sits in a mirror of the repo (see sandbox-mirror.mjs). The tool + * removes the mirror after each run, also when a signal or a crash stops it. * * Stryker config resolution (first match wins): - * 1. --config explicit override - * 2. /stryker.config.mjs package-local (e.g. workflow's vm carve-out) - * 3. scripts/mutation-health/stryker.cli.mjs packages/cli only (related-test - * discovery off — see stryker.cli.mjs) - * 4. scripts/mutation-health/stryker.default.mjs shared default (points at the - * package's own vitest.config.* — no - * bespoke vitest config required) + * 1. --config explicit override + * 2. /stryker.config.mjs package-local (e.g. workflow's vm carve-out) + * 3. stryker.cli.mjs packages/cli only (related-test discovery off) + * 4. stryker.default.mjs shared default (the package's own vitest config) * * Outputs (under /reports/mutation/): + * stryker.run.json — the exact config this run gave Stryker, to read. It can + * name a sandbox mirror that is gone after the run, so + * repeat a run with this script, not with `stryker run`. * raw.json — full Stryker Mutation Testing Elements report - * summary.json — compact actionable summary (this script). Emitted even on a - * non-zero / timed-out Stryker exit, as long as Stryker wrote - * a (partial) raw.json — a run that can't finish still surfaces - * the survivors it found rather than dying with nothing. - * - * Each summary file row also carries a `coverage` fraction in [0,1] — the share - * of mutants a test actually exercised. + * summary.json — compact actionable summary (this script), with a `coverage` + * fraction in [0,1] for each file. Also written when Stryker + * fails after a (partial) raw.json, so a run that cannot finish + * still shows the survivors it found. * * Gate semantics: * A run passes only when the score meets `STRYKER_THRESHOLD` and no mutant @@ -50,928 +56,59 @@ * 0 — the gate passed. * 1 — the gate failed. Iterate: read summary.json and strengthen the tests. * 2 — usage or config error. - * 3 — Stryker did not resolve or did not run. Never 1: a broken toolchain - * must stay distinct from a score of zero. + * 3 — Stryker did not resolve or did not run, or its test runner ran no + * test for a mutant that tests cover. Never 1: a broken toolchain must + * stay distinct from a score of zero. + * 130 / 143 — the run was cancelled with SIGINT / SIGTERM. */ -import { spawn, spawnSync } from 'node:child_process'; -import { readFile, writeFile, mkdir, rm } from 'node:fs/promises'; -import { existsSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from 'node:fs'; -import { createRequire } from 'node:module'; -import { fileURLToPath } from 'node:url'; +import { existsSync } from 'node:fs'; +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; +import { pathToFileURL } from 'node:url'; -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.resolve(__dirname, '../..'); +import { diffPlanLines, planFromDiff, planFromTarget } from './plan.mjs'; +import { createSandboxMirror } from './sandbox-mirror.mjs'; +import { + RAW_REPORT, + buildStrykerConfig, + exitCodeForSignal, + guardOutsideStryker, + loadBaseConfig, + resolveConfig, + resolveStrykerBin, + runStryker, + sandboxConfigError, + strykerRunArgv, + writeRunConfig, +} from './stryker.mjs'; +import { + buildNoTestsSummary, + buildSummary, + classifyRun, + mutantCountFromOutput, + overallGate, + reportLines, + testsNotRunLines, +} from './summary.mjs'; +import { + CLI_PACKAGE_DIR, + MutateError, + cliScopeError, + formatMutateArg, + parseTestFiles, + repoRoot, + toPackageRelative, +} from './targets.mjs'; const THRESHOLD = Number(process.env.STRYKER_THRESHOLD ?? 80); -function die(code, msg) { - process.stderr.write(`${msg}\n`); - process.exit(code); -} +// Where the sandbox mirrors of a repo go. git ignores `.stryker-tmp/`. +const mirrorParentFor = (root) => path.join(root, '.stryker-tmp'); -// --- pure helpers (exported for the unit tests; no I/O, no process state) --- - -function emptyCounts() { - return { - killed: 0, - survived: 0, - noCoverage: 0, - timeout: 0, - compileError: 0, - runtimeError: 0, - ignored: 0, - }; -} - -export function sliceFromLocation(source, loc) { - const lines = source.split('\n'); - const { start, end } = loc; - if (start.line === end.line) { - return lines[start.line - 1].slice(start.column, end.column); - } - return [ - lines[start.line - 1].slice(start.column), - ...lines.slice(start.line, end.line - 1), - lines[end.line - 1].slice(0, end.column), - ].join('\n'); -} - -export function scoreFromCounts(c) { - const detected = c.killed + c.timeout; - const valid = c.killed + c.timeout + c.survived + c.noCoverage; - return valid === 0 ? 0 : +((detected / valid) * 100).toFixed(2); -} - -/** - * Per-file line-coverage proxy distilled from the mutant census: the fraction - * of mutants a test actually exercised (anything that ran) over those that - * could be covered (ran + no-coverage). Ignored and compile-error mutants - * never ran for reasons unrelated to coverage, so they sit outside the ratio. - * - * Returns a fraction in [0,1]. The result is clamped. - */ -export function coverageFromCounts(c) { - const covered = (c.killed ?? 0) + (c.survived ?? 0) + (c.timeout ?? 0) + (c.runtimeError ?? 0); - const total = covered + (c.noCoverage ?? 0); - if (total === 0) return 0; - return +Math.min(1, Math.max(0, covered / total)).toFixed(4); -} - -/** - * What a finished Stryker run produced. `hasReport` must describe THIS run: - * the caller deletes the previous reports first, because a file left by an - * earlier run makes a crashed run look complete and report the earlier target. - * - * complete — the run finished and wrote a report. - * partial — the run wrote a report, then exited non-zero. Untested mutants - * can still be survivors, so this never passes the gate. - * no-tests — no test covers the target. A result, not an error: the score is - * zero and every mutant has no coverage. See DEVP-414. - * failed — no report. The caller reports a toolchain failure. - */ -export function classifyRun({ exitCode, output, hasReport }) { - if (!hasReport) { - return /no tests were executed/i.test(output) ? 'no-tests' : 'failed'; - } - return exitCode === 0 ? 'complete' : 'partial'; -} - -// A run is only "passing" when the score meets the floor AND every unkilled -// mutant has been explicitly justified (Ignored via a Stryker disable -// comment). Any Survived/NoCoverage mutant is unjustified by definition. -export function gatePassed(score, counts, threshold) { - return score >= threshold && counts.survived === 0 && counts.noCoverage === 0; -} - -/** - * Build the compact summary from a raw Stryker Mutation Testing Elements - * report. Pure: takes the parsed report plus run metadata, returns the summary - * object written to summary.json. - */ -export function buildSummary(raw, { threshold, target, generatedAt }) { - // test-id → test-name lookup so survivors can name the tests that covered - // the mutated line without killing the mutant. - const testIdToName = {}; - for (const info of Object.values(raw.testFiles ?? {})) { - for (const t of info.tests ?? []) { - testIdToName[t.id] = t.name; - } - } - - const filesSummary = []; - for (const [file, info] of Object.entries(raw.files)) { - const counts = emptyCounts(); - const survivors = []; - const ignored = []; - for (const m of info.mutants) { - switch (m.status) { - case 'Killed': - counts.killed++; - break; - case 'Survived': - counts.survived++; - break; - case 'NoCoverage': - counts.noCoverage++; - break; - case 'Timeout': - counts.timeout++; - break; - case 'CompileError': - counts.compileError++; - break; - case 'RuntimeError': - counts.runtimeError++; - break; - case 'Ignored': - counts.ignored++; - break; - } - if (m.status === 'Survived' || m.status === 'NoCoverage') { - survivors.push({ - id: m.id, - mutator: m.mutatorName, - status: m.status, - location: `${file}:${m.location.start.line}:${m.location.start.column}`, - line: m.location.start.line, - original: sliceFromLocation(info.source, m.location), - replacement: m.replacement, - coveringTests: (m.coveredBy ?? []).map((id) => testIdToName[id] ?? id), - }); - } - if (m.status === 'Ignored') { - ignored.push({ - id: m.id, - mutator: m.mutatorName, - location: `${file}:${m.location.start.line}:${m.location.start.column}`, - line: m.location.start.line, - reason: m.statusReason ?? '', - }); - } - } - survivors.sort((a, b) => a.line - b.line); - ignored.sort((a, b) => a.line - b.line); - const score = scoreFromCounts(counts); - filesSummary.push({ - file, - score, - coverage: coverageFromCounts(counts), - thresholdMet: gatePassed(score, counts, threshold), - counts, - survivors, - ignored, - }); - } - - const overallCounts = filesSummary.reduce((acc, f) => { - for (const k of Object.keys(acc)) acc[k] += f.counts[k]; - return acc; - }, emptyCounts()); - - const overallScore = scoreFromCounts(overallCounts); - return { - generatedAt, - threshold, - target, - overall: { - score: overallScore, - coverage: coverageFromCounts(overallCounts), - counts: overallCounts, - thresholdMet: gatePassed(overallScore, overallCounts, threshold), - }, - files: filesSummary, - }; -} - -/** - * Synthesise a score-0 red summary for the "No tests were executed" case — - * every mutant is no-coverage, so coverage is 0 too. See DEVP-414. - */ -export function buildNoTestsSummary({ threshold, target, noCoverage, generatedAt }) { - const counts = { ...emptyCounts(), noCoverage }; - const coverage = coverageFromCounts(counts); - return { - generatedAt, - threshold, - target, - overall: { score: 0, coverage, counts, thresholdMet: false }, - files: [ - { - file: target, - score: 0, - coverage, - thresholdMet: false, - counts, - survivors: [], - ignored: [], - }, - ], - }; -} - -// Walk up from a path to the nearest enclosing package.json (bounded by repoRoot). -function findPackageRoot(fromAbs) { - let dir = path.dirname(fromAbs); - while (dir === repoRoot || dir.startsWith(`${repoRoot}${path.sep}`)) { - if (existsSync(path.join(dir, 'package.json'))) return dir; - const parent = path.dirname(dir); - if (parent === dir) break; - dir = parent; - } - return null; -} - -// --- diff-mode planning (pure helpers exported for the unit tests) --- - -// Stryker's dry run stops with SIGABRT on the isolated-vm engine. See DEVP-257. -const BLOCKED_PACKAGES = new Set(['@n8n/expression-runtime']); - -const NON_SOURCE = [ - /\.d\.ts$/, - /\.(test|spec)\.[cm]?tsx?$/, - /(^|\/)__(tests|mocks)__\//, - /\.stories\.[cm]?tsx?$/, - /\.config\.[cm]?[jt]s$/, - /(^|\/)(dist|node_modules|coverage)\//, - /(^|\/)tests?\//, - /(^|\/)migrations\//, -]; - -// This is an exclusion list, not a `src/`-only allowlist. nodes-base and -// nodes-langchain keep their code in `nodes/` and `credentials/`. An allowlist -// drops the largest mutable surface in the repo. -export function isMutableSource(repoRelPath) { - if (!/\.[cm]?tsx?$/.test(repoRelPath)) return false; - return !NON_SOURCE.some((re) => re.test(repoRelPath)); -} - -export function changedTestFilesForPackage(changedFiles, packageDir) { - const prefix = `${toPosix(packageDir).replace(/\/$/, '')}/`; - return changedFiles - .map((file) => toPosix(file).replace(/^\.\//, '')) - .filter((file) => { - if (!file.startsWith(prefix)) return false; - const relative = file.slice(prefix.length); - return ( - (relative.startsWith('src/') || relative.startsWith('test/unit/')) && - /\.(test|spec)\.ts$/.test(relative) && - !/\.integration\.test\.ts$/.test(relative) - ); - }); -} - -// Merge overlapping and adjacent ranges. Stryker then gets one span per region. -export function mergeRanges(ranges) { - const out = []; - for (const r of [...ranges].sort((a, b) => a.start - b.start)) { - const last = out.at(-1); - if (last && r.start <= last.end + 1) last.end = Math.max(last.end, r.end); - else out.push({ ...r }); - } - return out; -} - -// Read the new-side line ranges from `git diff -U0` hunk headers. -// `@@ -12,0 +13,4 @@` gives `{ start: 13, end: 16 }`. A new-side count of zero -// is a deletion. No code stays there to mutate, so this drops it. -export function parseHunkRanges(diffText) { - const ranges = []; - for (const line of diffText.split('\n')) { - const m = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/.exec(line); - if (!m) continue; - const start = Number(m[1]); - const count = m[2] === undefined ? 1 : Number(m[2]); - if (count === 0) continue; - ranges.push({ start, end: start + count - 1 }); - } - return mergeRanges(ranges); -} - -// Join the targets with commas. Stryker keeps only the last `--mutate` flag, -// thus repeated flags mutate one target and the package pays for a dry run -// again for each file. -export function formatMutateArg(targets) { - return targets.join(','); -} - -export function splitRange(target) { - const m = /^(.*):(\d+)-(\d+)$/.exec(target); - return m ? { file: m[1], range: `${m[2]}-${m[3]}` } : { file: target, range: null }; -} - -// --- explicit test-file scoping (pure helpers exported for the unit tests) --- - -// `packages/cli` forks a process per test file and runs a global setup, thus -// Stryker's related-test discovery pulls in hundreds of files and no run -// finishes inside the timeout. An explicit `--test-files` list is required -// there, and the shared default config is swapped for the cli one. -export const CLI_PACKAGE_DIR = 'packages/cli'; - -// Path comparisons in this file are posix-shaped. `path.relative` gives -// backslashes on Windows, so normalise before matching a package dir. -function toPosix(p) { - return p.split(path.sep).join('/'); -} - -/** - * Flatten the raw `--test-files` values into one list. The flag repeats and - * each value also takes a comma-separated list, thus `--test-files a,b` and - * `--test-files a --test-files b` mean the same thing. Blanks are dropped and - * duplicates collapse, so a repeated path never runs its file twice. - */ -export function parseTestFiles(values) { - const out = []; - for (const value of values) { - for (const part of String(value).split(',')) { - const file = part.trim(); - if (file && !out.includes(file)) out.push(file); - } - } - return out; -} - -/** - * Stryker matches `testFiles` patterns against the files under its run cwd, - * which is the package dir. A repo-relative path (what the user types, and - * what `--diff` prints) therefore has to lose its package prefix. A path that - * is already package-relative, and a glob, both pass through untouched. - */ -export function toPackageRelative(file, packageDir) { - const normalised = toPosix(file).replace(/^\.\//, ''); - const prefix = `${toPosix(packageDir)}/`; - return normalised.startsWith(prefix) ? normalised.slice(prefix.length) : normalised; -} - -/** - * The Stryker config fields this run overrides on top of the resolved config - * file. Stryker merges its CLI options into the config object it runs with, - * thus every field here lands in that object. - */ -export function buildStrykerConfig({ targets, testFiles = [] }) { - const config = { mutate: [...targets] }; - if (testFiles.length > 0) config.testFiles = [...testFiles]; - return config; -} - -// Turn the config overrides into Stryker CLI flags. Stryker keeps only the last -// occurrence of a repeated flag, thus every list field goes over comma-joined. -export function strykerCliArgs(config) { - return Object.entries(config).flatMap(([field, value]) => [ - `--${field}`, - Array.isArray(value) ? value.join(',') : String(value), - ]); -} - -// Why a cli target may not run, or null when it may. Named so the message says -// what to do, not only what went wrong. -export function cliScopeError(packageDir, testFiles, diffMode = false) { - if (toPosix(packageDir) !== CLI_PACKAGE_DIR) return null; - if (testFiles.length > 0) return null; - if (diffMode) { - return ( - `Mutating ${CLI_PACKAGE_DIR} with --diff needs at least one changed test file.\n` + - 'Add or update a test that covers the changed source, or run a named target with --test-files.' - ); - } - return ( - `Mutating ${CLI_PACKAGE_DIR} needs --test-files.\n` + - 'Without it Stryker discovers every related test file in the package, forks a ' + - 'process for each one and never finishes. Name the test files that cover the target:\n' + - ' pnpm mutate packages/cli/src/foo.ts:10-40 --test-files packages/cli/src/__tests__/foo.test.ts' - ); -} - -// Which shared config a package gets when it ships none of its own. -export function defaultConfigNameFor(packageDir) { - return toPosix(packageDir) === CLI_PACKAGE_DIR ? 'stryker.cli.mjs' : 'stryker.default.mjs'; -} - -function git(args) { - const res = spawnSync('git', args, { - cwd: repoRoot, - encoding: 'utf8', - maxBuffer: 64 * 1024 * 1024, - }); - if (res.error) die(2, `git ${args[0]} failed to start: ${res.error.message}`); - return res; -} - -function packageNameOf(pkgRoot) { - try { - return JSON.parse(readFileSync(path.join(pkgRoot, 'package.json'), 'utf8')).name ?? ''; - } catch { - return ''; - } -} - -// Stryker runs the package's own vitest. If the `test` script runs something -// else, the package is skipped, not failed. -function packageUsesVitest(pkgRoot) { - try { - const pkg = JSON.parse(readFileSync(path.join(pkgRoot, 'package.json'), 'utf8')); - return /\bvitest\b/.test(pkg.scripts?.test ?? ''); - } catch { - return false; - } -} - -// Why a package cannot be scored, or null when it can. Both planners call this, -// so a named target and a --diff target always get the same answer. -function ineligibleReason(pkgRoot) { - const pkgName = packageNameOf(pkgRoot) || path.relative(repoRoot, pkgRoot); - if (BLOCKED_PACKAGES.has(packageNameOf(pkgRoot))) { - return `${pkgName} is blocked: the isolated-vm engine crashes Stryker's dry run (DEVP-257)`; - } - if (!packageUsesVitest(pkgRoot)) return `${pkgName} is not a vitest package`; - return null; -} - -function planFromDiff(base) { - // Diff the merge base against the working tree, not against HEAD. This also - // scores uncommitted edits. On a PR checkout the tree is clean, thus the - // result is the same as the branch diff. - const mergeBase = git(['merge-base', base, 'HEAD']); - if (mergeBase.status !== 0) { - die(2, `No merge base with '${base}' — is the ref fetched?\n${mergeBase.stderr.trim()}`); - } - const from = mergeBase.stdout.trim(); - - const names = git(['diff', '--name-only', from]); - if (names.status !== 0) { - die(2, `git diff against '${base}' failed.\n${names.stderr.trim()}`); - } - - const changedFiles = names.stdout - .split('\n') - .map((s) => s.trim()) - .filter(Boolean); - const byPackage = new Map(); - const skipped = []; - for (const file of changedFiles) { - if (!isMutableSource(file)) continue; - const abs = path.resolve(repoRoot, file); - if (!existsSync(abs)) continue; // the branch deleted the file - - const pkgRoot = findPackageRoot(abs); - if (!pkgRoot) { - skipped.push([file, 'no enclosing package']); - continue; - } - const reason = ineligibleReason(pkgRoot); - if (reason) { - skipped.push([file, reason]); - continue; - } - - const ranges = parseHunkRanges(git(['diff', '-U0', from, '--', file]).stdout); - if (ranges.length === 0) continue; - - const rel = path.relative(pkgRoot, abs); - const packageDir = path.relative(repoRoot, pkgRoot); - const job = byPackage.get(pkgRoot) ?? { pkgRoot, packageDir, targets: [] }; - for (const r of ranges) job.targets.push(`${rel}:${r.start}-${r.end}`); - byPackage.set(pkgRoot, job); - } - for (const job of byPackage.values()) { - if (toPosix(job.packageDir) === CLI_PACKAGE_DIR) { - job.testFiles = changedTestFilesForPackage(changedFiles, job.packageDir).filter((file) => - existsSync(path.resolve(repoRoot, file)), - ); - } - } - return { jobs: [...byPackage.values()], skipped }; -} - -// --- running --- - -function resolveConfig(pkgRoot, configArg) { - if (configArg) return path.resolve(repoRoot, configArg); - const local = path.join(pkgRoot, 'stryker.config.mjs'); - if (existsSync(local)) return local; - return path.join(__dirname, defaultConfigNameFor(path.relative(repoRoot, pkgRoot))); -} - -// Try the package's own copy first, then the root devDep. A package that pins -// Stryker thus gets the version it pinned. A miss is a broken checkout, not a -// red gate: exit 3 keeps it distinct from a score of zero. -function resolveStrykerBin(pkgRoot, packageDir) { - for (const from of [path.join(pkgRoot, 'package.json'), import.meta.url]) { - try { - const resolved = createRequire(from).resolve('@stryker-mutator/core/package.json'); - return path.join(path.dirname(resolved), 'bin/stryker.js'); - } catch { - continue; - } - } - return die( - 3, - `Could not resolve @stryker-mutator/core from ${packageDir} or the repo root. ` + - 'Run `pnpm install` — it is a root devDep.', - ); -} - -// --- working-tree snapshot and cleanup --- -// -// Runs use `--inPlace`, because the sandbox copy breaks each package whose -// vitest config finds a workspace dependency through a path alias. The alias -// does not stay correct in the copy. See the README for the failure. -// -// Stryker restores the files after a usual exit and after SIGINT, but not after -// a crash, a timeout, or a SIGTERM. Its preprocessing also reaches past the -// mutate targets: it instruments the related test graph and drops a -// `stryker-setup-.js` in the run cwd. A target-only snapshot therefore -// left mutated files and setup files behind. Snapshot the whole working tree -// instead, and run one cleanup routine on every exit path. - -// Glob `stryker-setup-*.js`, anchored to one path segment. -const STRYKER_SETUP_FILE = /^stryker-setup-[^/\\]*\.js$/; - -// Directories the walk never enters. `node_modules` alone makes a full-repo -// walk take minutes, and none of these hold working-tree state to clean. -const UNWALKED_DIRS = new Set([ - '.git', - 'node_modules', - 'dist', - 'coverage', - '.turbo', - '.stryker-tmp', -]); - -/** - * Every `stryker-setup-*.js` under `root`. The vitest runner writes one for - * each worker into the run cwd and does not always remove it, thus the search - * covers the whole repo, not only the package dir Stryker ran in. - */ -export function findStrykerSetupFiles(root) { - const found = []; - const queue = [root]; - while (queue.length > 0) { - const dir = queue.pop(); - let entries; - try { - entries = readdirSync(dir, { withFileTypes: true }); - } catch { - continue; // unreadable directory — nothing to collect - } - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - if (!UNWALKED_DIRS.has(entry.name)) queue.push(full); - } else if (STRYKER_SETUP_FILE.test(entry.name)) { - found.push(full); - } - } - } - return found; -} - -/** Delete every `stryker-setup-*.js` under `root`. Returns the paths removed. */ -export function removeStrykerSetupFiles(root) { - const removed = []; - for (const file of findStrykerSetupFiles(root)) { - try { - unlinkSync(file); - removed.push(file); - } catch { - // Already gone. There is nothing to remove. - } - } - return removed; -} - -/** Keep the exact bytes of each path, so a restore never needs git. */ -export function snapshotFiles(absPaths) { - const snap = new Map(); - for (const p of absPaths) { - try { - snap.set(p, readFileSync(p)); - } catch { - // The file is unreadable. There is nothing to restore. - } - } - return snap; -} - -/** Write the snapshot back where it differs. Returns the paths restored. */ -export function restoreFiles(snap) { - const restored = []; - for (const [p, original] of snap) { - try { - if (!readFileSync(p).equals(original)) { - writeFileSync(p, original); - restored.push(p); - } - } catch { - // The file is gone. There is nothing to restore. - } - } - return restored; -} - -/** - * Tracked files that were clean before the run and are dirty after it. Stryker - * changed them, and git holds their pre-run state, thus `git checkout --` is - * the correct undo. A file that was already dirty is excluded: it carries the - * user's own uncommitted work, which the byte snapshot restores instead. - */ -export function filesToCheckout(dirtyBefore, dirtyAfter) { - const before = new Set(dirtyBefore); - return dirtyAfter.filter((f) => !before.has(f)); -} - -/** - * The one cleanup routine every exit path shares: restore the working tree, - * then delete the `stryker-setup-*.js` files. Idempotent — the normal path, a - * crash, SIGINT and SIGTERM all call it, and only the first call does the work. - * A failing restore must not keep the setup files on disk, so each half is - * guarded on its own. - */ -export function createCleanup({ restore, removeSetupFiles, report }) { - let result; - return function cleanup() { - if (result) return result; - result = { restored: [], removed: [] }; - try { - result.restored = restore() ?? []; - } catch { - // Report what the second half did even when the restore fails. - } - try { - result.removed = removeSetupFiles() ?? []; - } catch { - // Nothing more to clean. - } - report?.(result); - return result; - }; -} - -/** - * Wire `cleanup` onto every exit path: the usual one (`exit`), a crash - * (`uncaughtException`), and a cancellation (`SIGINT` / `SIGTERM`). - * - * `onSignal` gets first refusal on a signal. It returns true when it told a - * live Stryker to stop; the run then unwinds and cleans up once the child is - * gone, because a restore that races Stryker's own writes fixes nothing. - * Otherwise this handler cleans up and leaves at once. - * - * `proc`, `exit` and `write` are injected so the unit tests can drive the - * handlers without signalling or ending the test runner. - */ -export function registerCleanupHandlers({ - cleanup, - onSignal, - proc = process, - exit = (code) => process.exit(code), - write = (msg) => process.stderr.write(msg), -}) { - const handleExit = () => cleanup(); - const handleUncaught = (err) => { - cleanup(); - write(`\n✗ mutate.mjs crashed: ${err?.stack ?? err}\n`); - exit(3); - }; - const handleSignal = (signal) => { - if (onSignal?.(signal)) return; - cleanup(); - exit(signal === 'SIGTERM' ? 143 : 130); - }; - const handleSigint = () => handleSignal('SIGINT'); - const handleSigterm = () => handleSignal('SIGTERM'); - - proc.on('exit', handleExit); - proc.on('uncaughtException', handleUncaught); - proc.on('SIGINT', handleSigint); - proc.on('SIGTERM', handleSigterm); - - return { - dispose() { - proc.off('exit', handleExit); - proc.off('uncaughtException', handleUncaught); - proc.off('SIGINT', handleSigint); - proc.off('SIGTERM', handleSigterm); - }, - }; -} - -// Tracked files that differ from HEAD, staged or not. Untracked files are left -// out: they have no pre-run state to go back to. This never dies — it runs -// inside the cleanup handlers, where exiting would skip the rest of the work. -function listDirtyFiles() { - const res = spawnSync('git', ['diff', '--name-only', 'HEAD'], { - cwd: repoRoot, - encoding: 'utf8', - maxBuffer: 64 * 1024 * 1024, - }); - if (res.error || res.status !== 0) return []; - return res.stdout - .split('\n') - .map((s) => s.trim()) - .filter(Boolean); -} - -// Bytes for every file that is already dirty plus the ones the run will touch, -// and the dirty set itself so the restore can tell Stryker's edits from the -// user's. -function snapshotWorkingTree(extraAbsPaths = []) { - const dirtyBefore = listDirtyFiles(); - const paths = new Set([...dirtyBefore.map((f) => path.join(repoRoot, f)), ...extraAbsPaths]); - return { bytes: snapshotFiles([...paths]), dirtyBefore }; -} - -function restoreWorkingTree({ bytes, dirtyBefore }) { - const restored = new Set(restoreFiles(bytes).map((p) => path.relative(repoRoot, p))); - const checkout = filesToCheckout(dirtyBefore, listDirtyFiles()); - if (checkout.length > 0) { - const res = spawnSync('git', ['checkout', '--', ...checkout], { - cwd: repoRoot, - encoding: 'utf8', - }); - if (!res.error && res.status === 0) for (const f of checkout) restored.add(f); - } - return [...restored]; -} - -async function runJob( - { pkgRoot, packageDir, targets, testFiles: plannedTestFiles = [] }, - { configArg, testFiles = [] }, -) { - const configPath = resolveConfig(pkgRoot, configArg); - const strykerBin = resolveStrykerBin(pkgRoot, packageDir); - const mutateArg = formatMutateArg(targets); - const selectedTestFiles = testFiles.length > 0 ? testFiles : plannedTestFiles; - const strykerConfig = buildStrykerConfig({ - targets, - testFiles: selectedTestFiles.map((f) => toPackageRelative(f, packageDir)), - }); - - const reportDir = path.join(pkgRoot, 'reports/mutation'); - const rawJsonPath = path.join(reportDir, 'raw.json'); - const summaryJsonPath = path.join(reportDir, 'summary.json'); - await mkdir(reportDir, { recursive: true }); - - // Delete the previous reports first. The code below reads "raw.json exists" - // as "this run wrote a report". A file from an earlier run makes a crashed - // run report the earlier target and its score, and it also hides the - // no-covering-tests result. After this, a report on disk is always this run's. - await Promise.all([rm(rawJsonPath, { force: true }), rm(summaryJsonPath, { force: true })]); - - process.stderr.write( - `\nRunning Stryker on ${packageDir} — ${targets.length} target(s) ` + - `(config: ${path.relative(repoRoot, configPath)}, threshold: ${THRESHOLD}%)\n`, - ); - if (strykerConfig.testFiles) { - process.stderr.write(` testFiles: ${strykerConfig.testFiles.join(', ')}\n`); - } - - const snapshot = snapshotWorkingTree([ - ...new Set(targets.map((t) => path.join(pkgRoot, splitRange(t).file))), - ]); - - const strykerOutputChunks = []; - let child; - let signalled; - const cleanup = createCleanup({ - restore: () => restoreWorkingTree(snapshot), - removeSetupFiles: () => removeStrykerSetupFiles(repoRoot), - report: ({ restored, removed }) => { - if (restored.length > 0) { - process.stderr.write( - `⚠ Stryker left changes behind; restored ${restored.length} file(s): ` + - `${restored.join(', ')}\n`, - ); - } - if (removed.length > 0) { - process.stderr.write( - ` removed ${removed.length} stryker-setup file(s): ` + - `${removed.map((p) => path.relative(repoRoot, p)).join(', ')}\n`, - ); - } - }, - }); - const handlers = registerCleanupHandlers({ - cleanup, - onSignal: (signal) => { - // Pass the signal to Stryker so it unwinds its own state, then let the - // `finally` below clean up once the child is gone. - if (!child || child.exitCode !== null || child.signalCode !== null) return false; - signalled = signal; - child.kill('SIGINT'); - return true; - }, - }); - let strykerExitCode; - try { - strykerExitCode = await new Promise((resolve) => { - child = spawn( - process.execPath, - [strykerBin, 'run', configPath, '--inPlace', ...strykerCliArgs(strykerConfig)], - { cwd: pkgRoot, stdio: ['inherit', 'pipe', 'pipe'] }, - ); - child.stdout.on('data', (chunk) => { - strykerOutputChunks.push(chunk); - process.stdout.write(chunk); - }); - child.stderr.on('data', (chunk) => { - strykerOutputChunks.push(chunk); - process.stderr.write(chunk); - }); - child.on('exit', (code) => resolve(code ?? 1)); - child.on('error', (err) => die(3, `Stryker failed to start: ${err.message}`)); - }); - } finally { - handlers.dispose(); - cleanup(); - // The run was cancelled. The tree is clean again, so leave with the shell's - // code for the signal instead of reporting on a run that never finished. - if (signalled) process.exit(signalled === 'SIGTERM' ? 143 : 130); - } - const strykerOutput = Buffer.concat(strykerOutputChunks).toString('utf8'); - const target = mutateArg; - - const outcome = classifyRun({ - exitCode: strykerExitCode, - output: strykerOutput, - hasReport: existsSync(rawJsonPath), - }); - - if (outcome === 'failed') { - process.stderr.write( - `✗ ${packageDir}: Stryker exited ${strykerExitCode} without producing ` + - `${path.relative(repoRoot, rawJsonPath)}\n`, - ); - return { packageDir, summaryJsonPath, failed: true }; - } - - if (outcome === 'no-tests') { - const mutantMatch = strykerOutput.match( - /Instrumented\s+\d+\s+source file\(s\)\s+with\s+(\d+)\s+mutant/i, - ); - const summary = buildNoTestsSummary({ - threshold: THRESHOLD, - target, - noCoverage: mutantMatch ? Number(mutantMatch[1]) : 0, - generatedAt: new Date().toISOString(), - }); - await writeFile(summaryJsonPath, JSON.stringify(summary, null, 2)); - return { packageDir, summaryJsonPath, summary, noTests: true }; - } - - const raw = JSON.parse(await readFile(rawJsonPath, 'utf8')); - const summary = buildSummary(raw, { - threshold: THRESHOLD, - target, - generatedAt: new Date().toISOString(), - }); - if (outcome === 'partial') summary.partial = true; - - await writeFile(summaryJsonPath, JSON.stringify(summary, null, 2)); - return { packageDir, summaryJsonPath, summary }; -} - -function reportJob({ packageDir, summaryJsonPath, summary, noTests, failed }) { - if (failed) return; - if (noTests) { - process.stderr.write( - `✗ ${packageDir} ${summary.target} 0.00% (no covering tests — recorded as score-0 red)\n`, - ); - process.stderr.write(` summary: ${path.relative(repoRoot, summaryJsonPath)}\n`); - return; - } - if (summary.partial) { - process.stderr.write( - `⚠ ${packageDir}: Stryker exited non-zero; summary built from a partial raw.json — ` + - 'results may be incomplete.\n', - ); - } - for (const f of summary.files) { - const mark = f.thresholdMet ? '✓' : '✗'; - process.stderr.write( - `${mark} ${f.file} ${f.score.toFixed(2)}% cov ${(f.coverage * 100).toFixed(0)}% ` + - `(killed ${f.counts.killed} / survived ${f.counts.survived} / no-cov ${f.counts.noCoverage} / timeout ${f.counts.timeout} / ignored ${f.counts.ignored})\n`, - ); - for (const s of f.survivors) { - process.stderr.write( - ` - ${s.status.toLowerCase().padEnd(10)} ${s.mutator.padEnd(22)} ${s.location}\n`, - ); - } - for (const ig of f.ignored) { - process.stderr.write( - ` · ${'ignored'.padEnd(10)} ${ig.mutator.padEnd(22)} ${ig.location}` + - `${ig.reason ? ` — ${ig.reason}` : ' — (no reason given)'}\n`, - ); - } - } - process.stderr.write(` summary: ${path.relative(repoRoot, summaryJsonPath)}\n`); -} - -// --- entry point --- - -const USAGE = `Usage: +export const USAGE = `Usage: node scripts/mutation-health/mutate.mjs [:-] [--package-dir ] [--config ] - [--test-files [,...]] + [--test-files [,...]] [--test-command ] node scripts/mutation-health/mutate.mjs --diff [--base ] [--config ] Options: @@ -980,9 +117,14 @@ Options: --base Branch point --diff measures against. Default origin/master. --diff Mutate every line this branch changed, one run per package. --test-files Test files Stryker runs, instead of the whole related-test - graph. Goes to Stryker's testFiles config field. Comma- - separated, and the flag repeats. Paths may be repo-relative - or package-relative. Required for ${CLI_PACKAGE_DIR} targets. + graph. Comma-separated, and the flag repeats. Paths may be + repo-relative or package-relative. Required for + ${CLI_PACKAGE_DIR} targets. + --test-command Shell command that runs the package's tests, for a package + whose \`test\` script does not run vitest. Runs in the + package dir once per mutant, with the --test-files paths + added at the end. Much slower than the default runner. + Needs a single target. # one file, whole node scripts/mutation-health/mutate.mjs packages/@n8n/crdt/src/utils.ts @@ -993,50 +135,29 @@ Options: # a cli target, scoped to the tests that must kill its mutants node scripts/mutation-health/mutate.mjs packages/cli/src/credentials/external-secrets.utils.ts:32-68 \\ --test-files packages/cli/src/credentials/__tests__/external-secrets.utils.test.ts + # a package with no vitest \`test\` script + node scripts/mutation-health/mutate.mjs packages/quality/testing/playwright/coverage-options.ts \\ + --test-files coverage-options.test.ts --test-command 'pnpm exec vitest run' # every line this branch changed, batched one Stryker run per package node scripts/mutation-health/mutate.mjs --diff --base origin/master`; -function planFromTarget(targetArg, packageDirArg) { - const { file, range } = splitRange(targetArg); +// Flags that take one value, and the field each value goes to. +const VALUE_FLAGS = new Map([ + ['--package-dir', 'packageDirArg'], + ['--config', 'configArg'], + ['--base', 'baseArg'], + ['--test-command', 'testCommand'], +]); - let pkgRoot; - let rel; - if (packageDirArg) { - pkgRoot = path.resolve(repoRoot, packageDirArg); - if (!existsSync(pkgRoot)) die(2, `Package dir not found: ${pkgRoot}`); - rel = path.isAbsolute(file) ? path.relative(pkgRoot, file) : file; - } else { - const abs = path.resolve(repoRoot, file); - if (!existsSync(abs)) die(2, `Target not found: ${abs}\n${USAGE}`); - pkgRoot = findPackageRoot(abs); - if (!pkgRoot) die(2, `Could not infer the package for ${file} — pass --package-dir.\n${USAGE}`); - rel = path.relative(pkgRoot, abs); - } - - if (rel.startsWith('..') || path.isAbsolute(rel)) { - die(2, `Target must live inside the package. Got: ${rel}`); - } - if (!existsSync(path.join(pkgRoot, rel))) { - die(2, `Target not found: ${path.join(pkgRoot, rel)}`); - } - if (!isMutableSource(rel)) { - die(2, `Not a mutable source file (test/declaration/config/build output): ${rel}`); - } - // --diff skips an ineligible package. A named target must refuse for the same - // reason, or it starts a run that is known to crash. - const reason = ineligibleReason(pkgRoot); - if (reason) die(2, `Cannot mutate ${rel}: ${reason}`); - - return { - pkgRoot, - packageDir: path.relative(repoRoot, pkgRoot), - targets: [range ? `${rel}:${range}` : rel], - }; -} +const SWITCHES = new Map([ + ['--diff', 'diffMode'], + ['--help', 'helpMode'], + ['-h', 'helpMode'], +]); /** * Read the command line. Pure, so the unit tests can assert what each flag - * parses to. Validation stays in `main`, which owns the exit codes. + * parses to. `usageError` checks the result, and `main` owns the exit codes. */ export function parseArgs(argv) { const parsed = { @@ -1046,97 +167,244 @@ export function parseArgs(argv) { baseArg: 'origin/master', diffMode: false, helpMode: false, - testFiles: [], + testCommand: undefined, }; const rawTestFiles = []; for (let i = 0; i < argv.length; i++) { const a = argv[i]; - if (a === '--package-dir') parsed.packageDirArg = argv[++i]; - else if (a === '--config') parsed.configArg = argv[++i]; - else if (a === '--base') parsed.baseArg = argv[++i]; + // A flag with no value after it reads as an empty value. An empty command + // is a usage error, so a run never starts on a command that was cut off. + if (VALUE_FLAGS.has(a)) parsed[VALUE_FLAGS.get(a)] = argv[++i] ?? ''; else if (a === '--test-files') rawTestFiles.push(argv[++i] ?? ''); - else if (a === '--diff') parsed.diffMode = true; - else if (a === '--help' || a === '-h') parsed.helpMode = true; + else if (SWITCHES.has(a)) parsed[SWITCHES.get(a)] = true; else if (!a.startsWith('--') && parsed.targetArg === undefined) parsed.targetArg = a; } - parsed.testFiles = parseTestFiles(rawTestFiles); - return parsed; + return { ...parsed, testFiles: parseTestFiles(rawTestFiles) }; } -async function main() { - const { packageDirArg, configArg, targetArg, baseArg, diffMode, helpMode, testFiles } = parseArgs( - process.argv.slice(2), - ); - - if (helpMode) { - process.stdout.write(`${USAGE}\n`); - process.exit(0); - } - if (diffMode && targetArg) die(2, `--diff takes no positional target.\n${USAGE}`); - if (!diffMode && !targetArg) die(2, `Missing mutate target.\n${USAGE}`); +const USAGE_RULES = [ + [(a) => a.diffMode && a.targetArg, '--diff takes no positional target.'], + [(a) => !a.diffMode && !a.targetArg, 'Missing mutate target.'], // --diff plans a run per package, so one test-file list cannot say which // package it belongs to. Refuse rather than apply it to all of them. - if (diffMode && testFiles.length > 0) { - die(2, `--test-files needs a single target; it does not work with --diff.\n${USAGE}`); - } + [ + (a) => a.diffMode && a.testFiles.length > 0, + '--test-files needs a single target; it does not work with --diff.', + ], + [ + (a) => a.testCommand !== undefined && a.testCommand.trim() === '', + "--test-command needs a command, for example --test-command 'pnpm exec vitest run'.", + ], + // A test command runs one package's tests, for the same reason as above. + [ + (a) => a.diffMode && a.testCommand !== undefined, + '--test-command needs a single target; it does not work with --diff.', + ], +]; - let jobs; - let skipped = []; - if (diffMode) { - ({ jobs, skipped } = planFromDiff(baseArg)); - for (const [file, why] of skipped) process.stderr.write(` skipped ${file} — ${why}\n`); - if (jobs.length === 0) { - process.stderr.write(`\nNothing mutable changed vs ${baseArg}.\n`); - process.exit(0); - } - const files = jobs.reduce((n, j) => n + j.targets.length, 0); - process.stderr.write( - `\nMutating ${files} changed range(s) across ${jobs.length} package(s) vs ${baseArg}.\n`, +/** The first problem with the parsed command line, or null. */ +export function usageError(args) { + return USAGE_RULES.find(([applies]) => applies(args))?.[1] ?? null; +} + +function selectedTestFiles(job, args) { + return args.testFiles.length > 0 ? args.testFiles : (job.testFiles ?? []); +} + +function planJobs(args, { log, git, repoRoot: root }) { + if (!args.diffMode) { + const customTestCommand = args.testCommand !== undefined; + return [ + planFromTarget(args.targetArg, args.packageDirArg, { customTestCommand, repoRoot: root }), + ]; + } + const plan = planFromDiff(args.baseArg, { git, repoRoot: root }); + for (const line of diffPlanLines(plan, args.baseArg)) log(line); + return plan.jobs; +} + +// Check the config and collect what the run needs. Nothing is written yet. +async function prepareRun(job, args, { resolveBin, root }) { + const { pkgRoot, packageDir } = job; + const configPath = resolveConfig(pkgRoot, args.configArg, root); + const configLabel = path.relative(root, configPath); + const base = await loadBaseConfig(configPath); + const configError = sandboxConfigError(base, configLabel); + if (configError) throw new MutateError(2, configError); + const testFiles = selectedTestFiles(job, args).map((f) => toPackageRelative(f, packageDir)); + return { base, configLabel, testFiles, strykerBin: resolveBin(pkgRoot, packageDir) }; +} + +function describeRun({ packageDir, targets }, { config, configLabel }) { + const lines = [ + `\nRunning Stryker on ${packageDir} — ${targets.length} target(s) ` + + `(config: ${configLabel}, runner: ${config.testRunner}, threshold: ${THRESHOLD}%)`, + ]; + if (config.commandRunner) lines.push(` command: ${config.commandRunner.command}`); + if (config.testFiles) lines.push(` testFiles: ${config.testFiles.join(', ')}`); + return lines; +} + +// Why a run gives no valid result, for the two toolchain failures. +function failureLines(outcome, { packageDir, run, rawJsonPath, report, root }) { + if (outcome === 'tests-not-run') return testsNotRunLines(packageDir, report); + return [ + `✗ ${packageDir}: Stryker exited ${run.exitCode} without producing ` + + `${path.relative(root, rawJsonPath)}`, + ]; +} + +async function summariseRun(job, { config, run, rawJsonPath, summaryJsonPath, root }, log) { + const { packageDir, targets } = job; + const report = existsSync(rawJsonPath) ? JSON.parse(await readFile(rawJsonPath, 'utf8')) : null; + const outcome = classifyRun({ exitCode: run.exitCode, output: run.output, report }); + const result = { packageDir, summaryPath: path.relative(root, summaryJsonPath) }; + if (outcome === 'failed' || outcome === 'tests-not-run') { + const failure = { packageDir, run, rawJsonPath, report, root }; + for (const line of failureLines(outcome, failure)) log(line); + return { ...result, failed: true }; + } + const meta = { + threshold: THRESHOLD, + target: formatMutateArg(targets), + generatedAt: new Date().toISOString(), + }; + const summary = + outcome === 'no-tests' + ? buildNoTestsSummary({ ...meta, noCoverage: mutantCountFromOutput(run.output) }) + : buildSummary(report, { ...meta, testRunner: config.testRunner }); + if (outcome === 'partial') summary.partial = true; + await writeFile(summaryJsonPath, JSON.stringify(summary, null, 2)); + return { ...result, summary, noTests: outcome === 'no-tests' }; +} + +// Clear the earlier reports, write the run config and run Stryker to the end. +// runStryker handles signals while Stryker runs, so the guard steps aside. +async function startRun(job, { config, strykerBin, guard }, io) { + const rawJsonPath = path.join(job.pkgRoot, RAW_REPORT); + const reportDir = path.dirname(rawJsonPath); + const summaryJsonPath = path.join(reportDir, 'summary.json'); + await mkdir(reportDir, { recursive: true }); + // Delete the previous reports first. The code below reads "raw.json exists" + // as "this run wrote a report". A file from an earlier run makes a crashed + // run report the earlier target and its score, and it also hides the + // no-covering-tests result. After this, a report on disk is always this run's. + await Promise.all([rm(rawJsonPath, { force: true }), rm(summaryJsonPath, { force: true })]); + const runConfigPath = await writeRunConfig(reportDir, config); + + const argv = strykerRunArgv(strykerBin, runConfigPath); + guard.release(); + const run = await runStryker({ argv, cwd: job.pkgRoot }, io).finally(() => guard.hold()); + if (run.cancelledBy) { + throw new MutateError( + exitCodeForSignal(run.cancelledBy), + `\nRun cancelled by ${run.cancelledBy}.`, ); - } else { - jobs = [planFromTarget(targetArg, packageDirArg)]; + } + return { run, rawJsonPath, summaryJsonPath }; +} + +/** + * Run Stryker for one planned job and summarise the result. `io` holds what the + * run uses outside this file, so the unit tests can run a job without Stryker: + * - `resolveStrykerBin`: finds Stryker's binary (default: resolveStrykerBin). + * - `repoRoot`: the repo that the config lookup, the printed paths and the + * sandbox mirror use (default: this repo). + * - everything else (`spawn`, the output streams, the signal-handling + * stand-ins) goes on to runStryker. + */ +export async function runJob(job, args, io = {}) { + const { + resolveStrykerBin: resolveBin = resolveStrykerBin, + repoRoot: root = repoRoot, + ...runIo + } = io; + const log = (line) => (runIo.stderr ?? process.stderr).write(`${line}\n`); + const prepared = await prepareRun(job, args, { resolveBin, root }); + const mirror = createSandboxMirror(job.pkgRoot, { + repoRoot: root, + parentDir: mirrorParentFor(root), + }); + // An exit on a signal or a crash skips `finally`, so the handlers remove the mirror. + const strykerIo = { ...runIo, onExit: () => mirror?.dispose() }; + const guard = guardOutsideStryker(strykerIo); + try { + const config = buildStrykerConfig({ + base: prepared.base, + targets: job.targets, + testFiles: prepared.testFiles, + testCommand: args.testCommand, + tempDir: mirror?.tempDir, + }); + for (const line of describeRun(job, { config, configLabel: prepared.configLabel })) log(line); + const run = { config, strykerBin: prepared.strykerBin, guard }; + const finished = await startRun(job, run, strykerIo); + return await summariseRun(job, { config, root, ...finished }, log); + } finally { + guard.release(); + mirror?.dispose(); + } +} + +/** Print the result of each run and the gate. Returns the exit code. */ +export function reportResults(results, log) { + log('\n=== Mutation summary ==='); + for (const r of results) for (const line of reportLines(r)) log(line); + + if (results.some((r) => r.failed)) { + log('\nGate: ERROR — at least one Stryker run produced no valid report.'); + return 3; + } + const overall = overallGate(results.map((r) => r.summary)); + const gateState = overall.passed ? 'PASS' : overall.partial ? 'FAIL (partial)' : 'FAIL'; + log( + `\nGate: ${gateState} • threshold: ${THRESHOLD}% • ` + + `unjustified survivors: ${overall.survived} • ignored (justified): ${overall.ignored}`, + ); + return overall.passed ? 0 : 1; +} + +/** + * The whole command. Returns the exit code, or throws a MutateError. `io.git` + * runs git for --diff (default: runGit). `io.repoRoot` is the repo that the + * plan reads (default: this repo). The rest of `io`, with `repoRoot`, goes to + * runJob. + */ +export async function main(argv, { git, ...io } = {}) { + const log = (line) => (io.stderr ?? process.stderr).write(`${line}\n`); + const args = parseArgs(argv); + if (args.helpMode) { + (io.stdout ?? process.stdout).write(`${USAGE}\n`); + return 0; + } + const problem = usageError(args); + if (problem) throw new MutateError(2, problem, { showUsage: true }); + + const jobs = planJobs(args, { log, git, repoRoot: io.repoRoot }); + if (jobs.length === 0) { + log(`\nNothing mutable changed vs ${args.baseArg}.`); + return 0; } for (const job of jobs) { - const selectedTestFiles = testFiles.length > 0 ? testFiles : (job.testFiles ?? []); - const scopeError = cliScopeError(job.packageDir, selectedTestFiles, diffMode); - if (scopeError) die(2, scopeError); + const scopeError = cliScopeError(job.packageDir, selectedTestFiles(job, args), args.diffMode); + if (scopeError) throw new MutateError(2, scopeError); } const results = []; - for (const job of jobs) { - results.push(await runJob(job, { configArg, testFiles })); - } - - process.stderr.write('\n=== Mutation summary ===\n'); - for (const r of results) reportJob(r); - - if (results.some((r) => r.failed)) { - process.stderr.write('\nGate: ERROR — at least one Stryker run produced no report.\n'); - process.exit(3); - } - - // A partial run never passes. The mutants it did not test can be survivors. - const overall = results.reduce( - (acc, r) => { - const c = r.summary.overall.counts; - acc.survived += c.survived + c.noCoverage; - acc.ignored += c.ignored; - acc.passed &&= r.summary.overall.thresholdMet && !r.summary.partial; - acc.partial ||= Boolean(r.summary.partial); - return acc; - }, - { survived: 0, ignored: 0, passed: true, partial: false }, - ); - - const gateState = overall.passed ? 'PASS' : overall.partial ? 'FAIL (partial)' : 'FAIL'; - process.stderr.write( - `\nGate: ${gateState} • threshold: ${THRESHOLD}% • ` + - `unjustified survivors: ${overall.survived} • ignored (justified): ${overall.ignored}\n`, - ); - process.exit(overall.passed ? 0 : 1); + for (const job of jobs) results.push(await runJob(job, args, io)); + return reportResults(results, log); } -const isCli = import.meta.url === `file://${process.argv[1]}`; -if (isCli) { - await main(); +// Print a stop the way the command line shows it, and return its exit code. +function reportStop(error) { + if (!(error instanceof MutateError)) { + process.stderr.write(`\n✗ mutate.mjs crashed: ${error?.stack ?? error}\n`); + return 3; + } + process.stderr.write(`${error.message}\n${error.showUsage ? `${USAGE}\n` : ''}`); + return error.exitCode; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exit(await main(process.argv.slice(2)).catch(reportStop)); } diff --git a/scripts/mutation-health/mutate.test.mjs b/scripts/mutation-health/mutate.test.mjs index a7e958e0085..1c8fe293524 100644 --- a/scripts/mutation-health/mutate.test.mjs +++ b/scripts/mutation-health/mutate.test.mjs @@ -1,437 +1,13 @@ -import { afterEach, beforeEach, describe, it } from 'node:test'; +import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; -import { EventEmitter } from 'node:events'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { - buildNoTestsSummary, - buildStrykerConfig, - buildSummary, - changedTestFilesForPackage, - classifyRun, - cliScopeError, - coverageFromCounts, - createCleanup, - defaultConfigNameFor, - filesToCheckout, - findStrykerSetupFiles, - formatMutateArg, - isMutableSource, - mergeRanges, - parseArgs, - parseHunkRanges, - parseTestFiles, - registerCleanupHandlers, - removeStrykerSetupFiles, - restoreFiles, - scoreFromCounts, - snapshotFiles, - splitRange, - strykerCliArgs, - toPackageRelative, -} from './mutate.mjs'; - -// A minimal Stryker Mutation Testing Elements report for one source file. Mix -// of statuses so coverage (anything that ran / ran + no-coverage) is a genuine -// fraction strictly between 0 and 1: 3 ran (killed/survived/timeout), 1 sat -// uncovered. -const RAW_FIXTURE = { - files: { - 'src/cron.ts': { - source: 'export const a = 1;\nexport const b = 2;\nexport const c = 3;\n', - mutants: [ - { - id: '1', - mutatorName: 'ArithmeticOperator', - status: 'Killed', - location: { start: { line: 1, column: 18 }, end: { line: 1, column: 19 } }, - replacement: '2', - }, - { - id: '2', - mutatorName: 'BooleanLiteral', - status: 'Survived', - location: { start: { line: 2, column: 18 }, end: { line: 2, column: 19 } }, - replacement: '3', - coveredBy: ['t1'], - }, - { - id: '3', - mutatorName: 'BlockStatement', - status: 'Timeout', - location: { start: { line: 3, column: 18 }, end: { line: 3, column: 19 } }, - replacement: '4', - }, - { - id: '4', - mutatorName: 'StringLiteral', - status: 'NoCoverage', - location: { start: { line: 3, column: 0 }, end: { line: 3, column: 5 } }, - replacement: '""', - }, - ], - }, - }, - testFiles: { - 'src/cron.test.ts': { - tests: [{ id: 't1', name: 'cron computes next run' }], - }, - }, -}; - -const RUN_META = { threshold: 80, target: 'src/cron.ts', generatedAt: '2026-06-21T00:00:00.000Z' }; - -function isFractionInUnitInterval(v) { - return typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= 1; -} - -describe('coverageFromCounts', () => { - it('is the share of mutants that ran (ran / ran + no-coverage)', () => { - // 3 ran (killed + timeout), 1 uncovered → 3/4 - const counts = { killed: 2, survived: 0, timeout: 1, noCoverage: 1, runtimeError: 0 }; - assert.equal(coverageFromCounts(counts), 0.75); - }); - - it('counts survived and runtime-error mutants as covered (they ran)', () => { - const counts = { killed: 0, survived: 1, timeout: 0, noCoverage: 1, runtimeError: 1 }; - assert.equal(coverageFromCounts(counts), 0.6667); - }); - - it('is 1 when every mutant was covered', () => { - assert.equal(coverageFromCounts({ killed: 5, survived: 0, timeout: 0, noCoverage: 0 }), 1); - }); - - it('is 0 when no mutant was covered', () => { - assert.equal(coverageFromCounts({ killed: 0, survived: 0, timeout: 0, noCoverage: 7 }), 0); - }); - - it('is 0 — never NaN — when there is nothing to cover', () => { - assert.equal(coverageFromCounts({ killed: 0, survived: 0, timeout: 0, noCoverage: 0 }), 0); - }); - - it('ignores compile-error and ignored mutants (they never ran for coverage reasons)', () => { - const counts = { - killed: 1, - survived: 0, - timeout: 0, - noCoverage: 1, - compileError: 3, - ignored: 4, - runtimeError: 0, - }; - // only killed (ran) + noCoverage count → 1/2 - assert.equal(coverageFromCounts(counts), 0.5); - }); - - it('always lands in [0,1]', () => { - for (const counts of [ - { killed: 1, survived: 2, timeout: 3, noCoverage: 4, runtimeError: 5 }, - { killed: 0, survived: 0, timeout: 0, noCoverage: 0 }, - { killed: 9, survived: 0, timeout: 0, noCoverage: 0 }, - ]) { - assert.ok(isFractionInUnitInterval(coverageFromCounts(counts))); - } - }); -}); - -describe('buildSummary', () => { - it('writes a per-file coverage fraction in [0,1] onto every file row', () => { - const summary = buildSummary(RAW_FIXTURE, RUN_META); - assert.equal(summary.files.length, 1); - const file = summary.files[0]; - assert.ok(isFractionInUnitInterval(file.coverage)); - // 3 ran (killed/survived/timeout) of 4 coverable → 0.75 - assert.equal(file.coverage, 0.75); - }); - - it('writes an overall coverage fraction in [0,1]', () => { - const summary = buildSummary(RAW_FIXTURE, RUN_META); - assert.ok(isFractionInUnitInterval(summary.overall.coverage)); - }); - - it('preserves the existing summary contract (score, counts, survivors)', () => { - const summary = buildSummary(RAW_FIXTURE, RUN_META); - const file = summary.files[0]; - assert.equal(file.score, scoreFromCounts(file.counts)); - assert.equal(file.counts.killed, 1); - assert.equal(file.counts.survived, 1); - assert.equal(file.counts.noCoverage, 1); - assert.equal(file.counts.timeout, 1); - // Survived + NoCoverage are unjustified survivors - assert.equal(file.survivors.length, 2); - // names the covering test for the survived mutant - const survived = file.survivors.find((s) => s.status === 'Survived'); - assert.deepEqual(survived.coveringTests, ['cron computes next run']); - }); -}); - -describe('buildNoTestsSummary (no covering tests)', () => { - it('reports coverage 0 in [0,1] when nothing covers the file', () => { - const summary = buildNoTestsSummary({ - threshold: 80, - target: 'src/cron.ts', - noCoverage: 12, - generatedAt: RUN_META.generatedAt, - }); - assert.equal(summary.files[0].coverage, 0); - assert.ok(isFractionInUnitInterval(summary.files[0].coverage)); - assert.ok(isFractionInUnitInterval(summary.overall.coverage)); - }); -}); - -describe('classifyRun', () => { - const DONE = 'Instrumented 1 source file(s) with 8 mutant(s)'; - const NO_TESTS = 'ERROR Stryker No tests were executed. Stryker will exit prematurely.'; - - it('is complete when the run wrote a report and exited zero', () => { - assert.equal(classifyRun({ exitCode: 0, output: DONE, hasReport: true }), 'complete'); - }); - - it('is partial when the run wrote a report and then exited non-zero', () => { - assert.equal(classifyRun({ exitCode: 1, output: DONE, hasReport: true }), 'partial'); - }); - - it('is no-tests when nothing covers the target', () => { - assert.equal(classifyRun({ exitCode: 1, output: NO_TESTS, hasReport: false }), 'no-tests'); - }); - - it('is failed when the run produced no report', () => { - assert.equal(classifyRun({ exitCode: 1, output: 'SIGABRT', hasReport: false }), 'failed'); - }); - - // The caller deletes the previous reports before each run. Without that, a - // crashed run finds the earlier report and is classified `partial`, so it - // reports the earlier target and its score instead of failing. - it('trusts hasReport as this run only — a report plus a crash is partial, never failed', () => { - assert.equal(classifyRun({ exitCode: 3, output: 'SIGABRT', hasReport: true }), 'partial'); - }); - - // Same trap for the no-tests path: a leftover report used to suppress it, and - // a genuine score-0 red was reported as the earlier run's passing score. - it('still detects no-tests when the run crashed without a report', () => { - assert.equal(classifyRun({ exitCode: 3, output: NO_TESTS, hasReport: false }), 'no-tests'); - }); -}); - -describe('isMutableSource', () => { - it('accepts product source wherever a package keeps it', () => { - assert.ok(isMutableSource('packages/workflow/src/cron.ts')); - // nodes-base has no src/. An allowlist drops the largest surface in the repo. - assert.ok(isMutableSource('packages/nodes-base/nodes/Slack/Slack.node.ts')); - assert.ok(isMutableSource('packages/nodes-base/credentials/SlackApi.credentials.ts')); - assert.ok(isMutableSource('packages/frontend/editor-ui/src/stores/ui.store.ts')); - // `[cm]?` in the extension test is there for the ESM/CJS variants. - assert.ok(isMutableSource('packages/@n8n/db/src/index.mts')); - assert.ok(isMutableSource('packages/@n8n/db/src/index.cts')); - }); - - it('rejects tests, declarations, configs and build output', () => { - assert.equal(isMutableSource('packages/workflow/src/cron.test.ts'), false); - assert.equal(isMutableSource('packages/workflow/src/cron.spec.ts'), false); - // The ESM/CJS variants are accepted as source, so they have to be - // excluded as tests too. - assert.equal(isMutableSource('packages/workflow/src/cron.test.mts'), false); - assert.equal(isMutableSource('packages/workflow/src/__tests__/cron.ts'), false); - assert.equal(isMutableSource('packages/workflow/src/__mocks__/cron.ts'), false); - assert.equal(isMutableSource('packages/workflow/src/types.d.ts'), false); - assert.equal(isMutableSource('packages/cli/vitest.config.ts'), false); - assert.equal(isMutableSource('packages/workflow/dist/cron.js'), false); - assert.equal(isMutableSource('packages/workflow/test/helper.ts'), false); - assert.equal(isMutableSource('packages/@n8n/db/src/migrations/sqlite/x.ts'), false); - assert.equal(isMutableSource('packages/design-system/src/Button.stories.ts'), false); - // The extension test is anchored: `.ts` has to end the path, not merely - // appear in it. Committed snapshots sit next to their source and would - // otherwise be handed to Stryker as mutable TypeScript. - assert.equal(isMutableSource('packages/cli/src/__snapshots__/foo.test.ts.snap'), false); - }); - - // .vue stays out. Each SFC package crashed Stryker's mutate step in the - // 2026-06 sweep, and the component layer gives little value. - it('rejects everything that is not TypeScript', () => { - assert.equal(isMutableSource('packages/frontend/editor-ui/src/App.vue'), false); - assert.equal(isMutableSource('packages/workflow/src/cron.js'), false); - assert.equal(isMutableSource('README.md'), false); - assert.equal(isMutableSource('packages/workflow/package.json'), false); - }); -}); - -describe('changedTestFilesForPackage', () => { - it('selects changed test files from the exact package', () => { - const changedFiles = [ - 'packages/@n8n/engine/src/runtime/__tests__/create-engine-runtime.test.ts', - 'packages/cli/src/modules/engine-v2/__tests__/engine-v2.runtime.test.ts', - 'packages/cli/src/modules/engine-v2/__tests__/in-memory-execution-response.test.ts', - 'packages/cli/src/modules/engine-v2/engine-v2.runtime.ts', - 'packages/cli-utils/src/foo.test.ts', - ]; - - assert.deepEqual(changedTestFilesForPackage(changedFiles, 'packages/cli'), [ - 'packages/cli/src/modules/engine-v2/__tests__/engine-v2.runtime.test.ts', - 'packages/cli/src/modules/engine-v2/__tests__/in-memory-execution-response.test.ts', - ]); - }); - - it('matches the CLI unit Vitest config roots and extensions', () => { - assert.deepEqual( - changedTestFilesForPackage( - [ - 'packages/cli/src/a.test.ts', - 'packages/cli/test/unit/b.spec.ts', - 'packages/cli/src/c.integration.test.ts', - 'packages/cli/test/integration/d.test.ts', - 'packages/cli/src/e.test.mts', - 'packages/cli/src/f.test.tsx', - ], - 'packages/cli/', - ), - ['packages/cli/src/a.test.ts', 'packages/cli/test/unit/b.spec.ts'], - ); - }); -}); - -describe('parseHunkRanges', () => { - it('reads new-side ranges out of `git diff -U0` headers', () => { - const diff = [ - 'diff --git a/src/cron.ts b/src/cron.ts', - '--- a/src/cron.ts', - '+++ b/src/cron.ts', - '@@ -12,0 +13,4 @@ export function tick() {', - '+const a = 1;', - '@@ -40,2 +44,2 @@', - '+const b = 2;', - // Counts run past one digit on both sides for any hunk of ten lines - // or more, which is most of them. - '@@ -80,12 +90,14 @@', - '+const c = 3;', - ].join('\n'); - assert.deepEqual(parseHunkRanges(diff), [ - { start: 13, end: 16 }, - { start: 44, end: 45 }, - { start: 90, end: 103 }, - ]); - }); - - // `git diff` of a file that itself talks about diffs (a patch fixture, this - // very test file) carries hunk-header text inside `+`/`-` content lines. - // Only a header at the start of a line is a header. - it('ignores hunk-header text that appears inside a content line', () => { - const diff = ['@@ -1,0 +5,1 @@', "+const H = '@@ -1,2 +300,4 @@';"].join('\n'); - assert.deepEqual(parseHunkRanges(diff), [{ start: 5, end: 5 }]); - }); - - it('treats a header with no new-side count as a single line', () => { - assert.deepEqual(parseHunkRanges('@@ -5 +7 @@'), [{ start: 7, end: 7 }]); - }); - - it('drops pure deletions — nothing survives there to mutate', () => { - assert.deepEqual(parseHunkRanges('@@ -10,4 +9,0 @@'), []); - }); - - it('returns nothing for a diff with no hunks', () => { - assert.deepEqual(parseHunkRanges(''), []); - }); -}); - -describe('mergeRanges', () => { - it('merges overlapping ranges', () => { - assert.deepEqual( - mergeRanges([ - { start: 1, end: 5 }, - { start: 3, end: 9 }, - ]), - [{ start: 1, end: 9 }], - ); - }); - - it('merges adjacent ranges so Stryker gets one span per region', () => { - assert.deepEqual( - mergeRanges([ - { start: 1, end: 4 }, - { start: 5, end: 8 }, - ]), - [{ start: 1, end: 8 }], - ); - }); - - it('keeps ranges with a real gap apart, and sorts them', () => { - assert.deepEqual( - mergeRanges([ - { start: 20, end: 22 }, - { start: 1, end: 4 }, - ]), - [ - { start: 1, end: 4 }, - { start: 20, end: 22 }, - ], - ); - }); - - it('leaves a fully-contained range absorbed', () => { - assert.deepEqual( - mergeRanges([ - { start: 1, end: 20 }, - { start: 5, end: 9 }, - ]), - [{ start: 1, end: 20 }], - ); - }); -}); - -describe('formatMutateArg', () => { - it('comma-joins every target into one flag value', () => { - assert.equal( - formatMutateArg(['src/a.ts:1-4', 'src/a.ts:20-22', 'src/b.ts']), - 'src/a.ts:1-4,src/a.ts:20-22,src/b.ts', - ); - }); -}); - -describe('splitRange', () => { - it('splits a trailing line range off the path', () => { - assert.deepEqual(splitRange('src/cron.ts:13-16'), { file: 'src/cron.ts', range: '13-16' }); - }); - - it('leaves a bare path alone', () => { - assert.deepEqual(splitRange('src/cron.ts'), { file: 'src/cron.ts', range: null }); - }); - - it('does not mistake a Windows drive letter or a colon in a dirname for a range', () => { - assert.deepEqual(splitRange('src/a:b/cron.ts'), { file: 'src/a:b/cron.ts', range: null }); - }); -}); - -describe('parseTestFiles', () => { - it('splits a comma-separated value', () => { - assert.deepEqual(parseTestFiles(['a.test.ts,b.test.ts']), ['a.test.ts', 'b.test.ts']); - }); - - it('collects a repeated flag', () => { - assert.deepEqual(parseTestFiles(['a.test.ts', 'b.test.ts']), ['a.test.ts', 'b.test.ts']); - }); - - it('accepts both forms at once, and trims the spaces around a comma', () => { - assert.deepEqual(parseTestFiles(['a.test.ts, b.test.ts', 'c.test.ts']), [ - 'a.test.ts', - 'b.test.ts', - 'c.test.ts', - ]); - }); - - // A duplicate would make Stryker run the same file twice for every mutant. - it('drops blanks and duplicates', () => { - assert.deepEqual(parseTestFiles(['a.test.ts,,a.test.ts', ' ', 'b.test.ts']), [ - 'a.test.ts', - 'b.test.ts', - ]); - }); - - it('returns nothing when the flag was not given', () => { - assert.deepEqual(parseTestFiles([]), []); - }); -}); +import { parseArgs, usageError } from './mutate.mjs'; +import { CLI_PACKAGE_DIR, isMutableSource, repoRoot } from './targets.mjs'; +import { SAFETY_GREP } from './test-doubles.mjs'; describe('parseArgs', () => { it('reads --test-files as a comma-separated list', () => { @@ -458,8 +34,20 @@ describe('parseArgs', () => { assert.deepEqual(parsed.testFiles, ['src/__tests__/foo.test.ts', 'src/__tests__/bar.test.ts']); }); - it('leaves testFiles empty when the flag is absent', () => { - assert.deepEqual(parseArgs(['src/foo.ts']).testFiles, []); + it('leaves testFiles empty and testCommand unset when the flags are absent', () => { + const parsed = parseArgs(['src/foo.ts']); + assert.deepEqual(parsed.testFiles, []); + assert.equal(parsed.testCommand, undefined); + }); + + it('reads --test-command as one value, spaces included', () => { + const parsed = parseArgs(['coverage-options.ts', '--test-command', 'pnpm exec vitest run']); + assert.equal(parsed.testCommand, 'pnpm exec vitest run'); + assert.equal(parsed.targetArg, 'coverage-options.ts'); + }); + + it('reads a --test-command with nothing after it as an empty command', () => { + assert.equal(parseArgs(['src/foo.ts', '--test-command']).testCommand, ''); }); it('keeps reading the other flags', () => { @@ -490,441 +78,197 @@ describe('parseArgs', () => { assert.equal(parseArgs(['-h']).helpMode, true); assert.equal(parseArgs(['src/foo.ts']).helpMode, false); }); + + it('reads a --test-files with nothing after it as no test file', () => { + assert.deepEqual(parseArgs(['src/foo.ts', '--test-files']).testFiles, []); + }); + + it('does not take an unknown flag as the target', () => { + assert.equal(parseArgs(['--verbose', 'src/a.ts']).targetArg, 'src/a.ts'); + }); + + it('takes the first positional argument as the target', () => { + assert.equal(parseArgs(['src/a.ts', 'src/b.ts']).targetArg, 'src/a.ts'); + }); }); -describe('toPackageRelative', () => { - // Stryker matches testFiles against the files under its run cwd, which is - // the package dir. - it('strips the package prefix off a repo-relative path', () => { +describe('usageError', () => { + const args = (argv) => usageError(parseArgs(argv)); + + it('accepts a named target and a bare --diff', () => { + assert.equal(args(['src/a.ts']), null); assert.equal( - toPackageRelative('packages/cli/src/__tests__/foo.test.ts', 'packages/cli'), - 'src/__tests__/foo.test.ts', + args(['src/a.ts', '--test-files', 'a.test.ts', '--test-command', 'pnpm test']), + null, + ); + assert.equal(args(['--diff']), null); + }); + + it('wants exactly one of a target and --diff', () => { + assert.match(args([]), /Missing mutate target/); + assert.match(args(['--diff', 'src/a.ts']), /--diff takes no positional target/); + }); + + it('refuses --test-files and --test-command with --diff', () => { + assert.match( + args(['--diff', '--test-files', 'a.test.ts']), + /--test-files needs a single target/, + ); + assert.match( + args(['--diff', '--test-command', 'pnpm test']), + /--test-command needs a single target/, ); }); - it('leaves an already package-relative path alone', () => { - assert.equal( - toPackageRelative('src/__tests__/foo.test.ts', 'packages/cli'), - 'src/__tests__/foo.test.ts', - ); - }); - - it('leaves a glob alone', () => { - assert.equal(toPackageRelative('src/**/*.test.ts', 'packages/cli'), 'src/**/*.test.ts'); - }); - - it('drops a leading ./', () => { - assert.equal(toPackageRelative('./src/foo.test.ts', 'packages/cli'), 'src/foo.test.ts'); - }); - - // `packages/cli-utils` must not lose its prefix to `packages/cli`. - it('only strips a whole path segment', () => { - assert.equal( - toPackageRelative('packages/cli-utils/src/foo.test.ts', 'packages/cli'), - 'packages/cli-utils/src/foo.test.ts', - ); + it('refuses an empty or blank test command', () => { + assert.match(args(['src/a.ts', '--test-command']), /--test-command needs a command/); + assert.match(args(['src/a.ts', '--test-command', ' ']), /--test-command needs a command/); }); }); -describe('buildStrykerConfig', () => { - it('puts the supplied test files in the testFiles config field', () => { - const config = buildStrykerConfig({ - targets: ['src/credentials/external-secrets.utils.ts:32-68'], - testFiles: ['src/credentials/__tests__/external-secrets.utils.test.ts'], - }); - assert.deepEqual(config.testFiles, [ - 'src/credentials/__tests__/external-secrets.utils.test.ts', - ]); - assert.deepEqual(config.mutate, ['src/credentials/external-secrets.utils.ts:32-68']); +// Run the command line in a new process. The timeout stops a case that starts +// a real Stryker run by mistake, so the case fails fast instead of hanging. +function run(args) { + return spawnSync(process.execPath, [path.join(import.meta.dirname, 'mutate.mjs'), ...args], { + cwd: path.resolve(import.meta.dirname, '../..'), + encoding: 'utf8', + timeout: 30_000, }); +} - // An empty `testFiles` is not the same as an absent one: Stryker reads a - // non-empty list as "run only these", so an empty one must not be sent. - it('leaves testFiles out when no test file was named', () => { - const config = buildStrykerConfig({ targets: ['src/cron.ts'] }); - assert.equal('testFiles' in config, false); - }); -}); +// Each case must stop before Stryker starts, so a mistyped command costs +// nothing instead of forking a process for each of the package's test files. +function assertStopsWith(res, status, pattern) { + assert.equal(res.signal, null, 'the command did not stop in time'); + assert.equal(res.status, status, res.stderr); + assert.match(res.stderr, pattern); + assert.doesNotMatch(res.stderr, /Running Stryker/); +} -describe('strykerCliArgs', () => { - it('forwards testFiles as one comma-joined flag', () => { - const args = strykerCliArgs( - buildStrykerConfig({ - targets: ['src/a.ts:1-4', 'src/b.ts'], - testFiles: ['src/__tests__/a.test.ts', 'src/__tests__/b.test.ts'], - }), - ); - assert.deepEqual(args, [ - '--mutate', - 'src/a.ts:1-4,src/b.ts', - '--testFiles', - 'src/__tests__/a.test.ts,src/__tests__/b.test.ts', - ]); - }); +const assertUsageError = (res, pattern) => assertStopsWith(res, 2, pattern); - it('sends no --testFiles flag when no test file was named', () => { - const args = strykerCliArgs(buildStrykerConfig({ targets: ['src/cron.ts'] })); - assert.deepEqual(args, ['--mutate', 'src/cron.ts']); - }); -}); - -describe('cliScopeError', () => { - it('refuses a packages/cli target with no --test-files', () => { - const error = cliScopeError('packages/cli', []); - assert.match(error, /--test-files/); - }); - - it('allows a packages/cli target once test files are named', () => { - assert.equal(cliScopeError('packages/cli', ['src/__tests__/foo.test.ts']), null); - }); - - it('explains how to scope a CLI diff with no changed test file', () => { - const error = cliScopeError('packages/cli', [], true); - assert.match(error, /changed test file/); - assert.match(error, /named target with --test-files/); - }); - - it('leaves every other package alone', () => { - assert.equal(cliScopeError('packages/workflow', []), null); - // A prefix match would sweep in an unrelated package. - assert.equal(cliScopeError('packages/cli-utils', []), null); - }); -}); - -describe('defaultConfigNameFor', () => { - it('gives packages/cli its own config, so related-test discovery stays off', () => { - assert.equal(defaultConfigNameFor('packages/cli'), 'stryker.cli.mjs'); - }); - - it('gives every other package the shared default', () => { - assert.equal(defaultConfigNameFor('packages/workflow'), 'stryker.default.mjs'); - assert.equal(defaultConfigNameFor('packages/@n8n/decorators'), 'stryker.default.mjs'); - }); -}); - -describe('the cli scope guard end to end', () => { - const wrapper = path.join(import.meta.dirname, 'mutate.mjs'); - const target = 'packages/cli/src/credentials/external-secrets.utils.ts:32-68'; - - function run(args) { - return spawnSync(process.execPath, [wrapper, ...args], { - cwd: path.resolve(import.meta.dirname, '../..'), - encoding: 'utf8', - }); +// Run `body` on a temp package with one source file and the given package.json. +function withTempPackage(packageJson, body) { + const pkgRoot = mkdtempSync(path.join(tmpdir(), 'mutate-cli-')); + try { + mkdirSync(path.join(pkgRoot, 'src')); + writeFileSync(path.join(pkgRoot, 'src/a.ts'), 'export const a = 1;\n'); + writeFileSync(path.join(pkgRoot, 'package.json'), JSON.stringify(packageJson)); + body(pkgRoot); + } finally { + rmSync(pkgRoot, { recursive: true, force: true }); } +} - // The run must stop before Stryker starts, so a mistyped command costs - // nothing instead of forking a process for each of the package's test files. - it('exits non-zero and names the flag when a cli target has no --test-files', () => { - const res = run([target]); - assert.equal(res.status, 2); - assert.match(res.stderr, /--test-files/); - // Stryker never started. - assert.doesNotMatch(res.stderr, /Running Stryker/); +describe('the cli end to end', () => { + // The command plans in this repo, so the target must exist. Any source file + // of packages/cli does: the run stops at the scope check, before Stryker. + const cliSource = readdirSync(path.join(repoRoot, CLI_PACKAGE_DIR, 'src')) + .map((name) => `${CLI_PACKAGE_DIR}/src/${name}`) + .find(isMutableSource); + const cliTarget = `${cliSource}:1-2`; + + // The message shows how to name the test files, so the usage text is not needed. + it('exits 2 and names the flag when a cli target has no --test-files', () => { + const res = run([cliTarget]); + assertUsageError(res, /^Mutating packages\/cli needs --test-files\./m); + assert.ok(res.stderr.endsWith(' --test-files packages/cli/src/__tests__/foo.test.ts\n')); + assert.doesNotMatch(res.stderr, /^Usage:/m); }); - it('refuses --test-files together with --diff', () => { - const res = run(['--diff', '--test-files', 'packages/cli/src/__tests__/foo.test.ts']); - assert.equal(res.status, 2); - assert.match(res.stderr, /--diff/); + it('refuses --test-files and --test-command together with --diff', () => { + assertUsageError( + run(['--diff', '--test-files', 'packages/cli/src/__tests__/foo.test.ts']), + /--diff/, + ); + assertUsageError( + run(['--diff', '--test-command', 'pnpm test']), + /--test-command needs a single target/, + ); }); - it('documents the flag in --help', () => { + it('refuses an empty test command and prints the usage', () => { + const res = run(['packages/@n8n/instance-ai/src/utils/model-config-id.ts', '--test-command']); + assertUsageError(res, /--test-command needs a command/); + assert.match(res.stderr, /^Usage:/m); + }); + + // Like the playwright package, which has only a `test:unit` script. A temp + // package keeps the check from starting Stryker if that package changes. + it('refuses a package without a vitest test script and points to --test-command', () => { + withTempPackage({ name: 'no-vitest', scripts: { 'test:unit': 'vitest run' } }, (pkgRoot) => { + const res = run(['src/a.ts', '--package-dir', pkgRoot]); + assertUsageError(res, /no-vitest is not a vitest package\n.*--test-command/); + }); + }); + + it('documents the flags in --help', () => { const res = run(['--help']); assert.equal(res.status, 0); assert.match(res.stdout, /--test-files/); + assert.match(res.stdout, /--test-command /); }); }); -describe('filesToCheckout', () => { - it('picks the files the run made dirty', () => { - assert.deepEqual(filesToCheckout(['a.ts'], ['a.ts', 'b.ts']), ['b.ts']); - }); +describe('the cli on a crash', () => { + // A config whose keys cannot be read makes the tool fail with an error that + // is not a planned stop: a broken tool, not a red gate. + const cases = { + 'an error': [ + "throw new TypeError('no keys')", + /\n✗ mutate\.mjs crashed: TypeError: no keys\n {4}at /, + ], + undefined: ['throw undefined', /\n✗ mutate\.mjs crashed: undefined\n$/], + }; - // A file that was dirty before the run holds the user's uncommitted work. - // `git checkout --` would throw it away; the byte snapshot restores it. - it('leaves the already-dirty files to the byte snapshot', () => { - assert.deepEqual(filesToCheckout(['a.ts', 'b.ts'], ['a.ts', 'b.ts']), []); - }); - - it('returns nothing when the run left the tree as it found it', () => { - assert.deepEqual(filesToCheckout([], []), []); - }); -}); - -describe('createCleanup', () => { - it('restores the tree and then removes the setup files', () => { - const calls = []; - const cleanup = createCleanup({ - restore: () => { - calls.push('restore'); - return ['src/cron.ts']; - }, - removeSetupFiles: () => { - calls.push('remove'); - return ['stryker-setup-0.js']; - }, + for (const [name, [thrown, message]] of Object.entries(cases)) { + it(`exits 3, never 1, and prints what was thrown when it throws ${name}`, () => { + withTempPackage({ name: 'pkg', scripts: { test: 'vitest run' } }, (pkgRoot) => { + const config = path.join(pkgRoot, 'crash.mjs'); + writeFileSync(config, `export default new Proxy({}, { ownKeys() { ${thrown}; } });\n`); + const res = run(['src/a.ts', '--package-dir', pkgRoot, '--config', config]); + assertStopsWith(res, 3, message); + }); }); - assert.deepEqual(cleanup(), { restored: ['src/cron.ts'], removed: ['stryker-setup-0.js'] }); - assert.deepEqual(calls, ['restore', 'remove']); - }); - - // Four exit paths are wired to the same routine, and more than one can fire - // (SIGINT, then `exit`). Doing the work twice would undo a restore the user - // made in between. - it('does the work once however many exit paths call it', () => { - let restores = 0; - let removals = 0; - const cleanup = createCleanup({ - restore: () => { - restores++; - return []; - }, - removeSetupFiles: () => { - removals++; - return []; - }, - }); - cleanup(); - cleanup(); - cleanup(); - assert.equal(restores, 1); - assert.equal(removals, 1); - }); - - it('still removes the setup files when the restore throws', () => { - let removals = 0; - const cleanup = createCleanup({ - restore: () => { - throw new Error('working tree is locked'); - }, - removeSetupFiles: () => { - removals++; - return ['stryker-setup-0.js']; - }, - }); - assert.deepEqual(cleanup(), { restored: [], removed: ['stryker-setup-0.js'] }); - assert.equal(removals, 1); - }); -}); - -describe('registerCleanupHandlers', () => { - // A stand-in process, so the tests can fire the handlers without signalling - // or ending the test runner. - function harness({ onSignal } = {}) { - const proc = new EventEmitter(); - const exits = []; - const writes = []; - let cleaned = 0; - const handlers = registerCleanupHandlers({ - cleanup: () => { - cleaned++; - return { restored: [], removed: [] }; - }, - onSignal, - proc, - exit: (code) => exits.push(code), - write: (msg) => writes.push(msg), - }); - return { proc, exits, writes, handlers, cleaned: () => cleaned }; } - - it('cleans up on the usual exit path', () => { - const h = harness(); - h.proc.emit('exit', 0); - assert.equal(h.cleaned(), 1); - }); - - it('cleans up and exits 130 on SIGINT', () => { - const h = harness({ onSignal: () => false }); - h.proc.emit('SIGINT'); - assert.equal(h.cleaned(), 1); - assert.deepEqual(h.exits, [130]); - }); - - it('cleans up and exits 143 on SIGTERM', () => { - const h = harness({ onSignal: () => false }); - h.proc.emit('SIGTERM'); - assert.equal(h.cleaned(), 1); - assert.deepEqual(h.exits, [143]); - }); - - // While Stryker is alive the run has to unwind first: a restore that races - // Stryker's own writes fixes nothing. The run's `finally` then cleans up. - it('defers to a live run, which still reaches cleanup', () => { - const seen = []; - const h = harness({ - onSignal: (signal) => { - seen.push(signal); - return true; - }, - }); - h.proc.emit('SIGINT'); - h.proc.emit('SIGTERM'); - assert.deepEqual(seen, ['SIGINT', 'SIGTERM']); - assert.equal(h.cleaned(), 0); - assert.deepEqual(h.exits, []); - - h.proc.emit('exit', 130); - assert.equal(h.cleaned(), 1); - }); - - it('cleans up on an uncaught exception and exits 3', () => { - const h = harness(); - h.proc.emit('uncaughtException', new Error('boom')); - assert.equal(h.cleaned(), 1); - assert.deepEqual(h.exits, [3]); - assert.match(h.writes.join(''), /boom/); - }); - - it('stops listening after dispose, so the next job owns its own snapshot', () => { - const h = harness({ onSignal: () => false }); - h.handlers.dispose(); - h.proc.emit('exit', 0); - h.proc.emit('SIGINT'); - h.proc.emit('SIGTERM'); - h.proc.emit('uncaughtException', new Error('boom')); - assert.equal(h.cleaned(), 0); - assert.deepEqual(h.exits, []); - }); }); -describe('working-tree cleanup', () => { - let root; +describe('the tool never writes to the working tree with git', () => { + const files = readdirSync(import.meta.dirname).filter((file) => file.endsWith('.mjs')); + // The files the tool runs. The test files and their doubles are not among them. + const sources = files.filter((file) => !/(\.test|^test-doubles)\.mjs$/.test(file)); + const read = (file) => readFileSync(path.join(import.meta.dirname, file), 'utf8'); - beforeEach(() => { - root = mkdtempSync(path.join(tmpdir(), 'mutate-cleanup-')); + // Comments may tell how the tool worked before. Only code counts. A `//` + // after a colon is part of a URL, not a comment. + const withoutComments = (text) => + text.replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/.*$/gm, '$1'); + + // git commands that overwrite files or move the work out of the tree. + const TREE_COMMANDS = '(checkout|restore|stash|reset|clean|switch)'; + const GIT_WRITE = [ + new RegExp(`\\bgit\\b.*\\b${TREE_COMMANDS}\\b`), + new RegExp(`['"\`]${TREE_COMMANDS}['"\`]`), + ]; + const writesTree = (code) => GIT_WRITE.some((pattern) => pattern.test(code)); + + // The command is built from parts, so this file passes the safety grep. + it('finds each form of a git command that writes to the tree', () => { + const command = ['git', 'checkout', '--'].join(' '); + assert.equal(writesTree(`execSync('${command} ' + f)`), true); + assert.equal(writesTree(`execSync(\`${command} \${f}\`)`), true); + assert.equal(writesTree("spawnSync('git', ['restore', '--', f])"), true); + assert.equal(writesTree("spawnSync('git', ['diff', '--name-only', from])"), false); + assert.equal(writesTree(withoutComments(`// ${command} was the undo`)), false); }); - afterEach(() => { - rmSync(root, { recursive: true, force: true }); + it('has no source file that starts such a command', () => { + assert.ok(sources.includes('mutate.mjs')); + for (const file of sources) assert.equal(writesTree(withoutComments(read(file))), false, file); }); - // A mutate target plus two files outside it. The old target-only snapshot - // restored the target and left the other two mutated. - function seedTree() { - mkdirSync(path.join(root, 'packages/pkg/src'), { recursive: true }); - mkdirSync(path.join(root, 'packages/other/src'), { recursive: true }); - const files = { - target: path.join(root, 'packages/pkg/src/cron.ts'), - sibling: path.join(root, 'packages/pkg/src/helper.ts'), - test: path.join(root, 'packages/other/src/cron.test.ts'), - }; - for (const [name, file] of Object.entries(files)) writeFileSync(file, `original ${name}\n`); - return files; - } - - // A stand-in for Stryker: a real child process that rewrites each file it is - // given and drops a setup file for each worker, the way the vitest runner - // does. `--inPlace` means those writes hit the working tree. - function runMockStryker({ mutates = [], setupFiles = [] }) { - const script = [ - "const { writeFileSync } = require('node:fs');", - `for (const f of ${JSON.stringify(mutates)}) writeFileSync(f, 'Stryker was here!\\n');`, - `for (const f of ${JSON.stringify(setupFiles)}) writeFileSync(f, '// stryker setup\\n');`, - ].join('\n'); - const res = spawnSync(process.execPath, ['-e', script], { encoding: 'utf8' }); - assert.equal(res.status, 0, res.stderr); - } - - it('restores every file the run modified, not only the mutate targets', () => { - const files = seedTree(); - const snap = snapshotFiles(Object.values(files)); - - runMockStryker({ mutates: Object.values(files) }); - for (const file of Object.values(files)) { - assert.equal(readFileSync(file, 'utf8'), 'Stryker was here!\n'); - } - - assert.deepEqual(new Set(restoreFiles(snap)), new Set(Object.values(files))); - for (const [name, file] of Object.entries(files)) { - assert.equal(readFileSync(file, 'utf8'), `original ${name}\n`); - } - }); - - it('reports only the files it had to write back', () => { - const files = seedTree(); - const snap = snapshotFiles(Object.values(files)); - - runMockStryker({ mutates: [files.sibling] }); - - assert.deepEqual(restoreFiles(snap), [files.sibling]); - }); - - it('skips a file the run deleted instead of throwing', () => { - const files = seedTree(); - const snap = snapshotFiles(Object.values(files)); - rmSync(files.test); - - assert.deepEqual(restoreFiles(snap), []); - }); - - it('deletes every stryker-setup-*.js under the repo root', () => { - const files = seedTree(); - const setupFiles = [ - path.join(root, 'stryker-setup-0.js'), - path.join(root, 'packages/pkg/stryker-setup-1.js'), - path.join(root, 'packages/other/src/stryker-setup-12.js'), - ]; - runMockStryker({ setupFiles }); - - assert.deepEqual(new Set(removeStrykerSetupFiles(root)), new Set(setupFiles)); - for (const file of setupFiles) assert.equal(existsSync(file), false); - assert.deepEqual(findStrykerSetupFiles(root), []); - // The sources stay where they are. - assert.ok(existsSync(files.target)); - }); - - it('leaves files that only look like a setup file alone', () => { - seedTree(); - const keep = [ - path.join(root, 'stryker-setup.js'), - path.join(root, 'stryker-setup-0.ts'), - path.join(root, 'my-stryker-setup-0.js'), - ]; - runMockStryker({ setupFiles: keep }); - - assert.deepEqual(removeStrykerSetupFiles(root), []); - for (const file of keep) assert.ok(existsSync(file)); - }); - - // A full-repo walk that enters node_modules takes minutes, and Stryker's own - // packaged `stryker-setup.js` lives there. - it('does not walk node_modules', () => { - mkdirSync(path.join(root, 'node_modules/@stryker-mutator'), { recursive: true }); - const inside = path.join(root, 'node_modules/@stryker-mutator/stryker-setup-0.js'); - runMockStryker({ setupFiles: [inside] }); - - assert.deepEqual(removeStrykerSetupFiles(root), []); - assert.ok(existsSync(inside)); - }); - - it('leaves neither mutants nor setup files behind when a run is interrupted', () => { - const files = seedTree(); - const setupFiles = [path.join(root, 'packages/pkg/stryker-setup-0.js')]; - const snap = snapshotFiles(Object.values(files)); - - const cleanup = createCleanup({ - restore: () => restoreFiles(snap), - removeSetupFiles: () => removeStrykerSetupFiles(root), - }); - const proc = new EventEmitter(); - const exits = []; - registerCleanupHandlers({ - cleanup, - onSignal: () => false, // Stryker is already gone - proc, - exit: (code) => exits.push(code), - write: () => {}, - }); - - runMockStryker({ mutates: Object.values(files), setupFiles }); - proc.emit('SIGINT'); // the user hits Ctrl-C mid-run - - for (const [name, file] of Object.entries(files)) { - assert.equal(readFileSync(file, 'utf8'), `original ${name}\n`); - } - assert.deepEqual(findStrykerSetupFiles(root), []); - assert.deepEqual(exits, [130]); + // The same check that a person does before a run: grep each .mjs file. + it('passes the safety grep in every file, tests included', () => { + for (const file of files) assert.doesNotMatch(read(file), SAFETY_GREP, file); }); }); diff --git a/scripts/mutation-health/plan-diff.test.mjs b/scripts/mutation-health/plan-diff.test.mjs new file mode 100644 index 00000000000..dbeff68d5fa --- /dev/null +++ b/scripts/mutation-health/plan-diff.test.mjs @@ -0,0 +1,268 @@ +import { afterEach, beforeEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { devNull, tmpdir } from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { diffPlanLines, planFromDiff, runGit } from './plan.mjs'; +import { MutateError } from './targets.mjs'; +import { SAMPLE_REPO, writeTree } from './test-doubles.mjs'; + +const instanceAiSource = 'packages/@n8n/instance-ai/src/utils/model-config-id.ts'; +const cliSource = 'packages/cli/src/credentials/external-secrets.utils.ts'; +const cliTest = 'packages/cli/src/credentials/__tests__/external-secrets.utils.test.ts'; +const HUNK = '@@ -1,0 +2,3 @@\n+x\n'; + +// A git stand-in that answers the read-only commands a plan needs and records +// every call. `hunks` maps a file to its `git diff -U0` output. +function stubGit({ mergeBaseStatus = 0, namesStatus = 0, names = [], hunks = {} } = {}) { + const calls = []; + const git = (args) => { + calls.push(args); + if (args[0] === 'merge-base') { + return { status: mergeBaseStatus, stdout: 'abc123\n', stderr: 'fatal: no such ref\n' }; + } + if (args[1] === '--name-only') { + return { + status: namesStatus, + stdout: `${names.join('\n')}\n`, + stderr: 'fatal: bad object\n', + }; + } + return { status: 0, stdout: hunks[args.at(-1)] ?? HUNK, stderr: '' }; + }; + return { git, calls }; +} + +// Each plan reads a temp copy of SAMPLE_REPO, so no test depends on the files of this repo. +let root; + +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), 'mutate-diff-')); + writeTree(root, SAMPLE_REPO); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +const plan = (base, git) => planFromDiff(base, { git, repoRoot: root }); + +// Run git in the temp repo. It ignores the user's git config, so no hook, +// signing or template applies. +function gitInRoot(args) { + const res = spawnSync( + 'git', + ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', ...args], + { + cwd: root, + encoding: 'utf8', + env: { ...process.env, GIT_CONFIG_GLOBAL: devNull, GIT_CONFIG_NOSYSTEM: '1' }, + }, + ); + assert.equal(res.status, 0, res.stderr); + return res.stdout; +} + +// Make the temp repo a git repo with SAMPLE_REPO committed on `main`. +function commitSampleRepo() { + gitInRoot(['init', '--quiet', '--initial-branch=main']); + gitInRoot(['add', '--all']); + gitInRoot(['commit', '--quiet', '--no-gpg-sign', '--message', 'sample']); +} + +function assertMutateError(fn, pattern) { + assert.throws(fn, (error) => { + assert.ok(error instanceof MutateError); + assert.equal(error.exitCode, 2); + assert.match(error.message, pattern); + return true; + }); +} + +describe('planFromDiff', () => { + it('plans one job per package from the changed line ranges', () => { + const { git } = stubGit({ names: [instanceAiSource, cliSource, cliTest] }); + const { jobs, skipped } = plan('origin/master', git); + assert.deepEqual(skipped, []); + const byDir = Object.fromEntries(jobs.map((job) => [job.packageDir, job])); + const instanceAi = byDir[path.join('packages', '@n8n', 'instance-ai')]; + assert.equal(instanceAi.pkgRoot, path.join(root, 'packages/@n8n/instance-ai')); + assert.deepEqual(instanceAi.targets, [ + `${path.join('src', 'utils', 'model-config-id.ts')}:2-4`, + ]); + // Only a cli job takes the changed cli tests as its test list. + assert.equal('testFiles' in instanceAi, false); + assert.deepEqual(byDir[path.join('packages', 'cli')].testFiles, [cliTest]); + }); + + it('leaves a deleted cli test out of the test list', () => { + const deleted = 'packages/cli/src/credentials/__tests__/gone.test.ts'; + const { git } = stubGit({ names: [cliSource, cliTest, deleted] }); + const [job] = plan('origin/master', git).jobs; + assert.deepEqual(job.testFiles, [cliTest]); + }); + + it('reads file names with spaces around them', () => { + const { git } = stubGit({ names: [` ${instanceAiSource} `, ''] }); + assert.equal(plan('origin/master', git).jobs.length, 1); + }); + + it('plans nothing for a changed file with no added lines', () => { + const { git } = stubGit({ + names: [instanceAiSource], + hunks: { [instanceAiSource]: '@@ -4,2 +3,0 @@\n' }, + }); + assert.deepEqual(plan('origin/master', git).jobs, []); + }); + + it('skips deleted files and non-source files without a word', () => { + const { git } = stubGit({ names: ['packages/gone/src/deleted.ts', 'README.md'] }); + assert.deepEqual(plan('origin/master', git), { jobs: [], skipped: [] }); + }); + + it('names each source file that it skips, and why', () => { + const blocked = 'packages/@n8n/expression-runtime/src/index.ts'; + const jest = 'packages/jest-pkg/src/a.ts'; + const loose = 'scripts/loose.ts'; + const { git } = stubGit({ names: [blocked, jest, loose] }); + const { jobs, skipped } = plan('origin/master', git); + assert.deepEqual(jobs, []); + assert.equal(skipped.length, 3); + assert.equal(skipped[0][0], blocked); + assert.match(skipped[0][1], /^@n8n\/expression-runtime is blocked: .*DEVP-257/); + assert.deepEqual(skipped.slice(1), [ + [jest, 'jest-pkg is not a vitest package'], + [loose, 'no enclosing package'], + ]); + }); +}); + +describe('planFromDiff git use', () => { + // A test file sits in a vitest package and has changed lines, so only the + // source filter keeps it out of the targets. + it('never makes a changed test file a target', () => { + const testFile = 'packages/@n8n/instance-ai/src/utils/__tests__/model-config-id.test.ts'; + const { git, calls } = stubGit({ names: [testFile] }); + assert.deepEqual(plan('origin/master', git), { jobs: [], skipped: [] }); + assert.equal( + calls.some((args) => args.includes('-U0')), + false, + ); + }); + + // Diffing the merge base against the working tree also scores uncommitted edits. + it('uses only git commands that read the repository, from the merge base', () => { + const { git, calls } = stubGit({ names: [instanceAiSource] }); + plan('upstream/master', git); + assert.deepEqual(calls, [ + ['merge-base', 'upstream/master', 'HEAD'], + ['diff', '--name-only', 'abc123'], + ['diff', '-U0', 'abc123', '--', instanceAiSource], + ]); + }); + + it('stops with a usage error when the base ref has no merge base', () => { + const { git } = stubGit({ mergeBaseStatus: 1 }); + assertMutateError( + () => plan('upstream/missing', git), + /^No merge base with 'upstream\/missing' — is the ref fetched\?\nfatal: no such ref$/, + ); + }); + + it('stops with a usage error when git cannot list the changed files', () => { + const { git } = stubGit({ namesStatus: 128 }); + assertMutateError( + () => plan('origin/master', git), + /^git diff against 'origin\/master' failed\.\nfatal: bad object$/, + ); + }); + + // The plan and its git data must come from the same repo. This repo has no + // `main` branch, so git from this repo has no merge base and the plan stops. + it('reads git in the repo root it is given when no git is injected', () => { + commitSampleRepo(); + writeFileSync( + path.join(root, instanceAiSource), + 'export const id = 1;\nexport const two = 2;\n', + ); + const { jobs, skipped } = planFromDiff('main', { repoRoot: root }); + assert.deepEqual(skipped, []); + assert.deepEqual( + jobs.map((job) => [job.pkgRoot, job.targets]), + [ + [ + path.join(root, 'packages/@n8n/instance-ai'), + [`${path.join('src', 'utils', 'model-config-id.ts')}:2-2`], + ], + ], + ); + }); +}); + +describe('runGit', () => { + it('runs git in the repo and returns its text output', () => { + const res = runGit(['rev-parse', '--show-toplevel']); + assert.equal(res.status, 0); + assert.equal(typeof res.stdout, 'string'); + assert.ok(res.stdout.trim().length > 0); + }); + + it('runs git in the root it is given', () => { + commitSampleRepo(); + const res = runGit(['rev-parse', '--show-toplevel'], root); + assert.equal(res.status, 0, res.stderr); + assert.equal(res.stdout.trim(), realpathSync(root)); + }); + + // A missing git is a setup problem, so it must not read as "nothing changed". + it('stops with a usage error that names the command when git cannot start', () => { + const emptyPath = mkdtempSync(path.join(tmpdir(), 'mutate-no-git-')); + try { + const planUrl = pathToFileURL(path.join(import.meta.dirname, 'plan.mjs')).href; + const script = [ + `import { runGit } from '${planUrl}';`, + "try { runGit(['merge-base', 'origin/master', 'HEAD']); } catch (error) {", + ' process.stdout.write(JSON.stringify([error.name, error.exitCode, error.message]));', + '}', + ].join('\n'); + const res = spawnSync(process.execPath, ['--input-type=module', '--eval', script], { + encoding: 'utf8', + env: { PATH: emptyPath }, + timeout: 20_000, + }); + const [name, exitCode, message] = JSON.parse(res.stdout || '[]'); + assert.equal(name, 'MutateError', res.stderr); + assert.equal(exitCode, 2); + assert.match(message, /^git merge-base failed to start: .*ENOENT/); + } finally { + rmSync(emptyPath, { recursive: true, force: true }); + } + }); +}); + +describe('diffPlanLines', () => { + const job = (targets) => ({ pkgRoot: '/repo/pkg', packageDir: 'pkg', targets }); + + it('counts the ranges of every package and names the base', () => { + const jobs = [job(['a.ts:1-2', 'b.ts:4-9']), job(['c.ts:3-3'])]; + assert.deepEqual(diffPlanLines({ jobs, skipped: [] }, 'upstream/master'), [ + '\nMutating 3 changed range(s) across 2 package(s) vs upstream/master.', + ]); + }); + + it('names each skipped file and why, before the count', () => { + const skipped = [['docs/a.ts', 'no enclosing package']]; + assert.deepEqual(diffPlanLines({ jobs: [job(['a.ts:1-2'])], skipped }, 'origin/master'), [ + ' skipped docs/a.ts — no enclosing package', + '\nMutating 1 changed range(s) across 1 package(s) vs origin/master.', + ]); + }); + + // main says that nothing changed. A count of zero runs would only add noise. + it('prints no count when no package has a run', () => { + assert.deepEqual(diffPlanLines({ jobs: [], skipped: [] }, 'origin/master'), []); + }); +}); diff --git a/scripts/mutation-health/plan.mjs b/scripts/mutation-health/plan.mjs new file mode 100644 index 00000000000..1c1b743e64d --- /dev/null +++ b/scripts/mutation-health/plan.mjs @@ -0,0 +1,224 @@ +/** + * Plan the Stryker runs: one job per package, with the targets to mutate and + * the test files to run. The planners read the file system and git. Git is + * injected so the unit tests can check which git commands a plan uses: only + * `merge-base` and `diff`, never one that writes. The repo root is also a + * parameter (default: this repo), so the unit tests plan in a temp repo. The + * default git runs in that same root, so a plan reads one repo only. + */ +import { spawnSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { + CLI_PACKAGE_DIR, + MutateError, + changedTestFilesForPackage, + isMutableSource, + parseHunkRanges, + repoRoot, + splitRange, + toPosix, +} from './targets.mjs'; + +// Stryker's dry run stops with SIGABRT on the isolated-vm engine. See DEVP-257. +const BLOCKED_PACKAGES = new Set(['@n8n/expression-runtime']); + +const TEST_COMMAND_HINT = + 'Name the command that runs its tests with --test-command, for example ' + + "--test-command 'pnpm exec vitest run'."; + +/** Run a read-only git command in `root` (default: this repo) and return its text output. */ +export function runGit(args, root = repoRoot) { + const res = spawnSync('git', args, { + cwd: root, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }); + if (res.error) throw new MutateError(2, `git ${args[0]} failed to start: ${res.error.message}`); + return res; +} + +function readPackageJson(pkgRoot) { + try { + return JSON.parse(readFileSync(path.join(pkgRoot, 'package.json'), 'utf8')); + } catch { + return {}; + } +} + +/** + * Why a package cannot be scored, or null when it can. Both planners call + * this, so a named target and a --diff target get the same answer. + * + * By default Stryker runs the package's own vitest, so the `test` script must + * run vitest. A custom test command replaces that script, so only the blocked + * list applies then. + */ +export function ineligibleReason( + pkgRoot, + { customTestCommand = false, repoRoot: root = repoRoot } = {}, +) { + const pkg = readPackageJson(pkgRoot); + const pkgName = pkg.name || path.relative(root, pkgRoot); + if (BLOCKED_PACKAGES.has(pkg.name)) { + return { + code: 'blocked', + message: `${pkgName} is blocked: the isolated-vm engine crashes Stryker's dry run (DEVP-257)`, + }; + } + if (customTestCommand || /\bvitest\b/.test(pkg.scripts?.test ?? '')) return null; + return { code: 'not-vitest', message: `${pkgName} is not a vitest package` }; +} + +// Walk up from a path to the nearest enclosing package.json (bounded by `root`). +export function findPackageRoot(fromAbs, root = repoRoot) { + let dir = path.dirname(fromAbs); + while (dir === root || dir.startsWith(`${root}${path.sep}`)) { + if (existsSync(path.join(dir, 'package.json'))) return dir; + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + return null; +} + +function changedFilesSince(base, git) { + // Diff the merge base against the working tree, not against HEAD. This also + // scores uncommitted edits. On a PR checkout the tree is clean, thus the + // result is the same as the branch diff. + const mergeBase = git(['merge-base', base, 'HEAD']); + if (mergeBase.status !== 0) { + throw new MutateError( + 2, + `No merge base with '${base}' — is the ref fetched?\n${mergeBase.stderr.trim()}`, + ); + } + const from = mergeBase.stdout.trim(); + const names = git(['diff', '--name-only', from]); + if (names.status !== 0) { + throw new MutateError(2, `git diff against '${base}' failed.\n${names.stderr.trim()}`); + } + const files = names.stdout + .split('\n') + .map((s) => s.trim()) + .filter(Boolean); + return { from, files }; +} + +// Add one changed file to its package's job, or record why it was skipped. +function planChangedFile(file, { from, git, root, byPackage, skipped }) { + if (!isMutableSource(file)) return; + const abs = path.resolve(root, file); + if (!existsSync(abs)) return; // the branch deleted the file + + const pkgRoot = findPackageRoot(abs, root); + if (!pkgRoot) { + skipped.push([file, 'no enclosing package']); + return; + } + const reason = ineligibleReason(pkgRoot, { repoRoot: root }); + if (reason) { + skipped.push([file, reason.message]); + return; + } + + const ranges = parseHunkRanges(git(['diff', '-U0', from, '--', file]).stdout); + if (ranges.length === 0) return; + + const rel = path.relative(pkgRoot, abs); + const packageDir = path.relative(root, pkgRoot); + const job = byPackage.get(pkgRoot) ?? { pkgRoot, packageDir, targets: [] }; + for (const r of ranges) job.targets.push(`${rel}:${r.start}-${r.end}`); + byPackage.set(pkgRoot, job); +} + +/** + * One job per package for every line this branch changed. `packages/cli` jobs + * also get the cli test files the patch changed, as their explicit test list. + * Without an injected `git`, git runs in `repoRoot`. + */ +export function planFromDiff( + base, + { repoRoot: root = repoRoot, git = (args) => runGit(args, root) } = {}, +) { + const { from, files } = changedFilesSince(base, git); + const byPackage = new Map(); + const skipped = []; + for (const file of files) planChangedFile(file, { from, git, root, byPackage, skipped }); + for (const job of byPackage.values()) { + if (toPosix(job.packageDir) === CLI_PACKAGE_DIR) { + job.testFiles = changedTestFilesForPackage(files, job.packageDir).filter((file) => + existsSync(path.resolve(root, file)), + ); + } + } + return { jobs: [...byPackage.values()], skipped }; +} + +/** What --diff prints about its plan: each skipped file, then the run count. */ +export function diffPlanLines({ jobs, skipped }, base) { + const lines = skipped.map(([file, why]) => ` skipped ${file} — ${why}`); + if (jobs.length === 0) return lines; + const ranges = jobs.reduce((n, job) => n + job.targets.length, 0); + const summary = `Mutating ${ranges} changed range(s) across ${jobs.length} package(s) vs ${base}.`; + return [...lines, `\n${summary}`]; +} + +// The package and the package-relative path of a named target. +function locateTarget(file, packageDirArg, root) { + if (packageDirArg) { + const pkgRoot = path.resolve(root, packageDirArg); + if (!existsSync(pkgRoot)) throw new MutateError(2, `Package dir not found: ${pkgRoot}`); + return { pkgRoot, rel: path.isAbsolute(file) ? path.relative(pkgRoot, file) : file }; + } + const abs = path.resolve(root, file); + if (!existsSync(abs)) throw new MutateError(2, `Target not found: ${abs}`, { showUsage: true }); + const pkgRoot = findPackageRoot(abs, root); + if (!pkgRoot) { + throw new MutateError(2, `Could not infer the package for ${file} — pass --package-dir.`, { + showUsage: true, + }); + } + return { pkgRoot, rel: path.relative(pkgRoot, abs) }; +} + +// Stop unless `rel` is a mutable source file inside the package. +function assertMutableTarget(pkgRoot, rel) { + if (rel.startsWith('..') || path.isAbsolute(rel)) { + throw new MutateError(2, `Target must live inside the package. Got: ${rel}`); + } + if (!existsSync(path.join(pkgRoot, rel))) { + throw new MutateError(2, `Target not found: ${path.join(pkgRoot, rel)}`); + } + if (!isMutableSource(rel)) { + throw new MutateError( + 2, + `Not a mutable source file (test/declaration/config/build output): ${rel}`, + ); + } +} + +/** The single job for a named target, or a MutateError that says what to fix. */ +export function planFromTarget( + targetArg, + packageDirArg, + { customTestCommand = false, repoRoot: root = repoRoot } = {}, +) { + const { file, range } = splitRange(targetArg); + const { pkgRoot, rel } = locateTarget(file, packageDirArg, root); + assertMutableTarget(pkgRoot, rel); + // --diff skips an ineligible package. A named target must refuse for the same + // reason, or it starts a run that is known to crash. + const reason = ineligibleReason(pkgRoot, { customTestCommand, repoRoot: root }); + if (reason) { + const hint = reason.code === 'not-vitest' ? `\n${TEST_COMMAND_HINT}` : ''; + throw new MutateError(2, `Cannot mutate ${rel}: ${reason.message}${hint}`); + } + + return { + pkgRoot, + packageDir: path.relative(root, pkgRoot), + targets: [range ? `${rel}:${range}` : rel], + }; +} diff --git a/scripts/mutation-health/plan.test.mjs b/scripts/mutation-health/plan.test.mjs new file mode 100644 index 00000000000..12d549f5134 --- /dev/null +++ b/scripts/mutation-health/plan.test.mjs @@ -0,0 +1,207 @@ +import { afterEach, beforeEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { findPackageRoot, ineligibleReason, planFromTarget } from './plan.mjs'; +import { MutateError } from './targets.mjs'; +import { SAMPLE_REPO, writeTree } from './test-doubles.mjs'; + +// Each test plans in this temp repo, so no test depends on the files of this repo. +let root; + +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), 'mutate-plan-')); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +// A package with one source file and the given package.json fields. +function seedPackage(fields) { + const pkgRoot = path.join(root, 'pkg'); + mkdirSync(path.join(pkgRoot, 'src'), { recursive: true }); + writeFileSync(path.join(pkgRoot, 'package.json'), JSON.stringify(fields)); + writeFileSync(path.join(pkgRoot, 'src/a.ts'), 'export const a = 1;\n'); + writeFileSync(path.join(pkgRoot, 'src/a.test.ts'), 'it("a", () => {});\n'); + return pkgRoot; +} + +function assertMutateError(fn, exitCode, pattern) { + assert.throws(fn, (error) => { + assert.ok(error instanceof MutateError); + assert.equal(error.exitCode, exitCode); + assert.match(error.message, pattern); + return true; + }); +} + +describe('ineligibleReason', () => { + it('accepts a package whose test script runs vitest', () => { + assert.equal( + ineligibleReason(seedPackage({ name: 'pkg', scripts: { test: 'vitest run' } })), + null, + ); + }); + + it('refuses a package whose test script runs something else', () => { + const reason = ineligibleReason(seedPackage({ name: 'pkg', scripts: { test: 'jest' } })); + assert.deepEqual(reason, { code: 'not-vitest', message: 'pkg is not a vitest package' }); + }); + + // The playwright package has no `test` script, only `test:unit`. + it('accepts a package with no vitest test script once a test command is given', () => { + const pkgRoot = seedPackage({ name: 'pkg', scripts: { 'test:unit': 'vitest run' } }); + assert.equal(ineligibleReason(pkgRoot).code, 'not-vitest'); + assert.equal(ineligibleReason(pkgRoot, { customTestCommand: true }), null); + }); + + it('refuses a package without scripts, or without a readable package.json', () => { + assert.equal(ineligibleReason(seedPackage({ name: 'pkg' })).code, 'not-vitest'); + const bare = path.join(root, 'packages/bare'); + mkdirSync(bare, { recursive: true }); + // A package without a name is named by its dir in the repo. + assert.deepEqual(ineligibleReason(bare, { repoRoot: root }), { + code: 'not-vitest', + message: 'packages/bare is not a vitest package', + }); + }); + + it('keeps a blocked package blocked, with or without a test command', () => { + const pkgRoot = seedPackage({ + name: '@n8n/expression-runtime', + scripts: { test: 'vitest run' }, + }); + for (const options of [{}, { customTestCommand: true }]) { + const reason = ineligibleReason(pkgRoot, options); + assert.equal(reason.code, 'blocked'); + assert.match(reason.message, /DEVP-257/); + } + }); +}); + +describe('planFromTarget', () => { + it('plans one job for a named target with a line range', () => { + const pkgRoot = seedPackage({ name: 'pkg', scripts: { test: 'vitest run' } }); + const job = planFromTarget('src/a.ts:3-9', pkgRoot); + assert.equal(job.pkgRoot, pkgRoot); + assert.deepEqual(job.targets, ['src/a.ts:3-9']); + }); + + it('reads a relative --package-dir from the repo root', () => { + const pkgRoot = seedPackage({ name: 'pkg', scripts: { test: 'vitest run' } }); + const job = planFromTarget('src/a.ts', 'pkg', { repoRoot: root }); + assert.equal(job.pkgRoot, pkgRoot); + assert.equal(job.packageDir, 'pkg'); + }); + + it('infers the package of a repo-relative target', () => { + writeTree(root, SAMPLE_REPO); + const target = 'packages/@n8n/instance-ai/src/utils/model-config-id.ts:4-9'; + const job = planFromTarget(target, undefined, { repoRoot: root }); + assert.equal(job.pkgRoot, path.join(root, 'packages/@n8n/instance-ai')); + assert.equal(job.packageDir, path.join('packages', '@n8n', 'instance-ai')); + assert.deepEqual(job.targets, [`${path.join('src', 'utils', 'model-config-id.ts')}:4-9`]); + }); + + it('refuses a package without a vitest test script and names --test-command', () => { + const pkgRoot = seedPackage({ name: 'pkg', scripts: {} }); + assertMutateError( + () => planFromTarget('src/a.ts', pkgRoot), + 2, + /Cannot mutate src\/a\.ts: pkg is not a vitest package\n.*--test-command/, + ); + }); + + it('names the command form in the hint', () => { + const pkgRoot = seedPackage({ name: 'pkg', scripts: {} }); + assertMutateError( + () => planFromTarget('src/a.ts', pkgRoot), + 2, + /\nName the command that runs its tests with --test-command, for example --test-command 'pnpm exec vitest run'\.$/, + ); + }); + + it('gives no test-command hint for a blocked package', () => { + const pkgRoot = seedPackage({ + name: '@n8n/expression-runtime', + scripts: { test: 'vitest run' }, + }); + assert.throws( + () => planFromTarget('src/a.ts', pkgRoot), + (error) => /DEVP-257\)$/.test(error.message) && !error.message.includes('--test-command'), + ); + }); + + it('plans that package once a test command is given', () => { + const pkgRoot = seedPackage({ name: 'pkg', scripts: {} }); + const job = planFromTarget('src/a.ts', pkgRoot, { customTestCommand: true }); + assert.deepEqual(job.targets, ['src/a.ts']); + }); +}); + +describe('planFromTarget refusals', () => { + it('refuses a target outside the package, a missing one and a test file', () => { + const pkgRoot = seedPackage({ name: 'pkg', scripts: { test: 'vitest run' } }); + assertMutateError(() => planFromTarget('../other.ts', pkgRoot), 2, /inside the package/); + assertMutateError(() => planFromTarget('src/missing.ts', pkgRoot), 2, /Target not found/); + assertMutateError(() => planFromTarget('src/a.test.ts', pkgRoot), 2, /Not a mutable source/); + }); + + it('stops when the package dir does not exist', () => { + assertMutateError( + () => planFromTarget('src/a.ts', path.join(root, 'missing')), + 2, + /^Package dir not found: .*missing$/, + ); + }); + + it('asks for --package-dir when no package holds the target', () => { + writeFileSync(path.join(root, 'loose.ts'), 'export const a = 1;\n'); + assert.throws( + () => planFromTarget(path.join(root, 'loose.ts'), undefined, { repoRoot: root }), + (error) => + error.exitCode === 2 && + error.showUsage === true && + /^Could not infer the package for .*loose\.ts — pass --package-dir\.$/.test(error.message), + ); + }); + + it('asks for the usage text when it cannot find the target to infer a package', () => { + assert.throws( + () => planFromTarget('packages/no-such-package/src/a.ts', undefined, { repoRoot: root }), + (error) => + error.exitCode === 2 && + error.showUsage === true && + /^Target not found: .*packages\/no-such-package\/src\/a\.ts$/.test(error.message), + ); + }); +}); + +describe('findPackageRoot', () => { + beforeEach(() => writeTree(root, SAMPLE_REPO)); + + it('finds the nearest package.json above a file', () => { + const file = path.join(root, 'packages/@n8n/instance-ai/src/utils/__tests__/a.test.ts'); + assert.equal(findPackageRoot(file, root), path.join(root, 'packages/@n8n/instance-ai')); + }); + + it('finds the repo root for a file at the top of the repo', () => { + writeTree(root, { 'package.json': { name: 'repo' } }); + assert.equal(findPackageRoot(path.join(root, 'knip.ts'), root), root); + }); + + it('finds no package for a file outside every package', () => { + assert.equal(findPackageRoot(path.join(root, 'scripts/loose.ts'), root), null); + }); + + // A package outside the repo is never planned. + it('does not look above the repo root', () => { + const file = path.join(root, 'packages/cli/src/credentials/a.ts'); + assert.equal(findPackageRoot(file, path.join(root, 'packages/cli/src')), null); + // The default repo is this one, and the temp repo lies outside it. + assert.equal(findPackageRoot(file), null); + }); +}); diff --git a/scripts/mutation-health/properties.test.mjs b/scripts/mutation-health/properties.test.mjs new file mode 100644 index 00000000000..d55582c375c --- /dev/null +++ b/scripts/mutation-health/properties.test.mjs @@ -0,0 +1,324 @@ +/** + * Generative checks of the pure helpers, with seeded samples (see samples.mjs). + * Each check states a rule that must hold for every input, so it also catches + * an off-by-one, an inverted condition or a missing branch that the example + * tests in the other files do not reach. + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { forEachSample } from './samples.mjs'; +import { toCommandLine } from './stryker.mjs'; +import { coverageFromCounts, emptyCounts, scoreFromCounts } from './summary.mjs'; +import { + mergeRanges, + parseHunkRanges, + parseTestFiles, + splitRange, + toPackageRelative, +} from './targets.mjs'; +import { toNestedNamePattern } from './vitest-compat.mjs'; + +// The pattern @stryker-mutator/vitest-runner 10.0.0 builds for a mutant (see +// vitest-compat.test.mjs, which also runs the installed runner). +function escapeRegExp(input) { + return input.replace(/[.*+\-?^${}()|[\]\\]/g, '\\$&'); +} +function runnerPattern(names) { + return new RegExp(names.map(escapeRegExp).join('|')); +} + +describe('toNestedNamePattern for any test names', () => { + // Name characters: spaces, `>` and every character a regex gives a meaning. + const NAME_CHARS = 'ab xy>.*+?^${}()|[]\\-'; + // Segment `i` (a suite or the test) starts with the digit `i`, and no other + // segment has that digit. A name that lacks a segment thus has no match. + const segments = (r) => + r.array(1, 5, () => '').map((_, i) => `${i}${r.string(NAME_CHARS, 0, 6)}`); + const vitest4 = (segs) => segs.join(' '); + const vitest5 = (segs) => segs.join(' > '); + + it('selects each covering test by its Vitest 5 and its Vitest 4 full name', () => { + forEachSample( + { seed: 101 }, + (r) => r.array(1, 3, () => segments(r)), + (tests) => { + const pattern = toNestedNamePattern(runnerPattern(tests.map(vitest4))); + for (const segs of tests) { + assert.equal(pattern.test(vitest5(segs)), true, vitest5(segs)); + assert.equal(pattern.test(vitest4(segs)), true, vitest4(segs)); + } + }, + ); + }); + + it('rejects a test of another suite and another test of the same suite', () => { + forEachSample( + { seed: 202 }, + (r) => ({ segs: segments(r), other: r.string('QJ', 1, 4) }), + ({ segs, other }) => { + const pattern = toNestedNamePattern(runnerPattern([vitest4(segs)])); + for (const index of segs.keys()) { + const changed = segs.with(index, other); + assert.equal(pattern.test(vitest5(changed)), false, vitest5(changed)); + } + }, + ); + }); + + it('returns a pattern that it made unchanged', () => { + forEachSample( + { seed: 303, count: 50 }, + (r) => segments(r), + (segs) => { + const nested = toNestedNamePattern(runnerPattern([vitest4(segs)])); + assert.equal(toNestedNamePattern(nested), nested); + }, + ); + }); +}); + +// Every line that a list of ranges covers. +function coveredLines(ranges) { + const lines = new Set(); + for (const { start, end } of ranges) for (let line = start; line <= end; line++) lines.add(line); + return [...lines].sort((a, b) => a - b); +} + +// Sorted, each range valid, and a gap of at least one line between ranges. +function assertMerged(ranges) { + for (const [index, range] of ranges.entries()) { + assert.ok(range.start <= range.end, JSON.stringify(range)); + const previous = ranges[index - 1]; + if (previous) assert.ok(range.start > previous.end + 1, JSON.stringify(ranges)); + } +} + +describe('mergeRanges for any ranges', () => { + const ranges = (r) => + r.array(0, 8, () => { + const start = r.int(1, 40); + return { start, end: start + r.int(0, 6) }; + }); + + it('covers the same lines, in sorted ranges with a gap between each two', () => { + forEachSample({ seed: 404 }, ranges, (input) => { + const merged = mergeRanges(input); + assertMerged(merged); + assert.deepEqual(coveredLines(merged), coveredLines(input)); + }); + }); + + it('leaves its input as it was', () => { + forEachSample({ seed: 505, count: 50 }, ranges, (input) => { + const copy = structuredClone(input); + mergeRanges(input); + assert.deepEqual(input, copy); + }); + }); +}); + +describe('parseHunkRanges for any diff', () => { + // A `git diff -U0` text with hunks between other diff lines. A count is + // left out at random: git writes `+12` for a count of one. + const diff = (r) => { + const hunks = r.array(0, 6, () => ({ start: r.int(1, 60), count: r.int(0, 5) })); + const lines = ['diff --git a/x.ts b/x.ts', '--- a/x.ts', '+++ b/x.ts']; + for (const { start, count } of hunks) { + const newSide = count === 1 && r.int(0, 1) === 0 ? `${start}` : `${start},${count}`; + lines.push(`@@ -${r.int(1, 60)},${r.int(0, 5)} +${newSide} @@ export function a() {`); + lines.push(...r.array(0, 2, () => `${r.pick(['+', '-'])}${r.string('ab @+-', 0, 6)}`)); + } + return { hunks, text: lines.join('\n') }; + }; + + it('covers exactly the added lines of every hunk, merged', () => { + forEachSample({ seed: 606 }, diff, ({ hunks, text }) => { + const ranges = parseHunkRanges(text); + assertMerged(ranges); + const added = hunks + .filter(({ count }) => count > 0) + .map(({ start, count }) => ({ start, end: start + count - 1 })); + assert.deepEqual(coveredLines(ranges), coveredLines(added)); + }); + }); +}); + +describe('toPackageRelative for any path', () => { + const segment = (r) => `${r.pick([...'abc'])}${r.string('abc@_-.', 0, 4)}`; + const sample = (r) => ({ + packageDir: r.array(1, 3, () => segment(r)).join('/'), + rel: r.array(1, 4, () => segment(r)).join('/'), + }); + + it('removes the package prefix of a repo-relative path, also after `./`', () => { + forEachSample({ seed: 707 }, sample, ({ packageDir, rel }) => { + assert.equal(toPackageRelative(`${packageDir}/${rel}`, packageDir), rel); + assert.equal(toPackageRelative(`./${packageDir}/${rel}`, packageDir), rel); + }); + }); + + it('keeps a path in a sibling dir whose name starts with the package name', () => { + forEachSample({ seed: 808 }, sample, ({ packageDir, rel }) => { + const sibling = `${packageDir}x/${rel}`; + assert.equal(toPackageRelative(sibling, packageDir), sibling); + }); + }); + + it('keeps a path that is already package-relative', () => { + forEachSample({ seed: 909 }, sample, ({ packageDir, rel }) => { + if (rel.startsWith(`${packageDir}/`)) return; + assert.equal(toPackageRelative(rel, packageDir), rel); + }); + }); +}); + +describe('parseTestFiles for any flag values', () => { + // The values of repeated --test-files flags: paths, commas and blanks. + const values = (r) => r.array(0, 5, () => r.string('ab/., \t', 0, 10)); + // Each non-blank path of the values, as typed, with repeats. + const pathsIn = (vals) => + vals + .flatMap((value) => value.split(',')) + .map((part) => part.trim()) + .filter(Boolean); + + it('keeps each typed path once, trimmed, in the order it first appears', () => { + forEachSample({ seed: 1401 }, values, (vals) => { + const files = parseTestFiles(vals); + const typed = pathsIn(vals); + assert.deepEqual(new Set(files), new Set(typed)); + assert.equal(new Set(files).size, files.length); + for (const file of files) + assert.ok(file !== '' && file === file.trim() && !file.includes(',')); + const firstSeen = files.map((file) => typed.indexOf(file)); + assert.deepEqual( + firstSeen, + [...firstSeen].sort((a, b) => a - b), + ); + }); + }); + + it('reads one comma-joined value as the repeated flag, and a second parse changes nothing', () => { + forEachSample({ seed: 1402 }, values, (vals) => { + const files = parseTestFiles(vals); + assert.deepEqual(parseTestFiles([vals.join(',')]), files); + assert.deepEqual(parseTestFiles(files), files); + assert.deepEqual(parseTestFiles([files.join(',')]), files); + }); + }); +}); + +describe('splitRange for any target', () => { + // Colons, dashes and digits in a file name look like parts of a range. + const FILE_CHARS = 'ab:-0123./'; + + it('splits off a trailing `:-` and keeps the rest as the file', () => { + forEachSample( + { seed: 1501 }, + (r) => ({ file: r.string(FILE_CHARS, 0, 10), start: r.int(0, 999), end: r.int(0, 999) }), + ({ file, start, end }) => { + const range = `${start}-${end}`; + assert.deepEqual(splitRange(`${file}:${range}`), { file, range }); + }, + ); + }); + + it('keeps a target that does not end in a whole range as the file', () => { + forEachSample( + { seed: 1502 }, + (r) => { + const [a, b] = [r.int(0, 99), r.int(0, 99)]; + const end = r.pick(['.ts', `:${a}`, `:${a}-`, `:-${b}`, `:${a}-${b}x`]); + return `${r.string(FILE_CHARS, 0, 10)}${end}`; + }, + (target) => assert.deepEqual(splitRange(target), { file: target, range: null }), + ); + }); +}); + +describe('toCommandLine for any file names', () => { + // Every character the shell gives a meaning, but no newline: the shell + // output below puts each sample on its own line. No program has a name made + // of q, j and Q, so a name that the shell runs as a command runs nothing. + const ARG_CHARS = 'qjQ _-./@%+=:,\'"$`\\*?[]{}~#!&;|<>()^'; + const skip = process.platform === 'win32' && 'the command runner uses a POSIX shell here'; + + it('gives the command each file name as one argument, as it was', { skip }, () => { + // Each special character alone and next to letters, then random names. + const samples = [...ARG_CHARS].map((c) => [c, `q${c}`, `${c}j`, `q${c}j`]); + forEachSample( + { seed: 1001, count: 100 }, + (r) => r.array(1, 4, () => r.string(ARG_CHARS, 0, 8)), + (args) => samples.push(args), + ); + // One shell for every sample. Each line prints its arguments with a NUL after each. + const script = samples + .map((args) => `${toCommandLine("printf '%s\\0'", args)}; printf '\\n'`) + .join('\n'); + // Files in the working dir let a glob that is not quoted expand. + const cwd = mkdtempSync(path.join(tmpdir(), 'mutate-shell-')); + try { + for (const file of ['q', 'j', 'qj']) writeFileSync(path.join(cwd, file), ''); + const res = spawnSync('sh', ['-c', script], { cwd, encoding: 'utf8', timeout: 20_000 }); + assert.equal(res.status, 0, res.stderr); + const printed = res.stdout.split('\n').slice(0, -1); + assert.equal(printed.length, samples.length); + for (const [index, args] of samples.entries()) { + assert.deepEqual(printed[index].split('\0').slice(0, -1), args, JSON.stringify(args)); + } + } finally { + rmSync(cwd, { recursive: true, force: true }); + } + }); +}); + +describe('scoreFromCounts and coverageFromCounts for any counts', () => { + const counts = (r) => { + const c = emptyCounts(); + for (const key of Object.keys(c)) c[key] = r.int(0, 12); + return c; + }; + const plus = (c, key) => ({ ...c, [key]: c[key] + 1 }); + + it('keeps the score in [0, 100], at 100 only when every valid mutant was detected', () => { + forEachSample({ seed: 1101 }, counts, (c) => { + const score = scoreFromCounts(c); + assert.ok(score >= 0 && score <= 100, String(score)); + const valid = c.killed + c.timeout + c.survived + c.noCoverage; + assert.equal(score === 100, valid > 0 && c.survived + c.noCoverage === 0); + assert.equal(score === 0, c.killed + c.timeout === 0); + }); + }); + + it('raises the score only for a detected mutant and lowers it only for a missed one', () => { + forEachSample({ seed: 1202 }, counts, (c) => { + const score = scoreFromCounts(c); + for (const key of ['killed', 'timeout']) assert.ok(scoreFromCounts(plus(c, key)) >= score); + for (const key of ['survived', 'noCoverage']) { + assert.ok(scoreFromCounts(plus(c, key)) <= score); + } + for (const key of ['ignored', 'compileError', 'runtimeError']) { + assert.equal(scoreFromCounts(plus(c, key)), score, key); + } + }); + }); + + it('keeps coverage in [0, 1], and only a mutant that no test ran lowers it', () => { + forEachSample({ seed: 1303 }, counts, (c) => { + const coverage = coverageFromCounts(c); + assert.ok(coverage >= 0 && coverage <= 1, String(coverage)); + assert.ok(coverageFromCounts(plus(c, 'noCoverage')) <= coverage); + for (const key of ['killed', 'survived', 'timeout', 'runtimeError']) { + assert.ok(coverageFromCounts(plus(c, key)) >= coverage, key); + } + for (const key of ['ignored', 'compileError']) { + assert.equal(coverageFromCounts(plus(c, key)), coverage, key); + } + }); + }); +}); diff --git a/scripts/mutation-health/run-job.test.mjs b/scripts/mutation-health/run-job.test.mjs new file mode 100644 index 00000000000..5e3d58b1a71 --- /dev/null +++ b/scripts/mutation-health/run-job.test.mjs @@ -0,0 +1,440 @@ +import { afterEach, beforeEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { runJob } from './mutate.mjs'; +import { COMPAT_RUNNER_PLUGIN, NO_TSCONFIG_REWRITE } from './stryker.mjs'; +import { MutateError, repoRoot, toPosix } from './targets.mjs'; +import { + FAKE_STRYKER_BIN, + IN_PLACE_KEY, + fakeProcess, + fakeSpawn, + namesInPlaceMode, + resolveFakeStrykerBin, + settledState, + sink, +} from './test-doubles.mjs'; + +// A Stryker report with one killed and one surviving mutant. +const RAW_REPORT = { + files: { + 'src/a.ts': { + source: 'export const a = 1 + 2;\n', + mutants: [ + { + id: '1', + mutatorName: 'ArithmeticOperator', + status: 'Killed', + location: { start: { line: 1, column: 17 }, end: { line: 1, column: 22 } }, + replacement: '1 - 2', + }, + { + id: '2', + mutatorName: 'NumericLiteral', + status: 'Survived', + location: { start: { line: 1, column: 21 }, end: { line: 1, column: 22 } }, + replacement: '3', + }, + ], + }, + }, +}; + +// A temp repo with the package under test and a sibling package that a config +// can reach with `..`. It is outside this repo, so a run gets a sandbox mirror +// only when a test passes the temp repo as `repoRoot`. +let root; +let pkgRoot; +const SIBLING = 'packages/@n8n/sib/x.ts'; +// The package dir in the temp repo, the way the planner gives it. +const PACKAGE_DIR = 'packages/@n8n/pkg'; +// A test file path as a user types it, and as `--diff` plans it: from the repo root. +const repoRel = (file) => toPosix(path.join(PACKAGE_DIR, file)); + +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), 'mutate-job-')); + pkgRoot = path.join(root, PACKAGE_DIR); + mkdirSync(path.join(pkgRoot, 'src'), { recursive: true }); + writeFileSync( + path.join(pkgRoot, 'package.json'), + '{ "name": "pkg", "scripts": { "test": "vitest run" } }', + ); + writeFileSync(path.join(pkgRoot, 'src/a.ts'), RAW_REPORT.files['src/a.ts'].source); + mkdirSync(path.dirname(path.join(root, SIBLING)), { recursive: true }); + writeFileSync(path.join(root, SIBLING), 'sibling\n'); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +const reportFile = (name) => path.join(pkgRoot, 'reports/mutation', name); + +// Run one job with a fake Stryker. `stryker` gets the fake child, the spawn +// call and the stand-in process, and decides what the run writes, prints and +// returns. +// `inRepo` makes the temp repo the repo of the run: the job's package dir is +// relative to it, and the sandbox mirror copies it. +// `onPrint` gets each text that the run prints to stderr, and the stand-in process. +function runFakeJob({ args = {}, stryker, inRepo = false, jobFields = {}, onPrint } = {}) { + const p = fakeProcess(); + const stderr = sink(); + if (onPrint) { + const write = stderr.write; + stderr.write = (chunk) => { + write(chunk); + onPrint(String(chunk), p); + }; + } + const doubles = fakeSpawn({ + onSpawn: (child, call) => setImmediate(() => stryker(child, call, p)), + }); + const job = { + pkgRoot, + packageDir: path.relative(inRepo ? root : repoRoot, pkgRoot), + targets: ['src/a.ts'], + ...jobFields, + }; + const fullArgs = { configArg: undefined, testFiles: [], testCommand: undefined, ...args }; + const io = { + resolveStrykerBin: resolveFakeStrykerBin, + ...(inRepo ? { repoRoot: root } : {}), + spawn: doubles.spawn, + stdout: sink(), + stderr, + proc: p.proc, + exit: p.exit, + write: p.write, + }; + const promise = runJob(job, fullArgs, io); + return { ...doubles, ...p, stderr, promise }; +} + +function writesReport(child) { + writeFileSync(reportFile('raw.json'), JSON.stringify(RAW_REPORT)); + child.finish(0); +} + +const runConfig = () => JSON.parse(readFileSync(reportFile('stryker.run.json'), 'utf8')); + +describe('runJob with the default runner', () => { + it('starts only Stryker, on the run config, in the package dir, with no in place flag', async () => { + const run = runFakeJob({ args: { testFiles: ['src/a.test.ts'] }, stryker: writesReport }); + await run.promise; + assert.equal(run.calls.length, 1); + const [{ command, args, options }] = run.calls; + assert.equal(command, process.execPath); + assert.deepEqual(args, [FAKE_STRYKER_BIN, 'run', reportFile('stryker.run.json')]); + assert.equal(args.some(namesInPlaceMode), false); + assert.equal(options.cwd, pkgRoot); + }); + + // Stryker runs in the package dir, so it matches test files from there. + it('gives Stryker the --test-files relative to the package dir', async () => { + const testFiles = [repoRel('src/a.test.ts'), `./${repoRel('src/b.test.ts')}`]; + await runFakeJob({ inRepo: true, args: { testFiles }, stryker: writesReport }).promise; + assert.deepEqual(runConfig().testFiles, ['src/a.test.ts', 'src/b.test.ts']); + }); + + it('passes package-relative --test-files and globs through unchanged', async () => { + const testFiles = ['src/a.test.ts', 'src/**/*.test.ts']; + await runFakeJob({ inRepo: true, args: { testFiles }, stryker: writesReport }).promise; + assert.deepEqual(runConfig().testFiles, testFiles); + }); + + it('gives Stryker a sandbox config with the vitest-compat runner and the test files', async () => { + const run = runFakeJob({ args: { testFiles: ['src/a.test.ts'] }, stryker: writesReport }); + await run.promise; + const config = runConfig(); + assert.equal(config.testRunner, 'vitest-compat'); + assert.ok(config.plugins.includes(COMPAT_RUNNER_PLUGIN)); + assert.deepEqual(config.mutate, ['src/a.ts']); + assert.deepEqual(config.testFiles, ['src/a.test.ts']); + assert.deepEqual(Object.keys(config).filter(namesInPlaceMode), []); + // A package outside the repo gets no mirror, so the config's temp dir stays. + assert.equal(config.tempDirName, '.stryker-tmp'); + assert.match( + run.stderr.text(), + /Running Stryker on .* \(config: .*stryker\.default\.mjs, runner: vitest-compat/, + ); + assert.match(run.stderr.text(), /^ {2}testFiles: src\/a\.test\.ts$/m); + }); + + // --diff gives a packages/cli job the cli tests that the patch changed, as + // paths from the repo root. + it('runs the test files that the plan gave the job when --test-files is absent', async () => { + const jobFields = { testFiles: [repoRel('src/a.test.ts'), repoRel('src/b.test.ts')] }; + const run = runFakeJob({ inRepo: true, jobFields, stryker: writesReport }); + await run.promise; + assert.deepEqual(runConfig().testFiles, ['src/a.test.ts', 'src/b.test.ts']); + assert.match(run.stderr.text(), /^ {2}testFiles: src\/a\.test\.ts, src\/b\.test\.ts$/m); + }); + + it('runs the --test-files instead of the test files of the plan', async () => { + const jobFields = { testFiles: [repoRel('src/a.test.ts')] }; + const args = { testFiles: [repoRel('src/c.test.ts')] }; + await runFakeJob({ inRepo: true, args, jobFields, stryker: writesReport }).promise; + assert.deepEqual(runConfig().testFiles, ['src/c.test.ts']); + }); +}); + +describe('runJob results', () => { + it("writes the summary from this run's report and keeps the gate", async () => { + const run = runFakeJob({ stryker: writesReport }); + const result = await run.promise; + assert.equal(result.packageDir, path.relative(repoRoot, pkgRoot)); + assert.match(result.summaryPath, /packages\/@n8n\/pkg\/reports\/mutation\/summary\.json$/); + assert.equal(result.noTests, false); + assert.equal(result.summary.overall.score, 50); + assert.equal(result.summary.overall.thresholdMet, false); + assert.equal(result.summary.testRunner, 'vitest-compat'); + assert.deepEqual(JSON.parse(readFileSync(reportFile('summary.json'), 'utf8')), result.summary); + }); + + // A report left by an earlier run must never stand in for this one. + it('fails a run that wrote no report, even when an earlier report was on disk', async () => { + mkdirSync(path.dirname(reportFile('raw.json')), { recursive: true }); + writeFileSync(reportFile('raw.json'), JSON.stringify(RAW_REPORT)); + writeFileSync(reportFile('summary.json'), '{}'); + const run = runFakeJob({ stryker: (child) => child.finish(1) }); + const result = await run.promise; + assert.equal(result.failed, true); + assert.equal(existsSync(reportFile('summary.json')), false); + assert.match( + run.stderr.text(), + /: Stryker exited 1 without producing .*packages\/@n8n\/pkg\/reports\/mutation\/raw\.json\n/, + ); + }); + + it('records a score-0 red result when no test covers the target', async () => { + const run = runFakeJob({ + stryker: (child) => { + child.stdout.emit('data', 'Instrumented 1 source file(s) with 7 mutant(s)\n'); + child.stdout.emit('data', 'ERROR No tests were executed.\n'); + child.finish(1); + }, + }); + const result = await run.promise; + assert.equal(result.noTests, true); + assert.equal(result.summary.overall.counts.noCoverage, 7); + }); +}); + +describe('runJob with a test command', () => { + // The command runs in the package dir, so it gets package-relative paths. + it('gives Stryker the command runner with the test files on the command', async () => { + const run = runFakeJob({ + inRepo: true, + args: { testFiles: [repoRel('src/a.test.ts')], testCommand: 'pnpm exec vitest run' }, + stryker: writesReport, + }); + const result = await run.promise; + const config = runConfig(); + assert.equal(config.testRunner, 'command'); + assert.equal(config.commandRunner.command, 'pnpm exec vitest run src/a.test.ts'); + assert.equal('testFiles' in config, false); + assert.equal(result.summary.testRunner, 'command'); + assert.match(run.stderr.text(), /^ {2}command: pnpm exec vitest run src\/a\.test\.ts$/m); + }); +}); + +describe('runJob guards and cancellation', () => { + const refusedConfigs = { + 'turns on in place mode': { [IN_PLACE_KEY]: true }, + 'uses the TypeScript checker': { checkers: ['typescript'] }, + }; + for (const [what, base] of Object.entries(refusedConfigs)) { + it(`refuses a config that ${what} before it writes anything`, async () => { + const configArg = path.join(root, 'refused.json'); + writeFileSync(configArg, JSON.stringify(base)); + const run = runFakeJob({ inRepo: true, args: { configArg }, stryker: writesReport }); + await assert.rejects( + run.promise, + (error) => + error instanceof MutateError && + error.exitCode === 2 && + error.message.startsWith('refused.json '), + ); + assert.equal(run.calls.length, 0); + assert.equal(existsSync(path.join(pkgRoot, 'reports')), false); + assert.equal(existsSync(path.join(root, '.stryker-tmp')), false); + }); + } + + it('exits with the signal code when the run is cancelled', async () => { + const run = runFakeJob({ stryker: cancels }); + await assert.rejects( + run.promise, + (error) => error.exitCode === 130 && error.message === '\nRun cancelled by SIGINT.', + ); + assert.deepEqual(run.children[0].kills, ['SIGINT']); + }); + + it('stops with exit 3 before it writes anything when Stryker is not installed', async () => { + const job = { pkgRoot, packageDir: 'packages/@n8n/pkg', targets: ['src/a.ts'] }; + const resolveStrykerBin = () => { + throw new MutateError(3, 'Could not resolve @stryker-mutator/core'); + }; + const doubles = fakeSpawn(); + await assert.rejects( + runJob(job, { testFiles: [] }, { resolveStrykerBin, repoRoot: root, spawn: doubles.spawn }), + (error) => error.exitCode === 3, + ); + assert.equal(doubles.calls.length, 0); + assert.equal(existsSync(path.join(pkgRoot, 'reports')), false); + assert.equal(existsSync(path.join(root, '.stryker-tmp')), false); + }); +}); + +// A user cancels the run while Stryker runs. +function cancels(child, _call, p) { + p.proc.emit('SIGINT'); + child.finish(130); +} + +describe('runJob in a repo, with the sandbox in a mirror', () => { + const mirrorParent = () => path.join(root, '.stryker-tmp'); + + // A fake Stryker that does what Stryker does with its config: it makes a + // sandbox in the temp dir. It records what a test in the sandbox would see. + function inspectsSandbox(seen, then = writesReport) { + return (child, call, p) => { + const config = JSON.parse(readFileSync(call.args[2], 'utf8')); + const sandbox = path.join(config.tempDirName, 'sandbox-1'); + mkdirSync(sandbox); + Object.assign(seen, { + config, + // The OS resolves `..` from the sandbox, as the test runner does. + sibling: readFileSync(`${sandbox}/../sib/x.ts`, 'utf8'), + mirrorExists: existsSync(mirrorParent()), + }); + then(child, call, p); + }; + } + + it('gives Stryker a temp dir in a mirror at the depth of the package', async () => { + const seen = {}; + await runFakeJob({ inRepo: true, stryker: inspectsSandbox(seen) }).promise; + assert.match( + path.relative(mirrorParent(), seen.config.tempDirName), + /^mirror-[^/]+\/packages\/@n8n$/, + ); + // The paths in the package's tsconfig are already right in the mirror. + assert.equal(seen.config.tsconfigFile, NO_TSCONFIG_REWRITE); + }); + + // The cli alias `../@n8n/telemetry/src` is one such path. + it('lets a path that leaves the sandbox with `..` reach the real sibling package', async () => { + const seen = {}; + await runFakeJob({ inRepo: true, stryker: inspectsSandbox(seen) }).promise; + assert.equal(seen.sibling, 'sibling\n'); + }); + + const exitPaths = { + 'a run that wrote a report': { finish: writesReport, outcome: 'resolved' }, + 'a run that wrote no report': { finish: (child) => child.finish(1), outcome: 'resolved' }, + 'a Stryker that did not start': { + finish: (child) => child.emit('error', new Error('spawn ENOENT')), + outcome: 3, + }, + 'a cancelled run': { finish: cancels, outcome: 130 }, + }; + + for (const [name, { finish, outcome }] of Object.entries(exitPaths)) { + it(`removes the mirror after ${name} and keeps every real file`, async () => { + const seen = {}; + const run = runFakeJob({ inRepo: true, stryker: inspectsSandbox(seen, finish) }); + const result = await run.promise.then( + () => 'resolved', + (error) => error.exitCode, + ); + assert.equal(result, outcome); + assert.equal(seen.mirrorExists, true); + assert.equal(existsSync(mirrorParent()), false); + assert.equal(readFileSync(path.join(root, SIBLING), 'utf8'), 'sibling\n'); + assert.ok(existsSync(path.join(pkgRoot, 'src/a.ts'))); + }); + } +}); + +// An exit on a signal or a crash skips every `finally`, so the handlers must +// remove the mirror before they exit. The stand-in `exit` only records the +// code, so each test records the state at the moment of the exit. +describe('runJob exits on a signal or a crash, with the sandbox in a mirror', () => { + const mirrorParent = () => path.join(root, '.stryker-tmp'); + const atExit = (p) => ({ exits: [...p.exits], mirrorExists: existsSync(mirrorParent()) }); + const handlerCounts = (p) => + ['SIGINT', 'SIGTERM', 'uncaughtException'].map((event) => p.proc.listenerCount(event)); + + const events = { SIGINT: 130, SIGTERM: 143, uncaughtException: 3 }; + const windows = { + 'before Stryker starts': { printed: /Running Stryker/, stryker: writesReport }, + 'after Stryker closed': { printed: /without producing/, stryker: (child) => child.finish(1) }, + }; + + for (const [when, { printed, stryker }] of Object.entries(windows)) { + for (const [event, code] of Object.entries(events)) { + it(`removes the mirror, then exits ${code}, on ${event} ${when}`, async () => { + const seen = {}; + const onPrint = (text, p) => { + if (seen.exits || !printed.test(text)) return; + p.proc.emit(event, new Error('boom')); + Object.assign(seen, atExit(p)); + }; + await runFakeJob({ inRepo: true, onPrint, stryker }).promise; + assert.deepEqual(seen, { exits: [code], mirrorExists: false }); + }); + } + } + + it('removes the mirror, then exits, on a signal after Stryker exited and before it closed', async () => { + const seen = {}; + const stryker = (child, _call, p) => { + child.exitCode = 0; + p.proc.emit('SIGTERM'); + Object.assign(seen, atExit(p), { kills: [...child.kills] }); + child.finish(0); + }; + await runFakeJob({ inRepo: true, stryker }).promise; + assert.deepEqual(seen, { exits: [143], mirrorExists: false, kills: [] }); + }); + + // Stryker can still work in its sandbox until it is gone. The job does not + // go on after the crash, so its promise never settles. + it('stops a live Stryker on a crash, then removes the mirror and exits 3 once it is gone', async () => { + const seen = {}; + const stryker = (child, _call, p) => { + p.proc.emit('uncaughtException', new Error('boom')); + Object.assign(seen, atExit(p), { kills: [...child.kills] }); + setImmediate(() => child.finish(130)); + }; + const run = runFakeJob({ inRepo: true, stryker }); + assert.equal(await run.exited, 3); + assert.deepEqual(seen, { exits: [], mirrorExists: true, kills: ['SIGINT'] }); + assert.equal(existsSync(mirrorParent()), false); + assert.equal(await settledState(run.promise), 'pending'); + }); + + // Two sets would both act on one signal: one would exit and remove the + // mirror while Stryker still runs in it. + it('listens with one set of handlers at a time, and with none after the job', async () => { + const counts = []; + const onPrint = (text, p) => { + if (/Running Stryker/.test(text)) counts.push(handlerCounts(p)); + }; + const stryker = (child, call, p) => { + counts.push(handlerCounts(p)); + writesReport(child, call, p); + }; + const run = runFakeJob({ inRepo: true, onPrint, stryker }); + await run.promise; + assert.deepEqual(counts, [ + [1, 1, 1], + [1, 1, 1], + ]); + assert.deepEqual(handlerCounts(run), [0, 0, 0]); + }); +}); diff --git a/scripts/mutation-health/run-outcome.test.mjs b/scripts/mutation-health/run-outcome.test.mjs new file mode 100644 index 00000000000..4a5926d4e8d --- /dev/null +++ b/scripts/mutation-health/run-outcome.test.mjs @@ -0,0 +1,155 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { classifyRun, survivorsWithNoTestRun, testsNotRunLines } from './summary.mjs'; + +// A mutant on line `line` of a report. `coveredBy` and `testsCompleted` are +// what Stryker records about its tests. `tests` can set either one to undefined. +function mutant(status, line, tests = {}) { + return { + id: String(line), + mutatorName: 'ConditionalExpression', + status, + location: { start: { line, column: 4 }, end: { line, column: 9 } }, + replacement: 'true', + coveredBy: ['t1'], + testsCompleted: 1, + ...tests, + }; +} + +const reportOf = (files) => ({ + files: Object.fromEntries( + Object.entries(files).map(([file, mutants]) => [file, { source: '', mutants }]), + ), +}); + +// A mutant that tests cover but that survived with no test run. +const notRun = (line) => mutant('Survived', line, { testsCompleted: 0 }); + +const HEALTHY = reportOf({ 'src/a.ts': [mutant('Killed', 1), mutant('Survived', 2)] }); + +describe('survivorsWithNoTestRun', () => { + it('lists each covered survivor with no test run, by its location', () => { + const report = reportOf({ 'src/a.ts': [mutant('Killed', 1), notRun(3), notRun(7)] }); + assert.deepEqual(survivorsWithNoTestRun(report), ['src/a.ts:3:4', 'src/a.ts:7:4']); + }); + + it('lists them across files, in report order', () => { + const report = reportOf({ 'src/b.ts': [notRun(2)], 'src/a.ts': [notRun(1)] }); + assert.deepEqual(survivorsWithNoTestRun(report), ['src/b.ts:2:4', 'src/a.ts:1:4']); + }); + + // Each case breaks exactly one of the three conditions. + const kept = { + 'a survivor whose covering tests ran': mutant('Survived', 1, { testsCompleted: 1 }), + 'a killed mutant with no test count': mutant('Killed', 1, { testsCompleted: 0 }), + 'a timeout': mutant('Timeout', 1, { testsCompleted: 0 }), + 'a survivor with no coverage data (the command runner)': mutant('Survived', 1, { + coveredBy: undefined, + testsCompleted: 0, + }), + 'a survivor that no test covers': mutant('Survived', 1, { coveredBy: [], testsCompleted: 0 }), + 'a survivor without a test count': mutant('Survived', 1, { testsCompleted: undefined }), + 'a mutant with no coverage': mutant('NoCoverage', 1, { coveredBy: [], testsCompleted: 0 }), + }; + for (const [what, m] of Object.entries(kept)) { + it(`leaves out ${what}`, () => { + assert.deepEqual(survivorsWithNoTestRun(reportOf({ 'src/a.ts': [m] })), []); + }); + } + + it('reads a report with no files or no mutants as empty', () => { + assert.deepEqual(survivorsWithNoTestRun({}), []); + assert.deepEqual(survivorsWithNoTestRun({ files: { 'src/a.ts': { source: '' } } }), []); + }); +}); + +describe('testsNotRunLines', () => { + const notRunReport = (count) => + reportOf({ 'src/a.ts': Array.from({ length: count }, (_, i) => notRun(i + 1)) }); + + it('names the package, the count and each location, then what it means', () => { + assert.deepEqual(testsNotRunLines('packages/a', notRunReport(2)), [ + '✗ packages/a: 2 mutant(s) survived with no test run, but tests cover them:', + ' - src/a.ts:1:4', + ' - src/a.ts:2:4', + ' The test runner did not run their covering tests, so the score is not valid.', + ' See "Vitest 5" in scripts/mutation-health/README.md.', + ]); + }); + + it('lists five locations and no remainder line when there are exactly five', () => { + const lines = testsNotRunLines('packages/a', notRunReport(5)); + assert.equal(lines.filter((line) => line.startsWith(' - ')).length, 5); + assert.equal( + lines.some((line) => line.includes('more')), + false, + ); + }); + + it('lists the first five locations and counts the rest', () => { + const lines = testsNotRunLines('packages/a', notRunReport(7)); + assert.equal( + lines[0], + '✗ packages/a: 7 mutant(s) survived with no test run, but tests cover them:', + ); + assert.deepEqual(lines.slice(1, 7), [ + ' - src/a.ts:1:4', + ' - src/a.ts:2:4', + ' - src/a.ts:3:4', + ' - src/a.ts:4:4', + ' - src/a.ts:5:4', + ' … and 2 more', + ]); + }); +}); + +describe('classifyRun', () => { + const DONE = 'Instrumented 1 source file(s) with 8 mutant(s)'; + const NO_TESTS = 'ERROR Stryker No tests were executed. Stryker will exit prematurely.'; + + it('is complete when the run wrote a report and exited zero', () => { + assert.equal(classifyRun({ exitCode: 0, output: DONE, report: HEALTHY }), 'complete'); + }); + + it('is partial when the run wrote a report and then exited non-zero', () => { + assert.equal(classifyRun({ exitCode: 1, output: DONE, report: HEALTHY }), 'partial'); + }); + + it('is no-tests when nothing covers the target', () => { + assert.equal(classifyRun({ exitCode: 1, output: NO_TESTS, report: null }), 'no-tests'); + }); + + // With the command runner, a test command that matches no file fails the + // dry run. Vitest says so, and the run is the same score-0 result. + it('is no-tests when the test command finds no test file', () => { + const output = 'No test files found, exiting with code 1'; + assert.equal(classifyRun({ exitCode: 1, output, report: null }), 'no-tests'); + }); + + it('is failed when the run produced no report', () => { + assert.equal(classifyRun({ exitCode: 1, output: 'SIGABRT', report: null }), 'failed'); + }); + + // The caller deletes the previous reports before each run. Without that, a + // crashed run finds the earlier report and is classified `partial`, so it + // reports the earlier target and its score instead of failing. + it('trusts the report as this run only — a report plus a crash is partial, never failed', () => { + assert.equal(classifyRun({ exitCode: 3, output: 'SIGABRT', report: HEALTHY }), 'partial'); + }); + + // Same trap for the no-tests path: a leftover report used to suppress it, and + // a genuine score-0 red was reported as the earlier run's passing score. + it('still detects no-tests when the run crashed without a report', () => { + assert.equal(classifyRun({ exitCode: 3, output: NO_TESTS, report: null }), 'no-tests'); + }); + + // The plain vitest runner under Vitest 5: every mutant survived and Stryker + // ran 0.00 tests per mutant. A score of zero would hide the broken runner. + it('is tests-not-run when a covered mutant survived with no test run', () => { + const report = reportOf({ 'src/a.ts': [mutant('Killed', 1), notRun(2)] }); + assert.equal(classifyRun({ exitCode: 0, output: DONE, report }), 'tests-not-run'); + assert.equal(classifyRun({ exitCode: 1, output: DONE, report }), 'tests-not-run'); + }); +}); diff --git a/scripts/mutation-health/run-stryker.test.mjs b/scripts/mutation-health/run-stryker.test.mjs new file mode 100644 index 00000000000..5624dc1689e --- /dev/null +++ b/scripts/mutation-health/run-stryker.test.mjs @@ -0,0 +1,182 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { runStryker } from './stryker.mjs'; +import { MutateError } from './targets.mjs'; +import { fakeProcess, fakeSpawn, namesInPlaceMode, settledState, sink } from './test-doubles.mjs'; + +const nextTurn = () => new Promise((resolve) => setImmediate(resolve)); + +// Start runStryker with a fake spawn and a stand-in process. +const argv = ['/bin/stryker.js', 'run', '/pkg/reports/mutation/stryker.run.json']; + +function start(onSpawn, { onExit, crashStopGraceMs } = {}) { + const doubles = fakeSpawn({ onSpawn }); + const p = fakeProcess(); + const out = sink(); + const err = sink(); + const io = { + spawn: doubles.spawn, + stdout: out, + stderr: err, + onExit, + crashStopGraceMs, + proc: p.proc, + exit: p.exit, + write: p.write, + }; + return { ...doubles, ...p, out, err, promise: runStryker({ argv, cwd: '/pkg' }, io) }; +} + +describe('runStryker', () => { + it('starts node on the Stryker binary in the package dir, with no in place flag', async () => { + const run = start((child) => setImmediate(() => child.finish(0))); + await run.promise; + assert.equal(run.calls.length, 1); + const [{ command, args, options }] = run.calls; + assert.equal(command, process.execPath); + assert.deepEqual(args, argv); + assert.equal(options.cwd, '/pkg'); + // Stryker reads no input. Its output is kept for the classification. + assert.deepEqual(options.stdio, ['inherit', 'pipe', 'pipe']); + assert.equal(args.some(namesInPlaceMode), false); + }); + + // The command list is the whole set of processes a run starts: no git, so + // nothing can check out files over the working tree. + it('starts no process other than Stryker', async () => { + const run = start((child) => setImmediate(() => child.finish(0))); + await run.promise; + assert.deepEqual( + run.calls.map((call) => call.command), + [process.execPath], + ); + }); + + it("streams Stryker's output through and keeps it for the caller", async () => { + const run = start((child) => + setImmediate(() => { + child.stdout.emit('data', Buffer.from('Instrumented 1 source file(s) with 3 mutant(s)\n')); + child.stderr.emit('data', Buffer.from('ERROR something\n')); + child.finish(1); + }), + ); + const result = await run.promise; + assert.deepEqual(result, { + exitCode: 1, + output: 'Instrumented 1 source file(s) with 3 mutant(s)\nERROR something\n', + cancelledBy: null, + }); + assert.equal(run.out.text(), 'Instrumented 1 source file(s) with 3 mutant(s)\n'); + assert.equal(run.err.text(), 'ERROR something\n'); + }); +}); + +describe('runStryker signals and start errors', () => { + it('passes a SIGINT on to a live Stryker and reports the run as cancelled', async () => { + const run = start(); + run.proc.emit('SIGINT'); + assert.deepEqual(run.children[0].kills, ['SIGINT']); + assert.deepEqual(run.exits, []); + run.children[0].finish(130); + assert.equal((await run.promise).cancelledBy, 'SIGINT'); + // The run is over, so its handlers are gone. + assert.equal(run.proc.listenerCount('SIGINT'), 0); + }); + + it('reports the exit code of Stryker, and 1 when a signal ended it', async () => { + for (const [code, expected] of [ + [2, 2], + [null, 1], + ]) { + const run = start((child) => setImmediate(() => child.emit('close', code))); + assert.equal((await run.promise).exitCode, expected); + } + }); + + it('does not forward a signal to a Stryker that a signal already ended', async () => { + const run = start(); + run.children[0].signalCode = 'SIGKILL'; + run.proc.emit('SIGINT'); + assert.deepEqual(run.exits, [130]); + assert.deepEqual(run.children[0].kills, []); + run.children[0].emit('close', null); + await run.promise; + }); + + it('runs the cleanup, then exits at once, on a signal that comes after Stryker is gone', async () => { + let cleanups = 0; + const run = start(undefined, { onExit: () => cleanups++ }); + run.children[0].exitCode = 0; + run.proc.emit('SIGTERM'); + assert.deepEqual(run.exits, [143]); + assert.equal(cleanups, 1); + assert.deepEqual(run.children[0].kills, []); + run.children[0].emit('close', 0); + await run.promise; + }); + + it('fails with exit code 3 when Stryker cannot start', async () => { + const run = start((child) => setImmediate(() => child.emit('error', new Error('ENOENT')))); + await assert.rejects(run.promise, (error) => { + assert.ok(error instanceof MutateError); + assert.equal(error.exitCode, 3); + assert.match(error.message, /Stryker failed to start: ENOENT/); + return true; + }); + assert.equal(run.proc.listenerCount('SIGINT'), 0); + }); +}); + +// The run does not go on after a crash, and the cleanup must not remove the +// mirror while Stryker still works in it. +describe('runStryker on a crash', () => { + it('stops a live Stryker on a crash, then runs the cleanup and exits 3 once it is gone', async () => { + const steps = []; + const run = start(undefined, { onExit: () => steps.push('cleanup') }); + run.children[0].kill = (signal) => steps.push(`kill ${signal}`); + run.proc.emit('uncaughtException', new Error('boom')); + await nextTurn(); + assert.deepEqual(steps, ['kill SIGINT']); + assert.deepEqual(run.exits, []); + assert.match( + run.writes.join(''), + /\nWaiting up to 10 s for Stryker to stop before the cleanup\.\n$/, + ); + run.children[0].finish(130); + assert.equal(await run.exited, 3); + assert.deepEqual(steps, ['kill SIGINT', 'cleanup']); + assert.equal(await settledState(run.promise), 'pending'); + }); + + // A second SIGINT makes Stryker exit at once. + it('stops Stryker again, then runs the cleanup and exits 3, when it is not gone in time', async () => { + const steps = []; + const run = start(undefined, { onExit: () => steps.push('cleanup'), crashStopGraceMs: 5 }); + run.children[0].kill = (signal) => steps.push(`kill ${signal}`); + run.proc.emit('uncaughtException', new Error('boom')); + assert.equal(await run.exited, 3); + assert.deepEqual(steps, ['kill SIGINT', 'kill SIGINT', 'cleanup']); + }); + + it('passes a signal during the wait on to Stryker, and exits 3 once Stryker is gone', async () => { + const run = start(); + run.proc.emit('uncaughtException', new Error('boom')); + run.proc.emit('SIGINT'); + assert.deepEqual(run.children[0].kills, ['SIGINT', 'SIGINT']); + assert.deepEqual(run.exits, []); + run.children[0].finish(130); + assert.equal(await run.exited, 3); + assert.deepEqual(run.exits, [3]); + }); + + it('does not stop a Stryker that is already gone on a crash, and exits 3 at once', async () => { + const run = start(); + run.children[0].exitCode = 1; + run.proc.emit('uncaughtException', new Error('boom')); + assert.deepEqual(run.children[0].kills, []); + assert.deepEqual(run.exits, [3]); + run.children[0].emit('close', 1); + assert.equal(await settledState(run.promise), 'pending'); + }); +}); diff --git a/scripts/mutation-health/samples.mjs b/scripts/mutation-health/samples.mjs new file mode 100644 index 00000000000..a8a8cf467a0 --- /dev/null +++ b/scripts/mutation-health/samples.mjs @@ -0,0 +1,46 @@ +/** + * Seeded generative checks for the unit tests. CI runs these tests with only + * the `.github/scripts` dependencies, so fast-check is not available. A fixed + * seed keeps each run the same, and a failure names the seed and the sample. + */ + +// mulberry32: a small 32-bit generator. It gives numbers in [0, 1). +function randomFrom(seed) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296; + }; +} + +/** Helpers that draw values from one seeded generator. */ +export function sampler(seed) { + const next = randomFrom(seed); + const int = (min, max) => min + Math.floor(next() * (max - min + 1)); + const pick = (items) => items[int(0, items.length - 1)]; + const string = (alphabet, minLength, maxLength) => + Array.from({ length: int(minLength, maxLength) }, () => pick([...alphabet])).join(''); + const array = (minLength, maxLength, draw) => + Array.from({ length: int(minLength, maxLength) }, () => draw()); + return { int, pick, string, array }; +} + +/** + * Run `check` on `count` samples. Sample `i` comes from `generate` with the + * seed `seed + i`. A failed check rethrows its error with the seed and the + * sample added to the message. + */ +export function forEachSample({ count = 200, seed = 1 }, generate, check) { + for (let i = 0; i < count; i++) { + const sample = generate(sampler(seed + i)); + try { + check(sample); + } catch (error) { + error.message = `${error.message}\nseed ${seed + i}, sample ${JSON.stringify(sample)}`; + throw error; + } + } +} diff --git a/scripts/mutation-health/sandbox-mirror.mjs b/scripts/mutation-health/sandbox-mirror.mjs new file mode 100644 index 00000000000..7e3a09763d0 --- /dev/null +++ b/scripts/mutation-health/sandbox-mirror.mjs @@ -0,0 +1,107 @@ +/** + * Give Stryker's sandbox the same place in the tree as the package it copies. + * + * Stryker copies the package to `/sandbox-/`. With the default + * temp dir that copy sits one level below the package dir, so a config path + * that leaves the package points into the temp dir and the test file does not + * load. Examples: the cli alias `path.resolve(__dirname, '../@n8n/telemetry/src')` + * and the `@nodes-testing` alias of nodes-base and nodes-langchain. + * + * The mirror fixes this. It is a chain of real directories along the + * package's path from the repo root, with a symbolic link to each real entry + * beside that path. Stryker gets the mirror of the package's parent dir as its + * temp dir, so `..` from the sandbox reaches the same files as `..` from the + * package. + * + * The mirror sits under `.stryker-tmp/` at the repo root, which git ignores. + * It holds only directories and links. Removing it never follows a link. + */ +import { + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + rmdirSync, + symlinkSync, + unlinkSync, +} from 'node:fs'; +import path from 'node:path'; + +// Entries the mirror never links. Without `.git`, git in the sandbox finds the +// real repository above the mirror, as it does from the package. A link to +// `.stryker-tmp` would put the mirror inside itself. +const UNLINKED = new Set(['.git', '.stryker-tmp']); + +function linkEntries(realDir, mirrorDir, skip) { + for (const name of readdirSync(realDir)) { + if (name === skip || UNLINKED.has(name)) continue; + symlinkSync(path.join(realDir, name), path.join(mirrorDir, name)); + } +} + +/** + * Remove a tree and never follow a link: a link is unlinked, not entered. A + * missing path is not an error, so a second call does nothing. + */ +export function removeTree(target) { + let stat; + try { + stat = lstatSync(target); + } catch { + return; + } + if (!stat.isDirectory()) { + unlinkSync(target); + return; + } + for (const name of readdirSync(target)) removeTree(path.join(target, name)); + rmdirSync(target); +} + +/** + * The path segments from the repo root to the package, or null when the + * package is the repo root or lies outside it. Such a package keeps Stryker's + * default temp dir. + */ +export function mirrorSegments(pkgRoot, repoRoot) { + const rel = path.relative(repoRoot, pkgRoot); + if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) return null; + return rel.split(path.sep); +} + +/** + * Build the mirror for the package at `pkgRoot` under `parentDir`. Returns the + * temp dir for Stryker and a `dispose` that removes the mirror, or null when + * the package needs no mirror. + */ +export function createSandboxMirror(pkgRoot, { repoRoot, parentDir }) { + const segments = mirrorSegments(pkgRoot, repoRoot); + if (!segments) return null; + mkdirSync(parentDir, { recursive: true }); + const mirrorRoot = mkdtempSync(path.join(parentDir, 'mirror-')); + let realDir = repoRoot; + let mirrorDir = mirrorRoot; + try { + for (const [index, segment] of segments.entries()) { + linkEntries(realDir, mirrorDir, segment); + if (index === segments.length - 1) break; + realDir = path.join(realDir, segment); + mirrorDir = path.join(mirrorDir, segment); + mkdirSync(mirrorDir); + } + } catch (error) { + removeMirror(mirrorRoot, parentDir); + throw error; + } + return { tempDir: mirrorDir, dispose: () => removeMirror(mirrorRoot, parentDir) }; +} + +// Remove the mirror, then its parent dir once no other run's mirror is in it. +function removeMirror(mirrorRoot, parentDir) { + removeTree(mirrorRoot); + try { + rmdirSync(parentDir); + } catch { + // Another run's mirror is still there, or the dir is already gone. + } +} diff --git a/scripts/mutation-health/sandbox-mirror.test.mjs b/scripts/mutation-health/sandbox-mirror.test.mjs new file mode 100644 index 00000000000..151f73664e7 --- /dev/null +++ b/scripts/mutation-health/sandbox-mirror.test.mjs @@ -0,0 +1,171 @@ +import { afterEach, beforeEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { createSandboxMirror, mirrorSegments, removeTree } from './sandbox-mirror.mjs'; + +let repo; + +beforeEach(() => { + repo = mkdtempSync(path.join(tmpdir(), 'mutate-mirror-')); +}); + +afterEach(() => { + rmSync(repo, { recursive: true, force: true }); +}); + +// A small repo: the package under test, two packages its config reaches with +// `..`, a root dependency and a git dir. +const REAL_FILES = { + pkgSource: 'packages/@n8n/pkg/src/a.ts', + telemetry: 'packages/@n8n/telemetry/src/index.ts', + nodesTesting: 'packages/core/nodes-testing/helper.ts', + rootDependency: 'node_modules/dep/index.js', + gitHead: '.git/HEAD', +}; + +function seedRepo() { + for (const [name, file] of Object.entries(REAL_FILES)) { + mkdirSync(path.dirname(path.join(repo, file)), { recursive: true }); + writeFileSync(path.join(repo, file), `${name}\n`); + } + return path.join(repo, 'packages/@n8n/pkg'); +} + +function mirrorFor(pkgRoot) { + return createSandboxMirror(pkgRoot, { + repoRoot: repo, + parentDir: path.join(repo, '.stryker-tmp'), + }); +} + +// Where Stryker would put the copy of the package. +function sandboxIn(tempDir) { + const sandbox = path.join(tempDir, 'sandbox-1'); + mkdirSync(sandbox); + return sandbox; +} + +describe('createSandboxMirror', () => { + it('puts the temp dir at the same depth as the package', () => { + const mirror = mirrorFor(seedRepo()); + assert.match(mirror.tempDir, /\.stryker-tmp\/mirror-[^/]+\/packages\/@n8n$/); + mirror.dispose(); + }); + + // The cli alias `../@n8n/telemetry/src` and the `@nodes-testing` alias + // `../../core/nodes-testing` both have to reach the real files. + it('lets `..` paths from the sandbox reach the same files as from the package', () => { + const mirror = mirrorFor(seedRepo()); + const sandbox = sandboxIn(mirror.tempDir); + const read = (rel) => readFileSync(path.resolve(sandbox, rel), 'utf8'); + assert.equal(read('../telemetry/src/index.ts'), 'telemetry\n'); + assert.equal(read('../../core/nodes-testing/helper.ts'), 'nodesTesting\n'); + assert.equal(read('../../../node_modules/dep/index.js'), 'rootDependency\n'); + mirror.dispose(); + }); + + // The sandbox is the package. git in the sandbox must find the real + // repository above the mirror, not a link to its git dir. + it('links neither the package itself, nor the git dir, nor the temp dir', () => { + const mirror = mirrorFor(seedRepo()); + const mirrorRoot = path.resolve(mirror.tempDir, '../..'); + assert.equal(existsSync(path.join(mirror.tempDir, 'pkg')), false); + assert.equal(existsSync(path.join(mirrorRoot, '.git')), false); + assert.equal(existsSync(path.join(mirrorRoot, '.stryker-tmp')), false); + mirror.dispose(); + }); + + it('needs no mirror for a package at the repo root or outside the repo', () => { + assert.equal(mirrorFor(repo), null); + assert.equal(mirrorFor(path.join(tmpdir(), 'elsewhere')), null); + }); + + it('removes a half-built mirror when a dir on the path cannot be read', () => { + seedRepo(); + assert.throws(() => mirrorFor(path.join(repo, 'packages/missing/pkg')), /ENOENT/); + assert.equal(existsSync(path.join(repo, '.stryker-tmp')), false); + }); +}); + +describe('the mirror dispose', () => { + it('removes the mirror and leaves every real file in place', () => { + const mirror = mirrorFor(seedRepo()); + mirror.dispose(); + assert.equal(existsSync(path.join(repo, '.stryker-tmp')), false); + for (const file of Object.values(REAL_FILES)) + assert.ok(existsSync(path.join(repo, file)), file); + }); + + // Stryker links `node_modules` into its sandbox. A crash can leave the + // sandbox behind, and removing it must not enter that link. + it('removes a sandbox left behind without following its links', () => { + const mirror = mirrorFor(seedRepo()); + const sandbox = sandboxIn(mirror.tempDir); + writeFileSync(path.join(sandbox, 'copy.ts'), 'copy\n'); + symlinkSync(path.join(repo, 'node_modules'), path.join(sandbox, 'node_modules')); + mirror.dispose(); + assert.equal(existsSync(sandbox), false); + assert.ok(existsSync(path.join(repo, REAL_FILES.rootDependency))); + }); + + // Runs in two packages can overlap. One run must not remove the other's mirror. + it("keeps the parent dir while another run's mirror is in it", () => { + const pkgRoot = seedRepo(); + const first = mirrorFor(pkgRoot); + const second = mirrorFor(pkgRoot); + first.dispose(); + assert.ok(existsSync(second.tempDir)); + second.dispose(); + assert.equal(existsSync(path.join(repo, '.stryker-tmp')), false); + }); + + it('does nothing the second time', () => { + const mirror = mirrorFor(seedRepo()); + mirror.dispose(); + mirror.dispose(); + assert.ok(existsSync(path.join(repo, REAL_FILES.telemetry))); + }); +}); + +describe('removeTree', () => { + it('unlinks a link to a directory and keeps the directory', () => { + seedRepo(); + const link = path.join(repo, 'link'); + symlinkSync(path.join(repo, 'packages'), link); + removeTree(link); + assert.throws(() => lstatSync(link), /ENOENT/); + assert.ok(existsSync(path.join(repo, REAL_FILES.pkgSource))); + }); + + it('ignores a path that does not exist', () => { + assert.doesNotThrow(() => removeTree(path.join(repo, 'missing'))); + }); +}); + +describe('mirrorSegments', () => { + it('gives the path from the repo root to the package', () => { + assert.deepEqual(mirrorSegments('/repo/packages/@n8n/pkg', '/repo'), [ + 'packages', + '@n8n', + 'pkg', + ]); + }); + + it('gives null for the repo root and for a path outside it', () => { + assert.equal(mirrorSegments('/repo', '/repo'), null); + assert.equal(mirrorSegments('/other/pkg', '/repo'), null); + assert.equal(mirrorSegments('/repository/pkg', '/repo'), null); + }); +}); diff --git a/scripts/mutation-health/stryker-process.test.mjs b/scripts/mutation-health/stryker-process.test.mjs new file mode 100644 index 00000000000..ff173c9c8a3 --- /dev/null +++ b/scripts/mutation-health/stryker-process.test.mjs @@ -0,0 +1,279 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { exitCodeForSignal, guardOutsideStryker, registerSignalHandlers } from './stryker.mjs'; +import { fakeProcess } from './test-doubles.mjs'; + +const nextTurn = () => new Promise((resolve) => setImmediate(resolve)); + +describe('exitCodeForSignal', () => { + it("gives the shell's code for each signal", () => { + assert.equal(exitCodeForSignal('SIGINT'), 130); + assert.equal(exitCodeForSignal('SIGTERM'), 143); + }); +}); + +// Handlers on a stand-in process, which record each exit and each message. +function harness(onSignal, onExit) { + const p = fakeProcess(); + const handlers = registerSignalHandlers({ + onSignal, + onExit, + proc: p.proc, + exit: p.exit, + write: p.write, + }); + return { ...p, handlers }; +} + +describe('registerSignalHandlers', () => { + it('exits 130 on SIGINT and 143 on SIGTERM when no run is alive', () => { + const h = harness(() => false); + h.proc.emit('SIGINT'); + h.proc.emit('SIGTERM'); + assert.deepEqual(h.exits, [130, 143]); + }); + + // A live Stryker gets the signal. The run ends once Stryker is gone. + it('defers to a live run instead of exiting', () => { + const seen = []; + const h = harness((signal) => { + seen.push(signal); + return true; + }); + h.proc.emit('SIGINT'); + h.proc.emit('SIGTERM'); + assert.deepEqual(seen, ['SIGINT', 'SIGTERM']); + assert.deepEqual(h.exits, []); + }); + + it('exits 3 on an uncaught exception, never 1, and prints its stack', () => { + const h = harness(() => false); + h.proc.emit('uncaughtException', new Error('boom')); + assert.deepEqual(h.exits, [3]); + assert.match(h.writes.join(''), /mutate\.mjs crashed: Error: boom\n {4}at /); + }); + + it('prints a thrown value that is not an Error', () => { + const h = harness(() => false); + h.proc.emit('uncaughtException', 'plain text'); + assert.match(h.writes.join(''), /mutate\.mjs crashed: plain text\n$/); + }); + + it('still exits 3 when the thrown value is undefined', () => { + const h = harness(() => false); + h.proc.emit('uncaughtException', undefined); + assert.deepEqual(h.exits, [3]); + assert.match(h.writes.join(''), /mutate\.mjs crashed: undefined\n$/); + }); + + it('exits on a signal when no run registered a signal callback', () => { + const h = harness(undefined); + h.proc.emit('SIGINT'); + assert.deepEqual(h.exits, [130]); + }); +}); + +describe('registerSignalHandlers cleanup', () => { + // An exit skips every `finally`, so the cleanup must run before it. + it('runs onExit before each exit it makes', () => { + const steps = []; + const p = fakeProcess(); + registerSignalHandlers({ + onSignal: () => false, + onExit: () => steps.push('cleanup'), + proc: p.proc, + exit: (code) => steps.push(`exit ${code}`), + write: p.write, + }); + p.proc.emit('SIGINT'); + p.proc.emit('SIGTERM'); + p.proc.emit('uncaughtException', new Error('boom')); + assert.deepEqual(steps, ['cleanup', 'exit 130', 'cleanup', 'exit 143', 'cleanup', 'exit 3']); + }); + + it('runs onExit only when it exits, not when a live run takes the signal', () => { + let cleanups = 0; + const h = harness( + () => true, + () => cleanups++, + ); + h.proc.emit('SIGINT'); + assert.equal(cleanups, 0); + assert.deepEqual(h.exits, []); + }); + + // The exit code contract holds even when the cleanup fails. + it('still exits with the same code when onExit throws, and prints why', () => { + const h = harness( + () => false, + () => { + throw new Error('EBUSY'); + }, + ); + h.proc.emit('SIGTERM'); + h.proc.emit('uncaughtException', new Error('boom')); + assert.deepEqual(h.exits, [143, 3]); + const printed = h.writes.join(''); + assert.equal(printed.match(/\n✗ Cleanup before exit failed: EBUSY\n/g).length, 2); + }); + + it('stops listening after dispose, so the next job owns its own handlers', () => { + const h = harness(() => false); + h.handlers.dispose(); + h.proc.emit('SIGINT'); + h.proc.emit('SIGTERM'); + h.proc.emit('uncaughtException', new Error('boom')); + assert.deepEqual(h.exits, []); + for (const event of ['SIGINT', 'SIGTERM', 'uncaughtException']) { + assert.equal(h.proc.listenerCount(event), 0); + } + }); +}); + +// A crash while Stryker runs waits for Stryker to stop before the cleanup. +describe('registerSignalHandlers with onCrash', () => { + function crashHarness(onCrash) { + const steps = []; + const p = fakeProcess(); + registerSignalHandlers({ + onSignal: () => false, + onCrash, + onExit: () => steps.push('cleanup'), + proc: p.proc, + exit: (code) => { + steps.push(`exit ${code}`); + p.exit(code); + }, + write: p.write, + }); + return { ...p, steps }; + } + + it('runs the cleanup and exits 3 at once when onCrash has nothing to wait for', () => { + const h = crashHarness(() => null); + h.proc.emit('uncaughtException', new Error('boom')); + assert.deepEqual(h.steps, ['cleanup', 'exit 3']); + }); + + it('runs the cleanup and exits 3 only after the promise of onCrash settles', async () => { + let stopped; + const h = crashHarness( + () => + new Promise((resolve) => { + stopped = resolve; + }), + ); + h.proc.emit('uncaughtException', new Error('boom')); + await nextTurn(); + assert.deepEqual(h.steps, []); + stopped(); + assert.equal(await h.exited, 3); + assert.deepEqual(h.steps, ['cleanup', 'exit 3']); + }); + + it('still runs the cleanup and exits 3 when the promise of onCrash rejects', async () => { + const h = crashHarness(() => Promise.reject(new Error('EPERM'))); + h.proc.emit('uncaughtException', new Error('boom')); + assert.equal(await h.exited, 3); + assert.deepEqual(h.steps, ['cleanup', 'exit 3']); + }); + + it('prints a second crash while it waits, but waits and exits only once', async () => { + let stopped; + let waits = 0; + const h = crashHarness(() => { + waits++; + return new Promise((resolve) => { + stopped = resolve; + }); + }); + h.proc.emit('uncaughtException', new Error('first')); + h.proc.emit('uncaughtException', new Error('second')); + stopped(); + await h.exited; + await nextTurn(); + assert.equal(waits, 1); + assert.deepEqual(h.steps, ['cleanup', 'exit 3']); + assert.match(h.writes.join(''), /crashed: Error: first\n[\s\S]*crashed: Error: second\n/); + }); +}); + +describe('guardOutsideStryker', () => { + const events = ['SIGINT', 'SIGTERM', 'uncaughtException']; + const counts = (p) => events.map((event) => p.proc.listenerCount(event)); + + function guarded() { + const p = fakeProcess(); + let cleanups = 0; + const guard = guardOutsideStryker({ + onExit: () => cleanups++, + proc: p.proc, + exit: p.exit, + write: p.write, + }); + return { ...p, guard, cleanups: () => cleanups }; + } + + it('listens from the start, and runs the cleanup before it exits', () => { + const h = guarded(); + assert.deepEqual(counts(h), [1, 1, 1]); + h.proc.emit('SIGINT'); + assert.deepEqual(h.exits, [130]); + assert.equal(h.cleanups(), 1); + }); + + it('stops listening on release and listens once more on hold', () => { + const h = guarded(); + h.guard.release(); + assert.deepEqual(counts(h), [0, 0, 0]); + h.proc.emit('SIGTERM'); + assert.deepEqual(h.exits, []); + h.guard.hold(); + h.guard.hold(); + assert.deepEqual(counts(h), [1, 1, 1]); + h.proc.emit('SIGTERM'); + assert.deepEqual(h.exits, [143]); + }); + + it('can release twice', () => { + const h = guarded(); + h.guard.release(); + h.guard.release(); + assert.deepEqual(counts(h), [0, 0, 0]); + }); +}); + +// Without stand-ins the handlers end the real process and write to its stderr. +describe('registerSignalHandlers in a real process', () => { + const strykerUrl = pathToFileURL(path.join(import.meta.dirname, 'stryker.mjs')).href; + + // Run `body` in a new node process after it registers the handlers. + function runWithHandlers(body) { + const script = [ + `import { registerSignalHandlers } from '${strykerUrl}';`, + 'registerSignalHandlers({ onSignal: () => false });', + body, + // Keep the process alive until a handler ends it. + 'setTimeout(() => {}, 10_000);', + ].join('\n'); + return spawnSync(process.execPath, ['--input-type=module', '--eval', script], { + encoding: 'utf8', + timeout: 20_000, + }); + } + + it('exits 130 on SIGINT and 143 on SIGTERM', () => { + assert.equal(runWithHandlers("process.kill(process.pid, 'SIGINT');").status, 130); + assert.equal(runWithHandlers("process.kill(process.pid, 'SIGTERM');").status, 143); + }); + + it('exits 3 and prints the stack to stderr on an uncaught exception', () => { + const res = runWithHandlers("setImmediate(() => { throw new Error('boom'); });"); + assert.equal(res.status, 3); + assert.match(res.stderr, /mutate\.mjs crashed: Error: boom\n {4}at /); + }); +}); diff --git a/scripts/mutation-health/stryker.default.mjs b/scripts/mutation-health/stryker.default.mjs index 8f4b54f3235..3930433db20 100644 --- a/scripts/mutation-health/stryker.default.mjs +++ b/scripts/mutation-health/stryker.default.mjs @@ -10,6 +10,11 @@ * * Reporter paths are relative to the run cwd, so reports land in * /reports/mutation/ and mutate.mjs reads raw.json from there. + * + * mutate.mjs runs this config with a few changes (see stryker.mjs): the + * `vitest-compat` runner in place of `vitest`, because the plain runner kills + * no mutant under Vitest 5 (see vitest-compat.mjs), and Stryker's sandbox in a + * mirror of the repo (see sandbox-mirror.mjs). */ /** @type {import('@stryker-mutator/api/core').PartialStrykerOptions} */ export default { @@ -26,7 +31,8 @@ export default { // static ones is an acceptable trade. Scores tick up slightly on // static-heavy files (smaller denominator) — expected, not a regression. ignoreStatic: true, - // Empty — mutate.mjs always passes --mutate . + // Empty: mutate.mjs sets `mutate` to the run's targets in the run config + // that it gives Stryker (reports/mutation/stryker.run.json). mutate: [], htmlReporter: { fileName: 'reports/mutation/raw.html' }, jsonReporter: { fileName: 'reports/mutation/raw.json' }, diff --git a/scripts/mutation-health/stryker.mjs b/scripts/mutation-health/stryker.mjs new file mode 100644 index 00000000000..76370c41693 --- /dev/null +++ b/scripts/mutation-health/stryker.mjs @@ -0,0 +1,410 @@ +/** + * Build and start one Stryker run. Runs always use Stryker's sandbox: Stryker + * copies the package into a temp dir under `.stryker-tmp/` and mutates the + * copy (see sandbox-mirror.mjs for where). The working tree is never written, + * so nothing has to be restored after a run. + * + * The run has one of two test runners: + * - `vitest-compat` (the default): Stryker's vitest runner, made to work with + * Vitest 5 (see vitest-compat.mjs). It records which tests cover each + * mutant and runs only those, so a run takes seconds. + * - `command` (`--test-command`): runs a shell command once per mutant. It + * works for a package without a vitest `test` script, but it runs every + * named test for each mutant and records no coverage, so it is slow. + */ +import { spawn as nodeSpawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { MutateError, defaultConfigNameFor, repoRoot } from './targets.mjs'; +import { COMPAT_RUNNER_NAME } from './vitest-compat.mjs'; + +const VITEST_RUNNER_PLUGIN = '@stryker-mutator/vitest-runner'; +export const COMPAT_RUNNER_PLUGIN = path.join(import.meta.dirname, 'vitest-compat-runner.mjs'); + +// mutate.mjs reads the raw report from here, relative to the package dir. +export const RAW_REPORT = 'reports/mutation/raw.json'; + +// Folders at the package root that no test reads. Without this, Stryker copies +// them into each sandbox. +const SANDBOX_IGNORE = [ + '/reports/mutation', + '/coverage', + '/test-results', + '/playwright-report', + '/.playwright-browsers', + '/ms-playwright-cache', +]; + +// Stryker's option for in place mode, matched in any letter case. +const IN_PLACE_OPTION = /^inplace$/i; + +const unique = (items) => [...new Set(items)]; + +/** + * The config file for a run: `--config`, then the package's own config, then + * the shared config for the package. `root` is the repo that a relative + * `--config` and the package dir are read from. + */ +export function resolveConfig(pkgRoot, configArg, root = repoRoot) { + if (configArg) return path.resolve(root, configArg); + const local = path.join(pkgRoot, 'stryker.config.mjs'); + if (existsSync(local)) return local; + return path.join(import.meta.dirname, defaultConfigNameFor(path.relative(root, pkgRoot))); +} + +// Try the package's own copy first, then the root devDep. A package that pins +// Stryker thus gets the version it pinned. A miss is a broken checkout, not a +// red gate: exit 3 keeps it distinct from a score of zero. `rootFrom` is the +// file the root devDep resolves from; the unit tests point it at a temp dir. +export function resolveStrykerBin(pkgRoot, packageDir, { rootFrom = import.meta.url } = {}) { + for (const from of [path.join(pkgRoot, 'package.json'), rootFrom]) { + try { + const resolved = createRequire(from).resolve('@stryker-mutator/core/package.json'); + return path.join(path.dirname(resolved), 'bin/stryker.js'); + } catch { + continue; + } + } + throw new MutateError( + 3, + `Could not resolve @stryker-mutator/core from ${packageDir} or the repo root. ` + + 'Run `pnpm install` — it is a root devDep.', + ); +} + +/** Read a Stryker config file the way Stryker does: JSON, or a module's default export. */ +export async function loadBaseConfig(configPath) { + if (!existsSync(configPath)) throw new MutateError(2, `Stryker config not found: ${configPath}`); + try { + if (configPath.endsWith('.json')) return JSON.parse(readFileSync(configPath, 'utf8')); + const module = await import(pathToFileURL(configPath).href); + return module.default ?? {}; + } catch (error) { + throw new MutateError(2, `Could not read the Stryker config ${configPath}: ${error.message}`); + } +} + +function workingTreeModeError(base, configLabel) { + const option = Object.keys(base).find((key) => IN_PLACE_OPTION.test(key)); + if (!option || !base[option]) return null; + return ( + `${configLabel} sets "${option}", which makes Stryker write mutants into the working tree.\n` + + 'mutate.mjs runs Stryker only in its sandbox copy of the package. Remove the option.' + ); +} + +// The TypeScript checker type-checks with the file that `tsconfigFile` names. +// A run in the mirror sets that option to a file that does not exist (see +// NO_TSCONFIG_REWRITE), so the checker cannot work there. +function typescriptCheckerError(base, configLabel) { + if (!Array.isArray(base.checkers) || !base.checkers.includes('typescript')) return null; + return ( + `${configLabel} uses the "typescript" checker, which mutate.mjs does not support.\n` + + "mutate.mjs turns off Stryker's tsconfig rewrite (tsconfigFile), and the checker needs that " + + 'file. Remove "typescript" from "checkers".' + ); +} + +/** Why a config cannot run, or null when it can. `configLabel` names the file. */ +export function sandboxConfigError(base, configLabel) { + return workingTreeModeError(base, configLabel) ?? typescriptCheckerError(base, configLabel); +} + +function shellQuote(arg) { + return /^[\w@%+=:,./-]+$/.test(arg) ? arg : `'${arg.replaceAll("'", "'\\''")}'`; +} + +/** The test command with its file arguments, each quoted for the shell. */ +export function toCommandLine(command, args = []) { + return [command.trim(), ...args.map(shellQuote)].join(' '); +} + +/** + * Keep each runner's incremental results in their own file. The runners name + * tests differently, so results from one cannot be reused by the other. This + * also drops the results that the plain vitest runner recorded under Vitest 5, + * when no mutant was ever killed. + */ +export function runnerIncrementalFile(file, runner) { + const ext = path.extname(file); + return `${file.slice(0, file.length - ext.length)}.${runner}${ext}`; +} + +function incrementalFields(base, runner) { + if (!base.incremental) return {}; + const file = base.incrementalFile ?? 'reports/stryker-incremental.json'; + return { incrementalFile: runnerIncrementalFile(file, runner) }; +} + +// Stryker rewrites each tsconfig path that leaves the package, because its +// default sandbox sits two levels below the package. A sandbox in the mirror +// (see sandbox-mirror.mjs) has the package's own depth, so those paths are +// already right. Stryker rewrites only the file that `tsconfigFile` names, so +// a name that matches no file turns the rewrite off. +export const NO_TSCONFIG_REWRITE = 'tsconfig.no-stryker-rewrite.json'; + +function sandboxFields(tempDir) { + return tempDir ? { tempDirName: tempDir, tsconfigFile: NO_TSCONFIG_REWRITE } : {}; +} + +function runFields(base, { targets, tempDir }) { + return { + mutate: [...targets], + ...sandboxFields(tempDir), + // Also remove the sandbox after a failed run: a copy of a large package + // costs disk space, and the report does not need it. + cleanTempDir: 'always', + ignorePatterns: unique([...(base.ignorePatterns ?? []), ...SANDBOX_IGNORE]), + reporters: unique([...(base.reporters ?? ['clear-text']), 'json']), + jsonReporter: { ...base.jsonReporter, fileName: RAW_REPORT }, + }; +} + +function commandRunnerFields(base, testCommand, testFiles) { + return { + ...incrementalFields(base, 'command'), + testRunner: 'command', + commandRunner: { command: toCommandLine(testCommand, testFiles) }, + // The command passes or fails as a whole, so there is no per-test coverage. + coverageAnalysis: 'off', + // Stryker finds static mutants only through per-test coverage. + ignoreStatic: false, + }; +} + +function usesVitestRunner(base) { + return [undefined, 'vitest', COMPAT_RUNNER_NAME].includes(base.testRunner); +} + +function vitestRunnerFields(base) { + return { + ...incrementalFields(base, COMPAT_RUNNER_NAME), + testRunner: COMPAT_RUNNER_NAME, + plugins: unique([...(base.plugins ?? []), VITEST_RUNNER_PLUGIN, COMPAT_RUNNER_PLUGIN]), + }; +} + +/** + * The whole config for one run: the resolved config file plus this run's + * fields. A config with the vitest runner gets the `vitest-compat` runner. + * Test files go to Stryker's `testFiles` field for that runner. The command + * runner does not accept `testFiles`, so they go onto the end of the command. + * `tempDir`, when given, is where Stryker puts its sandbox. + */ +export function buildStrykerConfig({ base = {}, targets, testFiles = [], testCommand, tempDir }) { + if (testCommand !== undefined) { + const { testFiles: _notForCommandRunner, ...rest } = base; + return { + ...rest, + ...runFields(base, { targets, tempDir }), + ...commandRunnerFields(base, testCommand, testFiles), + }; + } + const config = { ...base, ...runFields(base, { targets, tempDir }) }; + if (usesVitestRunner(base)) Object.assign(config, vitestRunnerFields(base)); + if (testFiles.length > 0) config.testFiles = [...testFiles]; + return config; +} + +/** + * Write the run config next to the reports. It is the exact config Stryker + * reads, so it can name a sandbox mirror that exists only during the run. + */ +export async function writeRunConfig(reportDir, config) { + const file = path.join(reportDir, 'stryker.run.json'); + await writeFile(file, `${JSON.stringify(config, null, 2)}\n`); + return file; +} + +/** The arguments for `node` that start Stryker on a run config, and nothing else. */ +export function strykerRunArgv(strykerBin, runConfigPath) { + return [strykerBin, 'run', runConfigPath]; +} + +export function exitCodeForSignal(signal) { + return signal === 'SIGTERM' ? 143 : 130; +} + +// How long the tool waits after a crash for a live Stryker to stop. On SIGINT +// Stryker saves its incremental results and exits, which takes about a second. +// The limit keeps a Stryker that does not stop from holding the tool. +export const CRASH_STOP_GRACE_MS = 10_000; + +const writeStderr = (msg) => process.stderr.write(msg); + +/** + * Handle a crash and a cancellation during a run. + * + * `onSignal` gets first refusal on SIGINT and SIGTERM. It returns true when it + * told a live Stryker to stop: the run then exits once Stryker is gone, and + * the caller's `finally` removes the mirror with Stryker's sandbox in it. + * Otherwise this handler exits at once. + * + * `onCrash` runs on an uncaught exception. When it returns a promise, the exit + * waits until the promise settles, for example until a live Stryker is gone. + * A second crash while it waits does not exit again. + * + * `onExit` runs before each exit that these handlers make. An exit skips every + * `finally` block, so `onExit` does the cleanup that a `finally` would do. A + * failed cleanup is printed and the exit code stays the same. + * + * `proc`, `exit` and `write` are injected so the unit tests can drive the + * handlers without signalling or ending the test runner. + */ +export function registerSignalHandlers({ + onSignal, + onCrash, + onExit, + proc = process, + exit = (code) => process.exit(code), + write = writeStderr, +}) { + const leave = (code) => { + try { + onExit?.(); + } catch (error) { + write(`\n✗ Cleanup before exit failed: ${error?.message ?? error}\n`); + } + exit(code); + }; + let waitingAfterCrash = false; + const handleUncaught = (err) => { + write(`\n✗ mutate.mjs crashed: ${err?.stack ?? err}\n`); + if (waitingAfterCrash) return; + const stopped = onCrash?.(); + if (!stopped) { + leave(3); + return; + } + waitingAfterCrash = true; + const leaveAfterCrash = () => leave(3); + stopped.then(leaveAfterCrash, leaveAfterCrash); + }; + const handleSignal = (signal) => { + if (onSignal?.(signal)) return; + leave(exitCodeForSignal(signal)); + }; + const handleSigint = () => handleSignal('SIGINT'); + const handleSigterm = () => handleSignal('SIGTERM'); + + proc.on('uncaughtException', handleUncaught); + proc.on('SIGINT', handleSigint); + proc.on('SIGTERM', handleSigterm); + + return { + dispose() { + proc.off('uncaughtException', handleUncaught); + proc.off('SIGINT', handleSigint); + proc.off('SIGTERM', handleSigterm); + }, + }; +} + +/** + * Handlers for the parts of a job before and after Stryker runs. There, a + * signal or a crash exits at once, after `io.onExit`. While Stryker runs, + * runStryker has its own handlers: call `release` just before runStryker and + * `hold` once it settles, so only one set of handlers listens at a time. + * `io` takes the same fields as registerSignalHandlers. + */ +export function guardOutsideStryker(io) { + let handlers = registerSignalHandlers(io); + return { + hold() { + handlers ??= registerSignalHandlers(io); + }, + release() { + handlers?.dispose(); + handlers = null; + }, + }; +} + +function isRunning(child) { + return child.exitCode === null && child.signalCode === null; +} + +// Tell a live Stryker to stop. Settles once it is gone, or after `graceMs`. +function stopWithin(child, graceMs) { + return new Promise((resolve) => { + const timer = setTimeout(resolve, graceMs); + child.once('close', () => { + clearTimeout(timer); + resolve(); + }); + child.kill('SIGINT'); + }); +} + +// The handlers while Stryker runs. `run` records a cancel and a crash. +function strykerHandlers(child, run, { onExit, crashStopGraceMs, ...handlerIo }) { + const write = handlerIo.write ?? writeStderr; + return registerSignalHandlers({ + ...handlerIo, + onSignal: (signal) => { + if (!isRunning(child)) return false; + run.cancelledBy = signal; + child.kill('SIGINT'); + return true; + }, + // The run does not go on after a crash. A live Stryker gets a short time + // to stop, so the cleanup does not remove the mirror while Stryker uses it. + onCrash: () => { + run.crashed = true; + if (!isRunning(child)) return null; + write(`Waiting up to ${crashStopGraceMs / 1000} s for Stryker to stop before the cleanup.\n`); + return stopWithin(child, crashStopGraceMs); + }, + // Stop a Stryker that is still alive, so it does not run on without the tool. + onExit: () => { + if (isRunning(child)) child.kill('SIGINT'); + onExit?.(); + }, + }); +} + +/** + * Start Stryker and wait until it is gone. Its output streams through and is + * also kept, so the caller can classify the run. `cancelledBy` names the + * signal that stopped the run, or is null. `spawn` and the streams are + * injected so the unit tests can check the command without starting Stryker. + * `onExit` is the caller's cleanup for an exit while Stryker runs (see + * registerSignalHandlers). After a crash the promise never settles: the crash + * handler ends the process once Stryker is gone or `crashStopGraceMs` passed. + */ +export function runStryker({ argv, cwd }, io = {}) { + const { + spawn = nodeSpawn, + stdout = process.stdout, + stderr = process.stderr, + crashStopGraceMs = CRASH_STOP_GRACE_MS, + ...handlerIo + } = io; + return new Promise((resolve, reject) => { + const chunks = []; + const run = { cancelledBy: null, crashed: false }; + const child = spawn(process.execPath, argv, { cwd, stdio: ['inherit', 'pipe', 'pipe'] }); + const handlers = strykerHandlers(child, run, { ...handlerIo, crashStopGraceMs }); + const keep = (sink) => (chunk) => { + chunks.push(Buffer.from(chunk)); + sink.write(chunk); + }; + child.stdout.on('data', keep(stdout)); + child.stderr.on('data', keep(stderr)); + child.on('error', (err) => { + if (run.crashed) return; + handlers.dispose(); + reject(new MutateError(3, `Stryker failed to start: ${err.message}`)); + }); + child.on('close', (code) => { + if (run.crashed) return; + handlers.dispose(); + const output = Buffer.concat(chunks).toString('utf8'); + resolve({ exitCode: code ?? 1, output, cancelledBy: run.cancelledBy }); + }); + }); +} diff --git a/scripts/mutation-health/stryker.test.mjs b/scripts/mutation-health/stryker.test.mjs new file mode 100644 index 00000000000..3208c61923f --- /dev/null +++ b/scripts/mutation-health/stryker.test.mjs @@ -0,0 +1,458 @@ +import { afterEach, beforeEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { + COMPAT_RUNNER_PLUGIN, + NO_TSCONFIG_REWRITE, + RAW_REPORT, + buildStrykerConfig, + loadBaseConfig, + resolveConfig, + resolveStrykerBin, + runnerIncrementalFile, + sandboxConfigError, + strykerRunArgv, + toCommandLine, + writeRunConfig, +} from './stryker.mjs'; +import defaultConfig from './stryker.default.mjs'; +import { MutateError, repoRoot } from './targets.mjs'; +import { IN_PLACE_KEY, namesInPlaceMode } from './test-doubles.mjs'; + +const VITEST_PLUGIN = '@stryker-mutator/vitest-runner'; +// The shape of the shared default config, for the fields this file overrides. +const SHARED_DEFAULT = { + testRunner: 'vitest', + plugins: [VITEST_PLUGIN], + reporters: ['progress', 'clear-text', 'html', 'json'], + coverageAnalysis: 'perTest', + ignoreStatic: true, + jsonReporter: { fileName: 'reports/mutation/raw.json' }, +}; + +describe('buildStrykerConfig with the vitest runner', () => { + it('puts the supplied test files in the testFiles config field', () => { + const config = buildStrykerConfig({ + targets: ['src/credentials/external-secrets.utils.ts:32-68'], + testFiles: ['src/credentials/__tests__/external-secrets.utils.test.ts'], + }); + assert.deepEqual(config.testFiles, [ + 'src/credentials/__tests__/external-secrets.utils.test.ts', + ]); + assert.deepEqual(config.mutate, ['src/credentials/external-secrets.utils.ts:32-68']); + }); + + // An empty `testFiles` is not the same as an absent one: Stryker reads a + // non-empty list as "run only these", so an empty one must not be sent. + it('leaves testFiles out when no test file was named', () => { + const config = buildStrykerConfig({ targets: ['src/cron.ts'] }); + assert.equal('testFiles' in config, false); + }); + + // The plain runner kills no mutant under Vitest 5. See vitest-compat.mjs. + it('swaps the plain vitest runner for vitest-compat and loads its plugin once', () => { + const config = buildStrykerConfig({ base: SHARED_DEFAULT, targets: ['src/a.ts'] }); + assert.equal(config.testRunner, 'vitest-compat'); + assert.deepEqual(config.plugins, [VITEST_PLUGIN, COMPAT_RUNNER_PLUGIN]); + assert.equal(config.coverageAnalysis, 'perTest'); + }); + + it('keeps the fields of a package config, such as its vitest config file', () => { + const base = { + ...SHARED_DEFAULT, + vitest: { configFile: 'vitest.stryker.config.ts' }, + timeoutMS: 15_000, + }; + const config = buildStrykerConfig({ base, targets: ['src/a.ts'] }); + assert.deepEqual(config.vitest, { configFile: 'vitest.stryker.config.ts' }); + assert.equal(config.timeoutMS, 15_000); + }); + + // A config can already name the compat runner, and Stryker still has to load it. + it('loads the compat plugin for a config that names the vitest-compat runner', () => { + const config = buildStrykerConfig({ + base: { testRunner: 'vitest-compat' }, + targets: ['src/a.ts'], + }); + assert.equal(config.testRunner, 'vitest-compat'); + assert.ok(config.plugins.includes(COMPAT_RUNNER_PLUGIN)); + }); + + it('leaves a config with another test runner on that runner', () => { + const config = buildStrykerConfig({ base: { testRunner: 'jest' }, targets: ['src/a.ts'] }); + assert.equal(config.testRunner, 'jest'); + assert.equal('plugins' in config, false); + }); +}); + +describe('buildStrykerConfig with a test command', () => { + const base = { ...SHARED_DEFAULT, testFiles: ['src/old.test.ts'] }; + + it('runs the command with the test files added at the end', () => { + const config = buildStrykerConfig({ + base, + targets: ['coverage-options.ts:1-40'], + testFiles: ['coverage-options.test.ts', 'utils/a b.test.ts'], + testCommand: 'pnpm exec vitest run ', + }); + assert.equal(config.testRunner, 'command'); + assert.deepEqual(config.commandRunner, { + command: "pnpm exec vitest run coverage-options.test.ts 'utils/a b.test.ts'", + }); + assert.deepEqual(config.mutate, ['coverage-options.ts:1-40']); + }); + + // The command runner throws on `testFiles`, and it records no per-test + // coverage, which `ignoreStatic` needs. + it('drops the fields that the command runner refuses', () => { + const config = buildStrykerConfig({ base, targets: ['a.ts'], testCommand: 'pnpm test' }); + assert.equal('testFiles' in config, false); + assert.equal(config.coverageAnalysis, 'off'); + assert.equal(config.ignoreStatic, false); + assert.equal(config.commandRunner.command, 'pnpm test'); + }); +}); + +// One run config for each runner, built from the shared default. +const configs = { + vitest: buildStrykerConfig({ base: SHARED_DEFAULT, targets: ['a.ts'] }), + command: buildStrykerConfig({ + base: SHARED_DEFAULT, + targets: ['a.ts'], + testCommand: 'pnpm test', + }), +}; + +describe('buildStrykerConfig for every runner', () => { + it('never turns on in place mode, so Stryker mutates only its sandbox copy', () => { + for (const config of Object.values(configs)) { + assert.deepEqual(Object.keys(config).filter(namesInPlaceMode), []); + } + }); + + // In the mirror the sandbox has the package's depth, so Stryker must not add + // `../..` to the tsconfig paths that leave the package. + it('puts the sandbox in the given temp dir and keeps tsconfig paths as they are', () => { + const tempDir = '/repo/.stryker-tmp/mirror-1/packages/@n8n'; + const base = { tempDirName: '.stryker-tmp' }; + for (const testCommand of [undefined, 'pnpm test']) { + const config = buildStrykerConfig({ base, targets: ['a.ts'], testCommand, tempDir }); + assert.equal(config.tempDirName, tempDir); + assert.equal(config.tsconfigFile, NO_TSCONFIG_REWRITE); + } + }); + + it('leaves the temp dir and the tsconfig rewrite of the config without a mirror', () => { + const config = buildStrykerConfig({ base: { tempDirName: '.stryker-tmp' }, targets: ['a.ts'] }); + assert.equal(config.tempDirName, '.stryker-tmp'); + assert.equal('tsconfigFile' in config, false); + }); + + it('removes the sandbox after every run and writes raw.json where mutate.mjs reads it', () => { + for (const config of Object.values(configs)) { + assert.equal(config.cleanTempDir, 'always'); + assert.equal(config.jsonReporter.fileName, RAW_REPORT); + assert.equal(config.reporters.filter((r) => r === 'json').length, 1); + } + }); +}); + +describe('buildStrykerConfig defaults', () => { + it('adds the json reporter to a config that has none', () => { + const config = buildStrykerConfig({ base: { reporters: ['progress'] }, targets: ['a.ts'] }); + assert.deepEqual(config.reporters, ['progress', 'json']); + assert.deepEqual(buildStrykerConfig({ targets: ['a.ts'] }).reporters, ['clear-text', 'json']); + }); + + it('keeps the reports and large artefact folders at the package root out of the sandbox', () => { + assert.deepEqual(buildStrykerConfig({ targets: ['a.ts'] }).ignorePatterns, [ + '/reports/mutation', + '/coverage', + '/test-results', + '/playwright-report', + '/.playwright-browsers', + '/ms-playwright-cache', + ]); + }); + + // Stryker loads the compat runner by this path in each test runner process. + it('names the compat runner plugin by the path of its file', () => { + assert.equal(path.basename(COMPAT_RUNNER_PLUGIN), 'vitest-compat-runner.mjs'); + assert.ok(statSync(COMPAT_RUNNER_PLUGIN).isFile()); + }); + + it('loads only the vitest runner and the compat plugin for a config without plugins', () => { + assert.deepEqual(buildStrykerConfig({ targets: ['a.ts'] }).plugins, [ + VITEST_PLUGIN, + COMPAT_RUNNER_PLUGIN, + ]); + }); + + // A tsconfig name that matches no file turns Stryker's rewrite off. Stryker + // reads the name from the package dir, so no dir of the repo may hold it. + it('names a tsconfig file that no package has', () => { + assert.match(NO_TSCONFIG_REWRITE, /^[\w.-]+\.json$/); + // In a git pathspec `*` also matches `/`, so this finds the file at any depth. + const res = spawnSync( + 'git', + ['ls-files', '--cached', '--others', '--exclude-standard', '--', `*${NO_TSCONFIG_REWRITE}`], + { cwd: repoRoot, encoding: 'utf8' }, + ); + assert.equal(res.status, 0, res.stderr); + assert.equal(res.stdout, ''); + }); + + it('keeps large artefact folders out of the sandbox copy, on top of the config list', () => { + const config = buildStrykerConfig({ + base: { ignorePatterns: ['/fixtures/big', '/coverage'] }, + targets: ['a.ts'], + }); + assert.equal(config.ignorePatterns[0], '/fixtures/big'); + assert.ok(config.ignorePatterns.includes('/playwright-report')); + assert.equal(config.ignorePatterns.filter((p) => p === '/coverage').length, 1); + }); + + // Incremental results recorded by one runner do not apply to the other. + it('keeps incremental results in a file for each runner', () => { + const base = { + incremental: true, + incrementalFile: 'reports/mutation/stryker-incremental.json', + }; + const vitest = buildStrykerConfig({ base, targets: ['a.ts'] }); + const command = buildStrykerConfig({ base, targets: ['a.ts'], testCommand: 'pnpm test' }); + assert.equal(vitest.incrementalFile, 'reports/mutation/stryker-incremental.vitest-compat.json'); + assert.equal(command.incrementalFile, 'reports/mutation/stryker-incremental.command.json'); + const noFile = buildStrykerConfig({ base: { incremental: true }, targets: ['a.ts'] }); + assert.equal(noFile.incrementalFile, 'reports/stryker-incremental.vitest-compat.json'); + assert.equal('incrementalFile' in configs.vitest, false); + }); +}); + +describe('runnerIncrementalFile', () => { + it('puts the runner name before the extension', () => { + assert.equal(runnerIncrementalFile('reports/x.json', 'command'), 'reports/x.command.json'); + assert.equal(runnerIncrementalFile('reports/x', 'command'), 'reports/x.command'); + }); +}); + +describe('sandboxConfigError', () => { + it('refuses a config that turns on in place mode, and names the file and the option', () => { + const error = sandboxConfigError({ [IN_PLACE_KEY]: true }, 'packages/pkg/stryker.config.mjs'); + assert.match(error, new RegExp(`packages/pkg/stryker.config.mjs sets "${IN_PLACE_KEY}"`)); + assert.match(error, /working tree/); + }); + + it('says what to do about the option', () => { + const error = sandboxConfigError({ [IN_PLACE_KEY]: true }, 'a.mjs'); + assert.match( + error, + /\nmutate\.mjs runs Stryker only in its sandbox copy of the package\. Remove the option\.$/, + ); + }); + + // Only the option itself switches the mode. A longer key is another option. + it('ignores keys that only contain the option name', () => { + const base = { [`${IN_PLACE_KEY}Backup`]: true, [`no${IN_PLACE_KEY}`]: true }; + assert.equal(sandboxConfigError(base, 'a.mjs'), null); + }); + + it('accepts a config that leaves in place mode off', () => { + assert.equal(sandboxConfigError({ [IN_PLACE_KEY]: false }, 'a.mjs'), null); + assert.equal(sandboxConfigError(SHARED_DEFAULT, 'a.mjs'), null); + }); + + // The run sets `tsconfigFile` to a file that does not exist, and the + // TypeScript checker type-checks with that file. + it('refuses a config that uses the TypeScript checker, and says what to do', () => { + const error = sandboxConfigError( + { checkers: ['other', 'typescript'] }, + 'packages/pkg/stryker.config.mjs', + ); + assert.match( + error, + /^packages\/pkg\/stryker\.config\.mjs uses the "typescript" checker, which mutate\.mjs does not support\.\n/, + ); + assert.match(error, /\(tsconfigFile\)/); + assert.match(error, /Remove "typescript" from "checkers"\.$/); + }); + + it('accepts a config with no checker or with only other checkers', () => { + for (const checkers of [undefined, [], ['other']]) { + assert.equal(sandboxConfigError({ checkers }, 'a.mjs'), null); + } + }); +}); + +describe('toCommandLine', () => { + it('leaves plain paths as they are', () => { + assert.equal( + toCommandLine('pnpm test', ['src/a.test.ts', '@scope/b.test.ts']), + 'pnpm test src/a.test.ts @scope/b.test.ts', + ); + }); + + it('quotes a path that the shell would split or expand', () => { + assert.equal(toCommandLine('pnpm test', ['a b.ts']), "pnpm test 'a b.ts'"); + assert.equal(toCommandLine('pnpm test', ['src/*.test.ts']), "pnpm test 'src/*.test.ts'"); + assert.equal(toCommandLine('pnpm test', ["it's.ts"]), "pnpm test 'it'\\''s.ts'"); + }); + + it('returns the command alone when no test file is named', () => { + assert.equal(toCommandLine(' pnpm exec vitest run '), 'pnpm exec vitest run'); + }); +}); + +describe('strykerRunArgv', () => { + it('starts Stryker on the run config with no other flag', () => { + const argv = strykerRunArgv('/bin/stryker.js', '/pkg/reports/mutation/stryker.run.json'); + assert.deepEqual(argv, ['/bin/stryker.js', 'run', '/pkg/reports/mutation/stryker.run.json']); + assert.equal(argv.some(namesInPlaceMode), false); + }); +}); + +describe('loadBaseConfig and resolveConfig', () => { + let root; + + beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), 'mutate-config-')); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it('reads a JSON config and the default export of a module config', async () => { + writeFileSync(path.join(root, 'a.json'), '{ "timeoutMS": 5 }'); + writeFileSync(path.join(root, 'b.mjs'), 'export default { timeoutMS: 6 };'); + assert.deepEqual(await loadBaseConfig(path.join(root, 'a.json')), { timeoutMS: 5 }); + assert.deepEqual(await loadBaseConfig(path.join(root, 'b.mjs')), { timeoutMS: 6 }); + }); + + it('stops with a config error for a missing or broken config', async () => { + writeFileSync(path.join(root, 'broken.mjs'), 'export default {'); + const cases = { + 'missing.mjs': /^Stryker config not found: .*missing\.mjs$/, + 'broken.mjs': /^Could not read the Stryker config .*broken\.mjs: /, + }; + for (const [file, message] of Object.entries(cases)) { + await assert.rejects( + loadBaseConfig(path.join(root, file)), + (error) => error.exitCode === 2 && message.test(error.message), + ); + } + }); + + it('writes the run config as JSON next to the reports and returns its path', async () => { + const file = await writeRunConfig(root, { mutate: ['src/a.ts'], concurrency: 2 }); + assert.equal(file, path.join(root, 'stryker.run.json')); + const text = readFileSync(file, 'utf8'); + assert.deepEqual(JSON.parse(text), { mutate: ['src/a.ts'], concurrency: 2 }); + assert.ok(text.endsWith('}\n')); + }); + + it('prefers --config, then the package config, then the shared default', () => { + const pkgRoot = path.join(root, 'pkg'); + mkdirSync(pkgRoot); + assert.equal(resolveConfig(pkgRoot), path.join(import.meta.dirname, 'stryker.default.mjs')); + writeFileSync(path.join(pkgRoot, 'stryker.config.mjs'), 'export default {};'); + assert.equal(resolveConfig(pkgRoot), path.join(pkgRoot, 'stryker.config.mjs')); + assert.equal(resolveConfig(pkgRoot, path.join(root, 'x.mjs')), path.join(root, 'x.mjs')); + }); + + it('reads a relative --config from the repo root', () => { + const pkgRoot = path.join(root, 'pkg'); + assert.equal(resolveConfig(pkgRoot, 'cfg/x.mjs', root), path.join(root, 'cfg/x.mjs')); + }); + + // Related-test discovery in packages/cli outlives any usable timeout. + it('gives packages/cli the cli config, with related-test discovery off', async () => { + const cliRoot = path.join(root, 'packages/cli'); + mkdirSync(cliRoot, { recursive: true }); + const cliConfig = path.join(import.meta.dirname, 'stryker.cli.mjs'); + assert.equal(resolveConfig(cliRoot, undefined, root), cliConfig); + // From another repo root the same dir is not packages/cli. + assert.equal(resolveConfig(cliRoot), path.join(import.meta.dirname, 'stryker.default.mjs')); + const { vitest, ...rest } = await loadBaseConfig(cliConfig); + assert.deepEqual(vitest, { related: false }); + assert.deepEqual(rest, defaultConfig); + writeFileSync(path.join(cliRoot, 'stryker.config.mjs'), 'export default {};'); + assert.equal(resolveConfig(cliRoot, undefined, root), path.join(cliRoot, 'stryker.config.mjs')); + }); +}); + +describe('resolveStrykerBin', () => { + let root; + let pkgRoot; + + beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), 'mutate-bin-')); + pkgRoot = path.join(root, 'pkg'); + mkdirSync(pkgRoot); + writeFileSync(path.join(pkgRoot, 'package.json'), '{ "name": "pkg" }'); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + // Put a stand-in copy of Stryker's core package in `dir`, and return where + // its binary would be. + function installFakeStryker(dir) { + const core = path.join(dir, 'node_modules/@stryker-mutator/core'); + mkdirSync(core, { recursive: true }); + writeFileSync(path.join(core, 'package.json'), '{ "name": "@stryker-mutator/core" }'); + return path.join(core, 'bin/stryker.js'); + } + + // The file that the root copy resolves from, in a stand-in repo root. + const fromRepo = () => path.join(root, 'repo/scripts/mutation-health/stryker.mjs'); + + it("uses the package's own copy, so a package gets the version it pins", () => { + const own = installFakeStryker(pkgRoot); + installFakeStryker(path.join(root, 'repo')); + assert.equal(resolveStrykerBin(pkgRoot, 'pkg', { rootFrom: fromRepo() }), own); + }); + + it('falls back to the root copy for a package without its own', () => { + const rootCopy = installFakeStryker(path.join(root, 'repo')); + assert.equal(resolveStrykerBin(pkgRoot, 'pkg', { rootFrom: fromRepo() }), rootCopy); + }); + + // A broken checkout must never read as a score of zero. + it('stops with exit 3 and says what to do when neither has a copy', () => { + assert.throws( + () => resolveStrykerBin(pkgRoot, 'packages/pkg', { rootFrom: fromRepo() }), + (error) => + error instanceof MutateError && + error.exitCode === 3 && + /^Could not resolve @stryker-mutator\/core from packages\/pkg or the repo root\. Run `pnpm install`/.test( + error.message, + ), + ); + }); + + // CI runs these tests without the root node_modules, so this check needs them. + const rootStryker = path.join(import.meta.dirname, '../../node_modules/@stryker-mutator/core'); + it( + "finds this repo's Stryker by default", + { skip: !existsSync(rootStryker) && 'the repo root has no node_modules' }, + () => { + // pnpm links the package, and the lookup gives the real path. + const expected = path.join(realpathSync(rootStryker), 'bin/stryker.js'); + assert.equal(resolveStrykerBin(pkgRoot, 'pkg'), expected); + }, + ); +}); diff --git a/scripts/mutation-health/summary-report.test.mjs b/scripts/mutation-health/summary-report.test.mjs new file mode 100644 index 00000000000..ce1ea0f4c40 --- /dev/null +++ b/scripts/mutation-health/summary-report.test.mjs @@ -0,0 +1,106 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { emptyCounts, overallGate, reportLines } from './summary.mjs'; + +function summary({ counts = {}, thresholdMet = true, partial, testRunner, files = [] } = {}) { + return { + target: 'src/a.ts', + ...(testRunner ? { testRunner } : {}), + ...(partial ? { partial } : {}), + overall: { counts: { ...emptyCounts(), ...counts }, thresholdMet }, + files, + }; +} + +function fileRow(overrides = {}) { + return { + file: 'src/a.ts', + score: 90, + coverage: 0.5, + thresholdMet: false, + counts: { ...emptyCounts(), killed: 9, survived: 1 }, + survivors: [{ status: 'Survived', mutator: 'EqualityOperator', location: 'src/a.ts:4:2' }], + ignored: [ + { mutator: 'StringLiteral', location: 'src/a.ts:9:1', reason: 'log text' }, + { mutator: 'BlockStatement', location: 'src/a.ts:12:1', reason: '' }, + ], + ...overrides, + }; +} + +describe('overallGate', () => { + it('passes when every package run passed', () => { + const gate = overallGate([summary(), summary({ counts: { ignored: 2 } })]); + assert.deepEqual(gate, { survived: 0, ignored: 2, passed: true, partial: false }); + }); + + it('fails when one package run fails, and counts uncovered mutants as survivors', () => { + const gate = overallGate([ + summary(), + summary({ thresholdMet: false, counts: { survived: 2, noCoverage: 1 } }), + ]); + assert.equal(gate.passed, false); + assert.equal(gate.survived, 3); + }); + + // The mutants a partial run never tested can be survivors. + it('never passes a partial run, even when its score met the gate', () => { + const gate = overallGate([summary({ partial: true })]); + assert.equal(gate.passed, false); + assert.equal(gate.partial, true); + }); +}); + +describe('reportLines', () => { + const base = { + packageDir: 'packages/pkg', + summaryPath: 'packages/pkg/reports/mutation/summary.json', + }; + + it('prints nothing for a failed run: the caller already said why', () => { + assert.deepEqual(reportLines({ ...base, failed: true }), []); + }); + + it('prints a score-0 red line when no test covers the target', () => { + const lines = reportLines({ ...base, noTests: true, summary: summary() }); + assert.match(lines[0], /^✗ packages\/pkg src\/a\.ts {2}0\.00% {2}\(no covering tests/); + assert.equal(lines.at(-1), ` summary: ${base.summaryPath}`); + }); + + it('prints the score, coverage and each unjustified and justified mutant', () => { + const lines = reportLines({ ...base, summary: summary({ files: [fileRow()] }) }); + assert.equal( + lines[0], + '✗ src/a.ts 90.00% cov 50% (killed 9 / survived 1 / no-cov 0 / timeout 0 / ignored 0)', + ); + assert.match(lines[1], /^ {3}- survived +EqualityOperator +src\/a\.ts:4:2$/); + assert.match(lines[2], /^ {3}· ignored +StringLiteral +src\/a\.ts:9:1 — log text$/); + assert.match(lines[3], /src\/a\.ts:12:1 — \(no reason given\)$/); + assert.equal(lines[4], ` summary: ${base.summaryPath}`); + }); + + it('marks a passing file with a tick', () => { + const row = fileRow({ thresholdMet: true, survivors: [], ignored: [] }); + const [line] = reportLines({ ...base, summary: summary({ files: [row] }) }); + assert.match(line, /^✓ src\/a\.ts/); + }); + + // The command runner runs every test for every mutant: a coverage figure + // would always read 100% and mean nothing. + it('prints no coverage figure for a command-runner run', () => { + const [line] = reportLines({ + ...base, + summary: summary({ testRunner: 'command', files: [fileRow()] }), + }); + assert.match(line, / cov n\/a /); + }); + + it('warns before the rows when the run was partial', () => { + const lines = reportLines({ ...base, summary: summary({ partial: true, files: [fileRow()] }) }); + assert.match( + lines[0], + /^⚠ packages\/pkg: Stryker exited non-zero.*results may be incomplete\.$/, + ); + }); +}); diff --git a/scripts/mutation-health/summary.mjs b/scripts/mutation-health/summary.mjs new file mode 100644 index 00000000000..6b43cd1d514 --- /dev/null +++ b/scripts/mutation-health/summary.mjs @@ -0,0 +1,327 @@ +/** + * Scoring, the gate and the summary that mutate.mjs writes. Pure: no I/O and + * no process state, so the unit tests can drive every rule directly. + */ + +// Stryker's mutant statuses and the count each one goes to. +const STATUS_COUNT = { + Killed: 'killed', + Survived: 'survived', + NoCoverage: 'noCoverage', + Timeout: 'timeout', + CompileError: 'compileError', + RuntimeError: 'runtimeError', + Ignored: 'ignored', +}; + +export function emptyCounts() { + return { + killed: 0, + survived: 0, + noCoverage: 0, + timeout: 0, + compileError: 0, + runtimeError: 0, + ignored: 0, + }; +} + +export function sliceFromLocation(source, loc) { + const lines = source.split('\n'); + const { start, end } = loc; + if (start.line === end.line) { + return lines[start.line - 1].slice(start.column, end.column); + } + return [ + lines[start.line - 1].slice(start.column), + ...lines.slice(start.line, end.line - 1), + lines[end.line - 1].slice(0, end.column), + ].join('\n'); +} + +export function scoreFromCounts(c) { + const detected = c.killed + c.timeout; + const valid = c.killed + c.timeout + c.survived + c.noCoverage; + return valid === 0 ? 0 : +((detected / valid) * 100).toFixed(2); +} + +/** + * Per-file line-coverage proxy distilled from the mutant census: the fraction + * of mutants a test actually exercised (anything that ran) over those that + * could be covered (ran + no-coverage). Ignored and compile-error mutants + * never ran for reasons unrelated to coverage, so they sit outside the ratio. + * + * Returns a fraction in [0,1]. The result is clamped. + */ +export function coverageFromCounts(c) { + const covered = (c.killed ?? 0) + (c.survived ?? 0) + (c.timeout ?? 0) + (c.runtimeError ?? 0); + const total = covered + (c.noCoverage ?? 0); + if (total === 0) return 0; + return +Math.min(1, Math.max(0, covered / total)).toFixed(4); +} + +// Stryker says this when the vitest runner finds no test. Vitest itself says +// the second one when a test command matches no test file. +const NO_TESTS_OUTPUT = /no tests were executed|no test files found/i; + +function mutantLocation(file, m) { + return `${file}:${m.location.start.line}:${m.location.start.column}`; +} + +/** + * The location of each mutant that tests cover but that survived with no test + * run. Stryker runs the covering tests of each mutant, so such a mutant means + * that the test runner selected none of them, and the survivor is not a real + * result. The plain vitest runner failed this way under Vitest 5 (see + * vitest-compat.mjs). A mutant without coverage data (the command runner) or + * without a test count is never in the list. + */ +export function survivorsWithNoTestRun(report) { + const locations = []; + for (const [file, info] of Object.entries(report.files ?? {})) { + for (const m of info.mutants ?? []) { + if (m.status === 'Survived' && m.coveredBy?.length > 0 && m.testsCompleted === 0) { + locations.push(mutantLocation(file, m)); + } + } + } + return locations; +} + +/** + * What a finished Stryker run produced. `report` is the parsed report of THIS + * run, or null when it wrote none. The caller deletes the previous reports + * first, because a file left by an earlier run makes a crashed run look + * complete and report the earlier target. + * + * complete — the run finished and wrote a report. + * partial — the run wrote a report, then exited non-zero. Untested + * mutants can still be survivors, so this never passes the gate. + * no-tests — no test covers the target. A result, not an error: the score + * is zero and every mutant has no coverage. See DEVP-414. + * tests-not-run — a mutant that tests cover survived with no test run (see + * survivorsWithNoTestRun). The score is not valid, so the + * caller reports a toolchain failure. + * failed — no report. The caller reports a toolchain failure. + */ +export function classifyRun({ exitCode, output, report }) { + if (!report) { + return NO_TESTS_OUTPUT.test(output) ? 'no-tests' : 'failed'; + } + if (survivorsWithNoTestRun(report).length > 0) return 'tests-not-run'; + return exitCode === 0 ? 'complete' : 'partial'; +} + +// A run with many such mutants lists only the first ones. +const LISTED_NOT_RUN = 5; + +/** What the tool prints for a `tests-not-run` result of the package at `packageDir`. */ +export function testsNotRunLines(packageDir, report) { + const locations = survivorsWithNoTestRun(report); + const more = locations.length - LISTED_NOT_RUN; + return [ + `✗ ${packageDir}: ${locations.length} mutant(s) survived with no test run, but tests cover them:`, + ...locations.slice(0, LISTED_NOT_RUN).map((location) => ` - ${location}`), + ...(more > 0 ? [` … and ${more} more`] : []), + ' The test runner did not run their covering tests, so the score is not valid.', + ' See "Vitest 5" in scripts/mutation-health/README.md.', + ]; +} + +// A run is only "passing" when the score meets the floor AND every unkilled +// mutant has been explicitly justified (Ignored via a Stryker disable +// comment). Any Survived/NoCoverage mutant is unjustified by definition. +export function gatePassed(score, counts, threshold) { + return score >= threshold && counts.survived === 0 && counts.noCoverage === 0; +} + +// Test id to test name, so a survivor can name the tests that ran the mutated +// line without killing the mutant. +function testNamesById(raw) { + const names = {}; + for (const info of Object.values(raw.testFiles ?? {})) { + for (const t of info.tests ?? []) names[t.id] = t.name; + } + return names; +} + +function survivorRow(file, source, m, testNames) { + return { + id: m.id, + mutator: m.mutatorName, + status: m.status, + location: mutantLocation(file, m), + line: m.location.start.line, + original: sliceFromLocation(source, m.location), + replacement: m.replacement, + // The command runner reports no per-test coverage, so this can be empty. + coveringTests: (m.coveredBy ?? []).map((id) => testNames[id] ?? id), + }; +} + +function ignoredRow(file, m) { + return { + id: m.id, + mutator: m.mutatorName, + location: mutantLocation(file, m), + line: m.location.start.line, + reason: m.statusReason ?? '', + }; +} + +function summariseFile(file, info, { threshold, testNames }) { + const counts = emptyCounts(); + const survivors = []; + const ignored = []; + for (const m of info.mutants) { + const key = STATUS_COUNT[m.status]; + if (key) counts[key]++; + if (m.status === 'Survived' || m.status === 'NoCoverage') { + survivors.push(survivorRow(file, info.source, m, testNames)); + } + if (m.status === 'Ignored') ignored.push(ignoredRow(file, m)); + } + survivors.sort((a, b) => a.line - b.line); + ignored.sort((a, b) => a.line - b.line); + const score = scoreFromCounts(counts); + return { + file, + score, + coverage: coverageFromCounts(counts), + thresholdMet: gatePassed(score, counts, threshold), + counts, + survivors, + ignored, + }; +} + +function sumCounts(rows) { + return rows.reduce((acc, row) => { + for (const k of Object.keys(acc)) acc[k] += row.counts[k]; + return acc; + }, emptyCounts()); +} + +/** + * Build the compact summary from a raw Stryker Mutation Testing Elements + * report. Pure: takes the parsed report plus run metadata, returns the summary + * object written to summary.json. `testRunner` records which Stryker runner + * scored the run, because the command runner measures no coverage. + */ +export function buildSummary(raw, { threshold, target, generatedAt, testRunner }) { + const testNames = testNamesById(raw); + const files = Object.entries(raw.files).map(([file, info]) => + summariseFile(file, info, { threshold, testNames }), + ); + const counts = sumCounts(files); + const score = scoreFromCounts(counts); + return { + generatedAt, + threshold, + target, + ...(testRunner ? { testRunner } : {}), + overall: { + score, + coverage: coverageFromCounts(counts), + counts, + thresholdMet: gatePassed(score, counts, threshold), + }, + files, + }; +} + +/** + * Synthesise a score-0 red summary for the "No tests were executed" case — + * every mutant is no-coverage, so coverage is 0 too. See DEVP-414. + */ +export function buildNoTestsSummary({ threshold, target, noCoverage, generatedAt }) { + const counts = { ...emptyCounts(), noCoverage }; + const coverage = coverageFromCounts(counts); + return { + generatedAt, + threshold, + target, + overall: { score: 0, coverage, counts, thresholdMet: false }, + files: [ + { + file: target, + score: 0, + coverage, + thresholdMet: false, + counts, + survivors: [], + ignored: [], + }, + ], + }; +} + +// Stryker prints the mutant count before it runs a test. A no-tests run has no +// report, so this line is the only count there is. +export function mutantCountFromOutput(output) { + const match = /Instrumented\s+\d+\s+source file\(s\)\s+with\s+(\d+)\s+mutant/i.exec(output); + return match ? Number(match[1]) : 0; +} + +/** + * The gate over every package run. A partial run never passes: the mutants it + * did not test can be survivors. + */ +export function overallGate(summaries) { + return summaries.reduce( + (acc, summary) => { + const c = summary.overall.counts; + acc.survived += c.survived + c.noCoverage; + acc.ignored += c.ignored; + acc.passed &&= summary.overall.thresholdMet && !summary.partial; + acc.partial ||= Boolean(summary.partial); + return acc; + }, + { survived: 0, ignored: 0, passed: true, partial: false }, + ); +} + +function fileLine(f, testRunner) { + const mark = f.thresholdMet ? '✓' : '✗'; + // The command runner runs every test for every mutant, so it measures no coverage. + const coverage = testRunner === 'command' ? 'n/a' : `${(f.coverage * 100).toFixed(0)}%`; + const c = f.counts; + return ( + `${mark} ${f.file} ${f.score.toFixed(2)}% cov ${coverage} ` + + `(killed ${c.killed} / survived ${c.survived} / no-cov ${c.noCoverage} / timeout ${c.timeout} / ignored ${c.ignored})` + ); +} + +function fileDetailLines(f, testRunner) { + const survivors = f.survivors.map( + (s) => ` - ${s.status.toLowerCase().padEnd(10)} ${s.mutator.padEnd(22)} ${s.location}`, + ); + const ignored = f.ignored.map( + (ig) => + ` · ${'ignored'.padEnd(10)} ${ig.mutator.padEnd(22)} ${ig.location}` + + `${ig.reason ? ` — ${ig.reason}` : ' — (no reason given)'}`, + ); + return [fileLine(f, testRunner), ...survivors, ...ignored]; +} + +/** + * The lines the summary prints for one package run. `summaryPath` is the + * repo-relative path of its summary.json. + */ +export function reportLines({ packageDir, summaryPath, summary, noTests, failed }) { + if (failed) return []; + if (noTests) { + return [ + `✗ ${packageDir} ${summary.target} 0.00% (no covering tests — recorded as score-0 red)`, + ` summary: ${summaryPath}`, + ]; + } + const partial = summary.partial + ? [ + `⚠ ${packageDir}: Stryker exited non-zero; summary built from a partial raw.json — ` + + 'results may be incomplete.', + ] + : []; + const files = summary.files.flatMap((f) => fileDetailLines(f, summary.testRunner)); + return [...partial, ...files, ` summary: ${summaryPath}`]; +} diff --git a/scripts/mutation-health/summary.test.mjs b/scripts/mutation-health/summary.test.mjs new file mode 100644 index 00000000000..ef02e341fa5 --- /dev/null +++ b/scripts/mutation-health/summary.test.mjs @@ -0,0 +1,353 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + buildNoTestsSummary, + buildSummary, + coverageFromCounts, + emptyCounts, + gatePassed, + mutantCountFromOutput, + scoreFromCounts, +} from './summary.mjs'; + +// A minimal Stryker Mutation Testing Elements report for one source file. Mix +// of statuses so coverage (anything that ran / ran + no-coverage) is a genuine +// fraction strictly between 0 and 1: 3 ran (killed/survived/timeout), 1 sat +// uncovered. +const RAW_FIXTURE = { + files: { + 'src/cron.ts': { + source: 'export const a = 1;\nexport const b = 2;\nexport const c = 3;\n', + mutants: [ + { + id: '1', + mutatorName: 'ArithmeticOperator', + status: 'Killed', + location: { start: { line: 1, column: 18 }, end: { line: 1, column: 19 } }, + replacement: '2', + }, + { + id: '2', + mutatorName: 'BooleanLiteral', + status: 'Survived', + location: { start: { line: 2, column: 18 }, end: { line: 2, column: 19 } }, + replacement: '3', + coveredBy: ['t1'], + }, + { + id: '3', + mutatorName: 'BlockStatement', + status: 'Timeout', + location: { start: { line: 3, column: 18 }, end: { line: 3, column: 19 } }, + replacement: '4', + }, + { + id: '4', + mutatorName: 'StringLiteral', + status: 'NoCoverage', + location: { start: { line: 3, column: 0 }, end: { line: 3, column: 5 } }, + replacement: '""', + }, + ], + }, + }, + testFiles: { + 'src/cron.test.ts': { + tests: [{ id: 't1', name: 'cron computes next run' }], + }, + }, +}; + +const RUN_META = { threshold: 80, target: 'src/cron.ts', generatedAt: '2026-06-21T00:00:00.000Z' }; + +function isFractionInUnitInterval(v) { + return typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= 1; +} + +function mutant(id, status, line, extra = {}) { + return { + id, + mutatorName: 'ConditionalExpression', + status, + location: { start: { line, column: 0 }, end: { line, column: 6 } }, + replacement: 'true', + ...extra, + }; +} + +describe('coverageFromCounts', () => { + it('is the share of mutants that ran (ran / ran + no-coverage)', () => { + // 3 ran (killed + timeout), 1 uncovered → 3/4 + const counts = { killed: 2, survived: 0, timeout: 1, noCoverage: 1, runtimeError: 0 }; + assert.equal(coverageFromCounts(counts), 0.75); + }); + + it('counts survived and runtime-error mutants as covered (they ran)', () => { + const counts = { killed: 0, survived: 1, timeout: 0, noCoverage: 1, runtimeError: 1 }; + assert.equal(coverageFromCounts(counts), 0.6667); + }); + + it('is 1 when every mutant was covered', () => { + assert.equal(coverageFromCounts({ killed: 5, survived: 0, timeout: 0, noCoverage: 0 }), 1); + }); + + it('is 0 when no mutant was covered', () => { + assert.equal(coverageFromCounts({ killed: 0, survived: 0, timeout: 0, noCoverage: 7 }), 0); + }); + + it('is 0 — never NaN — when there is nothing to cover', () => { + assert.equal(coverageFromCounts({ killed: 0, survived: 0, timeout: 0, noCoverage: 0 }), 0); + }); + + it('ignores compile-error and ignored mutants (they never ran for coverage reasons)', () => { + const counts = { + killed: 1, + survived: 0, + timeout: 0, + noCoverage: 1, + compileError: 3, + ignored: 4, + runtimeError: 0, + }; + // only killed (ran) + noCoverage count → 1/2 + assert.equal(coverageFromCounts(counts), 0.5); + }); + + it('always lands in [0,1]', () => { + for (const counts of [ + { killed: 1, survived: 2, timeout: 3, noCoverage: 4, runtimeError: 5 }, + { killed: 0, survived: 0, timeout: 0, noCoverage: 0 }, + { killed: 9, survived: 0, timeout: 0, noCoverage: 0 }, + ]) { + assert.ok(isFractionInUnitInterval(coverageFromCounts(counts))); + } + }); +}); + +describe('gatePassed', () => { + const clean = { survived: 0, noCoverage: 0 }; + + it('passes at the threshold with no unjustified survivor', () => { + assert.equal(gatePassed(80, clean, 80), true); + }); + + it('fails below the threshold', () => { + assert.equal(gatePassed(79.99, clean, 80), false); + }); + + // The score alone lets weak tests clear 80% and leave real gaps (DEVP-442). + it('fails on one survivor or one uncovered mutant, also at 100%', () => { + assert.equal(gatePassed(100, { survived: 1, noCoverage: 0 }, 80), false); + assert.equal(gatePassed(100, { survived: 0, noCoverage: 1 }, 80), false); + }); +}); + +describe('scoreFromCounts', () => { + // Timeouts count as detected. Uncovered mutants count against the score. + it('is detected (killed + timeout) over every mutant that could be detected', () => { + const counts = { killed: 2, timeout: 1, survived: 1, noCoverage: 1 }; + assert.equal(scoreFromCounts(counts), 60); + }); + + it('rounds to two decimals', () => { + assert.equal(scoreFromCounts({ killed: 2, timeout: 0, survived: 1, noCoverage: 0 }), 66.67); + }); + + it('is 0 — never NaN — when no mutant could be detected', () => { + assert.equal(scoreFromCounts({ killed: 0, timeout: 0, survived: 0, noCoverage: 0 }), 0); + }); +}); + +describe('buildSummary', () => { + it('writes a per-file coverage fraction in [0,1] onto every file row', () => { + const summary = buildSummary(RAW_FIXTURE, RUN_META); + assert.equal(summary.files.length, 1); + const file = summary.files[0]; + assert.ok(isFractionInUnitInterval(file.coverage)); + // 3 ran (killed/survived/timeout) of 4 coverable → 0.75 + assert.equal(file.coverage, 0.75); + }); + + it('writes an overall coverage fraction in [0,1]', () => { + const summary = buildSummary(RAW_FIXTURE, RUN_META); + assert.ok(isFractionInUnitInterval(summary.overall.coverage)); + }); + + it('preserves the existing summary contract (score, counts, survivors)', () => { + const summary = buildSummary(RAW_FIXTURE, RUN_META); + const file = summary.files[0]; + assert.equal(file.score, scoreFromCounts(file.counts)); + assert.equal(file.counts.killed, 1); + assert.equal(file.counts.survived, 1); + assert.equal(file.counts.noCoverage, 1); + assert.equal(file.counts.timeout, 1); + // Survived + NoCoverage are unjustified survivors + assert.equal(file.survivors.length, 2); + // names the covering test for the survived mutant + const survived = file.survivors.find((s) => s.status === 'Survived'); + assert.deepEqual(survived.coveringTests, ['cron computes next run']); + assert.equal(survived.location, 'src/cron.ts:2:18'); + }); +}); + +describe('buildSummary counts', () => { + it('counts compile errors and runtime errors apart, and leaves them out of the score', () => { + const raw = { + files: { + 'src/a.ts': { + source: 'a\n', + mutants: [ + mutant('1', 'Killed', 1), + mutant('2', 'CompileError', 1), + mutant('3', 'RuntimeError', 1), + mutant('4', 'RuntimeError', 1), + ], + }, + }, + }; + const file = buildSummary(raw, RUN_META).files[0]; + assert.equal(file.counts.compileError, 1); + assert.equal(file.counts.runtimeError, 2); + assert.equal(file.score, 100); + }); + + // A status from a newer Stryker must not add a count that nothing reads. + it('keeps exactly the known counts when a mutant has an unknown status', () => { + const raw = { files: { 'src/a.ts': { source: 'a\n', mutants: [mutant('1', 'Pending', 1)] } } }; + const file = buildSummary(raw, RUN_META).files[0]; + assert.deepEqual(file.counts, emptyCounts()); + }); +}); + +describe('buildSummary rows', () => { + it('quotes the mutated source, also across lines', () => { + const source = 'if (a === b) {\n\treturn 1;\n} // end\n'; + const raw = { + files: { + 'src/a.ts': { + source, + mutants: [ + { + ...mutant('1', 'Survived', 1), + location: { start: { line: 1, column: 4 }, end: { line: 1, column: 11 } }, + }, + { + ...mutant('2', 'Survived', 1), + location: { start: { line: 1, column: 13 }, end: { line: 3, column: 1 } }, + }, + ], + }, + }, + }; + const [single, multi] = buildSummary(raw, RUN_META).files[0].survivors; + assert.equal(single.original, 'a === b'); + assert.equal(multi.original, '{\n\treturn 1;\n}'); + }); + + it('lists survivors and justified mutants by line, with the reason for each ignore', () => { + const raw = { + files: { + 'src/a.ts': { + source: 'line1\nline2\nline3\n', + mutants: [ + mutant('9', 'Survived', 3), + mutant('8', 'Survived', 1), + mutant('7', 'Ignored', 2, { statusReason: 'equivalent: same output' }), + mutant('6', 'Ignored', 1), + ], + }, + }, + }; + const file = buildSummary(raw, RUN_META).files[0]; + assert.deepEqual( + file.survivors.map((s) => s.line), + [1, 3], + ); + assert.deepEqual( + file.ignored.map((ig) => [ig.line, ig.reason]), + [ + [1, ''], + [2, 'equivalent: same output'], + ], + ); + // Ignored mutants stay out of the score: 0 killed of 2 valid. + assert.equal(file.score, 0); + assert.equal(file.counts.ignored, 2); + }); + + it('adds the counts of every file into the overall row', () => { + const raw = { + files: { + 'src/a.ts': { source: 'a\n', mutants: [mutant('1', 'Killed', 1)] }, + 'src/b.ts': { + source: 'b\n', + mutants: [mutant('2', 'Killed', 1), mutant('3', 'Survived', 1)], + }, + }, + }; + const summary = buildSummary(raw, RUN_META); + assert.equal(summary.overall.counts.killed, 2); + assert.equal(summary.overall.counts.survived, 1); + assert.equal(summary.overall.score, 66.67); + assert.equal(summary.overall.thresholdMet, false); + }); + + // The command runner reports no `coveredBy`, so a survivor names no test. + it('records the test runner and accepts survivors without covering tests', () => { + const raw = { files: { 'src/a.ts': { source: 'a\n', mutants: [mutant('1', 'Survived', 1)] } } }; + const summary = buildSummary(raw, { ...RUN_META, testRunner: 'command' }); + assert.equal(summary.testRunner, 'command'); + assert.deepEqual(summary.files[0].survivors[0].coveringTests, []); + assert.equal('testRunner' in buildSummary(raw, RUN_META), false); + }); +}); + +describe('buildNoTestsSummary (no covering tests)', () => { + it('reports a score-0 red result with every mutant uncovered', () => { + const summary = buildNoTestsSummary({ + threshold: 80, + target: 'src/cron.ts', + noCoverage: 12, + generatedAt: RUN_META.generatedAt, + }); + assert.equal(summary.files[0].coverage, 0); + assert.equal(summary.overall.score, 0); + assert.equal(summary.overall.counts.noCoverage, 12); + assert.equal(summary.overall.thresholdMet, false); + assert.ok(isFractionInUnitInterval(summary.overall.coverage)); + }); + + it('writes one red file row for the target, with no survivor to list', () => { + const summary = buildNoTestsSummary({ threshold: 80, target: 'src/cron.ts', noCoverage: 3 }); + assert.deepEqual(summary.files, [ + { + file: 'src/cron.ts', + score: 0, + coverage: 0, + thresholdMet: false, + counts: { ...emptyCounts(), noCoverage: 3 }, + survivors: [], + ignored: [], + }, + ]); + }); +}); + +describe('mutantCountFromOutput', () => { + it("reads the mutant count from Stryker's instrumenter line", () => { + assert.equal( + mutantCountFromOutput('INFO Instrumenter Instrumented 1 source file(s) with 38 mutant(s)'), + 38, + ); + }); + + it('reads counts of any width, with any space between the words', () => { + const output = 'Instrumented 12 \tsource file(s)\n with 340 mutant(s)'; + assert.equal(mutantCountFromOutput(output), 340); + }); + + it('is 0 when Stryker never instrumented a file', () => { + assert.equal(mutantCountFromOutput('ERROR Could not read the config'), 0); + }); +}); diff --git a/scripts/mutation-health/targets.mjs b/scripts/mutation-health/targets.mjs new file mode 100644 index 00000000000..6d7f0b32c91 --- /dev/null +++ b/scripts/mutation-health/targets.mjs @@ -0,0 +1,159 @@ +/** + * Pure helpers that shape what a run mutates and which tests it runs: source + * filters, diff ranges, test-file lists and the packages/cli scope rule. No + * I/O, so the unit tests can drive every rule directly. + */ +import path from 'node:path'; + +export const repoRoot = path.resolve(import.meta.dirname, '../..'); + +/** + * A planned stop. `exitCode` follows the tool's contract: 2 for a usage or + * config error, 3 when Stryker cannot run. `showUsage` asks the entry point to + * print the usage text after the message. + */ +export class MutateError extends Error { + constructor(exitCode, message, { showUsage = false } = {}) { + super(message); + this.name = 'MutateError'; + this.exitCode = exitCode; + this.showUsage = showUsage; + } +} + +// `packages/cli` forks a process per test file and runs a global setup, thus +// Stryker's related-test discovery pulls in hundreds of files and no run +// finishes inside the timeout. An explicit `--test-files` list is required +// there, and the shared default config is swapped for the cli one. +export const CLI_PACKAGE_DIR = 'packages/cli'; + +// Path comparisons in this file are posix-shaped. `path.relative` gives +// backslashes on Windows, so normalise before matching a package dir. +export function toPosix(p) { + return p.split(path.sep).join('/'); +} + +const NON_SOURCE = [ + /\.d\.ts$/, + /\.(test|spec)\.[cm]?tsx?$/, + /(^|\/)__(tests|mocks)__\//, + /\.stories\.[cm]?tsx?$/, + /\.config\.[cm]?[jt]s$/, + /(^|\/)(dist|node_modules|coverage)\//, + /(^|\/)tests?\//, + /(^|\/)migrations\//, +]; + +// This is an exclusion list, not a `src/`-only allowlist. nodes-base and +// nodes-langchain keep their code in `nodes/` and `credentials/`. An allowlist +// drops the largest mutable surface in the repo. +export function isMutableSource(repoRelPath) { + if (!/\.[cm]?tsx?$/.test(repoRelPath)) return false; + return !NON_SOURCE.some((re) => re.test(repoRelPath)); +} + +export function changedTestFilesForPackage(changedFiles, packageDir) { + const prefix = `${toPosix(packageDir).replace(/\/$/, '')}/`; + return changedFiles + .map((file) => toPosix(file).replace(/^\.\//, '')) + .filter((file) => { + if (!file.startsWith(prefix)) return false; + const relative = file.slice(prefix.length); + return ( + (relative.startsWith('src/') || relative.startsWith('test/unit/')) && + /\.(test|spec)\.ts$/.test(relative) && + !/\.integration\.test\.ts$/.test(relative) + ); + }); +} + +// Merge overlapping and adjacent ranges. Stryker then gets one span per region. +export function mergeRanges(ranges) { + const out = []; + for (const r of [...ranges].sort((a, b) => a.start - b.start)) { + const last = out.at(-1); + if (last && r.start <= last.end + 1) last.end = Math.max(last.end, r.end); + else out.push({ ...r }); + } + return out; +} + +// Read the new-side line ranges from `git diff -U0` hunk headers. +// `@@ -12,0 +13,4 @@` gives `{ start: 13, end: 16 }`. A new-side count of zero +// is a deletion. No code stays there to mutate, so this drops it. +export function parseHunkRanges(diffText) { + const ranges = []; + for (const line of diffText.split('\n')) { + const m = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/.exec(line); + if (!m) continue; + const start = Number(m[1]); + const count = m[2] === undefined ? 1 : Number(m[2]); + if (count === 0) continue; + ranges.push({ start, end: start + count - 1 }); + } + return mergeRanges(ranges); +} + +// One label for every target of a run. The summary and the report print it. +export function formatMutateArg(targets) { + return targets.join(','); +} + +export function splitRange(target) { + const m = /^(.*):(\d+)-(\d+)$/.exec(target); + return m ? { file: m[1], range: `${m[2]}-${m[3]}` } : { file: target, range: null }; +} + +/** + * Flatten the raw `--test-files` values into one list. The flag repeats and + * each value also takes a comma-separated list, thus `--test-files a,b` and + * `--test-files a --test-files b` mean the same thing. Blanks are dropped and + * duplicates collapse, so a repeated path never runs its file twice. + */ +export function parseTestFiles(values) { + const out = []; + for (const value of values) { + for (const part of String(value).split(',')) { + const file = part.trim(); + if (file && !out.includes(file)) out.push(file); + } + } + return out; +} + +/** + * Stryker runs in the package dir, so it matches test files against paths + * under that dir, and a test command gets them as paths from there. A + * repo-relative path (what the user types, and what `--diff` prints) therefore + * has to lose its package prefix. A path that is already package-relative, and + * a glob, both pass through untouched. + */ +export function toPackageRelative(file, packageDir) { + const normalised = toPosix(file).replace(/^\.\//, ''); + const prefix = `${toPosix(packageDir)}/`; + return normalised.startsWith(prefix) ? normalised.slice(prefix.length) : normalised; +} + +// Why a cli target may not run, or null when it may. Named so the message says +// what to do, not only what went wrong. +export function cliScopeError(packageDir, testFiles, diffMode = false) { + if (toPosix(packageDir) !== CLI_PACKAGE_DIR) return null; + if (testFiles.length > 0) return null; + if (diffMode) { + return ( + `Mutating ${CLI_PACKAGE_DIR} with --diff needs at least one changed test file.\n` + + 'Add or update a test that covers the changed source, or run a named target with --test-files.' + ); + } + return ( + `Mutating ${CLI_PACKAGE_DIR} needs --test-files.\n` + + 'Without it Stryker discovers every related test file in the package, forks a ' + + 'process for each one and never finishes. Name the test files that cover the target:\n' + + ' pnpm mutate packages/cli/src/foo.ts:10-40 --test-files packages/cli/src/__tests__/foo.test.ts' + ); +} + +// Which shared config a package gets when it ships none of its own. +export function defaultConfigNameFor(packageDir) { + return toPosix(packageDir) === CLI_PACKAGE_DIR ? 'stryker.cli.mjs' : 'stryker.default.mjs'; +} diff --git a/scripts/mutation-health/targets.test.mjs b/scripts/mutation-health/targets.test.mjs new file mode 100644 index 00000000000..6a5da82f5af --- /dev/null +++ b/scripts/mutation-health/targets.test.mjs @@ -0,0 +1,377 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; + +import { + MutateError, + changedTestFilesForPackage, + cliScopeError, + defaultConfigNameFor, + formatMutateArg, + isMutableSource, + mergeRanges, + parseHunkRanges, + parseTestFiles, + splitRange, + repoRoot, + toPackageRelative, +} from './targets.mjs'; + +describe('repoRoot', () => { + // Plans, configs and the sandbox mirror all resolve from it. + it('is a folder above this tool', () => { + assert.ok(import.meta.dirname.startsWith(`${repoRoot}${path.sep}`)); + }); +}); + +describe('isMutableSource', () => { + it('accepts product source wherever a package keeps it', () => { + assert.ok(isMutableSource('packages/workflow/src/cron.ts')); + // nodes-base has no src/. An allowlist drops the largest surface in the repo. + assert.ok(isMutableSource('packages/nodes-base/nodes/Slack/Slack.node.ts')); + assert.ok(isMutableSource('packages/nodes-base/credentials/SlackApi.credentials.ts')); + assert.ok(isMutableSource('packages/frontend/editor-ui/src/stores/ui.store.ts')); + // `[cm]?` in the extension test is there for the ESM/CJS variants. + assert.ok(isMutableSource('packages/@n8n/db/src/index.mts')); + assert.ok(isMutableSource('packages/@n8n/db/src/index.cts')); + }); + + it('rejects tests, declarations, configs and build output', () => { + assert.equal(isMutableSource('packages/workflow/src/cron.test.ts'), false); + assert.equal(isMutableSource('packages/workflow/src/cron.spec.ts'), false); + // The ESM/CJS variants are accepted as source, so they have to be + // excluded as tests too. + assert.equal(isMutableSource('packages/workflow/src/cron.test.mts'), false); + assert.equal(isMutableSource('packages/workflow/src/__tests__/cron.ts'), false); + assert.equal(isMutableSource('packages/workflow/src/__mocks__/cron.ts'), false); + assert.equal(isMutableSource('packages/workflow/src/types.d.ts'), false); + assert.equal(isMutableSource('packages/cli/vitest.config.ts'), false); + assert.equal(isMutableSource('packages/workflow/dist/cron.js'), false); + assert.equal(isMutableSource('packages/workflow/test/helper.ts'), false); + assert.equal(isMutableSource('packages/@n8n/db/src/migrations/sqlite/x.ts'), false); + assert.equal(isMutableSource('packages/design-system/src/Button.stories.ts'), false); + // The extension test is anchored: `.ts` has to end the path, not merely + // appear in it. Committed snapshots sit next to their source and would + // otherwise be handed to Stryker as mutable TypeScript. + assert.equal(isMutableSource('packages/cli/src/__snapshots__/foo.test.ts.snap'), false); + }); + + // A named target is checked as a package-relative path, so a test or build + // folder can be the first segment. + it('rejects test, mock and build folders at the start of a package-relative path', () => { + for (const rel of [ + '__tests__/helper.ts', + '__mocks__/fs.ts', + 'test/helper.ts', + 'tests/helper.ts', + 'dist/index.ts', + 'coverage/x.ts', + 'migrations/1-init.ts', + ]) { + assert.equal(isMutableSource(rel), false, rel); + } + }); + + it('rejects the ESM and CJS variants of stories and configs', () => { + assert.equal(isMutableSource('src/Button.stories.mts'), false); + assert.equal(isMutableSource('vite.config.mts'), false); + assert.equal(isMutableSource('vitest.config.cts'), false); + }); + + // A name part such as `.config.js` in the middle of a name does not make a + // source file a config, a test, a story or a declaration. + it('reads only the end of a file name', () => { + for (const file of [ + 'src/schema.d.ts-builder.ts', + 'src/run.test.ts-helpers.ts', + 'src/button.stories.ts-loader.ts', + 'src/app.config.json-schema.ts', + ]) { + assert.equal(isMutableSource(file), true, file); + } + }); + + // .vue stays out. Each SFC package crashed Stryker's mutate step in the + // 2026-06 sweep, and the component layer gives little value. + it('rejects everything that is not TypeScript', () => { + assert.equal(isMutableSource('packages/frontend/editor-ui/src/App.vue'), false); + assert.equal(isMutableSource('packages/workflow/src/cron.js'), false); + assert.equal(isMutableSource('README.md'), false); + assert.equal(isMutableSource('packages/workflow/package.json'), false); + }); +}); + +describe('changedTestFilesForPackage', () => { + it('selects changed test files from the exact package', () => { + const changedFiles = [ + 'packages/@n8n/engine/src/runtime/__tests__/create-engine-runtime.test.ts', + 'packages/cli/src/modules/engine-v2/__tests__/engine-v2.runtime.test.ts', + 'packages/cli/src/modules/engine-v2/__tests__/in-memory-execution-response.test.ts', + 'packages/cli/src/modules/engine-v2/engine-v2.runtime.ts', + 'packages/cli-utils/src/foo.test.ts', + ]; + + assert.deepEqual(changedTestFilesForPackage(changedFiles, 'packages/cli'), [ + 'packages/cli/src/modules/engine-v2/__tests__/engine-v2.runtime.test.ts', + 'packages/cli/src/modules/engine-v2/__tests__/in-memory-execution-response.test.ts', + ]); + }); + + it('accepts paths with a leading ./', () => { + assert.deepEqual(changedTestFilesForPackage(['./packages/cli/src/a.test.ts'], 'packages/cli'), [ + 'packages/cli/src/a.test.ts', + ]); + }); + + // `packages/xyz/` has the same length as `packages/cli/`, so only the + // prefix check keeps its tests out. + it('leaves out the tests of another package', () => { + assert.deepEqual( + changedTestFilesForPackage(['packages/xyz/src/a.test.ts'], 'packages/cli'), + [], + ); + }); + + it('matches the CLI unit Vitest config roots and extensions', () => { + assert.deepEqual( + changedTestFilesForPackage( + [ + 'packages/cli/src/a.test.ts', + 'packages/cli/test/unit/b.spec.ts', + 'packages/cli/src/c.integration.test.ts', + 'packages/cli/test/integration/d.test.ts', + 'packages/cli/src/e.test.mts', + 'packages/cli/src/f.test.tsx', + ], + 'packages/cli/', + ), + ['packages/cli/src/a.test.ts', 'packages/cli/test/unit/b.spec.ts'], + ); + }); +}); + +describe('parseHunkRanges', () => { + it('reads new-side ranges out of `git diff -U0` headers', () => { + const diff = [ + 'diff --git a/src/cron.ts b/src/cron.ts', + '--- a/src/cron.ts', + '+++ b/src/cron.ts', + '@@ -12,0 +13,4 @@ export function tick() {', + '+const a = 1;', + '@@ -40,2 +44,2 @@', + '+const b = 2;', + // Counts run past one digit on both sides for any hunk of ten lines + // or more, which is most of them. + '@@ -80,12 +90,14 @@', + '+const c = 3;', + ].join('\n'); + assert.deepEqual(parseHunkRanges(diff), [ + { start: 13, end: 16 }, + { start: 44, end: 45 }, + { start: 90, end: 103 }, + ]); + }); + + // `git diff` of a file that itself talks about diffs (a patch fixture, this + // very test file) carries hunk-header text inside `+`/`-` content lines. + // Only a header at the start of a line is a header. + it('ignores hunk-header text that appears inside a content line', () => { + const diff = ['@@ -1,0 +5,1 @@', "+const H = '@@ -1,2 +300,4 @@';"].join('\n'); + assert.deepEqual(parseHunkRanges(diff), [{ start: 5, end: 5 }]); + }); + + it('treats a header with no new-side count as a single line', () => { + assert.deepEqual(parseHunkRanges('@@ -5 +7 @@'), [{ start: 7, end: 7 }]); + }); + + it('drops pure deletions — nothing survives there to mutate', () => { + assert.deepEqual(parseHunkRanges('@@ -10,4 +9,0 @@'), []); + }); + + it('returns nothing for a diff with no hunks', () => { + assert.deepEqual(parseHunkRanges(''), []); + }); +}); + +describe('mergeRanges', () => { + it('merges overlapping ranges', () => { + const ranges = [ + { start: 1, end: 5 }, + { start: 3, end: 9 }, + ]; + assert.deepEqual(mergeRanges(ranges), [{ start: 1, end: 9 }]); + }); + + it('merges adjacent ranges so Stryker gets one span per region', () => { + const ranges = [ + { start: 1, end: 4 }, + { start: 5, end: 8 }, + ]; + assert.deepEqual(mergeRanges(ranges), [{ start: 1, end: 8 }]); + }); + + it('keeps ranges with a real gap apart, and sorts them', () => { + const ranges = [ + { start: 20, end: 22 }, + { start: 1, end: 4 }, + ]; + assert.deepEqual(mergeRanges(ranges), [ + { start: 1, end: 4 }, + { start: 20, end: 22 }, + ]); + }); + + it('leaves a fully-contained range absorbed', () => { + const ranges = [ + { start: 1, end: 20 }, + { start: 5, end: 9 }, + ]; + assert.deepEqual(mergeRanges(ranges), [{ start: 1, end: 20 }]); + }); +}); + +describe('formatMutateArg', () => { + it('joins every target into one label', () => { + assert.equal( + formatMutateArg(['src/a.ts:1-4', 'src/a.ts:20-22', 'src/b.ts']), + 'src/a.ts:1-4,src/a.ts:20-22,src/b.ts', + ); + }); +}); + +describe('splitRange', () => { + it('splits a trailing line range off the path', () => { + assert.deepEqual(splitRange('src/cron.ts:13-16'), { file: 'src/cron.ts', range: '13-16' }); + }); + + it('leaves a bare path alone', () => { + assert.deepEqual(splitRange('src/cron.ts'), { file: 'src/cron.ts', range: null }); + }); + + it('reads a range only at the end of the target', () => { + assert.deepEqual(splitRange('src/a.ts:10-20.bak'), { file: 'src/a.ts:10-20.bak', range: null }); + }); + + it('does not mistake a Windows drive letter or a colon in a dirname for a range', () => { + assert.deepEqual(splitRange('src/a:b/cron.ts'), { file: 'src/a:b/cron.ts', range: null }); + }); +}); + +describe('parseTestFiles', () => { + it('splits a comma-separated value', () => { + assert.deepEqual(parseTestFiles(['a.test.ts,b.test.ts']), ['a.test.ts', 'b.test.ts']); + }); + + it('collects a repeated flag', () => { + assert.deepEqual(parseTestFiles(['a.test.ts', 'b.test.ts']), ['a.test.ts', 'b.test.ts']); + }); + + it('accepts both forms at once, and trims the spaces around a comma', () => { + assert.deepEqual(parseTestFiles(['a.test.ts, b.test.ts', 'c.test.ts']), [ + 'a.test.ts', + 'b.test.ts', + 'c.test.ts', + ]); + }); + + // A duplicate would make Stryker run the same file twice for every mutant. + it('drops blanks and duplicates', () => { + assert.deepEqual(parseTestFiles(['a.test.ts,,a.test.ts', ' ', 'b.test.ts']), [ + 'a.test.ts', + 'b.test.ts', + ]); + }); + + it('returns nothing when the flag was not given', () => { + assert.deepEqual(parseTestFiles([]), []); + }); +}); + +describe('toPackageRelative', () => { + // Stryker runs in the package dir, so test files are matched from there. + it('strips the package prefix off a repo-relative path', () => { + assert.equal( + toPackageRelative('packages/cli/src/__tests__/foo.test.ts', 'packages/cli'), + 'src/__tests__/foo.test.ts', + ); + }); + + it('leaves an already package-relative path alone', () => { + assert.equal( + toPackageRelative('src/__tests__/foo.test.ts', 'packages/cli'), + 'src/__tests__/foo.test.ts', + ); + }); + + it('leaves a glob alone', () => { + assert.equal(toPackageRelative('src/**/*.test.ts', 'packages/cli'), 'src/**/*.test.ts'); + }); + + it('drops a leading ./', () => { + assert.equal(toPackageRelative('./src/foo.test.ts', 'packages/cli'), 'src/foo.test.ts'); + }); + + it('drops ./ only at the start, so a path that leaves the package stays as it is', () => { + assert.equal(toPackageRelative('../shared/a.test.ts', 'packages/cli'), '../shared/a.test.ts'); + }); + + // `packages/cli-utils` must not lose its prefix to `packages/cli`. + it('only strips a whole path segment', () => { + assert.equal( + toPackageRelative('packages/cli-utils/src/foo.test.ts', 'packages/cli'), + 'packages/cli-utils/src/foo.test.ts', + ); + }); +}); + +describe('cliScopeError', () => { + it('refuses a packages/cli target with no --test-files and shows how to name them', () => { + const error = cliScopeError('packages/cli', []); + assert.match(error, /^Mutating packages\/cli needs --test-files\.\n/); + assert.match(error, /forks a process for each one and never finishes/); + assert.match( + error, + / {2}pnpm mutate packages\/cli\/src\/foo\.ts:10-40 --test-files packages\/cli\/src\/__tests__\/foo\.test\.ts$/, + ); + assert.doesNotMatch(error, /--diff/); + }); + + it('allows a packages/cli target once test files are named', () => { + assert.equal(cliScopeError('packages/cli', ['src/__tests__/foo.test.ts']), null); + }); + + it('explains how to scope a CLI diff with no changed test file', () => { + const error = cliScopeError('packages/cli', [], true); + assert.match(error, /changed test file/); + assert.match(error, /named target with --test-files/); + }); + + it('leaves every other package alone', () => { + assert.equal(cliScopeError('packages/workflow', []), null); + // A prefix match would sweep in an unrelated package. + assert.equal(cliScopeError('packages/cli-utils', []), null); + }); +}); + +describe('defaultConfigNameFor', () => { + it('gives packages/cli its own config, so related-test discovery stays off', () => { + assert.equal(defaultConfigNameFor('packages/cli'), 'stryker.cli.mjs'); + }); + + it('gives every other package the shared default', () => { + assert.equal(defaultConfigNameFor('packages/workflow'), 'stryker.default.mjs'); + assert.equal(defaultConfigNameFor('packages/@n8n/decorators'), 'stryker.default.mjs'); + }); +}); + +describe('MutateError', () => { + it('carries the exit code and prints no usage text unless asked', () => { + const error = new MutateError(3, 'Stryker did not start'); + assert.ok(error instanceof Error); + assert.equal(error.name, 'MutateError'); + assert.equal(error.exitCode, 3); + assert.equal(error.message, 'Stryker did not start'); + assert.equal(error.showUsage, false); + assert.equal(new MutateError(2, 'Missing target', { showUsage: true }).showUsage, true); + }); +}); diff --git a/scripts/mutation-health/test-doubles.mjs b/scripts/mutation-health/test-doubles.mjs new file mode 100644 index 00000000000..8cf9581fede --- /dev/null +++ b/scripts/mutation-health/test-doubles.mjs @@ -0,0 +1,140 @@ +/** + * Test doubles shared by the mutate.mjs unit tests: a `spawn` that starts no + * process and records each call, a stand-in process for signal handlers, a + * writable sink, a Stryker binary resolver and a small repo to plan in. + * + * CI runs these tests in a checkout without the root node_modules (the job + * installs only `.github/scripts`), so no test may need Stryker installed. + */ +import { EventEmitter } from 'node:events'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +/** + * A `spawn` stand-in. Each call returns a fake child with `stdout`, `stderr`, + * `kill` and `finish(code)`, which ends the child the way a real one ends. + * `onSpawn` runs for each call, for example to write a report and finish. + */ +export function fakeSpawn({ onSpawn } = {}) { + const calls = []; + const children = []; + const spawn = (command, args, options) => { + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.exitCode = null; + child.signalCode = null; + child.kills = []; + child.kill = (signal) => child.kills.push(signal); + child.finish = (code) => { + child.exitCode = code; + child.emit('close', code); + }; + calls.push({ command, args, options }); + children.push(child); + onSpawn?.(child, { command, args, options }); + return child; + }; + return { spawn, calls, children }; +} + +/** + * A stand-in process plus recorders for `exit` and `write`. `exited` settles + * with the code of the first exit, for a handler that exits after a wait. + */ +export function fakeProcess() { + const exits = []; + const writes = []; + let firstExit; + const exited = new Promise((resolve) => { + firstExit = resolve; + }); + return { + proc: new EventEmitter(), + exits, + writes, + exited, + exit: (code) => { + exits.push(code); + firstExit(code); + }, + write: (msg) => writes.push(msg), + }; +} + +/** 'resolved', 'rejected', or 'pending' when `promise` did not settle within one turn. */ +export function settledState(promise) { + return Promise.race([ + promise.then( + () => 'resolved', + () => 'rejected', + ), + new Promise((resolve) => setImmediate(() => resolve('pending'))), + ]); +} + +/** A writable stand-in that keeps what was written. */ +export function sink() { + const chunks = []; + return { + chunks, + write: (chunk) => chunks.push(String(chunk)), + text: () => chunks.join(''), + }; +} + +// A Stryker binary path for runs that a fake `spawn` starts. No file is there. +export const FAKE_STRYKER_BIN = '/fake/node_modules/@stryker-mutator/core/bin/stryker.js'; + +/** A `resolveStrykerBin` stand-in that needs no installed Stryker. */ +export const resolveFakeStrykerBin = () => FAKE_STRYKER_BIN; + +// Stryker's option name for in place mode. It is built from parts because a +// safety check greps this folder for the literal name: a match means that the +// tool still mutates the working tree. +export const IN_PLACE_KEY = ['in', 'Place'].join(''); + +/** True when an argument or config key switches on Stryker's in place mode. */ +export function namesInPlaceMode(text) { + return /^(--)?inplace$/i.test(text); +} + +// The safety check to do before a run: `grep -E` with this pattern on each .mjs +// file of this folder must find nothing. A match is a sign that the tool can +// write to the working tree. Built from parts, so this file passes the check. +export const SAFETY_GREP = new RegExp( + [ + IN_PLACE_KEY, + ['in', 'place'].join('-'), + ['git', 'checkout'].join(' '), + ['checkout', '--'].join(' '), + ].join('|'), +); + +/** Write each file of `files` (path: content) under `root`. An object is written as JSON. */ +export function writeTree(root, files) { + for (const [file, content] of Object.entries(files)) { + const abs = path.join(root, file); + mkdirSync(path.dirname(abs), { recursive: true }); + writeFileSync(abs, typeof content === 'string' ? content : JSON.stringify(content)); + } +} + +const vitestPackage = (name) => ({ name, scripts: { test: 'vitest run' } }); + +// A repo for the planners, with the package kinds that a plan treats apart. +// It has no package.json at its root, so a file outside the packages has no +// enclosing package. +export const SAMPLE_REPO = { + 'packages/@n8n/instance-ai/package.json': vitestPackage('@n8n/instance-ai'), + 'packages/@n8n/instance-ai/src/utils/model-config-id.ts': 'export const id = 1;\n', + 'packages/@n8n/instance-ai/src/utils/__tests__/model-config-id.test.ts': 'it("a", () => {});\n', + 'packages/cli/package.json': vitestPackage('n8n'), + 'packages/cli/src/credentials/external-secrets.utils.ts': 'export const a = 1;\n', + 'packages/cli/src/credentials/__tests__/external-secrets.utils.test.ts': 'it("a", () => {});\n', + 'packages/@n8n/expression-runtime/package.json': vitestPackage('@n8n/expression-runtime'), + 'packages/@n8n/expression-runtime/src/index.ts': 'export const b = 1;\n', + 'packages/jest-pkg/package.json': { name: 'jest-pkg', scripts: { test: 'jest' } }, + 'packages/jest-pkg/src/a.ts': 'export const c = 1;\n', + 'scripts/loose.ts': 'export const d = 1;\n', +}; diff --git a/scripts/mutation-health/vitest-compat-runner.mjs b/scripts/mutation-health/vitest-compat-runner.mjs new file mode 100644 index 00000000000..01a0ed5ccb8 --- /dev/null +++ b/scripts/mutation-health/vitest-compat-runner.mjs @@ -0,0 +1,15 @@ +/** + * Stryker plugin that declares the `vitest-compat` test runner. It is the + * runner of `@stryker-mutator/vitest-runner` with two fixes for Vitest 5 (see + * vitest-compat.mjs). mutate.mjs selects it for every config that uses the + * plain `vitest` runner, and Stryker loads this file by its path in each test + * runner process. + * + * Keep this file to the import: the logic is in vitest-compat.mjs, where the + * unit tests reach it without the runner installed. + */ +import { strykerPlugins as vitestRunnerPlugins } from '@stryker-mutator/vitest-runner'; + +import { compatPlugins } from './vitest-compat.mjs'; + +export const strykerPlugins = compatPlugins(vitestRunnerPlugins); diff --git a/scripts/mutation-health/vitest-compat.mjs b/scripts/mutation-health/vitest-compat.mjs new file mode 100644 index 00000000000..d3bc3eae9cd --- /dev/null +++ b/scripts/mutation-health/vitest-compat.mjs @@ -0,0 +1,117 @@ +/** + * Two fixes for `@stryker-mutator/vitest-runner` 10.0.0 under Vitest 5. Pure, + * so the unit tests can drive them without Stryker or Vitest. The runner is + * tested only against Vitest 4.1. + * + * 1. Test names. For each mutant the runner runs only the tests that cover + * it. It selects them with a `testNamePattern` regex built from names that + * join the suite and the test with a space ("suite test"). Vitest 4 matched + * the pattern against names in that form. Vitest 5 matches it against + * `fullTestName`, which joins them with " > " ("suite > test"). The pattern + * then misses every test inside a `describe`, no test runs, and every + * mutant survives with a score of 0. + * + * 2. Test files that do not load. The runner counts only tests. A test file + * that throws on import has no tests, so it drops out of the run without an + * error. Alone, it reads as "No tests were executed", which mutate.mjs then + * records as "no covering tests". + * + * Remove the first fix when the runner supports Vitest 5. + * + * `compatPlugins` builds the plugin from the runner's plugin list. The list is + * a parameter, so the unit tests run without the runner installed. + */ + +// The Stryker test runner that `vitest-compat-runner.mjs` declares. +export const COMPAT_RUNNER_NAME = 'vitest-compat'; + +// The two separators a full test name can have between a suite and a test. +const NAME_SEPARATOR = '(?: > | )'; + +const nestedPatterns = new WeakSet(); + +/** + * A pattern that selects the same tests with either separator. Any space in + * the runner's pattern can stand for a separator, so each one becomes an + * alternation. The runner escapes each name, so a space is never inside a + * character class. A pattern that this function made comes back unchanged. + */ +export function toNestedNamePattern(pattern) { + if (!(pattern instanceof RegExp) || nestedPatterns.has(pattern)) return pattern; + const nested = new RegExp(pattern.source.replaceAll(' ', NAME_SEPARATOR), pattern.flags); + nestedPatterns.add(nested); + return nested; +} + +/** + * Wrap a vitest runner so each test run selects tests by nested names. The + * runner sets `testNamePattern` on each project and then starts Vitest, so the + * wrap goes on `ctx.start`, after the runner has set the pattern. `init` + * creates a new Vitest context, so each `init` wraps the context it made. + */ +export function withNestedNamePatterns(runner) { + const init = runner.init.bind(runner); + runner.init = async (...args) => { + await init(...args); + const { ctx } = runner; + const start = ctx.start.bind(ctx); + ctx.start = (...startArgs) => { + for (const project of ctx.projects) { + project.config.testNamePattern = toNestedNamePattern(project.config.testNamePattern); + } + return start(...startArgs); + }; + }; + return runner; +} + +/** + * One line for each test file that failed before it collected a test, with + * the errors Vitest recorded. A file whose tests ran and failed is not here: + * the runner already reports failed tests. + */ +export function fileLoadErrors(files) { + return files + .filter((file) => file.result?.state === 'fail' && file.tasks.length === 0) + .map((file) => { + const messages = file.result.errors?.map((error) => error.message) ?? []; + return `${file.filepath ?? file.name}: ${messages.join('; ') || 'the file did not load'}`; + }); +} + +/** + * Wrap a vitest runner so a test file that does not load fails the dry run. + * Stryker then stops with the file's error instead of scoring the target as + * having no tests. + */ +export function withLoadErrors(runner) { + const dryRun = runner.dryRun.bind(runner); + runner.dryRun = async (...args) => { + const result = await dryRun(...args); + if (result.status !== 'complete') return result; + const errors = fileLoadErrors(runner.ctx.state.getFiles()); + if (errors.length === 0) return result; + return { status: 'error', errorMessage: `A test file did not load:\n${errors.join('\n')}` }; + }; + return runner; +} + +/** + * The Stryker plugins that declare the `vitest-compat` test runner, built from + * the plugin list of `@stryker-mutator/vitest-runner`. It throws when the list + * has no `vitest` test runner, so a runner upgrade that changes the export + * stops the run at load time with a clear message. + */ +export function compatPlugins(vitestRunnerPlugins) { + const vitestRunner = vitestRunnerPlugins.find( + (plugin) => plugin.kind === 'TestRunner' && plugin.name === 'vitest', + ); + if (!vitestRunner) { + throw new Error('@stryker-mutator/vitest-runner no longer declares the "vitest" test runner.'); + } + const factory = (injector) => + withLoadErrors(withNestedNamePatterns(vitestRunner.factory(injector))); + // Stryker reads the factory's dependencies from `inject`: the same as the wrapped one. + factory.inject = vitestRunner.factory.inject; + return [{ kind: vitestRunner.kind, name: COMPAT_RUNNER_NAME, factory }]; +} diff --git a/scripts/mutation-health/vitest-compat.test.mjs b/scripts/mutation-health/vitest-compat.test.mjs new file mode 100644 index 00000000000..e93a3d801d6 --- /dev/null +++ b/scripts/mutation-health/vitest-compat.test.mjs @@ -0,0 +1,409 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + COMPAT_RUNNER_NAME, + compatPlugins, + fileLoadErrors, + toNestedNamePattern, + withLoadErrors, + withNestedNamePatterns, +} from './vitest-compat.mjs'; + +// The pattern @stryker-mutator/vitest-runner 10.0.0 builds for a mutant: each +// covering test's name (suite and test joined by a space), escaped, joined by +// `|`. `escapeRegExp` is a copy of the runner's helper in @stryker-mutator/util. +function escapeRegExp(input) { + return input.replace(/[.*+\-?^${}()|[\]\\]/g, '\\$&'); +} +function runnerPattern(...names) { + return new RegExp(names.map(escapeRegExp).join('|')); +} + +describe('toNestedNamePattern', () => { + const name = 'modelConfigId returns plain-string model ids as-is'; + + // This is the Vitest 5 failure: the plain pattern misses the nested name. + it('selects a nested test by its Vitest 5 full name', () => { + const vitest5Name = 'modelConfigId > returns plain-string model ids as-is'; + assert.equal(runnerPattern(name).test(vitest5Name), false); + assert.equal(toNestedNamePattern(runnerPattern(name)).test(vitest5Name), true); + }); + + it('still selects the test by its Vitest 4 name', () => { + assert.equal(toNestedNamePattern(runnerPattern(name)).test(name), true); + }); + + it('selects tests nested at any depth', () => { + const pattern = toNestedNamePattern( + runnerPattern('scheduler lifecycle config rejects a jitter'), + ); + assert.equal(pattern.test('scheduler > lifecycle > config > rejects a jitter'), true); + }); + + it('selects every covering test and no other test', () => { + const pattern = toNestedNamePattern(runnerPattern('a reads x', 'b writes y')); + assert.equal(pattern.test('a > reads x'), true); + assert.equal(pattern.test('b > writes y'), true); + assert.equal(pattern.test('a > writes y'), false); + assert.equal(pattern.test('c > reads z'), false); + }); + + it('keeps regex characters in a test name literal', () => { + const pattern = toNestedNamePattern(runnerPattern('parser reads (a|b) as text.')); + assert.equal(pattern.test('parser > reads (a|b) as text.'), true); + assert.equal(pattern.test('parser > reads a as text.'), false); + assert.equal(pattern.test('parser > reads (a|b) as textX'), false); + }); + + it('keeps the flags of the pattern', () => { + assert.equal(toNestedNamePattern(/a b/i).flags, 'i'); + assert.equal(toNestedNamePattern(/a b/i).test('A > B'), true); + }); + + // Vitest can start twice on the same pattern. A second conversion would + // break the alternations of the first. + it('returns a pattern it made unchanged', () => { + const nested = toNestedNamePattern(runnerPattern(name)); + assert.equal(toNestedNamePattern(nested), nested); + }); + + it('leaves a missing pattern missing: the run then selects every test', () => { + assert.equal(toNestedNamePattern(undefined), undefined); + }); +}); + +describe('withNestedNamePatterns', () => { + // A stand-in for the vitest runner: `init` makes a Vitest context with two + // projects, and `start` records the patterns it runs with. + function fakeRunner() { + const started = []; + const runner = { + async init() { + const projects = [{ config: {} }, { config: {} }]; + this.ctx = { + projects, + start: async (files) => { + started.push({ files, patterns: projects.map((p) => p.config.testNamePattern) }); + }, + }; + }, + }; + return { runner, started }; + } + + // What the runner does for a mutant: set the pattern on each project, then start. + function runMutant(runner, pattern, files) { + for (const project of runner.ctx.projects) project.config.testNamePattern = pattern; + return runner.ctx.start(files); + } + + it('runs Vitest with patterns that select nested tests', async () => { + const { runner, started } = fakeRunner(); + assert.equal(withNestedNamePatterns(runner), runner); + await runner.init(); + await runMutant(runner, runnerPattern('suite reads x'), ['src/a.test.ts']); + assert.deepEqual(started[0].files, ['src/a.test.ts']); + for (const pattern of started[0].patterns) assert.equal(pattern.test('suite > reads x'), true); + }); + + it('runs Vitest without a pattern when the runner sets none', async () => { + const { runner, started } = fakeRunner(); + withNestedNamePatterns(runner); + await runner.init(); + await runMutant(runner, undefined, undefined); + assert.deepEqual(started[0].patterns, [undefined, undefined]); + }); + + // Stryker can init a runner again after it disposes it. + it('wraps the new Vitest context of each init', async () => { + const { runner, started } = fakeRunner(); + withNestedNamePatterns(runner); + await runner.init(); + await runner.init(); + await runMutant(runner, runnerPattern('suite reads x'), []); + assert.equal(started[0].patterns[0].test('suite > reads x'), true); + }); +}); + +// Vitest's record of a test file: `tasks` holds what it collected. +function testFile(filepath, state, { errors = [], tasks = 0 } = {}) { + return { + filepath, + tasks: Array.from({ length: tasks }, () => ({})), + result: { state, errors: errors.map((message) => ({ message })) }, + }; +} + +describe('fileLoadErrors', () => { + it('names each file that failed before it collected a test, with its errors', () => { + const files = [ + testFile('/pkg/a.test.ts', 'pass', { tasks: 3 }), + testFile('/pkg/b.test.ts', 'fail', { errors: ["Cannot find package '@n8n/telemetry'"] }), + testFile('/pkg/c.test.ts', 'fail', { errors: ['first', 'second'] }), + ]; + assert.deepEqual(fileLoadErrors(files), [ + "/pkg/b.test.ts: Cannot find package '@n8n/telemetry'", + '/pkg/c.test.ts: first; second', + ]); + }); + + // The runner reports failed tests itself. + it('leaves out a file that passed, or that has no result yet', () => { + const files = [testFile('/pkg/a.test.ts', 'pass'), { filepath: '/pkg/b.test.ts', tasks: [] }]; + assert.deepEqual(fileLoadErrors(files), []); + }); + + it('leaves out a file whose tests ran and failed', () => { + assert.deepEqual(fileLoadErrors([testFile('/pkg/a.test.ts', 'fail', { tasks: 2 })]), []); + }); + + it('still names a failed file that recorded no error message', () => { + const noErrorList = { filepath: '/pkg/b.test.ts', tasks: [], result: { state: 'fail' } }; + assert.deepEqual(fileLoadErrors([testFile('/pkg/a.test.ts', 'fail'), noErrorList]), [ + '/pkg/a.test.ts: the file did not load', + '/pkg/b.test.ts: the file did not load', + ]); + }); +}); + +describe('withLoadErrors', () => { + function fakeRunner(result, files) { + return { + ctx: { state: { getFiles: () => files } }, + dryRun: async () => result, + }; + } + + it('fails the dry run, with the error, when a test file did not load', async () => { + const files = [testFile('/pkg/a.test.ts', 'fail', { errors: ['Cannot find package'] })]; + const runner = withLoadErrors(fakeRunner({ status: 'complete', tests: [] }, files)); + assert.deepEqual(await runner.dryRun({}), { + status: 'error', + errorMessage: 'A test file did not load:\n/pkg/a.test.ts: Cannot find package', + }); + }); + + it('puts each file that did not load on its own line', async () => { + const files = [ + testFile('/pkg/a.test.ts', 'fail', { errors: ['first'] }), + testFile('/pkg/b.test.ts', 'fail', { errors: ['second'] }), + ]; + const runner = withLoadErrors(fakeRunner({ status: 'complete', tests: [] }, files)); + assert.equal( + (await runner.dryRun({})).errorMessage, + 'A test file did not load:\n/pkg/a.test.ts: first\n/pkg/b.test.ts: second', + ); + }); + + it('keeps a dry run where every file loaded', async () => { + const result = { status: 'complete', tests: [{ id: 't1' }] }; + const runner = withLoadErrors( + fakeRunner(result, [testFile('/pkg/a.test.ts', 'pass', { tasks: 1 })]), + ); + assert.equal(await runner.dryRun({}), result); + }); + + it('keeps a dry run that already failed for another reason', async () => { + const result = { status: 'timeout' }; + const runner = withLoadErrors(fakeRunner(result, [testFile('/pkg/a.test.ts', 'fail')])); + assert.equal(await runner.dryRun({}), result); + }); +}); + +describe('compatPlugins', () => { + // A stand-in for the plugin list of @stryker-mutator/vitest-runner. Its + // runner makes a Vitest context with one project and a set of test files. + function runnerPlugins(files = []) { + function factory(injector) { + return { + injector, + async init() { + const projects = [{ config: {} }]; + this.ctx = { + projects, + state: { getFiles: () => files }, + start: async () => projects.map((project) => project.config.testNamePattern), + }; + }, + async dryRun() { + return { status: 'complete', tests: [] }; + }, + }; + } + factory.inject = ['$injector']; + return [ + { kind: 'Reporter', name: 'vitest', factory: () => ({}) }, + { kind: 'TestRunner', name: 'vitest', factory }, + ]; + } + + it('declares one vitest-compat test runner with the dependencies of the vitest runner', () => { + const plugins = compatPlugins(runnerPlugins()); + assert.equal(plugins.length, 1); + const [plugin] = plugins; + assert.equal(plugin.kind, 'TestRunner'); + // The name shows in stryker.run.json and in the README. + assert.equal(plugin.name, 'vitest-compat'); + assert.equal(plugin.name, COMPAT_RUNNER_NAME); + assert.deepEqual(plugin.factory.inject, ['$injector']); + }); + + it('makes a vitest runner, from the same injector, that selects nested tests', async () => { + const [plugin] = compatPlugins(runnerPlugins()); + const injector = {}; + const runner = plugin.factory(injector); + assert.equal(runner.injector, injector); + await runner.init(); + runner.ctx.projects[0].config.testNamePattern = runnerPattern('suite reads x'); + const [pattern] = await runner.ctx.start([]); + assert.equal(pattern.test('suite > reads x'), true); + }); + + it('makes a vitest runner whose dry run fails when a test file did not load', async () => { + const files = [testFile('/pkg/a.test.ts', 'fail', { errors: ['Cannot find package'] })]; + const runner = compatPlugins(runnerPlugins(files))[0].factory({}); + await runner.init(); + assert.deepEqual(await runner.dryRun({}), { + status: 'error', + errorMessage: 'A test file did not load:\n/pkg/a.test.ts: Cannot find package', + }); + }); + + // A runner upgrade that renames the export must stop the run with a clear message. + it('throws when the list has no vitest test runner', () => { + const [reporter, testRunner] = runnerPlugins(); + const renamed = { ...testRunner, name: 'vitest-next' }; + for (const plugins of [[reporter], [renamed], []]) { + assert.throws( + () => compatPlugins(plugins), + /^Error: @stryker-mutator\/vitest-runner no longer declares the "vitest" test runner\.$/, + ); + } + }); +}); + +// CI runs these tests without the root node_modules, so these checks need them. +function unlessRunnerMissing(error) { + if ( + error.code === 'ERR_MODULE_NOT_FOUND' && + /'@stryker-mutator\/vitest-runner'/.test(error.message) + ) { + return null; + } + throw error; +} +const installedPlugins = await import('./vitest-compat-runner.mjs').then( + (module) => module.strykerPlugins, + unlessRunnerMissing, +); +const installedRunnerPlugins = await import('@stryker-mutator/vitest-runner').then( + (module) => module.strykerPlugins, + unlessRunnerMissing, +); + +// The injector that Stryker gives the runner's factory, with the runner options. +function injectorFor(options) { + return { + provideValue() { + return this; + }, + injectClass(RunnerClass) { + return new RunnerClass(options, {}, '__stryker__'); + }, + }; +} + +// A stand-in for the Vitest context that the runner's `init` makes. `start` +// records the files and the test name pattern of each test run. `files` is +// what Vitest recorded for the test files. +function fakeVitest(started, files = []) { + const projects = [{ config: {} }]; + return { + projects, + config: {}, + provide() {}, + state: { filesMap: new Map(), getFiles: () => files, errorsSet: new Set() }, + async start(files) { + started.push({ files, pattern: projects[0].config.testNamePattern }); + }, + }; +} + +describe('the vitest-compat plugin file', () => { + const skip = !installedPlugins && '@stryker-mutator/vitest-runner is not installed'; + + it( + 'declares the vitest-compat runner with the dependencies of the installed runner', + { skip }, + () => { + const vitestRunner = installedRunnerPlugins.find( + (plugin) => plugin.kind === 'TestRunner' && plugin.name === 'vitest', + ); + assert.equal(installedPlugins.length, 1); + const [plugin] = installedPlugins; + assert.equal(plugin.kind, 'TestRunner'); + assert.equal(plugin.name, COMPAT_RUNNER_NAME); + assert.deepEqual(plugin.factory.inject, vitestRunner.factory.inject); + }, + ); + + // The installed runner's own dry run, on a Vitest stand-in. The stand-in is + // set in place of `init`, which would start the real Vitest. + it( + 'makes the dry run of the installed runner fail when a test file did not load', + { skip }, + async () => { + // The runner reads the coverage of each file from `meta`. + const failed = { + ...testFile('/pkg/a.test.ts', 'fail', { errors: ['Cannot find package'] }), + meta: {}, + }; + const runner = installedPlugins[0].factory(injectorFor({ vitest: { related: false } })); + const started = []; + runner.ctx = fakeVitest(started, [failed]); + const result = await runner.dryRun({ files: ['/pkg/src/a.ts'] }); + assert.equal(started.length, 1); + assert.deepEqual(result, { + status: 'error', + errorMessage: 'A test file did not load:\n/pkg/a.test.ts: Cannot find package', + }); + }, + ); + + // The runner's own mutant run, with Vitest replaced by a stand-in. A runner + // upgrade that selects tests in another way fails here, and not as a run + // that kills no mutant. + it('makes the installed runner select a nested test for a mutant', { skip }, async () => { + const vitestRunner = installedRunnerPlugins.find( + (plugin) => plugin.kind === 'TestRunner' && plugin.name === 'vitest', + ); + const started = []; + function withFakeVitest(injector) { + const runner = vitestRunner.factory(injector); + // Only the start of Vitest is replaced. The fixes wrap this `init`. + runner.init = async () => { + runner.ctx = fakeVitest(started); + }; + return runner; + } + withFakeVitest.inject = vitestRunner.factory.inject; + const [plugin] = compatPlugins([ + { kind: vitestRunner.kind, name: 'vitest', factory: withFakeVitest }, + ]); + const runner = plugin.factory(injectorFor({ vitest: { related: true } })); + await runner.init(); + await runner.mutantRun({ + activeMutant: { id: '1' }, + testFilter: ['/pkg/a.test.ts#suite reads x'], + sandboxFileName: '/pkg/src/a.ts', + mutantActivation: 'runtime', + }); + assert.equal(started.length, 1); + const [{ files, pattern }] = started; + assert.deepEqual(files, ['/pkg/a.test.ts']); + assert.equal(pattern.test('suite > reads x'), true); + assert.equal(pattern.test('suite reads x'), true); + assert.equal(pattern.test('suite > reads y'), false); + }); +}); diff --git a/scripts/mutation-health/working-tree.test.mjs b/scripts/mutation-health/working-tree.test.mjs new file mode 100644 index 00000000000..cf37664540a --- /dev/null +++ b/scripts/mutation-health/working-tree.test.mjs @@ -0,0 +1,179 @@ +/** + * The main promise of the tool: it never writes the working tree. Stryker + * mutates a copy of the package, and the tool writes only its reports. So an + * edit that another agent makes while a run is in flight stays, whatever way + * the run ends. A tool that puts files back after a run (as in place mode + * needed) reverts such an edit, and fails here. + */ +import { afterEach, beforeEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + lstatSync, + mkdtempSync, + readFileSync, + readdirSync, + readlinkSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { runJob } from './mutate.mjs'; +import { toPosix } from './targets.mjs'; +import { fakeProcess, fakeSpawn, resolveFakeStrykerBin, sink, writeTree } from './test-doubles.mjs'; + +const PACKAGE_DIR = 'packages/@n8n/pkg'; +const TARGET = `${PACKAGE_DIR}/src/a.ts`; +// A file that the package's config can reach with `..`, through the mirror. +const SIBLING = 'packages/@n8n/sib/x.ts'; +const REPORTS = `${PACKAGE_DIR}/reports`; +// What another agent writes while the run is in flight. +const EDITS = { + [TARGET]: 'export const a = 40 + 2;\n', + [SIBLING]: 'edited during the run\n', +}; + +// A Stryker report with one killed mutant. +const REPORT = { + files: { + 'src/a.ts': { + source: 'export const a = 1 + 2;\n', + mutants: [ + { + id: '1', + mutatorName: 'ArithmeticOperator', + status: 'Killed', + location: { start: { line: 1, column: 17 }, end: { line: 1, column: 22 } }, + replacement: '1 - 2', + }, + ], + }, + }, +}; + +let root; + +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), 'mutate-tree-')); + writeTree(root, { + [`${PACKAGE_DIR}/package.json`]: { name: 'pkg', scripts: { test: 'vitest run' } }, + [TARGET]: REPORT.files['src/a.ts'].source, + [`${PACKAGE_DIR}/src/a.test.ts`]: 'it("a", () => {});\n', + [SIBLING]: 'sibling\n', + 'README.md': 'repo\n', + }); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +// Each entry under `dir`: a file maps to its content, a dir to '' and a +// link to its target. +function snapshotTree(dir, out = new Map()) { + for (const name of readdirSync(dir)) { + const abs = path.join(dir, name); + const rel = toPosix(path.relative(root, abs)); + const stat = lstatSync(abs); + if (stat.isSymbolicLink()) { + out.set(rel, `-> ${readlinkSync(abs)}`); + } else if (stat.isDirectory()) { + out.set(rel, ''); + snapshotTree(abs, out); + } else { + out.set(rel, readFileSync(abs, 'utf8')); + } + } + return out; +} + +// Each path that was added, removed or changed between two snapshots. +function changedPaths(before, after) { + const paths = new Set([...before.keys(), ...after.keys()]); + return [...paths].filter((p) => before.get(p) !== after.get(p)).sort(); +} + +const isReport = (p) => p === REPORTS || p.startsWith(`${REPORTS}/`); + +/** + * Run one job with a fake Stryker. While Stryker "runs", the edits are made, + * then `end` ends the run. Settles with how the job ended: 'resolved', the + * exit code of its error, or `exit ` when the tool exited after a crash + * (the job itself then never settles). + */ +async function runEditedJob(end) { + const p = fakeProcess(); + const doubles = fakeSpawn({ + onSpawn: (child, call) => + setImmediate(() => { + writeTree(root, EDITS); + end(child, call, p); + }), + }); + const job = { + pkgRoot: path.join(root, PACKAGE_DIR), + packageDir: PACKAGE_DIR, + targets: ['src/a.ts'], + }; + const args = { configArg: undefined, testFiles: ['src/a.test.ts'], testCommand: undefined }; + const io = { + resolveStrykerBin: resolveFakeStrykerBin, + repoRoot: root, + spawn: doubles.spawn, + stdout: sink(), + stderr: sink(), + proc: p.proc, + exit: p.exit, + write: p.write, + }; + const settled = runJob(job, args, io).then( + () => 'resolved', + (error) => error.exitCode, + ); + return await Promise.race([settled, p.exited.then((code) => `exit ${code}`)]); +} + +function writesReport(child, call) { + writeFileSync(path.join(call.options.cwd, 'reports/mutation/raw.json'), JSON.stringify(REPORT)); + child.finish(0); +} + +const ENDINGS = { + 'a run that wrote a report': { end: writesReport, ending: 'resolved' }, + 'a run that wrote no report': { end: (child) => child.finish(1), ending: 'resolved' }, + 'a Stryker that did not start': { + end: (child) => child.emit('error', new Error('spawn ENOENT')), + ending: 3, + }, + 'a run cancelled with SIGINT': { + end: (child, _call, p) => { + p.proc.emit('SIGINT'); + child.finish(130); + }, + ending: 130, + }, + 'a crash of the tool': { + end: (child, _call, p) => { + p.proc.emit('uncaughtException', new Error('boom')); + setImmediate(() => child.finish(130)); + }, + ending: 'exit 3', + }, +}; + +describe('runJob and the working tree', () => { + for (const [name, { end, ending }] of Object.entries(ENDINGS)) { + it(`keeps each edit made during ${name}, and writes only its reports`, async () => { + const before = snapshotTree(root); + assert.equal(await runEditedJob(end), ending); + const after = snapshotTree(root); + for (const [file, content] of Object.entries(EDITS)) { + assert.equal(after.get(file), content, file); + } + // The mirror is gone too: a `.stryker-tmp` entry shows up here. + const others = changedPaths(before, after).filter((p) => !(p in EDITS) && !isReport(p)); + assert.deepEqual(others, []); + }); + } +});