github-mirror/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-10-17 05:29:44 +02:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity Graph

feat(core): Make the vm expression engine the default (#35738)

Browse Source 
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Mike Repeć 2026-08-07 11:55:26 +02:00 committed by GitHub
parent 95e13260c4
commit 98e0952c36
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 110 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/WORKFLOWS.md vendored
View File

@ -71,7 +71,6 @@ Complete reference for n8n's `.github/` folder.
│ │ Schedule │───▶│ Nightly/Weekly Jobs │ ┌────────────┐ │
│ │ (cron) │ │ ├─ docker-build-push (nightly) │───▶│ Images │ │
│ └──────────┘ │ ├─ test-benchmark-nightly │───▶│ Metrics │ │
│ │ ├─ test-e2e-vm-expressions │ │
│ │ └─ test-e2e-coverage-weekly │ │
│ └──────────────────────────────────┘ │
│ │
@ -298,9 +297,6 @@ release-publish.yml
test-workflows-nightly.yml (manual dispatch only — nightly schedule disabled, DEVP-544)
└──────────────────────────▶ test-workflows-callable.yml
test-e2e-vm-expressions-nightly.yml
└──────────────────────────▶ test-e2e-reusable.yml
PR Comment Dispatchers (triggered by /command in PR comments):
test-workflows-pr-comment.yml
└──────────────────────────▶ test-workflows-callable.yml
@ -414,7 +410,6 @@ Push to master/1.x
| Daily 00:00 | `util-check-docs-urls.yml` | Doc link validation |
| Daily 01:30, 02:30, 03:30 | `test-benchmark-nightly.yml` | Performance benchmarks |
| Daily 02:00 | `test-get-n8n.yml` | get.n8n.io installer health |
| Daily 04:00 | `test-e2e-vm-expressions-nightly.yml`| VM expression E2E |
| Daily 05:00 | `test-benchmark-destroy-nightly.yml`| Cleanup benchmark env |
| Daily 06:00 | `util-sync-master-to-3x.yml` | Replay 3.x onto master (v3) |
| Daily 08:00 | `build-v3-nightly.yml` | Nightly v3 Docker images |

2
.github/workflows/test-e2e-reusable.yml vendored
View File

@ -43,7 +43,7 @@ on:
default: '[{"shard":1,"images":""}]'
type: string
n8n-env:
description: 'JSON string of n8n env vars to inject into test containers, e.g. {"N8N_EXPRESSION_ENGINE":"vm"}'
description: 'JSON string of n8n env vars to inject into test containers, e.g. {"N8N_EXPRESSION_ENGINE":"legacy"}'
required: false
default: ''
type: string

53
.github/workflows/test-e2e-vm-expressions-nightly.yml vendored
View File

@ -1,53 +0,0 @@
name: 'Test: E2E VM Expressions Nightly'
run-name: "Test: E2E VM Expressions (${{ inputs.branch || 'nightly' }})"
on:
schedule:
- cron: '0 4 * * *'
workflow_dispatch:
inputs:
branch:
description: 'GitHub branch/ref to test'
required: false
type: string
default: 'master'
jobs:
prepare-docker:
uses: ./.github/workflows/prepare-docker-reusable.yml
with:
branch: ${{ github.event_name == 'schedule' && 'master' || inputs.branch }}
secrets: inherit
e2e:
name: E2E
needs: [prepare-docker]
uses: ./.github/workflows/test-e2e-reusable.yml
with:
branch: ${{ github.event_name == 'schedule' && 'master' || inputs.branch }}
test-mode: docker-artifact
test-command: pnpm --filter=n8n-playwright test:container:multi-main:e2e
workers: '1'
pre-generated-matrix: '[{"shard":1},{"shard":2},{"shard":3},{"shard":4},{"shard":5},{"shard":6},{"shard":7},{"shard":8},{"shard":9},{"shard":10},{"shard":11},{"shard":12},{"shard":13},{"shard":14},{"shard":15},{"shard":16}]'
n8n-env: '{"N8N_EXPRESSION_ENGINE":"vm"}'
artifact-prefix: vm-expressions
currents-project-id: 'LRxcNt'
secrets: inherit
notify-on-failure:
name: Notify Slack on failure
runs-on: ubuntu-slim
needs: [e2e]
if: ${{ always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: .github/scripts/slack
sparse-checkout-cone-mode: false
- name: Notify Slack
env:
SLACK_TOKEN: ${{ secrets.QBOT_SLACK_TOKEN }}
run: |
node .github/scripts/slack/notify.mjs \
--channel '#project-secure-expression-evaluation' \
--text 'VM expression E2E nightly failed - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'

19
packages/@n8n/config/src/configs/__tests__/expression-engine.config.test.ts
View File

@ -10,6 +10,10 @@ describe('ExpressionEngineConfig', () => {
});
describe('defaults', () => {
test('engine defaults to vm', () => {
expect(Container.get(ExpressionEngineConfig).engine).toBe('vm');
});
test('bridgeTimeout defaults to 5000', () => {
expect(Container.get(ExpressionEngineConfig).bridgeTimeout).toBe(5000);
});
@ -19,6 +23,21 @@ describe('ExpressionEngineConfig', () => {
});
});
describe('N8N_EXPRESSION_ENGINE', () => {
test('overrides engine to legacy', () => {
vi.stubEnv('N8N_EXPRESSION_ENGINE', 'legacy');
expect(Container.get(ExpressionEngineConfig).engine).toBe('legacy');
});
test('falls back to default on invalid value', () => {
const consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
vi.stubEnv('N8N_EXPRESSION_ENGINE', 'not-an-engine');
expect(Container.get(ExpressionEngineConfig).engine).toBe('vm');
expect(consoleWarnSpy).toHaveBeenCalled();
});
});
describe('N8N_EXPRESSION_ENGINE_IDLE_TIMEOUT', () => {
test('overrides idleTimeout', () => {
vi.stubEnv('N8N_EXPRESSION_ENGINE_IDLE_TIMEOUT', '60');

8
packages/@n8n/config/src/configs/expression-engine.config.ts
View File

@ -8,13 +8,11 @@ const expressionEngineSchema = z.enum(['legacy', 'vm']);
export class ExpressionEngineConfig {
/**
* Which expression engine to use.
* - `legacy` runs expressions without isolation.
* - `vm` runs expressions in a V8 isolate.
*
* `vm` is currently **experimental**. Use at your own risk.
* - `vm` (default) runs expressions in a V8 isolate.
* - `legacy` runs expressions without isolation. Less secure and soon to be deprecated.
*/
@Env('N8N_EXPRESSION_ENGINE', expressionEngineSchema)
engine: 'legacy' | 'vm' = 'legacy';
engine: 'legacy' | 'vm' = 'vm';
/** Number of V8 isolates ready in the pool. */
@Env('N8N_EXPRESSION_ENGINE_POOL_SIZE')

2
packages/@n8n/config/test/config.test.ts
View File

@ -627,7 +627,7 @@ describe('GlobalConfig', () => {
trimOnStartUp: false,
},
expressionEngine: {
engine: 'legacy',
engine: 'vm',
poolSize: 1,
maxCodeCacheSize: 1024,
bridgeTimeout: 5000,

122
packages/@n8n/expression-runtime/README.md
View File

@ -4,18 +4,16 @@ Secure, isolated expression evaluation runtime for n8n workflows.
## Status
**In progress — landing as a series of incremental PRs.**
**Shipped — the `vm` engine is n8n's default expression engine.**
Implemented so far:
- ✅ TypeScript interfaces and architecture design (PR 1)
- ✅ Core architecture documentation (PR 1)
- ✅ Runtime bundle: extension functions, deep lazy proxy system (PR 2)
- ✅ `IsolatedVmBridge`: V8 isolate management via `isolated-vm` (PR 3)
- ✅ `ExpressionEvaluator`: tournament integration, expression code caching (PR 4)
- ✅ Integration tests (PR 4)
- ✅ TypeScript interfaces and architecture design
- ✅ Runtime bundle: extension functions, deep lazy proxy system
- ✅ `IsolatedVmBridge`: V8 isolate management via `isolated-vm`
- ✅ `ExpressionEvaluator`: tournament integration, expression code caching, isolate pooling
- ✅ Workflow integration — default engine; `N8N_EXPRESSION_ENGINE=legacy` opts out
- ✅ Observability (metrics, traces, logs) wired up in `packages/cli`
Coming in later PRs:
- 🚧 Workflow integration behind `N8N_EXPRESSION_ENGINE=vm` flag (PR 5)
Coming later:
- 🚧 Web Worker support (Phase 2+)
- 🚧 Performance optimizations (Phase 3)
@ -34,7 +32,7 @@ Future support (Phase 2+):
- 🔒 **Secure**: Expressions run in isolated V8 contexts with memory limits (128MB) and timeouts (5s)
- 🚀 **Performant**: Lazy data loading via proxies, script compilation caching, and expression code caching
- 📊 **Observable**: Built-in metrics, traces, and logs support (interfaces defined; providers coming later)
- 📊 **Observable**: Built-in metrics, traces, and logs support via `ObservabilityProvider`
- 🌐 **Universal**: Works in Node.js backend (browsers and task runners in Phase 2+)
- 🛡️ **AST Security**: Tournament AST hooks (`ThisSanitizer`, `PrototypeSanitizer`, `DollarSignValidator`) validate expressions before execution
@ -61,32 +59,33 @@ pnpm add @n8n/expression-runtime
```typescript
import { ExpressionEvaluator, IsolatedVmBridge } from '@n8n/expression-runtime';
// Create bridge
const bridge = new IsolatedVmBridge({
memoryLimit: 128,
timeout: 5000,
});
// Create evaluator
// Create evaluator with a bridge factory (bridges are pooled)
const evaluator = new ExpressionEvaluator({
bridge,
createBridge: () => new IsolatedVmBridge({ memoryLimit: 128, timeout: 5000 }),
maxCodeCacheSize: 1024,
});
// Initialize
await evaluator.initialize();
// Evaluate expression using {{ }} template syntax
// Acquire an isolate for a caller, evaluate, release
const caller = {};
await evaluator.acquire(caller);
const result = evaluator.evaluate(
'{{ $json.user.email }}',
{
$json: {
user: { email: 'test@example.com' }
}
}
},
caller,
);
console.log(result); // "test@example.com"
await evaluator.release(caller);
// Clean up
await evaluator.dispose();
```
@ -103,9 +102,9 @@ import {
DollarSignValidator,
} from 'n8n-workflow/expression-sandboxing';
const bridge = new IsolatedVmBridge({ timeout: 5000 });
const evaluator = new ExpressionEvaluator({
bridge,
createBridge: () => new IsolatedVmBridge({ timeout: 5000 }),
maxCodeCacheSize: 1024,
hooks: {
before: [ThisSanitizer],
after: [PrototypeSanitizer, DollarSignValidator],
@ -117,22 +116,19 @@ await evaluator.initialize();
When `hooks` is omitted the evaluator still runs tournament transformation (template parsing, `this` binding) but without AST security validation — suitable for development and testing.
### With Observability (Not Yet Implemented)
### With Observability
Pass an `ObservabilityProvider` implementation to emit metrics, traces, and logs for evaluations:
```typescript
import { OpenTelemetryProvider } from '@n8n/expression-runtime/observability';
const observability = new OpenTelemetryProvider({
serviceName: 'n8n-expressions',
});
const evaluator = new ExpressionEvaluator({
bridge,
createBridge: () => new IsolatedVmBridge({ timeout: 5000 }),
maxCodeCacheSize: 1024,
observability,
});
```
**Note**: Observability providers are not yet implemented. The `ObservabilityProvider` interface exists but no implementations are available yet.
This package defines the `ObservabilityProvider` interface; the production implementation lives in `packages/cli/src/expression-observability/expression-observability.provider.ts` and is wired up during backend startup. It is controlled via the `N8N_EXPRESSION_ENGINE_OBSERVABILITY_*` and `N8N_EXPRESSION_ENGINE_TRACES_*` environment variables (see below).
## API
@ -144,7 +140,9 @@ Main class for expression evaluation.
class ExpressionEvaluator {
constructor(config: EvaluatorConfig);
initialize(): Promise<void>;
evaluate(expression: string, data: WorkflowData, options?: EvaluateOptions): unknown;
acquire(owner: object): Promise<boolean>;
evaluate(expression: string, data: WorkflowData, caller: object, options?: EvaluateOptions): unknown;
release(owner: object): Promise<void>;
dispose(): Promise<void>;
isDisposed(): boolean;
}
@ -179,38 +177,41 @@ interface RuntimeBridge {
```typescript
interface EvaluatorConfig {
bridge: RuntimeBridge; // required
observability?: ObservabilityProvider; // optional - interfaces defined, providers not yet implemented
createBridge: () => RuntimeBridge; // required - factory, bridges are pooled
maxCodeCacheSize: number; // required - LRU size for tournament-transformed code
observability?: ObservabilityProvider; // optional - metrics/traces/logs provider
hooks?: TournamentHooks; // optional - AST security hooks for tournament
}
interface BridgeConfig {
memoryLimit?: number; // Default: 128 MB
timeout?: number; // Default: 5000 ms
debug?: boolean; // Default: false
poolSize?: number; // optional - pre-warmed bridges, default 1
idleTimeoutMs?: number; // optional - scale pool to 0 after idle period
logger?: Logger; // optional - falls back to no-op
}
```
## Environment Variables (Not Yet Implemented)
## Environment Variables
In n8n, the evaluator is configured via `ExpressionEngineConfig` (`@n8n/config`):
```bash
# Bridge configuration (not yet implemented)
N8N_EXPRESSION_MEMORY_LIMIT_MB=128
N8N_EXPRESSION_TIMEOUT_MS=5000
N8N_EXPRESSION_DEBUG=false
# Engine selection ('vm' is the default; 'legacy' opts out of isolation)
N8N_EXPRESSION_ENGINE=vm
# Code cache (not yet implemented - caches transformed code, not results)
N8N_EXPRESSION_CODE_CACHE_ENABLED=true
N8N_EXPRESSION_CODE_CACHE_MAX_SIZE=1000
# Isolate pool and code cache
N8N_EXPRESSION_ENGINE_POOL_SIZE=1
N8N_EXPRESSION_ENGINE_MAX_CODE_CACHE_SIZE=1024
N8N_EXPRESSION_ENGINE_IDLE_TIMEOUT= # seconds; unset = pool never scales to 0
# Observability (not yet implemented)
N8N_EXPRESSION_OBSERVABILITY_ENABLED=true
N8N_EXPRESSION_METRICS_ENABLED=true
N8N_EXPRESSION_TRACES_ENABLED=true
N8N_EXPRESSION_TRACE_SAMPLE_RATE=0.01
# Bridge limits
N8N_EXPRESSION_ENGINE_TIMEOUT=5000 # ms
N8N_EXPRESSION_ENGINE_MEMORY_LIMIT=128 # MB
# Observability
N8N_EXPRESSION_ENGINE_OBSERVABILITY_ENABLED=true
N8N_EXPRESSION_ENGINE_TRACES_ENABLED=true
N8N_EXPRESSION_ENGINE_SLOW_EVAL_THRESHOLD_MS=50
N8N_EXPRESSION_ENGINE_TRACES_SAMPLE_RATE=0.0
```
**Note**: Currently, configuration is passed via constructor options. Environment variable support will be added in future phases.
See `packages/@n8n/config/src/configs/expression-engine.config.ts` for the authoritative list and defaults.
## Development
@ -243,13 +244,18 @@ import { ExpressionEvaluator, IsolatedVmBridge } from '@n8n/expression-runtime';
describe('ExpressionEvaluator', () => {
it('evaluates simple expression', async () => {
const bridge = new IsolatedVmBridge({ timeout: 5000 });
const evaluator = new ExpressionEvaluator({ bridge });
const evaluator = new ExpressionEvaluator({
createBridge: () => new IsolatedVmBridge({ timeout: 5000 }),
maxCodeCacheSize: 1024,
});
await evaluator.initialize();
const result = evaluator.evaluate('{{ $json.value }}', { $json: { value: 42 } });
const caller = {};
await evaluator.acquire(caller);
const result = evaluator.evaluate('{{ $json.value }}', { $json: { value: 42 } }, caller);
expect(result).toBe(42);
await evaluator.release(caller);
await evaluator.dispose();
});

2
packages/cli/src/commands/__tests__/execute-batch.test.ts
View File

@ -19,6 +19,7 @@ import { DeprecationService } from '@/deprecation/deprecation.service';
import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus';
import { TelemetryEventRelay } from '@/events/relays/telemetry.event-relay';
import { WorkflowFailureNotificationEventRelay } from '@/events/relays/workflow-failure-notification.event-relay';
import { ExpressionObservabilityProvider } from '@/expression-observability/expression-observability.provider';
import { ExternalHooks } from '@/external-hooks';
import { License } from '@/license';
import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
@ -40,6 +41,7 @@ const loadNodesAndCredentials = mockInstance(LoadNodesAndCredentials);
const shutdownService = mockInstance(ShutdownService);
const deprecationService = mockInstance(DeprecationService);
mockInstance(MessageEventBus);
mockInstance(ExpressionObservabilityProvider);
const posthogClient = mockInstance(PostHogClient);
const telemetryEventRelay = mockInstance(TelemetryEventRelay);
const externalHooks = mockInstance(ExternalHooks);

2
packages/cli/src/commands/__tests__/execute.test.ts
View File

@ -18,6 +18,7 @@ import { DeprecationService } from '@/deprecation/deprecation.service';
import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus';
import { TelemetryEventRelay } from '@/events/relays/telemetry.event-relay';
import { WorkflowFailureNotificationEventRelay } from '@/events/relays/workflow-failure-notification.event-relay';
import { ExpressionObservabilityProvider } from '@/expression-observability/expression-observability.provider';
import { ExternalHooks } from '@/external-hooks';
import { License } from '@/license';
import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
@ -40,6 +41,7 @@ const loadNodesAndCredentials = mockInstance(LoadNodesAndCredentials);
const shutdownService = mockInstance(ShutdownService);
const deprecationService = mockInstance(DeprecationService);
mockInstance(MessageEventBus);
mockInstance(ExpressionObservabilityProvider);
const posthogClient = mockInstance(PostHogClient);
const telemetryEventRelay = mockInstance(TelemetryEventRelay);
const externalHooks = mockInstance(ExternalHooks);

2
packages/cli/src/deprecation/__tests__/deprecation.service.test.ts
View File

@ -67,6 +67,8 @@ describe('DeprecationService', () => {
['WEBHOOK_URL', 'https://example.com/', true],
['N8N_DEFAULT_BINARY_DATA_MODE', 'default', true],
['N8N_DEFAULT_BINARY_DATA_MODE', 'filesystem', false],
['N8N_EXPRESSION_ENGINE', 'legacy', true],
['N8N_EXPRESSION_ENGINE', 'vm', false],
])('should detect when %s is `%s`', (envVar, value, mustWarn) => {
toTest(envVar, value, mustWarn);
});

6
packages/cli/src/deprecation/deprecation.service.ts
View File

@ -91,6 +91,12 @@ export class DeprecationService {
'The default for this variable will be reduced from 5000 to 1000 in a future version. Set it explicitly to keep your current limit.',
checkValue: (value?: string) => value === undefined,
},
{
envVar: 'N8N_EXPRESSION_ENGINE',
message:
'The `legacy` expression engine runs expressions without isolation, is no longer considered secure, and will be removed in a future version. Remove this environment variable to use the default `vm` engine.',
checkValue: (value?: string) => value === 'legacy',
},
{
envVar: 'N8N_DEFAULT_BINARY_DATA_MODE',
message:

1
packages/testing/containers/services/n8n.ts
View File

@ -45,7 +45,6 @@ export class N8NStartupError extends Error {
const BASE_ENV: Record<string, string> = {
N8N_LOG_LEVEL: 'debug',
N8N_EXPRESSION_ENGINE: 'vm',
N8N_ENCRYPTION_KEY: process.env.N8N_ENCRYPTION_KEY ?? 'test-encryption-key',
E2E_TESTS: 'false',
QUEUE_HEALTH_CHECK_ACTIVE: 'true',

1
packages/testing/playwright/playwright.config.ts
View File

@ -66,7 +66,6 @@ if (BACKEND_URL && !SKIP_WEB_SERVER) {
N8N_METRICS: 'true',
N8N_RESTRICT_FILE_ACCESS_TO: '',
N8N_DYNAMIC_BANNERS_ENABLED: 'false',
N8N_EXPRESSION_ENGINE: 'vm',
...getTestEnv(),
},
});

4
packages/workflow/src/expression.ts
View File

@ -239,7 +239,7 @@ export class Expression {
}
/**
* Initialize the VM evaluator (if feature flag is enabled).
* Initialize the VM evaluator (no-op when the legacy engine is selected).
* Should be called once during application startup.
* Only available in Node.js environments (not in browser).
*/
@ -645,7 +645,7 @@ export class Expression {
if (Expression.expressionEngine === 'vm' && !IS_FRONTEND) {
if (!Expression.vmEvaluator) {
throw new UnexpectedError(
'N8N_EXPRESSION_ENGINE=vm is enabled but VM evaluator is not initialized. Call Expression.initExpressionEngine() during application startup.',
'The VM expression engine has not been initialized. Call Expression.initExpressionEngine() during application startup.',
);
}

4
packages/workflow/stryker.config.mjs
View File

@ -3,7 +3,7 @@ export default {
packageManager: 'pnpm',
testRunner: 'vitest',
plugins: ['@stryker-mutator/vitest-runner'],
// Use the Stryker-specific vitest config (vm-engine only) — see
// Use the Stryker-specific vitest config (legacy-engine only) — see
// vitest.stryker.config.ts for the rationale.
vitest: {
configFile: 'vitest.stryker.config.ts',
@ -23,7 +23,7 @@ export default {
jsonReporter: { fileName: 'reports/mutation/raw.json' },
// Cap low: a string/utils mutant running 15s+ is a spinning loop-condition mutant; fail it fast.
timeoutMS: 15_000,
// Each Stryker worker spawns vitest with one project (vm-engine, via
// Each Stryker worker spawns vitest with one project (legacy-engine, via
// vitest.stryker.config.ts). Default 4 is fine on CI runners; lower
// locally if you hit OOM during the initial dry run via
// `STRYKER_CONCURRENCY=1`.

7
packages/workflow/vitest.config.ts
View File

@ -14,14 +14,15 @@ export default defineConfig({
{
test: {
...sharedTestConfig,
name: 'legacy-engine',
name: 'vm-engine',
env: { N8N_EXPRESSION_ENGINE: 'vm' },
},
},
{
test: {
...sharedTestConfig,
name: 'vm-engine',
env: { N8N_EXPRESSION_ENGINE: 'vm' },
name: 'legacy-engine',
env: { N8N_EXPRESSION_ENGINE: 'legacy' },
},
},
],

3
packages/workflow/vitest.stryker.config.ts
View File

@ -2,7 +2,8 @@
//
// Runs the legacy expression engine, not vm/isolated-vm: the vm evaluator
// SIGABRTs in Stryker's worker on teardown (upstream isolated-vm #464, repros on
// Node 22 + 24). Trade-off: vm-only branches in expression.ts go unscored. Swap
// Node 22 + 24). Trade-off: vm-only branches in expression.ts — the production
// default engine — go unscored. Swap
// back to N8N_EXPRESSION_ENGINE=vm once #464 is fixed or we pnpm-patch the guard.
import { defineConfig } from 'vitest/config';