mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
test(engine): Add v2 infrastructure benchmarks (#38998)
This commit is contained in:
Declan Carroll
GitHub
parent
24e9125a7e
commit
0e7ccf6f74
|
|
@ -26,13 +26,14 @@ jobs:
|
|||
shards: '[{"shard":1,"images":""}]'
|
||||
- lane: webhook
|
||||
grep: '@bench:webhook'
|
||||
# 2 shards — webhook lane grew from 6 to 9 spec files with the
|
||||
# dedicated-proc topology (DEVP-200). Single shard pushed wall time
|
||||
# past 40 min; two shards halves that. Bump to 3 if specs grow further.
|
||||
shards: '[{"shard":1,"images":""},{"shard":2,"images":""}]'
|
||||
# Three shards keep the runtime comparison profiles from extending
|
||||
# this lane beyond the existing 25-minute target.
|
||||
shards: '[{"shard":1,"images":""},{"shard":2,"images":""},{"shard":3,"images":""}]'
|
||||
- lane: kafka
|
||||
grep: '@bench:kafka'
|
||||
shards: '[{"shard":1,"images":""},{"shard":2,"images":""}]'
|
||||
# The v2 profiles add two isolated stacks. Keep lane wall time near
|
||||
# the existing 22-minute target instead of serializing both profiles.
|
||||
shards: '[{"shard":1,"images":""},{"shard":2,"images":""},{"shard":3,"images":""}]'
|
||||
uses: ./.github/workflows/test-e2e-reusable.yml
|
||||
with:
|
||||
test-mode: docker-artifact
|
||||
|
|
@ -96,7 +97,7 @@ jobs:
|
|||
name: Notify Cats on failure
|
||||
runs-on: ubuntu-latest
|
||||
needs: [prepare-docker, benchmark, sizing-matrix]
|
||||
if: ${{ always() && contains(needs.*.result, 'failure') }}
|
||||
if: ${{ always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') }}
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
|
|
|
|||
58
AGENTS.md
58
AGENTS.md
|
|
@ -173,7 +173,7 @@ The monorepo is organized into these key packages:
|
|||
|
||||
- **Frontend:** Vue 3 + TypeScript + Vite + Pinia + Storybook UI Library
|
||||
- **Backend:** Node.js + TypeScript + Express + TypeORM
|
||||
- **Testing:** Vitest (unit) + Playwright (E2E)
|
||||
- **Testing:** Vitest (unit) + Playwright (UI, API, infrastructure, lifecycle, performance, and E2E orchestration)
|
||||
- **Database:** TypeORM with SQLite/PostgreSQL support
|
||||
- **Code Quality:** Biome (for formatting) + ESLint + lefthook git hooks
|
||||
|
||||
|
|
@ -327,33 +327,37 @@ extends one of those layers):
|
|||
- **data-testid must be a single value** (no spaces or multiple values)
|
||||
- Always use the `design-system` skill in reviews
|
||||
|
||||
### Testing Guidelines
|
||||
- **Always work from within the package directory** when running tests
|
||||
- **Mock all external dependencies** in unit tests
|
||||
- **Prefer reusing hoisted shared `mock<T>(...)` fixtures** when a typed mock is immutable and used across tests. This rule exists to avoid massive test slowdowns from repeatedly creating nested proxy mocks while preserving the type contract. Avoid replacing these with `as unknown as T` helpers for entities like `User`.
|
||||
- **Confirm test cases with user** before writing unit tests
|
||||
- **Typecheck is critical before committing** - always run `pnpm typecheck`
|
||||
- **When modifying pinia stores**, check for unused computed properties
|
||||
- **For Vitest packages that use `@n8n/di` decorators**, use `createVitestConfigWithDecorators` from `@n8n/vitest-config/node-decorators`. It enables SWC `decoratorMetadata` (esbuild doesn't emit it) and externalizes workspace packages that register services (`@n8n/di`, `@n8n/config`, `@n8n/constants`, `n8n-workflow`) so a single DI `Container` instance is shared across the runtime. Loading them through Vitest's pipeline alongside their CJS dist produces two `Container`s and `Container.get(...)` returns `undefined`.
|
||||
### Testing and Local Development
|
||||
|
||||
What we use for testing and writing tests:
|
||||
- For testing nodes and other backend components, we use Vitest for unit tests. Examples can be found in `packages/nodes-base/nodes/**/*test*`.
|
||||
- We use `nock` for server mocking
|
||||
- For frontend we use `vitest`
|
||||
- For E2E tests we use Playwright. Run with `pnpm --filter=n8n-playwright test:local`.
|
||||
See `packages/testing/playwright/README.md` for details.
|
||||
- **To iterate on a feature without docker rebuilds**, boot service containers
|
||||
and run the dev servers locally — `pnpm --filter n8n-containers services --services postgres,redis,mailpit,proxy`
|
||||
then `pnpm dev:be` (backend on 5678). For frontend hot reload, also run
|
||||
`pnpm dev:fe:editor` (8080). The root `pnpm dev` does not exist: it prints a
|
||||
notice and exits with code 0, thus `pnpm dev && …` looks successful but no
|
||||
server runs. See
|
||||
[Develop against running containers](packages/testing/playwright/README.md#develop-against-running-containers-avoid-docker-rebuilds).
|
||||
- **In a codespace agent session**, use `pnpm dev:up`. It installs the missing
|
||||
dependencies, starts the backend, waits for health, shares the port with the
|
||||
org, and prints the URL. See
|
||||
[.devcontainer/codespaces/README.md](.devcontainer/codespaces/README.md).
|
||||
- **For Playwright test maintenance/cleanup**, see `packages/testing/playwright/AGENTS.md` (includes janitor tool for static analysis, dead code removal, architecture enforcement, and TCR workflows).
|
||||
Choose the smallest runner that owns the behavior:
|
||||
|
||||
| Need | Use |
|
||||
|------|-----|
|
||||
| Unit or component behavior | Vitest from the owning package |
|
||||
| UI, API, lifecycle, topology, or performance orchestration | Playwright; read `packages/testing/playwright/AGENTS.md` |
|
||||
| Add a test service, capability, or managed stack | Read `packages/testing/containers/README.md` |
|
||||
|
||||
Testing rules:
|
||||
|
||||
- Run tests and `pnpm typecheck` from the owning package.
|
||||
- Confirm unit test cases with the user before you write them.
|
||||
- Mock external dependencies. Use `nock` for HTTP services.
|
||||
- Reuse immutable hoisted `mock<T>(...)` fixtures. Do not replace typed entity mocks with `as unknown as T`.
|
||||
- Use `createVitestConfigWithDecorators` for Vitest packages that use `@n8n/di` decorators.
|
||||
- Check for unused computed properties after you change a Pinia store.
|
||||
|
||||
Choose a local development path:
|
||||
|
||||
| Goal | Command |
|
||||
|------|---------|
|
||||
| Run product E2E against a local instance | `pnpm --filter=n8n-playwright test:local` |
|
||||
| Run backend with PostgreSQL, Redis, email, and proxy services | `pnpm --filter n8n-containers services --services postgres,redis,mailpit,proxy`, then `pnpm dev:be` |
|
||||
| Add editor hot reload | `pnpm dev:fe:editor` |
|
||||
| Start a Codespace backend and share its port | `pnpm dev:up` |
|
||||
|
||||
The root `pnpm dev` command does not start a server. See the
|
||||
[Playwright guide](packages/testing/playwright/README.md) and the
|
||||
[Codespaces guide](.devcontainer/codespaces/README.md) for details.
|
||||
|
||||
### Common Development Tasks
|
||||
|
||||
|
|
|
|||
98
packages/testing/containers/services/engine-postgres.ts
Normal file
98
packages/testing/containers/services/engine-postgres.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import { PostgreSqlContainer } from '@testcontainers/postgresql';
|
||||
import type { StartedNetwork } from 'testcontainers';
|
||||
|
||||
import { createSilentLogConsumer } from '../helpers/utils';
|
||||
import { TEST_CONTAINER_IMAGES } from '../test-containers';
|
||||
import { ENGINE_DATABASE } from './engine';
|
||||
import { PostgresHelper, type PostgresResult } from './postgres';
|
||||
import type { HelperContext, Service, StartContext } from './types';
|
||||
|
||||
const HOSTNAME = 'engine-postgres';
|
||||
const USERNAME = 'n8n_user';
|
||||
const PASSWORD = 'test_password';
|
||||
|
||||
export const enginePostgres: Service<PostgresResult> = {
|
||||
description: 'Engine PostgreSQL database',
|
||||
|
||||
async start(
|
||||
network: StartedNetwork,
|
||||
projectName: string,
|
||||
_options?: unknown,
|
||||
ctx?: StartContext,
|
||||
) {
|
||||
const { consumer, throwWithLogs } = createSilentLogConsumer();
|
||||
try {
|
||||
const container = await new PostgreSqlContainer(TEST_CONTAINER_IMAGES.postgres)
|
||||
.withNetwork(network)
|
||||
.withNetworkAliases(HOSTNAME)
|
||||
.withDatabase(ENGINE_DATABASE)
|
||||
.withUsername(USERNAME)
|
||||
.withPassword(PASSWORD)
|
||||
.withStartupTimeout(60000)
|
||||
.withLabels({
|
||||
'com.docker.compose.project': projectName,
|
||||
'com.docker.compose.service': HOSTNAME,
|
||||
})
|
||||
.withName(`${projectName}-${HOSTNAME}`)
|
||||
.withTmpFs({ '/var/lib/postgresql': 'rw' })
|
||||
.withCommand([
|
||||
'postgres',
|
||||
'-c',
|
||||
'fsync=off',
|
||||
'-c',
|
||||
'synchronous_commit=off',
|
||||
'-c',
|
||||
'full_page_writes=off',
|
||||
'-c',
|
||||
'max_connections=200',
|
||||
'-c',
|
||||
'shared_preload_libraries=pg_stat_statements',
|
||||
'-c',
|
||||
'pg_stat_statements.track=all',
|
||||
'-c',
|
||||
'pg_stat_statements.track_planning=on',
|
||||
'-c',
|
||||
'track_io_timing=on',
|
||||
])
|
||||
.withReuse()
|
||||
.withLogConsumer(consumer)
|
||||
.start();
|
||||
ctx?.registerContainer?.(container);
|
||||
await container.exec([
|
||||
'psql',
|
||||
'-U',
|
||||
USERNAME,
|
||||
'-d',
|
||||
ENGINE_DATABASE,
|
||||
'-c',
|
||||
'CREATE EXTENSION IF NOT EXISTS pg_stat_statements;',
|
||||
]);
|
||||
return {
|
||||
container,
|
||||
meta: { database: ENGINE_DATABASE, username: USERNAME, password: PASSWORD },
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
return throwWithLogs(error);
|
||||
}
|
||||
},
|
||||
|
||||
env(result: PostgresResult, external?: boolean) {
|
||||
const host = external ? result.container.getHost() : HOSTNAME;
|
||||
const port = external ? result.container.getMappedPort(5432) : 5432;
|
||||
return {
|
||||
N8N_ENGINE_DATABASE_URL: `postgres://${USERNAME}:${PASSWORD}@${host}:${port}/${ENGINE_DATABASE}`,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export function createEnginePostgresHelper(ctx: HelperContext): PostgresHelper | undefined {
|
||||
const result = ctx.serviceResults.enginePostgres as PostgresResult | undefined;
|
||||
if (!result) return undefined;
|
||||
return new PostgresHelper(result.container, result.meta);
|
||||
}
|
||||
|
||||
declare module './types' {
|
||||
interface ServiceHelpers {
|
||||
enginePostgres?: PostgresHelper;
|
||||
}
|
||||
}
|
||||
|
|
@ -70,6 +70,7 @@ export function applyEngineEnv(
|
|||
const modules = (env.N8N_ENABLED_MODULES ?? '').split(',').filter(Boolean);
|
||||
if (!modules.includes(ENGINE_MODULE)) modules.push(ENGINE_MODULE);
|
||||
env.N8N_ENABLED_MODULES = modules.join(',');
|
||||
if (env.N8N_ENGINE_DATABASE_URL) return;
|
||||
|
||||
const user = encodeURIComponent(env.DB_POSTGRESDB_USER);
|
||||
const password = encodeURIComponent(env.DB_POSTGRESDB_PASSWORD);
|
||||
|
|
|
|||
|
|
@ -211,6 +211,21 @@ export class PostgresHelper {
|
|||
await this.exec('SELECT pg_stat_statements_reset();');
|
||||
}
|
||||
|
||||
async countCompletedEngineV2Executions(workflowId: string): Promise<number> {
|
||||
if (!/^[A-Za-z0-9_-]+$/.test(workflowId)) {
|
||||
throw new Error(`Unexpected workflow ID: ${workflowId}`);
|
||||
}
|
||||
const { output } = await this.runIn(
|
||||
ENGINE_DATABASE,
|
||||
`
|
||||
SELECT COUNT(*)::bigint
|
||||
FROM workflow_execution
|
||||
WHERE status = 'completed' AND workflow_id = '${workflowId}';
|
||||
`,
|
||||
);
|
||||
return parseInt(output.trim(), 10);
|
||||
}
|
||||
|
||||
/** Top N statements ordered by call count descending. */
|
||||
async topStatements(limit = 15): Promise<
|
||||
Array<{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { cadvisor } from './cadvisor';
|
||||
import { cloudflared } from './cloudflared';
|
||||
import { createEnginePostgresHelper, enginePostgres } from './engine-postgres';
|
||||
import { gitea, createGiteaHelper } from './gitea';
|
||||
import { kafka, createKafkaHelper } from './kafka';
|
||||
import { kent, createKentHelper } from './kent';
|
||||
|
|
@ -25,6 +26,7 @@ import { victoriaMetrics } from './victoria-metrics';
|
|||
/** Service registry - must include all ServiceName entries */
|
||||
export const services: Record<ServiceName, Service<ServiceResult>> = {
|
||||
postgres,
|
||||
enginePostgres,
|
||||
redis,
|
||||
mailpit,
|
||||
gitea,
|
||||
|
|
@ -49,6 +51,7 @@ export const services: Record<ServiceName, Service<ServiceResult>> = {
|
|||
|
||||
export const helperFactories: Partial<HelperFactories> = {
|
||||
postgres: createPostgresHelper,
|
||||
enginePostgres: createEnginePostgresHelper,
|
||||
mailpit: createMailpitHelper,
|
||||
gitea: createGiteaHelper,
|
||||
keycloak: createKeycloakHelper,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ export const EXTERNAL_HOST = 'host.docker.internal';
|
|||
|
||||
export const SERVICE_NAMES = [
|
||||
'postgres',
|
||||
'enginePostgres',
|
||||
'redis',
|
||||
'mailpit',
|
||||
'gitea',
|
||||
|
|
|
|||
|
|
@ -46,6 +46,14 @@ describe('applyEngineEnv', () => {
|
|||
);
|
||||
});
|
||||
|
||||
test('keeps an explicit data plane database URL', () => {
|
||||
const env = { ...postgresEnv, N8N_ENGINE_DATABASE_URL: 'postgres://engine-postgres/db' };
|
||||
|
||||
applyEngineEnv(env, { engine: 'in-process', isQueueMode: false });
|
||||
|
||||
expect(env.N8N_ENGINE_DATABASE_URL).toBe('postgres://engine-postgres/db');
|
||||
});
|
||||
|
||||
test('rejects a stack without Postgres', () => {
|
||||
const env: Record<string, string> = { DB_TYPE: 'sqlite' };
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,27 @@
|
|||
# AGENTS.md
|
||||
|
||||
## Purpose
|
||||
|
||||
Playwright is n8n's general-purpose test orchestrator. Do not assume that every
|
||||
Playwright test drives the editor UI. This package also owns API tests, container
|
||||
topologies, process lifecycle tests, infrastructure validation, performance
|
||||
benchmarks, evaluation suites, and browser-backed harness contracts.
|
||||
|
||||
Choose the suite by the behavior under test:
|
||||
|
||||
| Behavior | Location | Runner |
|
||||
|----------|----------|--------|
|
||||
| Product UI or API journey | `tests/e2e/` | `test:local` or a container project |
|
||||
| Database, queue, multi-main, encryption, or process lifecycle | `tests/infrastructure/` | `test:infrastructure` or the named project |
|
||||
| Infrastructure throughput and resource use | `tests/infrastructure/benchmarks/` | `test:benchmark` |
|
||||
| Browser and canvas performance | `tests/performance/` | `test:performance` |
|
||||
| Fixture or harness contract | `tests/framework/` | `test:unit` or `test:harness` |
|
||||
| Evaluation scenario | Existing evaluation directory | Its named evaluation project |
|
||||
|
||||
Use Playwright when the test needs its worker lifecycle, fixtures, retries,
|
||||
artifacts, project matrix, browser context, or managed container stack. Use
|
||||
Vitest for browser-free unit and integration tests that need none of these.
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
|
|
@ -10,6 +32,9 @@ pnpm --filter=n8n-playwright test:local tests/e2e/credentials/crud.spec.ts
|
|||
# Run with container capabilities (requires pnpm build:docker first)
|
||||
pnpm --filter=n8n-playwright test:container:sqlite tests/e2e/auth/password-reset.spec.ts
|
||||
|
||||
# Run one infrastructure benchmark
|
||||
pnpm --filter=n8n-playwright test:benchmark tests/infrastructure/benchmarks/kafka/single-instance-ceiling.spec.ts
|
||||
|
||||
# Lint and typecheck
|
||||
pnpm --filter=n8n-playwright lint
|
||||
pnpm --filter=n8n-playwright typecheck
|
||||
|
|
@ -177,9 +202,11 @@ pnpm janitor --file=tests/my-new-test.spec.ts --verbose
|
|||
|
||||
See `packages/testing/janitor/README.md` for full documentation.
|
||||
|
||||
## Entry Points
|
||||
## UI Entry Points
|
||||
|
||||
All tests should start with `n8n.start.*` methods. See `composables/TestEntryComposer.ts`.
|
||||
UI journey tests should start with `n8n.start.*` methods. API, infrastructure,
|
||||
benchmark, lifecycle, and framework tests use their own fixtures and harnesses.
|
||||
See `composables/TestEntryComposer.ts` for UI entry points.
|
||||
|
||||
| Method | Use Case |
|
||||
|--------|----------|
|
||||
|
|
|
|||
|
|
@ -1,4 +1,21 @@
|
|||
# Playwright E2E Test Guide
|
||||
# Playwright Test Orchestration Guide
|
||||
|
||||
Playwright is not only the browser E2E runner. This package uses Playwright's
|
||||
worker lifecycle and project model to orchestrate browsers, APIs, n8n processes,
|
||||
service containers, deployment topologies, diagnostics, and benchmark artifacts.
|
||||
|
||||
## Choose a suite
|
||||
|
||||
| Goal | Start here |
|
||||
|------|------------|
|
||||
| Test a product UI or API journey | `tests/e2e/` |
|
||||
| Test PostgreSQL, queue mode, multi-main, encryption, or process lifecycle | `tests/infrastructure/` |
|
||||
| Measure infrastructure throughput or resource use | [`tests/infrastructure/benchmarks/README.md`](tests/infrastructure/benchmarks/README.md) |
|
||||
| Measure browser or canvas performance | `tests/performance/README.md` |
|
||||
| Test fixtures, startup, telemetry, or harness behavior | `tests/framework/` |
|
||||
|
||||
Read `AGENTS.md` before you add a test. It defines suite boundaries, fixtures,
|
||||
container capabilities, and required verification.
|
||||
|
||||
## Development setup
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -235,7 +235,11 @@ export const test = base.extend<
|
|||
await apiContext.dispose();
|
||||
|
||||
// The reset endpoint only reaches the control plane database.
|
||||
if (n8nStackConfig.engine) await n8nContainer.services.postgres.truncateEngineDatabase();
|
||||
if (n8nStackConfig.engine) {
|
||||
await (
|
||||
n8nContainer.services.enginePostgres ?? n8nContainer.services.postgres
|
||||
).truncateEngineDatabase();
|
||||
}
|
||||
}
|
||||
await use(undefined);
|
||||
},
|
||||
|
|
|
|||
|
|
@ -91,6 +91,8 @@ export interface BenchOptions {
|
|||
* collecting flamegraph data, not for clean ceiling numbers.
|
||||
*/
|
||||
tracing?: boolean;
|
||||
/** Runs engine v2 on the shared Postgres server or a separate server. */
|
||||
engine?: 'shared' | 'split';
|
||||
/** Additional env vars to merge over the base. */
|
||||
env?: Record<string, string>;
|
||||
}
|
||||
|
|
@ -114,6 +116,7 @@ export function benchConfig(isolation: string, opts: BenchOptions = {}): N8NConf
|
|||
const services = [...(BENCHMARK_CONFIG.services ?? [])];
|
||||
if (opts.kafka) services.push('kafka');
|
||||
if (opts.tracing) services.push('tracing');
|
||||
if (opts.engine === 'split') services.push('enginePostgres');
|
||||
|
||||
const env: Record<string, string> = {
|
||||
...BENCHMARK_CONFIG.env,
|
||||
|
|
@ -134,6 +137,7 @@ export function benchConfig(isolation: string, opts: BenchOptions = {}): N8NConf
|
|||
...(opts.mains !== undefined && { mains: opts.mains }),
|
||||
...(opts.workers !== undefined && { workers: opts.workers }),
|
||||
...(opts.webhooks !== undefined && { webhooks: opts.webhooks }),
|
||||
...(opts.engine !== undefined && { engine: 'in-process' as const }),
|
||||
env,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { appendFileSync } from 'fs';
|
|||
|
||||
interface BenchmarkRow {
|
||||
question: string;
|
||||
profile: string;
|
||||
variant: string;
|
||||
verdict: string;
|
||||
scenario: string;
|
||||
|
|
@ -16,6 +17,11 @@ interface Column {
|
|||
}
|
||||
|
||||
const COLUMNS: Column[] = [
|
||||
{
|
||||
header: 'complete',
|
||||
suffixes: ['completion-pct'],
|
||||
format: (v) => `${v.toFixed(1)}%`,
|
||||
},
|
||||
{
|
||||
header: 'exec/s',
|
||||
suffixes: ['throughput', 'exec-per-sec'],
|
||||
|
|
@ -56,6 +62,21 @@ const COLUMNS: Column[] = [
|
|||
suffixes: ['event-loop-lag'],
|
||||
format: (v) => `${(v * 1000).toFixed(0)}ms`,
|
||||
},
|
||||
{
|
||||
header: 'CPU',
|
||||
suffixes: ['main-cpu-avg'],
|
||||
format: (v) => `${v.toFixed(0)}%`,
|
||||
},
|
||||
{
|
||||
header: 'RAM',
|
||||
suffixes: ['main-memory-mb'],
|
||||
format: (v) => `${v.toFixed(0)}MB`,
|
||||
},
|
||||
{
|
||||
header: 'PG CPU',
|
||||
suffixes: ['pg-cpu-avg'],
|
||||
format: (v) => `${v.toFixed(0)}%`,
|
||||
},
|
||||
{
|
||||
header: 'pg tx/s',
|
||||
suffixes: ['pg-tx-rate'],
|
||||
|
|
@ -106,6 +127,7 @@ interface QuestionShape {
|
|||
columns: Column[];
|
||||
showVariant: boolean;
|
||||
showVerdict: boolean;
|
||||
showProfile: boolean;
|
||||
}
|
||||
|
||||
function shapeFor(rows: BenchmarkRow[]): QuestionShape {
|
||||
|
|
@ -113,6 +135,7 @@ function shapeFor(rows: BenchmarkRow[]): QuestionShape {
|
|||
columns: COLUMNS.filter((col) => rows.some((r) => col.suffixes.some((s) => r.metrics.has(s)))),
|
||||
showVariant: rows.some((r) => r.variant.length > 0),
|
||||
showVerdict: rows.some((r) => r.verdict.length > 0),
|
||||
showProfile: rows.some((r) => r.profile.length > 0),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -156,6 +179,7 @@ class BenchmarkSummaryReporter implements Reporter {
|
|||
|
||||
this.rows.push({
|
||||
question,
|
||||
profile: String(dimensions.runtime_profile ?? ''),
|
||||
variant: String(dimensions.variant ?? ''),
|
||||
verdict: String(dimensions.verdict ?? ''),
|
||||
scenario,
|
||||
|
|
@ -204,12 +228,14 @@ class BenchmarkSummaryReporter implements Reporter {
|
|||
const scenario = rows[0]?.scenario ?? '';
|
||||
|
||||
const headerCells: string[] = [];
|
||||
if (shape.showProfile) headerCells.push('Profile');
|
||||
if (shape.showVariant) headerCells.push('Variant');
|
||||
if (shape.showVerdict) headerCells.push('Verdict');
|
||||
for (const col of shape.columns) headerCells.push(col.header);
|
||||
|
||||
const rowCells: string[][] = rows.map((row) => {
|
||||
const cells: string[] = [];
|
||||
if (shape.showProfile) cells.push(row.profile || '—');
|
||||
if (shape.showVariant) cells.push(row.variant || '—');
|
||||
if (shape.showVerdict) cells.push(row.verdict || '—');
|
||||
for (const col of shape.columns) cells.push(formatCell(row, col));
|
||||
|
|
@ -225,8 +251,9 @@ class BenchmarkSummaryReporter implements Reporter {
|
|||
leftAlign ? padRight(s, w) : pad(s, w);
|
||||
|
||||
// Variant + Verdict are left-aligned text; metric columns are right-aligned numbers.
|
||||
const isText = (i: number) =>
|
||||
(shape.showVariant && i === 0) || (shape.showVerdict && i === (shape.showVariant ? 1 : 0));
|
||||
const textColumns =
|
||||
Number(shape.showProfile) + Number(shape.showVariant) + Number(shape.showVerdict);
|
||||
const isText = (i: number) => i < textColumns;
|
||||
|
||||
console.log('');
|
||||
console.log(`▎ ${question}`);
|
||||
|
|
@ -256,6 +283,7 @@ class BenchmarkSummaryReporter implements Reporter {
|
|||
lines.push('');
|
||||
|
||||
const headers: string[] = [];
|
||||
if (shape.showProfile) headers.push('Profile');
|
||||
if (shape.showVariant) headers.push('Variant');
|
||||
if (shape.showVerdict) headers.push('Verdict');
|
||||
for (const col of shape.columns) headers.push(col.header);
|
||||
|
|
@ -265,6 +293,7 @@ class BenchmarkSummaryReporter implements Reporter {
|
|||
|
||||
for (const row of rows) {
|
||||
const cells: string[] = [];
|
||||
if (shape.showProfile) cells.push(row.profile || '—');
|
||||
if (shape.showVariant) cells.push(row.variant || '—');
|
||||
if (shape.showVerdict) cells.push(row.verdict || '—');
|
||||
for (const col of shape.columns) cells.push(formatCell(row, col));
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ const SERVICE_IMAGES = {
|
|||
mailpit: ['mailpit'],
|
||||
mysql: ['mysql'],
|
||||
ngrok: ['ngrok'],
|
||||
enginePostgres: ['postgres'],
|
||||
postgres: ['postgres'],
|
||||
postgresExporter: ['postgresExporter'],
|
||||
proxy: ['mockserver'],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
# Infrastructure Benchmark Instructions
|
||||
|
||||
Read `README.md` before you change this suite.
|
||||
|
||||
## Purpose
|
||||
|
||||
These Playwright specs use the worker lifecycle as a benchmark orchestrator.
|
||||
They create isolated container stacks, generate load, sample completions and
|
||||
resources, attach structured reports, and render the GitHub job summary. Most
|
||||
specs do not use a browser.
|
||||
|
||||
## Rules
|
||||
|
||||
- Give each spec one scaling question.
|
||||
- Configure topology with `benchConfig()` in the spec.
|
||||
- Reuse the Kafka or webhook harness. Do not copy orchestration logic into a spec.
|
||||
- Add an owner and a unique `question` annotation.
|
||||
- Set `dimensions.variant` when one summary contains multiple rows.
|
||||
- Keep runtime comparison profiles out of the deployment sizing matrix.
|
||||
- Treat v2 saturation profiles as measurements. Require progress, not full drain.
|
||||
- Keep exact completion checks for v1 delivery baselines.
|
||||
- Record a new comparison axis as a metric dimension.
|
||||
- Add a shard when a lane would exceed its existing wall-time target.
|
||||
- Send Slack failure notifications only for scheduled runs.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
pnpm lint
|
||||
pnpm typecheck
|
||||
pnpm test:unit
|
||||
pnpm janitor
|
||||
pnpm test:benchmark <changed-spec>
|
||||
```
|
||||
|
||||
Run commands from `packages/testing/playwright`. Build the n8n image first when
|
||||
the benchmark must include product-code changes.
|
||||
|
|
@ -16,9 +16,16 @@ The architectural ceiling. No queue tax, no worker dispatch. What's the absolute
|
|||
|---------|------|----------|
|
||||
| kafka | `single-instance-ceiling.spec.ts` | How much can we process on a single instance? |
|
||||
| kafka | `single-instance-ceiling-lazy-cache.spec.ts` | How much can lazy expression isolates process? |
|
||||
| kafka | `single-instance-ceiling-engine-v2.spec.ts` | How much can engine v2 process with eager expression isolates? |
|
||||
| kafka | `single-instance-ceiling-engine-v2-lazy-cache.spec.ts` | How much can engine v2 process with lazy expression isolates? |
|
||||
| kafka | `single-instance-ceiling-engine-v2-split-db.spec.ts` | Does a separate engine database server change v2 throughput? |
|
||||
| kafka | `steady-rate-breaking-point.spec.ts` | At what input rate does the system fall behind? |
|
||||
| webhook | `webhook-single-instance.spec.ts` | What is the single-instance webhook ingestion ceiling? |
|
||||
| webhook | `webhook-single-instance-lazy-cache.spec.ts` | What is the lazy expression-isolate webhook ceiling? |
|
||||
| webhook | `webhook-single-instance-engine-v2.spec.ts` | What is the engine v2 webhook ceiling with eager isolates? |
|
||||
| webhook | `webhook-single-instance-engine-v2-lazy-cache.spec.ts` | What is the engine v2 webhook ceiling with lazy isolates? |
|
||||
| webhook | `webhook-single-instance-engine-v2-split-db.spec.ts` | Does a separate engine database server change v2 webhook throughput? |
|
||||
| webhook | `webhook-sync-latency-floor.spec.ts` | How does connection count affect synchronous webhook latency? |
|
||||
|
||||
### Actual — `1m + 1wp + 1w queue mode`
|
||||
|
||||
|
|
@ -64,26 +71,33 @@ Webhook-trigger specs in **Actual** and **Scaling** use the production-canonical
|
|||
|
||||
All specs share a single env profile aligned with internal n8n production defaults — connection-pool, lock-duration, and Bull/Redis tuning from real deployments. See `BENCHMARK_CONFIG` in `playwright-projects.ts`.
|
||||
|
||||
### Expression engine profiles
|
||||
### Runtime comparison profiles
|
||||
|
||||
The direct Kafka and webhook baselines run with two explicit VM expression-engine profiles:
|
||||
The direct Kafka and webhook baselines run with two explicit VM expression-engine profiles and an execution engine v2 comparison:
|
||||
|
||||
| Profile | Lazy acquisition | Compile cache | Purpose |
|
||||
|---------|------------------|---------------|---------|
|
||||
| `vm-eager` | Off | Off | Tracks the current default execution path. |
|
||||
| `vm-lazy-cache` | On | On | Tracks the optimized no-expression execution path. |
|
||||
| Profile | Execution engine | Expression engine | Lazy acquisition | Compile cache | Purpose |
|
||||
|---------|------------------|-------------------|------------------|---------------|---------|
|
||||
| `vm-eager` | v1 | VM | Off | Off | Tracks the current default execution path. |
|
||||
| `vm-lazy-cache` | v1 | VM | On | On | Tracks the optimized no-expression execution path. |
|
||||
| `engine-v2` | v2 | VM | Off | Off | Tracks the new workflow execution engine with the same expression settings as `vm-eager`. |
|
||||
| `engine-v2-vm-lazy-cache` | v2 | VM | On | On | Measures the combined execution engine v2 and optimized expression-isolate path. |
|
||||
| `engine-v2-split-db-vm-lazy-cache` | v2 | VM | On | On | Runs the data plane on a separate PostgreSQL server. |
|
||||
|
||||
Each metric records `expression_engine`, `expression_lazy_acquire`, `expression_compile_cache`, and `expression_profile` dimensions. The lazy comparison runs remain in benchmark telemetry and run-report artifacts, but do not feed the deployment sizing matrix while eager VM is the default runtime.
|
||||
Each metric records `execution_engine`, `expression_engine`, `expression_lazy_acquire`, `expression_compile_cache`, and `expression_profile` dimensions. The lazy and engine v2 comparison runs remain in benchmark telemetry and run-report artifacts, but do not feed the deployment sizing matrix while execution engine v1 with eager VM is the default runtime.
|
||||
|
||||
These NoOp workflows do not evaluate expressions. Lazy mode therefore avoids acquiring an isolate, so these comparisons primarily measure lazy acquisition. Use the expression-engine microbenchmarks to measure compile-cache behavior directly.
|
||||
|
||||
The engine v2 webhook comparison uses the same 120-second ingestion load as the v1 profiles. It does not wait for the accepted backlog to drain. Compare its tail execution rate and backlog growth instead of its completion ratio.
|
||||
|
||||
The engine v2 Kafka comparisons report partial completion without enforcing a completion ratio. They require only observable progress because these runs measure throughput under saturation. The v1 baselines still require exact completion.
|
||||
|
||||
## Running
|
||||
|
||||
```bash
|
||||
# Build n8n image first (skip if you only changed test code).
|
||||
pnpm build:docker
|
||||
|
||||
# Full suite — all 15 specs sequentially (each spawns its own container).
|
||||
# Full suite — all 21 specs sequentially (each spawns its own container).
|
||||
pnpm --filter=n8n-playwright test:benchmark
|
||||
|
||||
# One spec.
|
||||
|
|
@ -117,14 +131,20 @@ Every run prints a per-test `[DIAG]` block and emits a Benchmark Summary table a
|
|||
| Column | Meaning |
|
||||
|--------|---------|
|
||||
| `exec/s` | Workflow executions per second across the active window |
|
||||
| `complete` | Percentage of the published finite load observed as completed |
|
||||
| `tail/s` | Throughput across the final 60s of the run — closest to the architectural ceiling |
|
||||
| `actions/s` | `exec/s × nodeCount` — total node executions per second |
|
||||
| `p50/p99` | Per-execution duration percentiles (when execution data is saved) |
|
||||
| `req/s` | HTTP requests per second (webhook specs only) |
|
||||
| `ev lag` | Node.js event loop lag (sum across mains/workers) |
|
||||
| `CPU` | Average CPU use across main containers |
|
||||
| `RAM` | Average memory use across main containers |
|
||||
| `PG CPU` | Average control-plane PostgreSQL container CPU use |
|
||||
| `pg tx/s` | Postgres `xact_commit` rate from postgres-exporter |
|
||||
| `queue` | Bull jobs waiting (queue specs only) |
|
||||
|
||||
Runtime comparison rows include a `Profile` column such as `v1 eager`, `v1 lazy/cache`, or `v2 lazy/cache, split DB`.
|
||||
|
||||
Tail and staged rates require at least three distinct counter samples and 80% coverage of the requested window. Duplicate polls between VictoriaMetrics scrapes do not count as samples. A short or incomplete window omits the tail metric instead of reporting a whole-run fallback or zero. Kafka stage rates use the publisher's actual boundaries.
|
||||
|
||||
For deeper PG analysis, every spec also logs a top-N `pg_stat_statements` breakdown ranked by total ms/s of work (calls/s × avg ms), plus a `[PG SATURATION]` block (total query CPU including planner overhead and the long tail, buffer hit ratio, bgwriter / WAL pressure, `pg_stat_io` per-backend-type IO) and a `[CONTAINERS]` block (per-container CPU/memory/IO from cAdvisor or `docker stats` sampler). Each run also attaches a `run-report.json` artifact with the full structured report — feedable directly to an LLM for bottleneck analysis.
|
||||
|
|
@ -158,6 +178,8 @@ Adding a new trigger type requires one driver + one or more spec files. The harn
|
|||
|
||||
## Adding a spec
|
||||
|
||||
Agents must also follow `AGENTS.md` in this directory.
|
||||
|
||||
1. Pick a question that isn't already answered by an existing spec.
|
||||
2. Decide which tier it belongs to: **Peak** (no workers), **Actual** (1m+1w), or **Scaling** (2m+2w).
|
||||
3. Create `kafka/<question>.spec.ts` or `webhook/<question>.spec.ts`.
|
||||
|
|
@ -168,3 +190,4 @@ Adding a new trigger type requires one driver + one or more spec files. The harn
|
|||
- Scaling: `benchConfig('<slug>', { mains: 2, workers: 2 })` (kafka adds `kafka: true`)
|
||||
5. Wire the trigger driver (`kafkaDriver` or `setupWebhook`) and a harness (`runLoadTest` or `runWebhookThroughputTest`).
|
||||
6. Annotate with `{ type: 'question', description: '<slug>' }` so the question is searchable in test metadata.
|
||||
7. Set `variant` when the question emits multiple rows. The summary reporter does not infer row labels from other dimensions.
|
||||
|
|
|
|||
|
|
@ -7,9 +7,11 @@ import {
|
|||
kafkaDriver,
|
||||
resolveMetricQuery,
|
||||
waitForThroughput,
|
||||
WORKFLOW_SUCCESS_QUERY,
|
||||
} from '../../../../utils/benchmark';
|
||||
import type {
|
||||
BenchmarkDimensions,
|
||||
CompletionCounterReader,
|
||||
ExecutionMetrics,
|
||||
LoadProfile,
|
||||
TriggerSetupContext,
|
||||
|
|
@ -30,7 +32,9 @@ interface KafkaLoadOptions {
|
|||
variant?: string;
|
||||
minTailRateEfficiency?: number;
|
||||
requireKeptUpStage?: boolean;
|
||||
requireTailMeasurement?: boolean;
|
||||
dimensions?: BenchmarkDimensions;
|
||||
counterReader?: CompletionCounterReader;
|
||||
}
|
||||
|
||||
export async function runKafkaLoadTest(options: KafkaLoadOptions): Promise<ExecutionMetrics> {
|
||||
|
|
@ -53,11 +57,13 @@ export async function runKafkaLoadTest(options: KafkaLoadOptions): Promise<Execu
|
|||
variant: options.variant,
|
||||
minTailRateEfficiency: options.minTailRateEfficiency,
|
||||
requireKeptUpStage: options.requireKeptUpStage,
|
||||
requireTailMeasurement: options.requireTailMeasurement,
|
||||
dimensions: options.dimensions,
|
||||
warmUp: async () => {
|
||||
counterReader: options.counterReader,
|
||||
warmUp: async ({ counterReader }) => {
|
||||
const metrics = options.services.observability.metrics;
|
||||
const metricQuery = resolveMetricQuery(options.testInfo);
|
||||
const baselineCounter = await getBaselineCounter(metrics, metricQuery);
|
||||
const baselineCounter = await getBaselineCounter(metrics, metricQuery, counterReader);
|
||||
const published = await handle.preload(WARMUP_MESSAGE_COUNT);
|
||||
const result = await waitForThroughput(metrics, {
|
||||
expectedCount: published.totalPublished,
|
||||
|
|
@ -65,6 +71,7 @@ export async function runKafkaLoadTest(options: KafkaLoadOptions): Promise<Execu
|
|||
timeoutMs: 60_000,
|
||||
baselineValue: baselineCounter,
|
||||
metricQuery,
|
||||
counterReader,
|
||||
});
|
||||
if (result.totalCompleted !== published.totalPublished) {
|
||||
throw new Error(
|
||||
|
|
@ -76,11 +83,22 @@ export async function runKafkaLoadTest(options: KafkaLoadOptions): Promise<Execu
|
|||
}
|
||||
|
||||
export async function runKafkaBacklogTest(
|
||||
options: Omit<KafkaLoadOptions, 'scenario' | 'load'> & { messageCount: number },
|
||||
options: Omit<KafkaLoadOptions, 'scenario' | 'load'> & {
|
||||
messageCount: number;
|
||||
},
|
||||
): Promise<void> {
|
||||
await runKafkaLoadTest({
|
||||
...options,
|
||||
scenario: { nodeCount: 1, payloadSize: '1KB', nodeOutputSize: 'noop', partitions: 3 },
|
||||
counterReader:
|
||||
options.dimensions?.execution_engine === 'v1'
|
||||
? async () => await options.api.metrics.getCounter(WORKFLOW_SUCCESS_QUERY)
|
||||
: undefined,
|
||||
scenario: {
|
||||
nodeCount: 1,
|
||||
payloadSize: '1KB',
|
||||
nodeOutputSize: 'noop',
|
||||
partitions: 3,
|
||||
},
|
||||
load: { type: 'preloaded', count: options.messageCount },
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,6 +50,8 @@ export interface LoadTestOptions {
|
|||
trigger: TriggerType;
|
||||
/** PromQL metric to track workflow completions. Defaults to resolveMetricQuery(testInfo). */
|
||||
metricQuery?: string;
|
||||
/** Direct completion reader for single-instance comparisons. */
|
||||
counterReader?: SetupContext['counterReader'];
|
||||
/** Additional dimensions attached to every metric and the run report. */
|
||||
dimensions?: BenchmarkDimensions;
|
||||
/** When provided, the result log includes a resource breakdown (mode/main/workers/total). */
|
||||
|
|
@ -69,6 +71,8 @@ export interface LoadTestOptions {
|
|||
minTailRateEfficiency?: number;
|
||||
/** Fail unless one stage keeps up with at least 95% of its requested rate. */
|
||||
requireKeptUpStage?: boolean;
|
||||
/** Fail unless the sampler produces a covered tail-rate measurement. */
|
||||
requireTailMeasurement?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -97,6 +101,7 @@ export async function runLoadTest(options: LoadTestOptions): Promise<ExecutionMe
|
|||
testInfo,
|
||||
handle,
|
||||
metricQuery: options.metricQuery,
|
||||
counterReader: options.counterReader,
|
||||
warmUp: options.warmUp,
|
||||
});
|
||||
|
||||
|
|
@ -105,6 +110,7 @@ export async function runLoadTest(options: LoadTestOptions): Promise<ExecutionMe
|
|||
metrics: services.observability.metrics,
|
||||
baselineCounter: setup.baselineCounter,
|
||||
metricQuery: setup.metricQuery,
|
||||
counterReader: setup.counterReader,
|
||||
timeoutMs,
|
||||
nodeCount,
|
||||
});
|
||||
|
|
@ -117,8 +123,8 @@ export async function runLoadTest(options: LoadTestOptions): Promise<ExecutionMe
|
|||
);
|
||||
}
|
||||
|
||||
// Duration sampling — empty when EXECUTIONS_DATA_SAVE_ON_SUCCESS=none.
|
||||
// Completion count comes from VictoriaMetrics regardless.
|
||||
// Duration sampling is empty when EXECUTIONS_DATA_SAVE_ON_SUCCESS=none.
|
||||
// Completion count comes from the configured counter reader.
|
||||
const durations = await sampleExecutionDurations(api.workflows, setup.workflowId);
|
||||
const metrics = buildMetrics(exec.throughputResult.totalCompleted, 0, totalDurationMs, durations);
|
||||
|
||||
|
|
@ -130,6 +136,13 @@ export async function runLoadTest(options: LoadTestOptions): Promise<ExecutionMe
|
|||
await attachStagedResults(testInfo, dimensions, exec.throughputResult, stagedLoad);
|
||||
} else {
|
||||
await attachLoadTestResults(testInfo, dimensions, metrics);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
'completion-pct',
|
||||
(exec.throughputResult.totalCompleted / exec.expectedExecutions) * 100,
|
||||
'%',
|
||||
dimensions,
|
||||
);
|
||||
await attachPhaseMetrics(testInfo, dimensions, exec.throughputResult);
|
||||
// Tail rate (last 60s) — closest to the architectural ceiling. Reporter
|
||||
// surfaces this as the `tail/s` column. Skipped for staged runs where the
|
||||
|
|
@ -214,6 +227,9 @@ export async function runLoadTest(options: LoadTestOptions): Promise<ExecutionMe
|
|||
});
|
||||
expect(keptUp).toBe(true);
|
||||
}
|
||||
if (options.requireTailMeasurement) {
|
||||
expect(exec.throughputResult.tailExecPerSec).toBeGreaterThan(0);
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import {
|
|||
} from '../../../../utils/benchmark';
|
||||
import type {
|
||||
BenchmarkDimensions,
|
||||
CompletionCounterReader,
|
||||
ContainerStat,
|
||||
DiagnosticsResult,
|
||||
PostgresMetrics,
|
||||
|
|
@ -55,6 +56,8 @@ export interface SetupContext {
|
|||
handle: BenchmarkHandle;
|
||||
/** Override for the PromQL completion metric. Defaults to `resolveMetricQuery(testInfo)`. */
|
||||
metricQuery?: string;
|
||||
/** Direct completion reader for profiles that must avoid metrics scrape latency. */
|
||||
counterReader?: CompletionCounterReader;
|
||||
/** Forwarded to `createWorkflowFromDefinition`. `makeUnique` defaults to true. */
|
||||
createOptions?: CreateWorkflowOptions;
|
||||
/**
|
||||
|
|
@ -62,7 +65,11 @@ export interface SetupContext {
|
|||
* and before the baseline counter is captured. Anything the warm-up
|
||||
* triggers won't pollute the measurement window.
|
||||
*/
|
||||
warmUp?: (setup: { workflowId: string; webhookPath?: string }) => Promise<void>;
|
||||
warmUp?: (setup: {
|
||||
workflowId: string;
|
||||
webhookPath?: string;
|
||||
counterReader?: CompletionCounterReader;
|
||||
}) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface SetupResult {
|
||||
|
|
@ -72,6 +79,7 @@ export interface SetupResult {
|
|||
webhookPath?: string;
|
||||
baselineCounter: number;
|
||||
metricQuery: string;
|
||||
counterReader?: CompletionCounterReader;
|
||||
/** Wall-clock when the workflow finished activating — useful as a fallback timer. */
|
||||
activationStart: number;
|
||||
/** Cumulative `pg_stat_wal` counters captured post-warm-up. Diffed at end-of-run. */
|
||||
|
|
@ -102,6 +110,13 @@ export async function setupBenchmarkRun(ctx: SetupContext): Promise<SetupResult>
|
|||
makeUnique: ctx.createOptions?.makeUnique ?? true,
|
||||
webhookPrefix: ctx.createOptions?.webhookPrefix,
|
||||
});
|
||||
const counterReader =
|
||||
ctx.api.options.workflowSettings?.engineType === 'v2'
|
||||
? async () =>
|
||||
await (
|
||||
ctx.services.enginePostgres ?? ctx.services.postgres
|
||||
).countCompletedEngineV2Executions(workflowId)
|
||||
: ctx.counterReader;
|
||||
|
||||
// VictoriaMetrics needs at least one scrape before queries return data.
|
||||
await obs.metrics.waitForMetric('n8n_version_info', {
|
||||
|
|
@ -116,10 +131,10 @@ export async function setupBenchmarkRun(ctx: SetupContext): Promise<SetupResult>
|
|||
await ctx.handle.waitForReady({ timeoutMs: 30_000 });
|
||||
}
|
||||
if (ctx.warmUp) {
|
||||
await ctx.warmUp({ workflowId, webhookPath });
|
||||
await ctx.warmUp({ workflowId, webhookPath, counterReader });
|
||||
}
|
||||
|
||||
const baselineCounter = await getBaselineCounter(obs.metrics, metricQuery);
|
||||
const baselineCounter = await getBaselineCounter(obs.metrics, metricQuery, counterReader);
|
||||
|
||||
// Best-effort: the postgres helper may not be wired or pg_stat_statements
|
||||
// may not be available on a custom postgres image.
|
||||
|
|
@ -151,6 +166,7 @@ export async function setupBenchmarkRun(ctx: SetupContext): Promise<SetupResult>
|
|||
webhookPath,
|
||||
baselineCounter,
|
||||
metricQuery,
|
||||
counterReader,
|
||||
activationStart,
|
||||
walBaseline,
|
||||
dockerStatsSampler,
|
||||
|
|
@ -631,6 +647,17 @@ export async function attachReportMetrics(
|
|||
if (mainCpu !== undefined) {
|
||||
await attachMetric(testInfo, 'main-cpu-avg', mainCpu, '%', dimensions);
|
||||
}
|
||||
const mainContainers = report.containers.filter((container) =>
|
||||
container.name.startsWith('n8n-main'),
|
||||
);
|
||||
if (mainContainers.length > 0) {
|
||||
const mainMemoryMb =
|
||||
mainContainers.reduce((sum, container) => sum + (container.memBytes ?? 0), 0) /
|
||||
mainContainers.length /
|
||||
1024 /
|
||||
1024;
|
||||
await attachMetric(testInfo, 'main-memory-mb', mainMemoryMb, 'MB', dimensions);
|
||||
}
|
||||
const workerCpu = avgCpuFor('n8n-worker');
|
||||
if (workerCpu !== undefined) {
|
||||
await attachMetric(testInfo, 'worker-cpu-avg', workerCpu, '%', dimensions);
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import type { ServiceHelpers } from 'n8n-containers/services/types';
|
|||
|
||||
import type { ApiHelpers } from '../../../../services/api-helper';
|
||||
import type { BenchmarkDimensions } from '../../../../utils/benchmark';
|
||||
import { WORKFLOW_SUCCESS_QUERY } from '../../../../utils/benchmark';
|
||||
import { setupWebhook } from '../../../../utils/benchmark/webhook-driver';
|
||||
import { runWebhookThroughputTest } from './webhook-throughput-harness';
|
||||
|
||||
|
|
@ -16,7 +17,10 @@ export async function runSingleInstanceWebhookBenchmark(options: {
|
|||
testInfo: TestInfo;
|
||||
baseUrl: string;
|
||||
dimensions: BenchmarkDimensions;
|
||||
drainTimeoutSeconds?: number;
|
||||
}): Promise<void> {
|
||||
const drainTimeoutSeconds =
|
||||
options.drainTimeoutSeconds ?? SINGLE_INSTANCE_WEBHOOK_DRAIN_TIMEOUT_SECONDS;
|
||||
const handle = setupWebhook({
|
||||
scenario: {
|
||||
nodeCount: 1,
|
||||
|
|
@ -36,14 +40,14 @@ export async function runSingleInstanceWebhookBenchmark(options: {
|
|||
pipelining: 1,
|
||||
warmupSeconds: 0,
|
||||
durationSeconds: SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS,
|
||||
timeoutMs:
|
||||
(SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS +
|
||||
SINGLE_INSTANCE_WEBHOOK_DRAIN_TIMEOUT_SECONDS +
|
||||
60) *
|
||||
1000,
|
||||
drainTimeoutSeconds: SINGLE_INSTANCE_WEBHOOK_DRAIN_TIMEOUT_SECONDS,
|
||||
timeoutMs: (SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS + drainTimeoutSeconds + 60) * 1000,
|
||||
drainTimeoutSeconds,
|
||||
maxErrorRatePct: 1,
|
||||
minCompletedResponseRatio: 0.95,
|
||||
minCompletedResponseRatio: drainTimeoutSeconds === 0 ? undefined : 0.95,
|
||||
counterReader:
|
||||
options.api.options.workflowSettings?.engineType === 'v2'
|
||||
? undefined
|
||||
: async () => await options.api.metrics.getCounter(WORKFLOW_SUCCESS_QUERY),
|
||||
dimensions: options.dimensions,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import {
|
|||
resolveMetricQuery,
|
||||
waitForThroughput,
|
||||
} from '../../../../utils/benchmark';
|
||||
import type { BenchmarkDimensions } from '../../../../utils/benchmark';
|
||||
import type { BenchmarkDimensions, CompletionCounterReader } from '../../../../utils/benchmark';
|
||||
import type { WebhookHandle } from '../../../../utils/benchmark/webhook-driver';
|
||||
import { attachMetric } from '../../../../utils/performance-helper';
|
||||
|
||||
|
|
@ -39,8 +39,12 @@ export interface WebhookThroughputOptions {
|
|||
drainTimeoutSeconds?: number;
|
||||
/** PromQL metric to track workflow completions. Defaults to resolveMetricQuery(testInfo). */
|
||||
metricQuery?: string;
|
||||
/** Direct completion reader for single-instance comparisons. */
|
||||
counterReader?: CompletionCounterReader;
|
||||
/** Additional dimensions attached to every metric and the run report. */
|
||||
dimensions?: BenchmarkDimensions;
|
||||
/** Human-readable label for sibling runs in one benchmark summary. */
|
||||
variant?: string;
|
||||
/**
|
||||
* Seconds of discarded load BEFORE the measurement window. Warms V8 JIT,
|
||||
* PG connection pool, webhook cache, and BullMQ worker hot paths so the
|
||||
|
|
@ -63,7 +67,7 @@ export interface WebhookThroughputOptions {
|
|||
|
||||
/**
|
||||
* Runs a webhook throughput test using autocannon for HTTP load generation
|
||||
* and VictoriaMetrics for workflow completion tracking.
|
||||
* and the configured counter reader for workflow completion tracking.
|
||||
*
|
||||
* Webhook differs from kafka harnesses: load is generated by autocannon (HTTP)
|
||||
* rather than via a TriggerHandle.publishAtRate. The shared `executeLoad`
|
||||
|
|
@ -95,6 +99,7 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions
|
|||
duration_s: durationSeconds,
|
||||
};
|
||||
if (nodeOutputSize !== undefined) dimensions.output = nodeOutputSize;
|
||||
if (options.variant !== undefined) dimensions.variant = options.variant;
|
||||
|
||||
// Warm-up runs inside setupBenchmarkRun so the baseline counter is captured
|
||||
// AFTER the probe POST — otherwise warm-up traffic would offset the measurement.
|
||||
|
|
@ -104,6 +109,7 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions
|
|||
testInfo,
|
||||
handle,
|
||||
metricQuery,
|
||||
counterReader: options.counterReader,
|
||||
createOptions: { webhookPrefix: 'bench' },
|
||||
warmUp: async ({ webhookPath }) => {
|
||||
if (!webhookPath) throw new Error('Webhook path missing — workflow has no webhook node?');
|
||||
|
|
@ -141,6 +147,7 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions
|
|||
` Workflow: ${nodeCount} nodes (${nodeOutputSize})`,
|
||||
);
|
||||
|
||||
const loadEndAt = Date.now() + durationSeconds * 1000;
|
||||
const [cannonResult, throughputResult] = await Promise.all([
|
||||
autocannon({
|
||||
url: webhookUrl,
|
||||
|
|
@ -161,6 +168,7 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions
|
|||
timeoutMs: (durationSeconds + drainTimeoutSeconds) * 1000,
|
||||
baselineValue: setup.baselineCounter,
|
||||
metricQuery,
|
||||
counterReader: setup.counterReader,
|
||||
}),
|
||||
]);
|
||||
|
||||
|
|
@ -267,6 +275,8 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions
|
|||
successfulResponses > 0 ? throughputResult.totalCompleted / successfulResponses : 0;
|
||||
expect(completionRatio).toBeGreaterThanOrEqual(options.minCompletedResponseRatio);
|
||||
} else {
|
||||
expect(throughputResult.totalCompleted).toBeGreaterThan(0);
|
||||
expect(throughputResult.tailExecPerSec ?? 0).toBeGreaterThan(0);
|
||||
const lastActive = throughputResult.samples.findLast((sample) => sample.delta > 0);
|
||||
expect(lastActive?.timestamp ?? 0).toBeGreaterThanOrEqual(loadEndAt - 10_000);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
import { test } from '../../../../fixtures/base';
|
||||
import { BENCHMARK_MAIN_RESOURCES, benchConfig } from '../../../../playwright-projects';
|
||||
import { ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE } from '../../../../utils/benchmark';
|
||||
import { runKafkaBacklogTest } from '../harness/kafka-backlog-harness';
|
||||
|
||||
const MESSAGE_COUNT = 10_000;
|
||||
|
||||
test.use({
|
||||
capability: benchConfig(
|
||||
`single-instance-ceiling-${ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.isolationSuffix}`,
|
||||
{
|
||||
kafka: true,
|
||||
engine: ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.engineDatabase,
|
||||
env: ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.env,
|
||||
},
|
||||
),
|
||||
});
|
||||
|
||||
test.describe(
|
||||
'How much can execution engine v2 process with lazy expression isolates on a single instance?',
|
||||
{
|
||||
tag: '@bench:kafka',
|
||||
annotation: [
|
||||
{ type: 'owner', description: 'Catalysts' },
|
||||
{
|
||||
type: 'question',
|
||||
description: 'single-instance-throughput-ceiling-engine-v2-vm-lazy-cache',
|
||||
},
|
||||
],
|
||||
},
|
||||
() => {
|
||||
test('Kafka trigger + 1 noop, 1KB payload, 10k msgs (engine v2, VM lazy + compile cache)', async ({
|
||||
api,
|
||||
services,
|
||||
}, testInfo) => {
|
||||
await runKafkaBacklogTest({
|
||||
api,
|
||||
services,
|
||||
testInfo,
|
||||
messageCount: MESSAGE_COUNT,
|
||||
timeoutMs: 600_000,
|
||||
resourceSummary: { plan: BENCHMARK_MAIN_RESOURCES },
|
||||
dimensions: ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.dimensions,
|
||||
requireTailMeasurement: true,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import { test } from '../../../../fixtures/base';
|
||||
import { BENCHMARK_MAIN_RESOURCES, benchConfig } from '../../../../playwright-projects';
|
||||
import { ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE } from '../../../../utils/benchmark';
|
||||
import { runKafkaBacklogTest } from '../harness/kafka-backlog-harness';
|
||||
|
||||
test.use({
|
||||
capability: benchConfig(
|
||||
`single-instance-ceiling-${ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE.isolationSuffix}`,
|
||||
{
|
||||
kafka: true,
|
||||
engine: ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE.engineDatabase,
|
||||
env: ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE.env,
|
||||
},
|
||||
),
|
||||
});
|
||||
|
||||
test.describe(
|
||||
'Engine v2 split database Kafka throughput',
|
||||
{
|
||||
annotation: [
|
||||
{ type: 'owner', description: 'Catalysts' },
|
||||
{ type: 'question', description: 'engine-v2-split-db-kafka' },
|
||||
],
|
||||
},
|
||||
() => {
|
||||
test('Kafka trigger + 1 noop, 1KB payload, 10k msgs (engine v2, split DB, VM lazy + cache) @bench:kafka', async ({
|
||||
api,
|
||||
services,
|
||||
}, testInfo) => {
|
||||
await runKafkaBacklogTest({
|
||||
api,
|
||||
services,
|
||||
testInfo,
|
||||
messageCount: 10_000,
|
||||
timeoutMs: 600_000,
|
||||
resourceSummary: { plan: BENCHMARK_MAIN_RESOURCES },
|
||||
dimensions: ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE.dimensions,
|
||||
requireTailMeasurement: true,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
import { test } from '../../../../fixtures/base';
|
||||
import { BENCHMARK_MAIN_RESOURCES, benchConfig } from '../../../../playwright-projects';
|
||||
import { ENGINE_V2_BENCHMARK_PROFILE } from '../../../../utils/benchmark';
|
||||
import { runKafkaBacklogTest } from '../harness/kafka-backlog-harness';
|
||||
|
||||
const MESSAGE_COUNT = 5_000;
|
||||
|
||||
test.use({
|
||||
capability: benchConfig(
|
||||
`single-instance-ceiling-${ENGINE_V2_BENCHMARK_PROFILE.isolationSuffix}`,
|
||||
{
|
||||
kafka: true,
|
||||
engine: ENGINE_V2_BENCHMARK_PROFILE.engineDatabase,
|
||||
env: ENGINE_V2_BENCHMARK_PROFILE.env,
|
||||
},
|
||||
),
|
||||
});
|
||||
|
||||
test.describe(
|
||||
'How much can execution engine v2 process on a single instance?',
|
||||
{
|
||||
tag: '@bench:kafka',
|
||||
annotation: [
|
||||
{ type: 'owner', description: 'Catalysts' },
|
||||
{ type: 'question', description: 'single-instance-throughput-ceiling-engine-v2' },
|
||||
],
|
||||
},
|
||||
() => {
|
||||
test('Kafka trigger + 1 noop, 1KB payload, 5k msgs (engine v2)', async ({
|
||||
api,
|
||||
services,
|
||||
}, testInfo) => {
|
||||
await runKafkaBacklogTest({
|
||||
api,
|
||||
services,
|
||||
testInfo,
|
||||
messageCount: MESSAGE_COUNT,
|
||||
timeoutMs: 300_000,
|
||||
resourceSummary: { plan: BENCHMARK_MAIN_RESOURCES },
|
||||
dimensions: ENGINE_V2_BENCHMARK_PROFILE.dimensions,
|
||||
requireTailMeasurement: true,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
|
@ -3,7 +3,7 @@ import { BENCHMARK_MAIN_RESOURCES, benchConfig } from '../../../../playwright-pr
|
|||
import { VM_LAZY_CACHE_BENCHMARK_PROFILE } from '../../../../utils/benchmark';
|
||||
import { runKafkaBacklogTest } from '../harness/kafka-backlog-harness';
|
||||
|
||||
const MESSAGE_COUNT = 2_000;
|
||||
const MESSAGE_COUNT = 20_000;
|
||||
|
||||
test.use({
|
||||
capability: benchConfig(
|
||||
|
|
@ -22,7 +22,7 @@ test.describe(
|
|||
],
|
||||
},
|
||||
() => {
|
||||
test('Kafka trigger + 1 noop, 1KB payload, 2k msgs (VM lazy + compile cache)', async ({
|
||||
test('Kafka trigger + 1 noop, 1KB payload, 20k msgs (VM lazy + compile cache)', async ({
|
||||
api,
|
||||
services,
|
||||
}, testInfo) => {
|
||||
|
|
@ -31,7 +31,7 @@ test.describe(
|
|||
services,
|
||||
testInfo,
|
||||
messageCount: MESSAGE_COUNT,
|
||||
timeoutMs: 180_000,
|
||||
timeoutMs: 600_000,
|
||||
resourceSummary: { plan: BENCHMARK_MAIN_RESOURCES },
|
||||
minimumCompletionRatio: 1,
|
||||
dimensions: VM_LAZY_CACHE_BENCHMARK_PROFILE.dimensions,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { BENCHMARK_MAIN_RESOURCES, benchConfig } from '../../../../playwright-pr
|
|||
import { VM_EAGER_BENCHMARK_PROFILE } from '../../../../utils/benchmark';
|
||||
import { runKafkaBacklogTest } from '../harness/kafka-backlog-harness';
|
||||
|
||||
const MESSAGE_COUNT = 2_000;
|
||||
const MESSAGE_COUNT = 5_000;
|
||||
|
||||
test.use({
|
||||
capability: benchConfig(`single-instance-ceiling-${VM_EAGER_BENCHMARK_PROFILE.isolationSuffix}`, {
|
||||
|
|
@ -22,13 +22,13 @@ test.describe(
|
|||
],
|
||||
},
|
||||
() => {
|
||||
test('Kafka trigger + 1 noop, 1KB payload, 2k msgs', async ({ api, services }, testInfo) => {
|
||||
test('Kafka trigger + 1 noop, 1KB payload, 5k msgs', async ({ api, services }, testInfo) => {
|
||||
await runKafkaBacklogTest({
|
||||
api,
|
||||
services,
|
||||
testInfo,
|
||||
messageCount: MESSAGE_COUNT,
|
||||
timeoutMs: 180_000,
|
||||
timeoutMs: 300_000,
|
||||
resourceSummary: { plan: BENCHMARK_MAIN_RESOURCES },
|
||||
minimumCompletionRatio: 1,
|
||||
dimensions: VM_EAGER_BENCHMARK_PROFILE.dimensions,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
import { test } from '../../../../fixtures/base';
|
||||
import { benchConfig } from '../../../../playwright-projects';
|
||||
import { ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE } from '../../../../utils/benchmark';
|
||||
import {
|
||||
runSingleInstanceWebhookBenchmark,
|
||||
SINGLE_INSTANCE_WEBHOOK_CONNECTIONS,
|
||||
SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS,
|
||||
} from '../harness/single-instance-webhook-harness';
|
||||
|
||||
test.use({
|
||||
capability: benchConfig(
|
||||
`webhook-single-instance-${ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.isolationSuffix}`,
|
||||
{
|
||||
engine: ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.engineDatabase,
|
||||
env: ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.env,
|
||||
},
|
||||
),
|
||||
});
|
||||
|
||||
test.describe(
|
||||
'What is the single-instance webhook ingestion ceiling with execution engine v2 and lazy expression isolates?',
|
||||
{
|
||||
tag: '@bench:webhook',
|
||||
annotation: [
|
||||
{ type: 'owner', description: 'Catalysts' },
|
||||
{
|
||||
type: 'question',
|
||||
description: 'webhook-single-instance-engine-v2-vm-lazy-cache',
|
||||
},
|
||||
],
|
||||
},
|
||||
() => {
|
||||
test(`Async webhook + 1 noop, 1KB payload, ${SINGLE_INSTANCE_WEBHOOK_CONNECTIONS} connections × ${SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS}s (1 main, no workers, engine v2, VM lazy + compile cache)`, async ({
|
||||
api,
|
||||
services,
|
||||
backendUrl,
|
||||
}, testInfo) => {
|
||||
await runSingleInstanceWebhookBenchmark({
|
||||
api,
|
||||
services,
|
||||
testInfo,
|
||||
baseUrl: backendUrl,
|
||||
dimensions: ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.dimensions,
|
||||
drainTimeoutSeconds: 0,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import { test } from '../../../../fixtures/base';
|
||||
import { benchConfig } from '../../../../playwright-projects';
|
||||
import { ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE } from '../../../../utils/benchmark';
|
||||
import {
|
||||
runSingleInstanceWebhookBenchmark,
|
||||
SINGLE_INSTANCE_WEBHOOK_CONNECTIONS,
|
||||
SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS,
|
||||
} from '../harness/single-instance-webhook-harness';
|
||||
|
||||
test.use({
|
||||
capability: benchConfig(
|
||||
`webhook-single-instance-${ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE.isolationSuffix}`,
|
||||
{
|
||||
engine: ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE.engineDatabase,
|
||||
env: ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE.env,
|
||||
},
|
||||
),
|
||||
});
|
||||
|
||||
test.describe(
|
||||
'Engine v2 split database webhook throughput',
|
||||
{
|
||||
annotation: [
|
||||
{ type: 'owner', description: 'Catalysts' },
|
||||
{ type: 'question', description: 'engine-v2-split-db-webhook' },
|
||||
],
|
||||
},
|
||||
() => {
|
||||
test(`Async webhook + 1 noop, 1KB payload, ${SINGLE_INSTANCE_WEBHOOK_CONNECTIONS} connections × ${SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS}s (engine v2, split DB, VM lazy + cache) @bench:webhook`, async ({
|
||||
api,
|
||||
services,
|
||||
backendUrl,
|
||||
}, testInfo) => {
|
||||
await runSingleInstanceWebhookBenchmark({
|
||||
api,
|
||||
services,
|
||||
testInfo,
|
||||
baseUrl: backendUrl,
|
||||
dimensions: ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE.dimensions,
|
||||
drainTimeoutSeconds: 0,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
import { test } from '../../../../fixtures/base';
|
||||
import { benchConfig } from '../../../../playwright-projects';
|
||||
import { ENGINE_V2_BENCHMARK_PROFILE } from '../../../../utils/benchmark';
|
||||
import {
|
||||
runSingleInstanceWebhookBenchmark,
|
||||
SINGLE_INSTANCE_WEBHOOK_CONNECTIONS,
|
||||
SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS,
|
||||
} from '../harness/single-instance-webhook-harness';
|
||||
|
||||
test.use({
|
||||
capability: benchConfig(
|
||||
`webhook-single-instance-${ENGINE_V2_BENCHMARK_PROFILE.isolationSuffix}`,
|
||||
{
|
||||
engine: ENGINE_V2_BENCHMARK_PROFILE.engineDatabase,
|
||||
env: ENGINE_V2_BENCHMARK_PROFILE.env,
|
||||
},
|
||||
),
|
||||
});
|
||||
|
||||
test.describe(
|
||||
'What is the single-instance webhook ingestion ceiling with execution engine v2?',
|
||||
{
|
||||
tag: '@bench:webhook',
|
||||
annotation: [
|
||||
{ type: 'owner', description: 'Catalysts' },
|
||||
{ type: 'question', description: 'webhook-single-instance-engine-v2' },
|
||||
],
|
||||
},
|
||||
() => {
|
||||
test(`Async webhook + 1 noop, 1KB payload, ${SINGLE_INSTANCE_WEBHOOK_CONNECTIONS} connections × ${SINGLE_INSTANCE_WEBHOOK_DURATION_SECONDS}s (1 main, no workers, engine v2)`, async ({
|
||||
api,
|
||||
services,
|
||||
backendUrl,
|
||||
}, testInfo) => {
|
||||
await runSingleInstanceWebhookBenchmark({
|
||||
api,
|
||||
services,
|
||||
testInfo,
|
||||
baseUrl: backendUrl,
|
||||
dimensions: ENGINE_V2_BENCHMARK_PROFILE.dimensions,
|
||||
drainTimeoutSeconds: 0,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
|
@ -58,6 +58,7 @@ test.describe(
|
|||
testInfo,
|
||||
baseUrl: backendUrl,
|
||||
connections,
|
||||
variant: `${connections} connection${connections === 1 ? '' : 's'}`,
|
||||
pipelining: 1,
|
||||
warmupSeconds: WARMUP_SECONDS,
|
||||
durationSeconds: DURATION_SECONDS,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
import type { BenchmarkDimensions } from './types';
|
||||
|
||||
interface ExpressionBenchmarkProfile {
|
||||
interface RuntimeBenchmarkProfile {
|
||||
isolationSuffix: string;
|
||||
env: Record<string, string>;
|
||||
dimensions: BenchmarkDimensions;
|
||||
engineDatabase?: 'shared' | 'split';
|
||||
}
|
||||
|
||||
export const VM_EAGER_BENCHMARK_PROFILE: ExpressionBenchmarkProfile = {
|
||||
export const VM_EAGER_BENCHMARK_PROFILE: RuntimeBenchmarkProfile = {
|
||||
isolationSuffix: 'vm-eager',
|
||||
env: {
|
||||
N8N_EXPRESSION_ENGINE: 'vm',
|
||||
|
|
@ -14,6 +15,8 @@ export const VM_EAGER_BENCHMARK_PROFILE: ExpressionBenchmarkProfile = {
|
|||
N8N_EXPRESSION_ENGINE_COMPILE_CACHE: 'false',
|
||||
},
|
||||
dimensions: {
|
||||
runtime_profile: 'v1 eager',
|
||||
execution_engine: 'v1',
|
||||
expression_engine: 'vm',
|
||||
expression_lazy_acquire: 0,
|
||||
expression_compile_cache: 0,
|
||||
|
|
@ -21,7 +24,7 @@ export const VM_EAGER_BENCHMARK_PROFILE: ExpressionBenchmarkProfile = {
|
|||
},
|
||||
};
|
||||
|
||||
export const VM_LAZY_CACHE_BENCHMARK_PROFILE: ExpressionBenchmarkProfile = {
|
||||
export const VM_LAZY_CACHE_BENCHMARK_PROFILE: RuntimeBenchmarkProfile = {
|
||||
isolationSuffix: 'vm-lazy-cache',
|
||||
env: {
|
||||
N8N_EXPRESSION_ENGINE: 'vm',
|
||||
|
|
@ -29,9 +32,56 @@ export const VM_LAZY_CACHE_BENCHMARK_PROFILE: ExpressionBenchmarkProfile = {
|
|||
N8N_EXPRESSION_ENGINE_COMPILE_CACHE: 'true',
|
||||
},
|
||||
dimensions: {
|
||||
runtime_profile: 'v1 lazy/cache',
|
||||
execution_engine: 'v1',
|
||||
expression_engine: 'vm',
|
||||
expression_lazy_acquire: 1,
|
||||
expression_compile_cache: 1,
|
||||
expression_profile: 'vm-lazy-cache',
|
||||
},
|
||||
};
|
||||
|
||||
export const ENGINE_V2_BENCHMARK_PROFILE: RuntimeBenchmarkProfile = {
|
||||
isolationSuffix: 'engine-v2',
|
||||
env: {
|
||||
...VM_EAGER_BENCHMARK_PROFILE.env,
|
||||
},
|
||||
engineDatabase: 'shared',
|
||||
dimensions: {
|
||||
runtime_profile: 'v2 eager',
|
||||
execution_engine: 'v2',
|
||||
database_topology: 'shared-postgres',
|
||||
database_metrics_scope: 'combined',
|
||||
expression_engine: 'vm',
|
||||
expression_lazy_acquire: 0,
|
||||
expression_compile_cache: 0,
|
||||
expression_profile: 'vm-eager',
|
||||
},
|
||||
};
|
||||
|
||||
export const ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE: RuntimeBenchmarkProfile = {
|
||||
isolationSuffix: 'engine-v2-vm-lazy-cache',
|
||||
env: {
|
||||
...VM_LAZY_CACHE_BENCHMARK_PROFILE.env,
|
||||
},
|
||||
engineDatabase: 'shared',
|
||||
dimensions: {
|
||||
...VM_LAZY_CACHE_BENCHMARK_PROFILE.dimensions,
|
||||
runtime_profile: 'v2 lazy/cache',
|
||||
execution_engine: 'v2',
|
||||
database_topology: 'shared-postgres',
|
||||
database_metrics_scope: 'combined',
|
||||
},
|
||||
};
|
||||
|
||||
export const ENGINE_V2_SPLIT_DB_LAZY_CACHE_BENCHMARK_PROFILE: RuntimeBenchmarkProfile = {
|
||||
...ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE,
|
||||
isolationSuffix: 'engine-v2-split-db-vm-lazy-cache',
|
||||
engineDatabase: 'split',
|
||||
dimensions: {
|
||||
...ENGINE_V2_LAZY_CACHE_BENCHMARK_PROFILE.dimensions,
|
||||
runtime_profile: 'v2 lazy/cache, split DB',
|
||||
database_topology: 'split-postgres',
|
||||
database_metrics_scope: 'control-plane',
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
import type { MetricsHelper } from 'n8n-containers';
|
||||
|
||||
import { measureStageWindows, measureSteadyPhases, waitForThroughput } from './throughput-measure';
|
||||
import type { ThroughputResult } from './throughput-measure';
|
||||
import type { CompletionCounterReader, ThroughputResult } from './throughput-measure';
|
||||
import type { LoadProfile, TriggerHandle } from './types';
|
||||
|
||||
/** Context shared by every executor — metrics + measurement plumbing, no load specifics. */
|
||||
|
|
@ -20,6 +20,7 @@ export interface ExecutorContext {
|
|||
metrics: MetricsHelper;
|
||||
baselineCounter: number;
|
||||
metricQuery: string;
|
||||
counterReader?: CompletionCounterReader;
|
||||
timeoutMs: number;
|
||||
nodeCount: number;
|
||||
}
|
||||
|
|
@ -59,6 +60,7 @@ async function runPreloaded(load: PreloadedLoad, ctx: ExecutorContext): Promise<
|
|||
timeoutMs: ctx.timeoutMs,
|
||||
baselineValue: ctx.baselineCounter,
|
||||
metricQuery: ctx.metricQuery,
|
||||
counterReader: ctx.counterReader,
|
||||
});
|
||||
|
||||
return {
|
||||
|
|
@ -91,6 +93,7 @@ async function runSteady(load: SteadyLoad, ctx: ExecutorContext): Promise<Execut
|
|||
timeoutMs: ctx.timeoutMs,
|
||||
baselineValue: ctx.baselineCounter,
|
||||
metricQuery: ctx.metricQuery,
|
||||
counterReader: ctx.counterReader,
|
||||
publishEndAt,
|
||||
}),
|
||||
]);
|
||||
|
|
@ -150,6 +153,7 @@ async function runStaged(load: StagedLoad, ctx: ExecutorContext): Promise<Execut
|
|||
timeoutMs: ctx.timeoutMs,
|
||||
baselineValue: ctx.baselineCounter,
|
||||
metricQuery: ctx.metricQuery,
|
||||
counterReader: ctx.counterReader,
|
||||
publishEndAt,
|
||||
stageBoundaries,
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -174,9 +174,14 @@ function groupByCell(
|
|||
): Map<Scale, CellGroupEntry[]> {
|
||||
const byScale = new Map<Scale, CellGroupEntry[]>();
|
||||
for (const { path, report } of reports) {
|
||||
// The sizing matrix represents the default runtime. Optimized comparison
|
||||
// profiles remain available in benchmark metrics and run-report artifacts.
|
||||
if (report.scenario.dimensions['expression_profile'] === 'vm-lazy-cache') continue;
|
||||
// The sizing matrix represents the default runtime. Comparison profiles
|
||||
// remain available in benchmark metrics and run-report artifacts.
|
||||
if (
|
||||
report.scenario.dimensions['expression_profile'] === 'vm-lazy-cache' ||
|
||||
report.scenario.dimensions['execution_engine'] === 'v2'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const cellCoord = findMapping(path, report, mapping);
|
||||
if (!cellCoord) {
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ export function measureSteadyPhases(
|
|||
|
||||
export const WORKFLOW_SUCCESS_QUERY = 'n8n_workflow_success_total';
|
||||
export const QUEUE_JOBS_COMPLETED_QUERY = 'n8n_scaling_mode_queue_jobs_completed';
|
||||
export type CompletionCounterReader = () => Promise<number>;
|
||||
|
||||
/**
|
||||
* Returns the completion metric for the current Playwright project.
|
||||
|
|
@ -257,6 +258,7 @@ export async function waitForThroughput(
|
|||
* per-stage measurements so a ramp test can identify the breaking point.
|
||||
*/
|
||||
stageBoundaries?: number[];
|
||||
counterReader?: CompletionCounterReader;
|
||||
},
|
||||
): Promise<ThroughputResult> {
|
||||
const {
|
||||
|
|
@ -269,6 +271,7 @@ export async function waitForThroughput(
|
|||
stallThresholdMs = 60_000,
|
||||
publishEndAt,
|
||||
stageBoundaries,
|
||||
counterReader,
|
||||
} = options;
|
||||
|
||||
const samples: ThroughputSample[] = [];
|
||||
|
|
@ -282,9 +285,14 @@ export async function waitForThroughput(
|
|||
const remaining = deadline - Date.now();
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, remaining)));
|
||||
|
||||
let results;
|
||||
let current: number;
|
||||
try {
|
||||
results = await metrics.query(`sum(last_over_time(${metricQuery}[5m]))`);
|
||||
if (counterReader) {
|
||||
current = await counterReader();
|
||||
} else {
|
||||
const results = await metrics.query(`sum(last_over_time(${metricQuery}[5m]))`);
|
||||
current = results.length > 0 ? results.reduce((sum, r) => sum + r.value, 0) : 0;
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(
|
||||
`[THROUGHPUT] Query error: ${error instanceof Error ? error.message : String(error)}`,
|
||||
|
|
@ -292,8 +300,6 @@ export async function waitForThroughput(
|
|||
continue;
|
||||
}
|
||||
|
||||
const current = results.length > 0 ? results.reduce((sum, r) => sum + r.value, 0) : 0;
|
||||
|
||||
// Monotonic guard: counters should never decrease.
|
||||
// If VictoriaMetrics returns a stale/missing value, skip this sample.
|
||||
if (current < highWaterMark) {
|
||||
|
|
@ -346,7 +352,9 @@ export async function waitForThroughput(
|
|||
export async function getBaselineCounter(
|
||||
metrics: MetricsHelper,
|
||||
metricQuery: string = WORKFLOW_SUCCESS_QUERY,
|
||||
counterReader?: CompletionCounterReader,
|
||||
): Promise<number> {
|
||||
if (counterReader) return await counterReader();
|
||||
try {
|
||||
const results = await metrics.query(`sum(last_over_time(${metricQuery}[5m]))`);
|
||||
return results.length > 0 ? results.reduce((sum, r) => sum + r.value, 0) : 0;
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user