mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
refactor: Move idle memory baselines to efficiency (#41037)
This commit is contained in:
Declan Carroll
GitHub
parent
df7b3a19c6
commit
0650b9cce0
2
.github/WORKFLOWS.md
vendored
2
.github/WORKFLOWS.md
vendored
|
|
@ -587,7 +587,7 @@ Push to master
|
|||
| Daily 03:00 | `sec-sync-bundle-branches.yml` | Merge the base into `bundle/*` |
|
||||
| Daily 00:00 | `docker-build-push.yml` | Nightly Docker images |
|
||||
| Daily 00:00 | `test-db.yml` | Database compatibility |
|
||||
| Daily 00:00 | `test-e2e-performance-reusable.yml`| Performance E2E |
|
||||
| Daily 00:00 | `test-e2e-performance-reusable.yml`| Browser performance and idle memory baselines |
|
||||
| Daily 00:00 | `release-storybook.yml` | Storybook deploy |
|
||||
| Daily 00:00 | `release-chromatic.yml` | Visual regression |
|
||||
| Daily 00:00 | `util-check-docs-urls.yml` | Doc link validation |
|
||||
|
|
|
|||
7
.github/workflows/ci-pull-requests.yml
vendored
7
.github/workflows/ci-pull-requests.yml
vendored
|
|
@ -147,10 +147,17 @@ jobs:
|
|||
cubic.yaml
|
||||
.agents/review-rules/**
|
||||
e2e-performance:
|
||||
packages/quality/efficiency/memory/idle/**
|
||||
packages/quality/efficiency/playwright.config.ts
|
||||
packages/quality/efficiency/package.json
|
||||
packages/quality/testing/playwright/package.json
|
||||
packages/quality/testing/playwright/quality-*.ts
|
||||
packages/quality/testing/playwright/fixtures/**
|
||||
packages/quality/testing/playwright/tests/performance/**
|
||||
packages/quality/testing/playwright/utils/performance-helper.ts
|
||||
packages/quality/environments/containers/**
|
||||
.github/workflows/test-e2e-performance-reusable.yml
|
||||
.github/workflows/test-e2e-reusable.yml
|
||||
# Whole directories, not the `engine-*` files alone. Engine v2
|
||||
# routing lives in ordinary control plane code: `workflow-runner.ts`
|
||||
# and the webhook and trigger paths decide whether a run reaches the
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
name: 'Test: E2E Performance'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
workflow_dispatch:
|
||||
|
|
@ -24,6 +27,17 @@ jobs:
|
|||
artifact-prefix: performance
|
||||
secrets: inherit
|
||||
|
||||
idle-memory-baselines:
|
||||
needs: [prepare-docker]
|
||||
uses: ./.github/workflows/test-e2e-reusable.yml
|
||||
with:
|
||||
test-mode: docker-artifact
|
||||
test-command: pnpm --filter=@n8n/efficiency test:memory
|
||||
workers: '1'
|
||||
currents-project-id: 'O9BJaN'
|
||||
artifact-prefix: memory-performance
|
||||
secrets: inherit
|
||||
|
||||
canvas-perf-sentinels:
|
||||
name: Canvas Perf Sentinels
|
||||
# Sentinels are nightly only — wall-clock browser perf has 15-30% variance,
|
||||
|
|
|
|||
7
.github/workflows/test-e2e-reusable.yml
vendored
7
.github/workflows/test-e2e-reusable.yml
vendored
|
|
@ -195,9 +195,10 @@ jobs:
|
|||
with:
|
||||
name: ${{ inputs.artifact-prefix }}-shard-${{ matrix.shard }}
|
||||
path: |
|
||||
packages/quality/testing/playwright/test-results/
|
||||
packages/quality/testing/playwright/test-results.json
|
||||
packages/quality/testing/playwright/playwright-report/
|
||||
packages/quality/*/test-results*
|
||||
packages/quality/*/playwright-report/
|
||||
packages/quality/*/*/test-results*
|
||||
packages/quality/*/*/playwright-report/
|
||||
packages/quality/testing/playwright/coverage/lcov.info
|
||||
packages/quality/testing/playwright/coverage/by-spec/
|
||||
packages/quality/testing/playwright/coverage/index.html
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Read the repository [AGENTS.md](../../AGENTS.md) first. Use the [quality enginee
|
|||
| Policy | Repository rules and static analysis | [`rules-engine`](policy/rules-engine/), [`code-health`](policy/code-health/) |
|
||||
| Testing | Test selection, architecture checks, and test orchestration | [`test-impact`](testing/test-impact/), [`janitor`](testing/janitor/), [`playwright`](testing/playwright/) |
|
||||
| Environments | Test services and deployment stacks | [`containers`](environments/containers/) |
|
||||
| Efficiency | Speed, memory, load, latency, throughput, capacity, and resource use | [Efficiency workspace](efficiency/), [microbenchmarks](efficiency/microbenchmarks/), [Playwright performance tests](testing/playwright/tests/performance/), [infrastructure benchmarks](testing/playwright/tests/infrastructure/benchmarks/), [benchmark CLI](efficiency/scale/benchmark/) |
|
||||
| Efficiency | Speed, memory, load, latency, throughput, capacity, and resource use | [Efficiency workspace](efficiency/), [idle memory](efficiency/memory/idle/), [microbenchmarks](efficiency/microbenchmarks/), [Playwright performance tests](testing/playwright/tests/performance/), [infrastructure benchmarks](testing/playwright/tests/infrastructure/benchmarks/), [benchmark CLI](efficiency/scale/benchmark/) |
|
||||
|
||||
## Working rules
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@ Its configuration owns the efficiency projects and their scopes. Suites live in
|
|||
| Tool | Use it to | Guide |
|
||||
| --- | --- | --- |
|
||||
| `@n8n/efficiency` | Measure Docker image size and layer waste with Dive | [Image measurements](images/README.md) |
|
||||
| `@n8n/efficiency` | Measure idle memory with fixed container profiles | [Idle memory baselines](memory/idle/README.md) |
|
||||
| `@n8n/performance` | Compare code paths without building an n8n Docker image | [Microbenchmarks](microbenchmarks/README.md) |
|
||||
| `@n8n/n8n-benchmark` | Run load scenarios against an n8n instance | [Benchmark CLI](scale/benchmark/README.md) |
|
||||
|
||||
For idle memory, canvas performance, throughput, and local soak tests, use the [Playwright suite map](../testing/playwright/README.md#choose-a-suite). These specs need Playwright fixtures, projects, metrics, or managed containers.
|
||||
For memory retention, canvas performance, throughput, and local soak tests, use the [Playwright suite map](../testing/playwright/README.md#choose-a-suite).
|
||||
|
||||
Run image measurements with `pnpm --filter @n8n/efficiency test:images`.
|
||||
Run idle memory baselines with `pnpm --filter @n8n/efficiency test:memory`.
|
||||
The nested microbenchmark and benchmark CLI workspaces keep their own commands and configurations.
|
||||
|
|
|
|||
42
packages/quality/efficiency/memory/idle/README.md
Normal file
42
packages/quality/efficiency/memory/idle/README.md
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# Measure idle memory
|
||||
|
||||
Use these baselines to compare idle memory across module configurations.
|
||||
You need Docker and an n8n test image with the E2E endpoints.
|
||||
The suite uses `n8nio/n8n:local` by default. Set `TEST_IMAGE_N8N` to use another test image.
|
||||
Each profile limits n8n to 0.75 GB of memory and 0.5 CPU.
|
||||
|
||||
1. Build the test image when you need to measure local product changes:
|
||||
|
||||
```bash
|
||||
pnpm build:docker
|
||||
```
|
||||
|
||||
Reuse an existing test image when product code has not changed.
|
||||
|
||||
2. Run the baselines from the repository root:
|
||||
|
||||
```bash
|
||||
pnpm --filter @n8n/efficiency test:memory
|
||||
```
|
||||
|
||||
Select one profile with a spec path:
|
||||
|
||||
```bash
|
||||
pnpm --filter @n8n/efficiency test:memory memory-consumption-cloud.spec.ts
|
||||
```
|
||||
|
||||
3. Open the native Playwright report:
|
||||
|
||||
```bash
|
||||
pnpm --filter @n8n/efficiency report
|
||||
```
|
||||
|
||||
**Verify:** Each profile passes and attaches five memory measurements.
|
||||
The suite resets and seeds the database before each worker measures memory.
|
||||
It requests GC and waits for stable heap readings through the shared `getStableHeap()` helper.
|
||||
It reports heap used, heap total, RSS, PSS, and non-heap overhead in the existing `MB` series.
|
||||
Compare results on the same machine with the same image and resource limits.
|
||||
Repeat measurements before you attribute a difference to a module.
|
||||
|
||||
CI runs this suite in `test-e2e-performance-reusable.yml` with Currents project `O9BJaN` and the shared QA metrics reporter.
|
||||
Keep the metric identifiers unchanged. PR comments consume the `memory`, `instance-ai`, and `agents` baseline series.
|
||||
|
|
@ -1,13 +1,16 @@
|
|||
import { test, expect } from '../../fixtures/base';
|
||||
import { attachMetric, getStableHeap } from '../../utils/performance-helper';
|
||||
import { test, expect } from 'n8n-playwright/containers';
|
||||
import { getStableHeap } from 'n8n-playwright/memory';
|
||||
import { attachMetric } from 'n8n-playwright/metrics';
|
||||
|
||||
export function runMemoryBaseline({ name, owner }: { name: string; owner: string }) {
|
||||
test.describe(
|
||||
`Module Memory Impact · ${name}`,
|
||||
{ annotation: [{ type: 'owner', description: owner }] },
|
||||
() => {
|
||||
test(`Idle baseline · ${name}`, async ({ n8nContainer, services }, testInfo) => {
|
||||
const obs = services.observability;
|
||||
test(`Idle baseline · ${name}`, async ({ n8nContainer, dbSetup }, testInfo) => {
|
||||
// Keep the seeded instance state used by the original baseline.
|
||||
void dbSetup;
|
||||
const obs = n8nContainer.services.observability;
|
||||
const result = await getStableHeap(n8nContainer.baseUrl, obs.metrics);
|
||||
|
||||
await attachMetric(testInfo, `${name}-heap-used-baseline`, result.heapUsedMB, 'MB');
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { runMemoryBaseline } from './memory-baseline';
|
||||
import { test } from '../../fixtures/base';
|
||||
import { runMemoryBaseline } from './memory-baseline.test.utils';
|
||||
import { test } from 'n8n-playwright/containers';
|
||||
|
||||
test.use({
|
||||
capability: {
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { runMemoryBaseline } from './memory-baseline';
|
||||
import { test } from '../../fixtures/base';
|
||||
import { runMemoryBaseline } from './memory-baseline.test.utils';
|
||||
import { test } from 'n8n-playwright/containers';
|
||||
|
||||
// Emits `memory-*-baseline` series consumed by .github/workflows/ci-pull-requests.yml — do not rename without updating that workflow.
|
||||
test.use({
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { runMemoryBaseline } from './memory-baseline';
|
||||
import { test } from '../../fixtures/base';
|
||||
import { runMemoryBaseline } from './memory-baseline.test.utils';
|
||||
import { test } from 'n8n-playwright/containers';
|
||||
|
||||
test.use({
|
||||
capability: {
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { runMemoryBaseline } from './memory-baseline';
|
||||
import { test } from '../../fixtures/base';
|
||||
import { runMemoryBaseline } from './memory-baseline.test.utils';
|
||||
import { test } from 'n8n-playwright/containers';
|
||||
|
||||
test.use({
|
||||
capability: {
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { runMemoryBaseline } from './memory-baseline';
|
||||
import { test } from '../../fixtures/base';
|
||||
import { runMemoryBaseline } from './memory-baseline.test.utils';
|
||||
import { test } from 'n8n-playwright/containers';
|
||||
|
||||
test.use({
|
||||
capability: {
|
||||
|
|
@ -4,10 +4,11 @@
|
|||
"scripts": {
|
||||
"test:all": "playwright test",
|
||||
"test:images": "playwright test --project='image-efficiency:*'",
|
||||
"test:memory": "playwright test --project=memory-baselines",
|
||||
"report": "playwright show-report",
|
||||
"lint": "oxlint --quiet playwright.config.ts images",
|
||||
"lint": "oxlint --quiet playwright.config.ts images memory/idle",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"format:check": "biome ci playwright.config.ts images"
|
||||
"format:check": "biome ci playwright.config.ts images memory/idle"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "catalog:"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
/* eslint-disable import-x/no-default-export */
|
||||
import { defineQualityConfig } from 'n8n-playwright/config';
|
||||
import type { ContainerWorkerFixtures } from 'n8n-playwright/containers';
|
||||
|
||||
const images = [
|
||||
{ name: 'n8n', ref: process.env.TEST_IMAGE_N8N ?? 'n8nio/n8n:local' },
|
||||
|
|
@ -10,15 +11,24 @@ const images = [
|
|||
},
|
||||
];
|
||||
|
||||
export default defineQualityConfig({
|
||||
export default defineQualityConfig<object, ContainerWorkerFixtures>({
|
||||
testMatch: '**/*.spec.ts',
|
||||
workers: 1,
|
||||
timeout: 300_000,
|
||||
retries: 0,
|
||||
use: { trace: 'off', video: 'off', screenshot: 'off' },
|
||||
projects: images.map((image) => ({
|
||||
name: `image-efficiency:${image.name}`,
|
||||
testDir: './images',
|
||||
metadata: { image },
|
||||
})),
|
||||
projects: [
|
||||
...images.map((image) => ({
|
||||
name: `image-efficiency:${image.name}`,
|
||||
testDir: './images',
|
||||
metadata: { image },
|
||||
})),
|
||||
{
|
||||
name: 'memory-baselines',
|
||||
testDir: './memory/idle',
|
||||
use: {
|
||||
containerConfig: { resourceQuota: { memory: 0.75, cpu: 0.5 }, env: { E2E_TESTS: 'true' } },
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,5 +6,5 @@
|
|||
"lib": ["esnext", "dom"],
|
||||
"types": ["@playwright/test", "node"]
|
||||
},
|
||||
"include": ["playwright.config.ts", "images/**/*.ts"]
|
||||
"include": ["playwright.config.ts", "images/**/*.ts", "memory/idle/**/*.ts"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,8 @@ Commands in this table run from the repository root. Container-backed tests need
|
|||
| Test a product UI or API journey | `e2e` (local) or `<mode>:e2e` (container) | `pnpm --filter n8n-playwright test:local` | [E2E tests](tests/e2e/) |
|
||||
| Test PostgreSQL, queue mode, multi-main, or process lifecycle | `<mode>:infrastructure` | `pnpm --filter n8n-playwright exec playwright test --project=postgres:infrastructure` | [Infrastructure tests](tests/infrastructure/) |
|
||||
| Check accessibility within a product journey | E2E project with the `a11y` fixture | `pnpm --filter n8n-playwright test:local` | [Accessibility checks](docs/ACCESSIBILITY.md) |
|
||||
| Measure idle memory, retention, or canvas performance | `performance` | `pnpm --filter n8n-playwright test:performance` | [Performance guide](tests/performance/README.md) |
|
||||
| Measure memory retention or canvas performance | `performance` | `pnpm --filter n8n-playwright test:performance` | [Performance guide](tests/performance/README.md) |
|
||||
| Measure idle memory | `@n8n/efficiency` memory project | `pnpm --filter @n8n/efficiency test:memory` | [Idle memory baselines](../../efficiency/memory/idle/README.md) |
|
||||
| Measure Docker image size and layer waste | `@n8n/efficiency` image projects | `pnpm --filter @n8n/efficiency test:images` | [Image efficiency](../../efficiency/images/README.md) |
|
||||
| Measure infrastructure throughput and resource use | `benchmarking:infrastructure` | `pnpm --filter n8n-playwright test:benchmark` | [Infrastructure benchmarks](tests/infrastructure/benchmarks/README.md) |
|
||||
| Investigate Instance AI memory with a local-only, Docker-backed run | `benchmark-memory-instanceai:infrastructure` | `pnpm --filter n8n-playwright exec playwright test --project=benchmark-memory-instanceai:infrastructure` | [Local benchmark setup](tests/infrastructure/benchmarks-local/README.md) |
|
||||
|
|
@ -34,11 +35,18 @@ Other private workspaces can declare `n8n-playwright` and the catalog-pinned `@p
|
|||
| `n8n-playwright/test` | `test` and `expect` with the base Currents fixtures |
|
||||
| `n8n-playwright/cli` | CLI fixture with command steps, output attachments, exit-code checks, and process cleanup |
|
||||
| `n8n-playwright/metrics` | `attachMetric()` for QA metrics and Currents analytics |
|
||||
| `n8n-playwright/containers` | Container configuration, worker lifecycle, database preparation, keepalive, and failure diagnostics |
|
||||
| `n8n-playwright/memory` | `getStableHeap()` with the existing GC and memory sampling method |
|
||||
|
||||
The configuration factory accepts standard Playwright options. It does not start a browser, application, or container stack.
|
||||
Reporter paths resolve from this package. Test paths and report outputs resolve from the consuming workspace.
|
||||
Supply `reporter` to replace the defaults, or append suite-specific reporters to `qualityReporters()`.
|
||||
The existing container and product fixtures remain in `fixtures/`.
|
||||
The product fixtures reuse the shared container lifecycle from `fixtures/containers.ts`.
|
||||
|
||||
The container export accepts `containerConfig` at project scope and `capability` through `test.use()`.
|
||||
It keeps the same configuration precedence as the product fixtures.
|
||||
Request `dbSetup` when a suite needs the reset and seeded database.
|
||||
The container fixtures do not launch a browser.
|
||||
|
||||
### CLI fixture
|
||||
|
||||
|
|
|
|||
|
|
@ -2,16 +2,12 @@ import type { CurrentsFixtures, CurrentsWorkerFixtures } from '@currents/playwri
|
|||
import { fixtures as currentsFixtures } from '@currents/playwright';
|
||||
import { test as base, expect, request } from '@playwright/test';
|
||||
import type { ServiceHelpers } from 'n8n-containers/services/types';
|
||||
import type { N8NConfig, N8NProcessUrl, N8NStack } from 'n8n-containers/stack';
|
||||
import { createN8NStack } from 'n8n-containers/stack';
|
||||
import type { N8NProcessUrl } from 'n8n-containers/stack';
|
||||
|
||||
import { a11yFixtures, type A11yTestFixtures } from './a11y';
|
||||
import {
|
||||
CAPABILITIES,
|
||||
shouldSkipContainerRequirement,
|
||||
type CapabilityOption,
|
||||
} from './capabilities';
|
||||
import { shouldSkipContainerRequirement } from './capabilities';
|
||||
import { consoleErrorFixtures } from './console-error-monitor';
|
||||
import { containerFixtures, type ContainerWorkerFixtures } from './containers';
|
||||
import { engineParityDisposition, workflowSettingsFor } from './engine-parity';
|
||||
import { N8N_AUTH_COOKIE } from '../config/constants';
|
||||
import { setupDefaultInterceptors } from '../config/intercepts';
|
||||
|
|
@ -61,38 +57,13 @@ type TestFixtures = {
|
|||
engineParity: undefined;
|
||||
};
|
||||
|
||||
type WorkerFixtures = {
|
||||
type WorkerFixtures = ContainerWorkerFixtures & {
|
||||
n8nUrl: string;
|
||||
backendUrl: string;
|
||||
frontendUrl: string;
|
||||
internalUrl: string;
|
||||
dbSetup: undefined;
|
||||
n8nStackConfig: N8NConfig;
|
||||
n8nContainer: N8NStack;
|
||||
capability?: CapabilityOption;
|
||||
};
|
||||
|
||||
type ProjectUse = { containerConfig?: N8NConfig };
|
||||
|
||||
function parseGlobalTestEnv(): Record<string, string> {
|
||||
const raw = process.env.N8N_TEST_ENV;
|
||||
if (!raw) return {};
|
||||
try {
|
||||
return JSON.parse(raw) as Record<string, string>;
|
||||
} catch {
|
||||
console.warn('[base.ts] Failed to parse N8N_TEST_ENV');
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function logKeepalive(container: N8NStack): void {
|
||||
console.log('\n=== KEEPALIVE: Containers left running for debugging ===');
|
||||
console.log(` URL: ${container.baseUrl}`);
|
||||
console.log(` Project: ${container.projectName}`);
|
||||
console.log(' Cleanup: pnpm --filter n8n-containers stack:clean:all');
|
||||
console.log('=========================================================\n');
|
||||
}
|
||||
|
||||
export const test = base.extend<
|
||||
TestFixtures &
|
||||
CurrentsFixtures &
|
||||
|
|
@ -105,14 +76,12 @@ export const test = base.extend<
|
|||
...v8CoverageFixtures,
|
||||
...backendV8CoverageFixtures,
|
||||
...currentsFixtures.actionFixtures,
|
||||
...containerFixtures,
|
||||
...observabilityFixtures,
|
||||
...consoleErrorFixtures,
|
||||
...quarantineFixtures,
|
||||
...a11yFixtures,
|
||||
|
||||
// Option for test.use({ capability: 'proxy' }) - transformed into N8NStack by n8nContainer
|
||||
capability: [undefined, { scope: 'worker', option: true }],
|
||||
|
||||
// Rejects an unknown @engine:* tag anywhere; only an engine v2 stack skips or
|
||||
// expects failure. See fixtures/engine-parity.ts for the tags.
|
||||
engineParity: [
|
||||
|
|
@ -137,63 +106,6 @@ export const test = base.extend<
|
|||
{ auto: true },
|
||||
],
|
||||
|
||||
// Resolves the effective N8NConfig from project.containerConfig (base) +
|
||||
// capability (override) + N8N_TEST_ENV (global). Topology-neutral: it
|
||||
// always produces a config, even when a container will not be provisioned.
|
||||
n8nStackConfig: [
|
||||
async ({ capability }, use, workerInfo) => {
|
||||
const { containerConfig: base = {} } = workerInfo.project.use as ProjectUse;
|
||||
const override: N8NConfig = !capability
|
||||
? {}
|
||||
: typeof capability === 'string'
|
||||
? CAPABILITIES[capability]
|
||||
: capability;
|
||||
|
||||
const globalEnv = parseGlobalTestEnv();
|
||||
|
||||
const config: N8NConfig = {
|
||||
...base,
|
||||
...override,
|
||||
services: [...new Set([...(base.services ?? []), ...(override.services ?? [])])],
|
||||
env: {
|
||||
...globalEnv,
|
||||
...base.env,
|
||||
...override.env,
|
||||
E2E_TESTS: 'true',
|
||||
N8N_RESTRICT_FILE_ACCESS_TO: '',
|
||||
},
|
||||
// Coverage pipeline opt-in: when the coverage runner sets N8N_COVERAGE_DIR,
|
||||
// bridge it to the stack's typed config so containers collect V8 coverage.
|
||||
...(process.env.N8N_COVERAGE_DIR ? { coverageHostDir: process.env.N8N_COVERAGE_DIR } : {}),
|
||||
};
|
||||
|
||||
await use(config);
|
||||
},
|
||||
{ scope: 'worker', box: true },
|
||||
],
|
||||
|
||||
// Creates container from n8nStackConfig.
|
||||
// When N8N_BASE_URL is set, skips container creation for local testing.
|
||||
n8nContainer: [
|
||||
async ({ n8nStackConfig }, use) => {
|
||||
if (getBackendUrl()) {
|
||||
await use(null!);
|
||||
return;
|
||||
}
|
||||
|
||||
const container = await createN8NStack(n8nStackConfig);
|
||||
await use(container);
|
||||
|
||||
if (process.env.N8N_CONTAINERS_KEEPALIVE === 'true') {
|
||||
logKeepalive(container);
|
||||
return;
|
||||
}
|
||||
|
||||
await container.stop();
|
||||
},
|
||||
{ scope: 'worker', box: true },
|
||||
],
|
||||
|
||||
n8nUrl: [
|
||||
async ({ n8nContainer }, use) => {
|
||||
const envBaseURL = process.env.N8N_BASE_URL ?? n8nContainer?.baseUrl;
|
||||
|
|
@ -230,27 +142,6 @@ export const test = base.extend<
|
|||
{ scope: 'worker' },
|
||||
],
|
||||
|
||||
dbSetup: [
|
||||
async ({ n8nContainer, n8nStackConfig }, use) => {
|
||||
if (n8nContainer) {
|
||||
console.log('Resetting database for new container');
|
||||
const apiContext = await request.newContext({ baseURL: n8nContainer.baseUrl });
|
||||
const api = new ApiHelpers(apiContext);
|
||||
await api.resetDatabase();
|
||||
await apiContext.dispose();
|
||||
|
||||
// The reset endpoint only reaches the control plane database.
|
||||
if (n8nStackConfig.engine) {
|
||||
await (
|
||||
n8nContainer.services.enginePostgres ?? n8nContainer.services.postgres
|
||||
).truncateEngineDatabase();
|
||||
}
|
||||
}
|
||||
await use(undefined);
|
||||
},
|
||||
{ scope: 'worker' },
|
||||
],
|
||||
|
||||
baseURL: async ({ frontendUrl, dbSetup }, use) => {
|
||||
void dbSetup; // Ensure dbSetup runs first
|
||||
await use(frontendUrl);
|
||||
|
|
|
|||
106
packages/quality/testing/playwright/fixtures/containers.ts
Normal file
106
packages/quality/testing/playwright/fixtures/containers.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
import type { Fixtures } from '@playwright/test';
|
||||
import { request } from '@playwright/test';
|
||||
import { createN8NStack } from 'n8n-containers/stack';
|
||||
import type { N8NConfig, N8NStack } from 'n8n-containers/stack';
|
||||
|
||||
import { CAPABILITIES, type CapabilityOption } from './capabilities';
|
||||
import { ApiHelpers } from '../services/api-helper';
|
||||
import { getBackendUrl } from '../utils/url-helper';
|
||||
|
||||
export type ContainerWorkerFixtures = {
|
||||
containerConfig: N8NConfig;
|
||||
capability?: CapabilityOption;
|
||||
n8nStackConfig: N8NConfig;
|
||||
n8nContainer: N8NStack;
|
||||
dbSetup: undefined;
|
||||
};
|
||||
|
||||
function parseGlobalTestEnv(): Record<string, string> {
|
||||
const raw = process.env.N8N_TEST_ENV;
|
||||
if (!raw) return {};
|
||||
try {
|
||||
return JSON.parse(raw) as Record<string, string>;
|
||||
} catch {
|
||||
console.warn('[containers.ts] Failed to parse N8N_TEST_ENV');
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export const containerFixtures: Fixtures<object, ContainerWorkerFixtures> = {
|
||||
containerConfig: [{}, { scope: 'worker', option: true }],
|
||||
capability: [undefined, { scope: 'worker', option: true }],
|
||||
// Keep the same config precedence for product and efficiency suites.
|
||||
n8nStackConfig: [
|
||||
async ({ capability, containerConfig: base }, use) => {
|
||||
const override: N8NConfig = !capability
|
||||
? {}
|
||||
: typeof capability === 'string'
|
||||
? CAPABILITIES[capability]
|
||||
: capability;
|
||||
|
||||
const globalEnv = parseGlobalTestEnv();
|
||||
|
||||
const config: N8NConfig = {
|
||||
...base,
|
||||
...override,
|
||||
services: [...new Set([...(base.services ?? []), ...(override.services ?? [])])],
|
||||
env: {
|
||||
...globalEnv,
|
||||
...base.env,
|
||||
...override.env,
|
||||
E2E_TESTS: 'true',
|
||||
N8N_RESTRICT_FILE_ACCESS_TO: '',
|
||||
},
|
||||
// Pass the coverage runner's output directory to the stack.
|
||||
...(process.env.N8N_COVERAGE_DIR ? { coverageHostDir: process.env.N8N_COVERAGE_DIR } : {}),
|
||||
};
|
||||
|
||||
await use(config);
|
||||
},
|
||||
{ scope: 'worker', box: true },
|
||||
],
|
||||
n8nContainer: [
|
||||
async ({ n8nStackConfig }, use) => {
|
||||
if (getBackendUrl()) {
|
||||
await use(null!);
|
||||
return;
|
||||
}
|
||||
|
||||
const container = await createN8NStack(n8nStackConfig);
|
||||
try {
|
||||
await use(container);
|
||||
} finally {
|
||||
if (process.env.N8N_CONTAINERS_KEEPALIVE === 'true') {
|
||||
console.log('\n=== KEEPALIVE: Containers left running for debugging ===');
|
||||
console.log(` URL: ${container.baseUrl}`);
|
||||
console.log(` Project: ${container.projectName}`);
|
||||
console.log(' Cleanup: pnpm --filter n8n-containers stack:clean:all');
|
||||
console.log('=========================================================\n');
|
||||
} else {
|
||||
await container.stop();
|
||||
}
|
||||
}
|
||||
},
|
||||
{ scope: 'worker', box: true },
|
||||
],
|
||||
dbSetup: [
|
||||
async ({ n8nContainer, n8nStackConfig }, use) => {
|
||||
if (n8nContainer) {
|
||||
console.log('Resetting database for new container');
|
||||
const apiContext = await request.newContext({ baseURL: n8nContainer.baseUrl });
|
||||
const api = new ApiHelpers(apiContext);
|
||||
await api.resetDatabase();
|
||||
await apiContext.dispose();
|
||||
|
||||
// The reset endpoint only reaches the control plane database.
|
||||
if (n8nStackConfig.engine) {
|
||||
await (
|
||||
n8nContainer.services.enginePostgres ?? n8nContainer.services.postgres
|
||||
).truncateEngineDatabase();
|
||||
}
|
||||
}
|
||||
await use(undefined);
|
||||
},
|
||||
{ scope: 'worker' },
|
||||
],
|
||||
};
|
||||
|
|
@ -5,7 +5,9 @@
|
|||
"./config": "./quality-config.ts",
|
||||
"./test": "./quality-test.ts",
|
||||
"./cli": "./fixtures/cli.ts",
|
||||
"./metrics": "./utils/metrics.ts"
|
||||
"./metrics": "./utils/metrics.ts",
|
||||
"./containers": "./quality-containers.ts",
|
||||
"./memory": "./quality-memory.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "N8N_BASE_URL=http://localhost:5678 N8N_EDITOR_URL=http://localhost:8080 RESET_E2E_DB=true playwright test --project=e2e",
|
||||
|
|
|
|||
12
packages/quality/testing/playwright/quality-containers.ts
Normal file
12
packages/quality/testing/playwright/quality-containers.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
import { containerFixtures, type ContainerWorkerFixtures } from './fixtures/containers';
|
||||
import { observabilityFixtures, type ObservabilityTestFixtures } from './fixtures/observability';
|
||||
import { test as base } from './quality-test';
|
||||
|
||||
export { expect } from './quality-test';
|
||||
export type { ContainerWorkerFixtures } from './fixtures/containers';
|
||||
export type { N8NConfig, N8NStack } from 'n8n-containers/stack';
|
||||
|
||||
export const test = base.extend<ObservabilityTestFixtures, ContainerWorkerFixtures>({
|
||||
...containerFixtures,
|
||||
...observabilityFixtures,
|
||||
});
|
||||
2
packages/quality/testing/playwright/quality-memory.ts
Normal file
2
packages/quality/testing/playwright/quality-memory.ts
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
export { getStableHeap } from './utils/performance-helper';
|
||||
export type { StableHeapOptions, StableHeapResult } from './utils/performance-helper';
|
||||
|
|
@ -114,36 +114,8 @@ await test.info().attach('performance-metrics', {
|
|||
|
||||
### "I want to measure how much memory a module adds to the instance"
|
||||
|
||||
Idle heap baselines are collected by the `memory-consumption-*.spec.ts` files via the
|
||||
shared `runMemoryBaseline()` helper. Each spec disables one or more modules through
|
||||
`N8N_DISABLED_MODULES` and stabilises memory post-GC, so the diff between specs
|
||||
reveals a module's footprint.
|
||||
|
||||
```typescript
|
||||
// memory-consumption-no-mcp-only.spec.ts
|
||||
import { runMemoryBaseline } from './memory-baseline';
|
||||
import { test } from '../../fixtures/base';
|
||||
|
||||
test.use({
|
||||
capability: {
|
||||
services: ['victoriaLogs', 'victoriaMetrics', 'vector'],
|
||||
env: { N8N_DISABLED_MODULES: 'mcp' },
|
||||
},
|
||||
});
|
||||
|
||||
runMemoryBaseline({ name: 'no-mcp-only', owner: 'AI' });
|
||||
```
|
||||
|
||||
Run a focused subset and compare `heap-used` (the only metric stable enough for
|
||||
single-shot diffs — `heap-total` / `RSS` / `non-heap` are noisy):
|
||||
|
||||
```bash
|
||||
pnpm test:performance --grep "no-mcp|memory"
|
||||
```
|
||||
|
||||
Each spec emits `${name}-heap-used-baseline`, `${name}-rss-baseline`, etc. Add the
|
||||
series to `.github/workflows/ci-pull-requests.yml` if you want it surfaced in PR
|
||||
comments. Run twice and average to shake out variance before quoting numbers.
|
||||
Use the [idle memory baseline guide](../../../../efficiency/memory/idle/README.md).
|
||||
Run `pnpm --filter @n8n/efficiency test:memory` from the repository root.
|
||||
|
||||
## Canvas benchmark suite (`tests/performance/canvas/`)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user