From 06e78f39bee331295f7205f1dae6c0ff940557da Mon Sep 17 00:00:00 2001 From: Declan Carroll Date: Wed, 20 May 2026 11:16:00 +0100 Subject: [PATCH] test: Consolidate memory baseline specs and add no-mcp impact test (#30745) Co-authored-by: Claude Opus 4.7 (1M context) --- .../playwright/tests/performance/README.md | 32 ++++++++++++++ .../tests/performance/memory-baseline.ts | 37 ++++++++++++++++ .../memory-consumption-agents.spec.ts | 43 ++----------------- .../memory-consumption-cloud.spec.ts | 37 ++-------------- .../memory-consumption-instance-ai.spec.ts | 43 ++----------------- .../memory-consumption-no-mcp-only.spec.ts | 13 ++++++ ...memory-consumption-no-mcp-registry.spec.ts | 13 ++++++ 7 files changed, 105 insertions(+), 113 deletions(-) create mode 100644 packages/testing/playwright/tests/performance/memory-baseline.ts create mode 100644 packages/testing/playwright/tests/performance/memory-consumption-no-mcp-only.spec.ts create mode 100644 packages/testing/playwright/tests/performance/memory-consumption-no-mcp-registry.spec.ts diff --git a/packages/testing/playwright/tests/performance/README.md b/packages/testing/playwright/tests/performance/README.md index 9b0b427780e..9d8a41b3eeb 100644 --- a/packages/testing/playwright/tests/performance/README.md +++ b/packages/testing/playwright/tests/performance/README.md @@ -101,6 +101,38 @@ await test.info().attach('performance-metrics', { }); ``` +### "I want to measure how much memory a module adds to the instance" +Idle heap baselines are collected by the `memory-consumption-*.spec.ts` files via the +shared `runMemoryBaseline()` helper. Each spec disables one or more modules through +`N8N_DISABLED_MODULES` and stabilises memory post-GC, so the diff between specs +reveals a module's footprint. + +```typescript +// memory-consumption-no-mcp-only.spec.ts +import { runMemoryBaseline } from './memory-baseline'; +import { test } from '../../fixtures/base'; + +test.use({ + capability: { + services: ['victoriaLogs', 'victoriaMetrics', 'vector'], + env: { N8N_DISABLED_MODULES: 'mcp' }, + }, +}); + +runMemoryBaseline({ name: 'no-mcp-only', owner: 'AI' }); +``` + +Run a focused subset and compare `heap-used` (the only metric stable enough for +single-shot diffs — `heap-total` / `RSS` / `non-heap` are noisy): + +```bash +pnpm test:performance --grep "no-mcp|memory" +``` + +Each spec emits `${name}-heap-used-baseline`, `${name}-rss-baseline`, etc. Add the +series to `.github/workflows/ci-pull-requests.yml` if you want it surfaced in PR +comments. Run twice and average to shake out variance before quoting numbers. + ## API Reference ### `measurePerformance(page, actionName, actionFn)` diff --git a/packages/testing/playwright/tests/performance/memory-baseline.ts b/packages/testing/playwright/tests/performance/memory-baseline.ts new file mode 100644 index 00000000000..d146f4f65bc --- /dev/null +++ b/packages/testing/playwright/tests/performance/memory-baseline.ts @@ -0,0 +1,37 @@ +import { test, expect } from '../../fixtures/base'; +import { attachMetric, getStableHeap } from '../../utils/performance-helper'; + +export function runMemoryBaseline({ name, owner }: { name: string; owner: string }) { + test.describe( + `Module Memory Impact · ${name} @capability:observability`, + { annotation: [{ type: 'owner', description: owner }] }, + () => { + test(`Idle baseline · ${name}`, async ({ n8nContainer, services }, testInfo) => { + const obs = services.observability; + const result = await getStableHeap(n8nContainer.baseUrl, obs.metrics); + + await attachMetric(testInfo, `${name}-heap-used-baseline`, result.heapUsedMB, 'MB'); + await attachMetric(testInfo, `${name}-heap-total-baseline`, result.heapTotalMB, 'MB'); + await attachMetric(testInfo, `${name}-rss-baseline`, result.rssMB, 'MB'); + await attachMetric(testInfo, `${name}-pss-baseline`, result.pssMB ?? 0, 'MB'); + await attachMetric( + testInfo, + `${name}-non-heap-overhead-baseline`, + result.nonHeapOverheadMB, + 'MB', + ); + + expect(result.heapUsedMB).toBeGreaterThan(0); + expect(result.heapTotalMB).toBeGreaterThan(0); + expect(result.rssMB).toBeGreaterThan(0); + + console.log( + `[${name.toUpperCase()} IDLE] Heap used: ${result.heapUsedMB.toFixed(1)} MB | ` + + `Heap total: ${result.heapTotalMB.toFixed(1)} MB | ` + + `RSS: ${result.rssMB.toFixed(1)} MB | ` + + `Non-heap overhead: ${result.nonHeapOverheadMB.toFixed(1)} MB`, + ); + }); + }, + ); +} diff --git a/packages/testing/playwright/tests/performance/memory-consumption-agents.spec.ts b/packages/testing/playwright/tests/performance/memory-consumption-agents.spec.ts index 8705267dfe6..cdfeb9f74ff 100644 --- a/packages/testing/playwright/tests/performance/memory-consumption-agents.spec.ts +++ b/packages/testing/playwright/tests/performance/memory-consumption-agents.spec.ts @@ -1,5 +1,5 @@ -import { test, expect } from '../../fixtures/base'; -import { attachMetric, getStableHeap } from '../../utils/performance-helper'; +import { runMemoryBaseline } from './memory-baseline'; +import { test } from '../../fixtures/base'; test.use({ capability: { @@ -11,41 +11,4 @@ test.use({ }, }); -test.describe( - 'Agents Memory Consumption @capability:observability', - { - annotation: [{ type: 'owner', description: 'Agent' }], - }, - () => { - test('Idle baseline with Agents module loaded', async ({ - n8nContainer, - services, - }, testInfo) => { - const obs = services.observability; - - const result = await getStableHeap(n8nContainer.baseUrl, obs.metrics); - - await attachMetric(testInfo, 'agents-heap-used-baseline', result.heapUsedMB, 'MB'); - await attachMetric(testInfo, 'agents-heap-total-baseline', result.heapTotalMB, 'MB'); - await attachMetric(testInfo, 'agents-rss-baseline', result.rssMB, 'MB'); - await attachMetric(testInfo, 'agents-pss-baseline', result.pssMB ?? 0, 'MB'); - await attachMetric( - testInfo, - 'agents-non-heap-overhead-baseline', - result.nonHeapOverheadMB, - 'MB', - ); - - expect(result.heapUsedMB).toBeGreaterThan(0); - expect(result.heapTotalMB).toBeGreaterThan(0); - expect(result.rssMB).toBeGreaterThan(0); - - console.log( - `[AGENTS IDLE] Heap used: ${result.heapUsedMB.toFixed(1)} MB | ` + - `Heap total: ${result.heapTotalMB.toFixed(1)} MB | ` + - `RSS: ${result.rssMB.toFixed(1)} MB | ` + - `Non-heap overhead: ${result.nonHeapOverheadMB.toFixed(1)} MB`, - ); - }); - }, -); +runMemoryBaseline({ name: 'agents', owner: 'AI' }); diff --git a/packages/testing/playwright/tests/performance/memory-consumption-cloud.spec.ts b/packages/testing/playwright/tests/performance/memory-consumption-cloud.spec.ts index ebf1c4a7610..ac15b426556 100644 --- a/packages/testing/playwright/tests/performance/memory-consumption-cloud.spec.ts +++ b/packages/testing/playwright/tests/performance/memory-consumption-cloud.spec.ts @@ -1,6 +1,7 @@ -import { test, expect } from '../../fixtures/base'; -import { attachMetric, getStableHeap } from '../../utils/performance-helper'; +import { runMemoryBaseline } from './memory-baseline'; +import { test } from '../../fixtures/base'; +// Emits `memory-*-baseline` series consumed by .github/workflows/ci-pull-requests.yml — do not rename without updating that workflow. test.use({ capability: { resourceQuota: { memory: 0.75, cpu: 0.5 }, @@ -8,34 +9,4 @@ test.use({ }, }); -test.describe( - 'Memory Consumption @capability:observability', - { - annotation: [{ type: 'owner', description: 'Catalysts' }], - }, - () => { - test('Memory consumption baseline with starter plan resources', async ({ - n8nContainer, - services, - }, testInfo) => { - const obs = services.observability; - - const result = await getStableHeap(n8nContainer.baseUrl, obs.metrics); - - await attachMetric(testInfo, 'memory-heap-used-baseline', result.heapUsedMB, 'MB'); - await attachMetric(testInfo, 'memory-heap-total-baseline', result.heapTotalMB, 'MB'); - await attachMetric(testInfo, 'memory-rss-baseline', result.rssMB, 'MB'); - await attachMetric(testInfo, 'memory-pss-baseline', result.pssMB ?? 0, 'MB'); - await attachMetric( - testInfo, - 'memory-non-heap-overhead-baseline', - result.nonHeapOverheadMB, - 'MB', - ); - - expect(result.heapUsedMB).toBeGreaterThan(0); - expect(result.heapTotalMB).toBeGreaterThan(0); - expect(result.rssMB).toBeGreaterThan(0); - }); - }, -); +runMemoryBaseline({ name: 'memory', owner: 'Catalysts' }); diff --git a/packages/testing/playwright/tests/performance/memory-consumption-instance-ai.spec.ts b/packages/testing/playwright/tests/performance/memory-consumption-instance-ai.spec.ts index 2fea40b9386..b3cabbbb1c6 100644 --- a/packages/testing/playwright/tests/performance/memory-consumption-instance-ai.spec.ts +++ b/packages/testing/playwright/tests/performance/memory-consumption-instance-ai.spec.ts @@ -1,5 +1,5 @@ -import { test, expect } from '../../fixtures/base'; -import { attachMetric, getStableHeap } from '../../utils/performance-helper'; +import { runMemoryBaseline } from './memory-baseline'; +import { test } from '../../fixtures/base'; test.use({ capability: { @@ -12,41 +12,4 @@ test.use({ }, }); -test.describe( - 'Instance AI Memory Consumption @capability:observability', - { - annotation: [{ type: 'owner', description: 'Catalysts' }], - }, - () => { - test('Idle baseline with Instance AI module loaded', async ({ - n8nContainer, - services, - }, testInfo) => { - const obs = services.observability; - - const result = await getStableHeap(n8nContainer.baseUrl, obs.metrics); - - await attachMetric(testInfo, 'instance-ai-heap-used-baseline', result.heapUsedMB, 'MB'); - await attachMetric(testInfo, 'instance-ai-heap-total-baseline', result.heapTotalMB, 'MB'); - await attachMetric(testInfo, 'instance-ai-rss-baseline', result.rssMB, 'MB'); - await attachMetric(testInfo, 'instance-ai-pss-baseline', result.pssMB ?? 0, 'MB'); - await attachMetric( - testInfo, - 'instance-ai-non-heap-overhead-baseline', - result.nonHeapOverheadMB, - 'MB', - ); - - expect(result.heapUsedMB).toBeGreaterThan(0); - expect(result.heapTotalMB).toBeGreaterThan(0); - expect(result.rssMB).toBeGreaterThan(0); - - console.log( - `[INSTANCE AI IDLE] Heap used: ${result.heapUsedMB.toFixed(1)} MB | ` + - `Heap total: ${result.heapTotalMB.toFixed(1)} MB | ` + - `RSS: ${result.rssMB.toFixed(1)} MB | ` + - `Non-heap overhead: ${result.nonHeapOverheadMB.toFixed(1)} MB`, - ); - }); - }, -); +runMemoryBaseline({ name: 'instance-ai', owner: 'Instance AI' }); diff --git a/packages/testing/playwright/tests/performance/memory-consumption-no-mcp-only.spec.ts b/packages/testing/playwright/tests/performance/memory-consumption-no-mcp-only.spec.ts new file mode 100644 index 00000000000..998de89f7c7 --- /dev/null +++ b/packages/testing/playwright/tests/performance/memory-consumption-no-mcp-only.spec.ts @@ -0,0 +1,13 @@ +import { runMemoryBaseline } from './memory-baseline'; +import { test } from '../../fixtures/base'; + +test.use({ + capability: { + services: ['victoriaLogs', 'victoriaMetrics', 'vector'], + env: { + N8N_DISABLED_MODULES: 'mcp', + }, + }, +}); + +runMemoryBaseline({ name: 'no-mcp-only', owner: 'AI' }); diff --git a/packages/testing/playwright/tests/performance/memory-consumption-no-mcp-registry.spec.ts b/packages/testing/playwright/tests/performance/memory-consumption-no-mcp-registry.spec.ts new file mode 100644 index 00000000000..b1551b5c478 --- /dev/null +++ b/packages/testing/playwright/tests/performance/memory-consumption-no-mcp-registry.spec.ts @@ -0,0 +1,13 @@ +import { runMemoryBaseline } from './memory-baseline'; +import { test } from '../../fixtures/base'; + +test.use({ + capability: { + services: ['victoriaLogs', 'victoriaMetrics', 'vector'], + env: { + N8N_DISABLED_MODULES: 'mcp-registry', + }, + }, +}); + +runMemoryBaseline({ name: 'no-mcp-registry', owner: 'AI' });