diff --git a/.github/CI-TELEMETRY.md b/.github/CI-TELEMETRY.md index ed8379b27bb..723250eb78e 100644 --- a/.github/CI-TELEMETRY.md +++ b/.github/CI-TELEMETRY.md @@ -85,6 +85,7 @@ return 'blacksmith'; | Accessibility buckets | `packages/quality/testing/playwright/reporters/a11y-reporter.ts` | Per-bucket axe score, violated rules, violating elements | | Build stats | `.github/scripts/send-build-stats.mjs` | Per-package build duration, cache hit/miss, run total | | Docker stats | `.github/scripts/send-docker-stats.mjs` | Image size per platform, docker build duration | +| Dive image efficiency | `packages/quality/efficiency/images/image-efficiency.spec.ts` | Layer content size, estimated waste, efficiency score, and layer count through the shared Playwright reporter | | Container stack | `packages/quality/environments/containers/telemetry.ts` | E2E stack startup times per service | ## Secrets diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index e06bc7a3fff..309e8569768 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -411,6 +411,7 @@ Run-workflow dropdown, which only a user with write access can pick. |-----------------------------|---------------------------------------------------------| | `util-codespace-preview.yml`| Wake, re-serve or delete a PR preview instance by hand | | `util-data-tooling.yml` | SQLite/PostgreSQL export/import validation (manual) | +| `test-image-efficiency.yml` | Measure n8n and runners image size and layer waste with Dive | | `util-probe-registry.yml` | Diagnose slow npm metadata fetches (temporary) | | `sec-sync-bundle-branches.yml` | Sync both bundle branches on demand, including before a bundle cut | diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index 8b1760f09f4..930a6eeaeaa 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -80,6 +80,7 @@ jobs: !**/*.test.ts !**/*.spec.ts !packages/quality/testing/playwright/** + !packages/quality/efficiency/** !packages/frontend/@n8n/storybook/** !.devcontainer/** !scripts/agent-setup.mjs diff --git a/.github/workflows/prepare-docker-reusable.yml b/.github/workflows/prepare-docker-reusable.yml index 77b2e2b854b..0705c35f638 100644 --- a/.github/workflows/prepare-docker-reusable.yml +++ b/.github/workflows/prepare-docker-reusable.yml @@ -9,6 +9,16 @@ name: 'Prepare n8n Docker (reusable)' on: workflow_call: + secrets: + QA_METRICS_WEBHOOK_URL: + description: 'QA metrics webhook URL.' + required: false + QA_METRICS_WEBHOOK_USER: + description: 'QA metrics webhook username.' + required: false + QA_METRICS_WEBHOOK_PASSWORD: + description: 'QA metrics webhook password.' + required: false inputs: build-variant: description: 'standard, coverage, or pc' diff --git a/.github/workflows/test-image-efficiency.yml b/.github/workflows/test-image-efficiency.yml new file mode 100644 index 00000000000..85898e33435 --- /dev/null +++ b/.github/workflows/test-image-efficiency.yml @@ -0,0 +1,71 @@ +name: 'Test: Image Efficiency' + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + prepare-docker: + uses: ./.github/workflows/prepare-docker-reusable.yml + secrets: + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} + + image-efficiency: + name: Image Efficiency + needs: prepare-docker + runs-on: ${{ vars.RUNNER_PROVIDER == 'github' && 'ubuntu-latest' || 'blacksmith-2vcpu-ubuntu-2204' }} + timeout-minutes: 20 + env: + CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }} + CURRENTS_PROJECT_ID: 'nHHLA5' + CURRENTS_CI_BUILD_ID: ${{ github.repository }}-${{ github.run_id }}-${{ github.run_attempt }}-images + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} + steps: + - name: Checkout + uses: useblacksmith/checkout@bcec731f1eb1367240608d1c889a75b96db6ec53 # v1.5.0 + + - name: Setup Environment + uses: ./.github/actions/setup-nodejs + with: + build-command: pnpm turbo run build --filter=@n8n/utils... + + - name: Load images from cache + uses: ./.github/actions/load-n8n-docker + + - name: Install Dive + shell: bash + run: | + set -euo pipefail + # Keep this version in sync with efficiency/images/dive-results.ts. + VERSION=0.13.1 + DIR=$(mktemp -d "$RUNNER_TEMP/dive.XXXXXX") + ARCHIVE="$DIR/dive.tar.gz" + curl --fail --location --retry 3 \ + "https://github.com/wagoodman/dive/releases/download/v${VERSION}/dive_${VERSION}_linux_amd64.tar.gz" \ + --output "$ARCHIVE" + echo "0970549eb4a306f8825a84145a2534153badb4d7dcf3febd1967c706367c3d0e $ARCHIVE" | sha256sum --check + tar -xzf "$ARCHIVE" -C "$DIR" dive + echo "$DIR" >> "$GITHUB_PATH" + + - name: Measure image layer efficiency + run: >- + pnpm --filter=@n8n/efficiency exec playwright test + --project=image-efficiency:n8n --project=image-efficiency:runners + + - name: Upload image efficiency reports + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: image-efficiency-${{ github.run_attempt }} + path: | + packages/quality/efficiency/test-results/ + packages/quality/efficiency/test-results.json + packages/quality/efficiency/playwright-report/ + retention-days: 7 + if-no-files-found: ignore diff --git a/packages/quality/AGENTS.md b/packages/quality/AGENTS.md index dec5c9a6b8f..73b146fab22 100644 --- a/packages/quality/AGENTS.md +++ b/packages/quality/AGENTS.md @@ -7,11 +7,12 @@ Read the repository [AGENTS.md](../../AGENTS.md) first. Use the [quality enginee | Policy | Repository rules and static analysis | [`rules-engine`](policy/rules-engine/), [`code-health`](policy/code-health/) | | Testing | Test selection, architecture checks, and test orchestration | [`test-impact`](testing/test-impact/), [`janitor`](testing/janitor/), [`playwright`](testing/playwright/) | | Environments | Test services and deployment stacks | [`containers`](environments/containers/) | -| Efficiency | Speed, memory, load, latency, throughput, capacity, and resource use | [Microbenchmarks](efficiency/microbenchmarks/), [Playwright performance tests](testing/playwright/tests/performance/), [infrastructure benchmarks](testing/playwright/tests/infrastructure/benchmarks/), [benchmark CLI](efficiency/scale/benchmark/) | +| Efficiency | Speed, memory, load, latency, throughput, capacity, and resource use | [Efficiency workspace](efficiency/), [microbenchmarks](efficiency/microbenchmarks/), [Playwright performance tests](testing/playwright/tests/performance/), [infrastructure benchmarks](testing/playwright/tests/infrastructure/benchmarks/), [benchmark CLI](efficiency/scale/benchmark/) | ## Working rules - Read the instructions in the owning package before you change its files. +- Keep suites in their owning area. Reuse exported runner configuration and fixtures instead of copying them. - Keep benchmark methods and results comparable when you move files. Do not change behavior during a path migration. - Keep existing package names and `pnpm --filter` commands when you move workspaces. - Update workspace discovery, CI paths, test-impact rules, ownership, documentation, and the lockfile when you move a package. Verify that each moved workspace still runs its intended checks. diff --git a/packages/quality/README.md b/packages/quality/README.md index 7d0ad2dc469..7bdd6e7ac3c 100644 --- a/packages/quality/README.md +++ b/packages/quality/README.md @@ -1,12 +1,12 @@ # Quality engineering -Each area owns workspaces. A test stays with its runner, even when it measures another quality concern. +Each area owns workspaces and suites. Suites can reuse shared runner configuration and fixtures across workspace boundaries. | Area | Owns | Start here | | --- | --- | --- | | Policy | Repository rules and static checks | [Policy](policy/README.md) | | Testing | Test selection, architecture checks, and the Playwright runner | [Testing](testing/README.md) | | Environments | Services and deployment stacks for tests | [Environments](environments/README.md) | -| Efficiency | Standalone microbenchmark and load-test tools | [Efficiency](efficiency/README.md) | +| Efficiency | Resource measurements, microbenchmarks, and load-test tools | [Efficiency](efficiency/README.md) | -Use the [Playwright suite map](testing/playwright/README.md#choose-a-suite) to find browser, accessibility, memory, and infrastructure tests. Their specs remain in Playwright because they use its fixtures and projects. +Use the [Playwright suite map](testing/playwright/README.md#choose-a-suite) to find existing browser, accessibility, memory, and infrastructure suites. diff --git a/packages/quality/efficiency/.gitignore b/packages/quality/efficiency/.gitignore new file mode 100644 index 00000000000..5aeb1c02819 --- /dev/null +++ b/packages/quality/efficiency/.gitignore @@ -0,0 +1,2 @@ +playwright-report/ +results.xml diff --git a/packages/quality/efficiency/AGENTS.md b/packages/quality/efficiency/AGENTS.md index 8f67508ca3d..183df869628 100644 --- a/packages/quality/efficiency/AGENTS.md +++ b/packages/quality/efficiency/AGENTS.md @@ -6,3 +6,5 @@ Read the parent [quality instructions](../AGENTS.md) before you change a benchma - Do not change benchmark methods during a path migration. - Keep local measurements possible without an n8n Docker image rebuild. State when a benchmark needs an image. - Put task steps and result interpretation in the owning benchmark's README. +- Keep shared Playwright reporting and fixtures in `../testing/playwright/`. Import them through the package exports. +- Keep root checks separate from nested workspaces. diff --git a/packages/quality/efficiency/README.md b/packages/quality/efficiency/README.md index e16948783f9..0981c16bc6d 100644 --- a/packages/quality/efficiency/README.md +++ b/packages/quality/efficiency/README.md @@ -1,10 +1,16 @@ # Efficiency tooling -This area owns tools that measure speed and scale outside the Playwright runner. +This area owns resource measurements, microbenchmarks, and load-test tools. +The root `@n8n/efficiency` workspace uses Playwright with shared reporting from `n8n-playwright`. +Its configuration owns the efficiency projects and their scopes. Suites live in folders such as `images/`. | Tool | Use it to | Guide | | --- | --- | --- | +| `@n8n/efficiency` | Measure Docker image size and layer waste with Dive | [Image measurements](images/README.md) | | `@n8n/performance` | Compare code paths without building an n8n Docker image | [Microbenchmarks](microbenchmarks/README.md) | | `@n8n/n8n-benchmark` | Run load scenarios against an n8n instance | [Benchmark CLI](scale/benchmark/README.md) | For idle memory, canvas performance, throughput, and local soak tests, use the [Playwright suite map](../testing/playwright/README.md#choose-a-suite). These specs need Playwright fixtures, projects, metrics, or managed containers. + +Run image measurements with `pnpm --filter @n8n/efficiency test:images`. +The nested microbenchmark and benchmark CLI workspaces keep their own commands and configurations. diff --git a/packages/quality/efficiency/images/README.md b/packages/quality/efficiency/images/README.md new file mode 100644 index 00000000000..3180e153a1e --- /dev/null +++ b/packages/quality/efficiency/images/README.md @@ -0,0 +1,65 @@ +# Measure Docker image efficiency + +Use the `image-efficiency:*` Playwright projects to measure built images with Dive 0.13.1. +These projects use the existing artifact and metrics reporters. They do not start a browser or n8n instance. + +## Before you start + +- Install Dive 0.13.1 from its [release](https://github.com/wagoodman/dive/releases/tag/v0.13.1). +- Start Docker. +- Use existing local images, or build current images with `DOCKER_BUILD_DISTROLESS=true pnpm build:docker`. + Redirect build output to a log file. + +## Run the measurements + +1. Run all three image projects from the repository root: + + ```bash + pnpm --filter=@n8n/efficiency test:images + ``` + + To measure only n8n, select its project: + + ```bash + pnpm --filter=@n8n/efficiency exec playwright test --project=image-efficiency:n8n + ``` + + The defaults are `n8nio/n8n:local`, `n8nio/runners:local`, and `n8nio/runners:local-distroless`. + Override them with `TEST_IMAGE_N8N`, `TEST_IMAGE_RUNNERS`, and `TEST_IMAGE_RUNNERS_DISTROLESS`. + Use `DIVE_BINARY` if Dive is not on your `PATH`. + +2. Open the Playwright report: + + ```bash + pnpm --filter=@n8n/efficiency report + ``` + +3. Read `image-efficiency-summary` for layer sizes and the largest wasted files. + Use `dive-report` for the full file inventory. + Use `image-identity` to confirm the image ID, platform, and Dive version. + Open the command step to read stdout, stderr, and the exit status when analysis fails. + Image, platform, and Dive version annotations appear on the test result. + +4. Compare results for the same image type, platform, and Dive version. + The metrics reporter sends measurements when the `QA_METRICS_WEBHOOK_*` variables are configured. + The `platform` dimension comes from the image, such as `linux/arm64` or `linux/amd64`, not the host OS. + Docker images built on Apple Silicon usually use `linux/arm64`. + + - `docker-image-layer-size-` measures uncompressed layer content in MiB. + - `docker-image-wasted-size-` measures Dive's estimated wasted content in MiB. + - `docker-image-efficiency-` records Dive's efficiency score as a percentage. + - `docker-image-layer-count-` records the number of filesystem layers. + + Layer size is not registry download size or final filesystem size. + A high efficiency score can still describe a large image. + +## CI results + +Run `Test: Image Efficiency` through GitHub Actions with a manual dispatch. +The standalone workflow prepares or restores the cached n8n and runners images on amd64. +Run the distroless project locally when that image is available. +The image job uploads a Playwright report and test attachments. +Currents stores results and attachments when its record key is configured. + +Measurements have no size or waste budgets yet. Analysis errors and invalid reports fail the test. +Dive's JSON export bypasses its CI rules, so `.dive-ci` does not enforce budgets in these projects. diff --git a/packages/quality/efficiency/images/dive-results.ts b/packages/quality/efficiency/images/dive-results.ts new file mode 100644 index 00000000000..37fc16a67d8 --- /dev/null +++ b/packages/quality/efficiency/images/dive-results.ts @@ -0,0 +1,63 @@ +import { z } from 'zod'; + +export const DIVE_VERSION = '0.13.1'; + +export const diveReportSchema = z.object({ + layer: z + .array( + z.object({ + index: z.number().int().nonnegative(), + digestId: z.string(), + sizeBytes: z.number().int().nonnegative(), + command: z.string(), + }), + ) + .min(1), + image: z.object({ + sizeBytes: z.number().int().positive(), + inefficientBytes: z.number().int().nonnegative(), + efficiencyScore: z.number().min(0).max(1), + fileReference: z.array( + z.object({ + count: z.number().int().positive(), + sizeBytes: z.number().int().nonnegative(), + file: z.string(), + }), + ), + }), +}); + +export function summarizeDiveReport(report: z.infer) { + return { + sizeBytes: report.image.sizeBytes, + wastedBytes: report.image.inefficientBytes, + efficiencyPercent: report.image.efficiencyScore * 100, + layerCount: report.layer.length, + largestLayers: [...report.layer].sort((a, b) => b.sizeBytes - a.sizeBytes).slice(0, 10), + largestWastedFiles: [...report.image.fileReference] + .sort((a, b) => b.sizeBytes - a.sizeBytes) + .slice(0, 20), + }; +} + +export function formatDiveSummary( + image: string, + platform: string, + summary: ReturnType, +): string { + const mib = (bytes: number) => `${(bytes / 1024 / 1024).toFixed(2)} MiB`; + return [ + `Image: ${image}`, + `Platform: ${platform}`, + `Layer content: ${mib(summary.sizeBytes)}`, + `Estimated waste: ${mib(summary.wastedBytes)}`, + `Efficiency: ${summary.efficiencyPercent.toFixed(2)}%`, + `Filesystem layers: ${summary.layerCount}`, + '', + 'Largest layers:', + ...summary.largestLayers.map((layer) => `${mib(layer.sizeBytes)} ${layer.command}`), + '', + 'Largest wasted files:', + ...summary.largestWastedFiles.map((file) => `${mib(file.sizeBytes)} ${file.file}`), + ].join('\n'); +} diff --git a/packages/quality/efficiency/images/image-efficiency.spec.ts b/packages/quality/efficiency/images/image-efficiency.spec.ts new file mode 100644 index 00000000000..83ff9f69018 --- /dev/null +++ b/packages/quality/efficiency/images/image-efficiency.spec.ts @@ -0,0 +1,104 @@ +import { expect, test } from 'n8n-playwright/cli'; +import { attachMetric } from 'n8n-playwright/metrics'; +import { readFile } from 'node:fs/promises'; +import { z } from 'zod'; + +import { + DIVE_VERSION, + diveReportSchema, + formatDiveSummary, + summarizeDiveReport, +} from './dive-results'; + +const dive = process.env.DIVE_BINARY ?? 'dive'; +const imageMetadataSchema = z.object({ image: z.object({ name: z.string(), ref: z.string() }) }); +const imageIdentitySchema = z.object({ + Id: z.string().startsWith('sha256:'), + Os: z.string(), + Architecture: z.string(), +}); + +test( + 'image layer efficiency', + { annotation: [{ type: 'owner', description: 'Developer Platform' }] }, + async ({ cli }, testInfo) => { + const { image } = imageMetadataSchema.parse(testInfo.project.metadata); + const { stdout: version } = await cli.run(dive, ['--version'], { timeout: 10_000 }); + expect(version.trim(), 'Use the pinned Dive version for comparable measurements').toBe( + `dive ${DIVE_VERSION}`, + ); + + // Inspect first so a missing local image fails instead of pulling a different image. + const { stdout: identityJson } = await cli.run( + 'docker', + [ + 'image', + 'inspect', + image.ref, + '--format', + '{"Id":{{json .Id}},"Os":{{json .Os}},"Architecture":{{json .Architecture}}}', + ], + { timeout: 30_000 }, + ); + const identity = imageIdentitySchema.parse(JSON.parse(identityJson)); + const reportPath = testInfo.outputPath('dive.json'); + const platform = `${identity.Os}/${identity.Architecture}`; + testInfo.annotations.push( + { type: 'image', description: image.ref }, + { type: 'platform', description: platform }, + { type: 'tool', description: `Dive ${DIVE_VERSION}` }, + ); + + await testInfo.attach('image-identity', { + body: JSON.stringify({ ...image, ...identity, diveVersion: DIVE_VERSION }, null, 2), + contentType: 'application/json', + }); + + await cli.run(dive, [identity.Id, '--source', 'docker', '--json', reportPath], { + title: 'Analyze image layers with Dive', + timeout: 240_000, + }); + + await testInfo.attach('dive-report', { path: reportPath, contentType: 'application/json' }); + const report = diveReportSchema.parse(JSON.parse(await readFile(reportPath, 'utf8'))); + const summary = summarizeDiveReport(report); + await testInfo.attach('image-efficiency-summary', { + body: formatDiveSummary(image.ref, platform, summary), + contentType: 'text/plain', + }); + + const dimensions = { + image: image.name, + platform, + dive_version: DIVE_VERSION, + }; + await attachMetric( + testInfo, + `docker-image-layer-size-${image.name}`, + summary.sizeBytes / 1024 / 1024, + 'MiB', + dimensions, + ); + await attachMetric( + testInfo, + `docker-image-wasted-size-${image.name}`, + summary.wastedBytes / 1024 / 1024, + 'MiB', + dimensions, + ); + await attachMetric( + testInfo, + `docker-image-efficiency-${image.name}`, + summary.efficiencyPercent, + '%', + dimensions, + ); + await attachMetric( + testInfo, + `docker-image-layer-count-${image.name}`, + summary.layerCount, + 'count', + dimensions, + ); + }, +); diff --git a/packages/quality/efficiency/oxlint.config.mts b/packages/quality/efficiency/oxlint.config.mts new file mode 100644 index 00000000000..b50c92512db --- /dev/null +++ b/packages/quality/efficiency/oxlint.config.mts @@ -0,0 +1,7 @@ +import { backendConfig } from '@n8n/oxlint-config/backend'; +import { defineConfig } from 'oxlint'; + +export default defineConfig({ + extends: [backendConfig], + options: { typeAware: true }, +}); diff --git a/packages/quality/efficiency/package.json b/packages/quality/efficiency/package.json new file mode 100644 index 00000000000..d9f9cdf2e4b --- /dev/null +++ b/packages/quality/efficiency/package.json @@ -0,0 +1,25 @@ +{ + "name": "@n8n/efficiency", + "private": true, + "scripts": { + "test:all": "playwright test", + "test:images": "playwright test --project='image-efficiency:*'", + "report": "playwright show-report", + "lint": "oxlint --quiet playwright.config.ts images", + "typecheck": "tsc --noEmit", + "format:check": "biome ci playwright.config.ts images" + }, + "peerDependencies": { + "zod": "catalog:" + }, + "devDependencies": { + "@n8n/oxlint-config": "workspace:*", + "@playwright/test": "catalog:e2e", + "n8n-playwright": "workspace:*", + "oxlint": "catalog:", + "oxlint-tsgolint": "catalog:", + "typescript": "catalog:typescript", + "zod": "catalog:" + }, + "license": "LicenseRef-n8n-sustainable-use" +} diff --git a/packages/quality/efficiency/playwright.config.ts b/packages/quality/efficiency/playwright.config.ts new file mode 100644 index 00000000000..430ae20b921 --- /dev/null +++ b/packages/quality/efficiency/playwright.config.ts @@ -0,0 +1,24 @@ +/* eslint-disable import-x/no-default-export */ +import { defineQualityConfig } from 'n8n-playwright/config'; + +const images = [ + { name: 'n8n', ref: process.env.TEST_IMAGE_N8N ?? 'n8nio/n8n:local' }, + { name: 'runners', ref: process.env.TEST_IMAGE_RUNNERS ?? 'n8nio/runners:local' }, + { + name: 'runners-distroless', + ref: process.env.TEST_IMAGE_RUNNERS_DISTROLESS ?? 'n8nio/runners:local-distroless', + }, +]; + +export default defineQualityConfig({ + testMatch: '**/*.spec.ts', + workers: 1, + timeout: 300_000, + retries: 0, + use: { trace: 'off', video: 'off', screenshot: 'off' }, + projects: images.map((image) => ({ + name: `image-efficiency:${image.name}`, + testDir: './images', + metadata: { image }, + })), +}); diff --git a/packages/quality/efficiency/tsconfig.json b/packages/quality/efficiency/tsconfig.json new file mode 100644 index 00000000000..ae46d95a8d3 --- /dev/null +++ b/packages/quality/efficiency/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "sourceMap": false, + "declaration": false, + "lib": ["esnext", "dom"], + "types": ["@playwright/test", "node"] + }, + "include": ["playwright.config.ts", "images/**/*.ts"] +} diff --git a/packages/quality/testing/janitor/src/rules/valid-owner-annotation.rule.ts b/packages/quality/testing/janitor/src/rules/valid-owner-annotation.rule.ts index c33786e0741..bf6303f9efe 100644 --- a/packages/quality/testing/janitor/src/rules/valid-owner-annotation.rule.ts +++ b/packages/quality/testing/janitor/src/rules/valid-owner-annotation.rule.ts @@ -17,6 +17,7 @@ const CANONICAL_OWNERS = new Set([ 'Lifecycle & Governance', 'Docs', 'Catalysts', + 'Developer Platform', 'NODES', 'Payday', 'Adore', diff --git a/packages/quality/testing/playwright/AGENTS.md b/packages/quality/testing/playwright/AGENTS.md index f086a5a1596..baaf6d5ad5a 100644 --- a/packages/quality/testing/playwright/AGENTS.md +++ b/packages/quality/testing/playwright/AGENTS.md @@ -2,7 +2,10 @@ Read the [quality](../../AGENTS.md) and [testing](../AGENTS.md) instructions first. Use the [suite map](README.md#choose-a-suite) to choose a project. -## Place tests by runner +## Place suites and shared tooling + +- Efficiency suites can live in `../../efficiency/` and reuse the `n8n-playwright` exports. +- Keep shared reporting and lifecycle fixtures in this package. Do not import the product configuration from another workspace. - Keep product UI and API journeys in `tests/e2e/`. - Keep database, queue, multi-main, encryption, and process lifecycle tests in `tests/infrastructure/`. diff --git a/packages/quality/testing/playwright/README.md b/packages/quality/testing/playwright/README.md index 2392cf31767..91620ee783c 100644 --- a/packages/quality/testing/playwright/README.md +++ b/packages/quality/testing/playwright/README.md @@ -14,6 +14,7 @@ Commands in this table run from the repository root. Container-backed tests need | Test PostgreSQL, queue mode, multi-main, or process lifecycle | `:infrastructure` | `pnpm --filter n8n-playwright exec playwright test --project=postgres:infrastructure` | [Infrastructure tests](tests/infrastructure/) | | Check accessibility within a product journey | E2E project with the `a11y` fixture | `pnpm --filter n8n-playwright test:local` | [Accessibility checks](docs/ACCESSIBILITY.md) | | Measure idle memory, retention, or canvas performance | `performance` | `pnpm --filter n8n-playwright test:performance` | [Performance guide](tests/performance/README.md) | +| Measure Docker image size and layer waste | `@n8n/efficiency` image projects | `pnpm --filter @n8n/efficiency test:images` | [Image efficiency](../../efficiency/images/README.md) | | Measure infrastructure throughput and resource use | `benchmarking:infrastructure` | `pnpm --filter n8n-playwright test:benchmark` | [Infrastructure benchmarks](tests/infrastructure/benchmarks/README.md) | | Investigate Instance AI memory with a local-only, Docker-backed run | `benchmark-memory-instanceai:infrastructure` | `pnpm --filter n8n-playwright exec playwright test --project=benchmark-memory-instanceai:infrastructure` | [Local benchmark setup](tests/infrastructure/benchmarks-local/README.md) | | Test workflow execution and schemas | `cli-workflows` | `pnpm --filter n8n-playwright test:workflows` | [Workflow tests](tests/cli-workflows/README.md) | @@ -21,7 +22,46 @@ Commands in this table run from the repository root. Container-backed tests need | Check browser-backed harness contracts | Harness Vitest config | `pnpm --filter n8n-playwright test:harness` | [Test-writing patterns](docs/TESTING_PATTERNS.md) | | Run evaluation scenarios | `eval` | `pnpm --filter n8n-playwright test:evals` | [Evaluation tests](tests/evals/) | -The Playwright package owns the runner and its specs. Standalone measurement tools live in [efficiency](../../efficiency/README.md). Read [AGENTS.md](AGENTS.md) before you change a test. The examples below run from this package unless noted. +The Playwright package owns shared test tooling and product suites. Efficiency suites can reuse its exports from the [efficiency workspace](../../efficiency/README.md). Read [AGENTS.md](AGENTS.md) before you change a test. The examples below run from this package unless noted. + +## Shared setup exports + +Other private workspaces can declare `n8n-playwright` and the catalog-pinned `@playwright/test` as development dependencies. + +| Export | Provides | +| --- | --- | +| `n8n-playwright/config` | `defineQualityConfig()` and `qualityReporters()` for Currents, QA metrics, and standard reports | +| `n8n-playwright/test` | `test` and `expect` with the base Currents fixtures | +| `n8n-playwright/cli` | CLI fixture with command steps, output attachments, exit-code checks, and process cleanup | +| `n8n-playwright/metrics` | `attachMetric()` for QA metrics and Currents analytics | + +The configuration factory accepts standard Playwright options. It does not start a browser, application, or container stack. +Reporter paths resolve from this package. Test paths and report outputs resolve from the consuming workspace. +Supply `reporter` to replace the defaults, or append suite-specific reporters to `qualityReporters()`. +The existing container and product fixtures remain in `fixtures/`. + +### CLI fixture + +```typescript +import { test } from 'n8n-playwright/cli'; + +test('analyze an image', async ({ cli }, testInfo) => { + const report = testInfo.outputPath('analysis.json'); + await cli.run('dive', ['n8nio/n8n:local', '--json', report], { + title: 'Analyze image layers', + timeout: 240_000, + }); + await testInfo.attach('analysis', { path: report, contentType: 'application/json' }); +}); +``` + +`cli.run()` returns the Execa result. It expects exit code zero unless `expectedExitCode` is supplied. +Pass arguments as an array. Use `cwd`, `env`, and `timeout` for command options. +Values in `env` override inherited environment variables. +Commands run directly without a shell. Output keeps its trailing newlines. +Each command creates a native step with command details, stdout, and stderr attachments. +The fixture scrubs secrets from recorded text. It does not record environment variables. +Fixture teardown stops unfinished processes. Await each command before reading its output files. ## Development setup ```bash diff --git a/packages/quality/testing/playwright/fixtures/cli.ts b/packages/quality/testing/playwright/fixtures/cli.ts new file mode 100644 index 00000000000..16bec44fcd4 --- /dev/null +++ b/packages/quality/testing/playwright/fixtures/cli.ts @@ -0,0 +1,69 @@ +import { scrubSecretsInText } from '@n8n/utils/scrub-secrets'; +import type { Options, Result, Subprocess } from 'execa'; + +import { expect, test as base } from '../quality-test'; + +export { expect } from '../quality-test'; + +export type CliOptions = Pick & { + title?: string; + expectedExitCode?: number; +}; + +export interface Cli { + run(command: string, args?: string[], options?: CliOptions): Promise>; +} + +export const test = base.extend<{ cli: Cli }>({ + cli: async ({}, use) => { + const processes = new Set>(); + const run: Cli['run'] = async (command, args = [], options = {}) => { + const { title = command, expectedExitCode = 0, ...processOptions } = options; + return await test.step(scrubSecretsInText(title), async (step) => { + const { execa } = await import('execa'); + const child = execa(command, args, { + ...processOptions, + reject: false, + stripFinalNewline: false, + killDescendants: true, + }); + processes.add(child); + const result = await child.finally(() => processes.delete(child)); + + await step.attach('command', { + body: scrubSecretsInText( + JSON.stringify( + { + command, + args, + exitCode: result.exitCode, + signal: result.signal, + error: result.shortMessage, + }, + null, + 2, + ), + ), + contentType: 'application/json', + }); + await step.attach('command-stdout', { + body: scrubSecretsInText(result.stdout), + contentType: 'text/plain', + }); + await step.attach('command-stderr', { + body: scrubSecretsInText(result.stderr), + contentType: 'text/plain', + }); + expect(result.exitCode, `${command} exit code`).toBe(expectedExitCode); + return result; + }); + }; + + try { + await use({ run }); + } finally { + for (const child of processes) child.kill('SIGKILL'); + await Promise.allSettled(processes); + } + }, +}); diff --git a/packages/quality/testing/playwright/oxlint.config.mts b/packages/quality/testing/playwright/oxlint.config.mts index 9c1f4bb20f7..52e742b7d06 100644 --- a/packages/quality/testing/playwright/oxlint.config.mts +++ b/packages/quality/testing/playwright/oxlint.config.mts @@ -173,6 +173,7 @@ export default defineConfig({ { files: [ 'fixtures/langsmith.ts', + 'fixtures/cli.ts', 'fixtures/quarantine.ts', 'tests/cli-workflows/workflow-tests.spec.ts', 'tests/e2e/chat-hub/fixtures.ts', diff --git a/packages/quality/testing/playwright/package.json b/packages/quality/testing/playwright/package.json index 1d1637926ab..7cc5f2b0b95 100644 --- a/packages/quality/testing/playwright/package.json +++ b/packages/quality/testing/playwright/package.json @@ -1,6 +1,12 @@ { "name": "n8n-playwright", "private": true, + "exports": { + "./config": "./quality-config.ts", + "./test": "./quality-test.ts", + "./cli": "./fixtures/cli.ts", + "./metrics": "./utils/metrics.ts" + }, "scripts": { "dev": "N8N_BASE_URL=http://localhost:5678 N8N_EDITOR_URL=http://localhost:8080 RESET_E2E_DB=true playwright test --project=e2e", "test:dev-server-smoke": "N8N_BASE_URL=http://localhost:5678 N8N_EDITOR_URL=http://localhost:8080 RESET_E2E_DB=true playwright test --project=dev-server-smoke", @@ -83,6 +89,7 @@ "axe-html-reporter": "^2.2.11", "debug": "^4.3.4", "eslint-plugin-playwright": "catalog:e2e", + "execa": "10.1.0", "flatted": "catalog:", "generate-schema": "2.6.0", "jsonwebtoken": "catalog:", diff --git a/packages/quality/testing/playwright/playwright.config.ts b/packages/quality/testing/playwright/playwright.config.ts index 59dfd35ef8b..e694fc0b500 100644 --- a/packages/quality/testing/playwright/playwright.config.ts +++ b/packages/quality/testing/playwright/playwright.config.ts @@ -1,13 +1,10 @@ /* eslint-disable import-x/no-default-export */ -import type { CurrentsFixtures, CurrentsWorkerFixtures } from '@currents/playwright'; -import { currentsReporter } from '@currents/playwright'; -import { defineConfig } from '@playwright/test'; import type { PlaywrightTestConfig } from '@playwright/test'; import os from 'os'; import path from 'path'; -import currentsConfig from './currents.config'; import { getProjects } from './playwright-projects'; +import { defineQualityConfig, qualityReporters } from './quality-config'; import { getBackendUrl, getFrontendUrl, getPortFromUrl, getReadinessUrl } from './utils/url-helper'; const IS_CI = !!process.env.CI; @@ -115,7 +112,7 @@ if (IS_DEV && FRONTEND_URL) { }); } -export default defineConfig({ +export default defineQualityConfig({ globalSetup: './global-setup.ts', globalTeardown: IS_DEV ? './global-teardown.ts' : undefined, forbidOnly: IS_CI, @@ -143,28 +140,9 @@ export default defineConfig({ currentsFixturesEnabled: !!process.env.CI, }, - reporter: IS_CI - ? [ - ['list'], - ['junit', { outputFile: process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME ?? 'results.xml' }], - ['html', { open: 'never' }], - ['json', { outputFile: 'test-results.json' }], - ...(process.env.CURRENTS_RECORD_KEY ? [currentsReporter(currentsConfig)] : []), - ['./reporters/metrics-reporter.ts'], - ['./reporters/benchmark-summary-reporter.ts'], - ...(process.env.PLAYWRIGHT_A11Y_REPORT - ? ([['./reporters/a11y-reporter.ts']] as const) - : []), - ...(process.env.LANGSMITH_API_KEY ? ([['./reporters/langsmith-eval.ts']] as const) : []), - ] - : [ - ['html'], - ['./reporters/metrics-reporter.ts'], - ['./reporters/benchmark-summary-reporter.ts'], - ...(process.env.PLAYWRIGHT_A11Y_REPORT - ? ([['./reporters/a11y-reporter.ts']] as const) - : []), - ['list'], - ...(process.env.LANGSMITH_API_KEY ? ([['./reporters/langsmith-eval.ts']] as const) : []), - ], + reporter: [ + ...qualityReporters(), + ...(process.env.PLAYWRIGHT_A11Y_REPORT ? ([['./reporters/a11y-reporter.ts']] as const) : []), + ...(process.env.LANGSMITH_API_KEY ? ([['./reporters/langsmith-eval.ts']] as const) : []), + ], }); diff --git a/packages/quality/testing/playwright/quality-config.ts b/packages/quality/testing/playwright/quality-config.ts new file mode 100644 index 00000000000..8109e32e1d4 --- /dev/null +++ b/packages/quality/testing/playwright/quality-config.ts @@ -0,0 +1,52 @@ +import type { CurrentsFixtures, CurrentsWorkerFixtures } from '@currents/playwright'; +import { currentsReporter } from '@currents/playwright'; +import { defineConfig } from '@playwright/test'; +import type { PlaywrightTestConfig, ReporterDescription } from '@playwright/test'; +import { resolve } from 'node:path'; + +import currentsConfig from './currents.config'; + +export function qualityReporters(): ReporterDescription[] { + const shared: ReporterDescription[] = [ + [resolve(__dirname, 'reporters/metrics-reporter.ts')], + [resolve(__dirname, 'reporters/benchmark-summary-reporter.ts')], + ]; + + if (!process.env.CI) { + return [['html'], ...shared, ['list']]; + } + + return [ + ['list'], + ['junit', { outputFile: process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME ?? 'results.xml' }], + ['html', { open: 'never' }], + ['json', { outputFile: 'test-results.json' }], + ...(process.env.CURRENTS_RECORD_KEY ? [currentsReporter(currentsConfig)] : []), + ...shared, + ]; +} + +/** Share reporting without starting a browser, application, or container stack. */ +export function defineQualityConfig< + TestFixtures extends object = object, + WorkerFixtures extends object = object, +>( + config: PlaywrightTestConfig< + TestFixtures & CurrentsFixtures, + WorkerFixtures & CurrentsWorkerFixtures + >, +) { + return defineConfig({ + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + ...config, + reporter: config.reporter ?? qualityReporters(), + use: Object.assign( + { + currentsConfigOptions: currentsConfig, + currentsFixturesEnabled: !!process.env.CI && !!process.env.CURRENTS_RECORD_KEY, + }, + config.use, + ), + }); +} diff --git a/packages/quality/testing/playwright/quality-test.ts b/packages/quality/testing/playwright/quality-test.ts new file mode 100644 index 00000000000..68d86255321 --- /dev/null +++ b/packages/quality/testing/playwright/quality-test.ts @@ -0,0 +1,7 @@ +import type { CurrentsFixtures, CurrentsWorkerFixtures } from '@currents/playwright'; +import { fixtures } from '@currents/playwright'; +import { test as base } from '@playwright/test'; + +export { expect } from '@playwright/test'; + +export const test = base.extend(fixtures.baseFixtures); diff --git a/packages/quality/testing/playwright/reporters/metrics-reporter.ts b/packages/quality/testing/playwright/reporters/metrics-reporter.ts index 4a16935090f..563938bfef6 100644 --- a/packages/quality/testing/playwright/reporters/metrics-reporter.ts +++ b/packages/quality/testing/playwright/reporters/metrics-reporter.ts @@ -30,7 +30,7 @@ const LOG_PREFIX = '[MetricsReporter]'; * If your test contains a testInfo.attach() call with a name starting with 'metric:', the metric * will be collected and sent as a single batched payload at the end of the run. * - * See utils/performance-helper.ts for the attachMetric() helper. + * See utils/metrics.ts for the attachMetric() helper. */ class MetricsReporter implements Reporter { private readonly webhook: CiMetricsWebhook; diff --git a/packages/quality/testing/playwright/utils/metrics.ts b/packages/quality/testing/playwright/utils/metrics.ts new file mode 100644 index 00000000000..43b6c91f853 --- /dev/null +++ b/packages/quality/testing/playwright/utils/metrics.ts @@ -0,0 +1,24 @@ +import type { TestInfo } from '@playwright/test'; + +/** Attach a metric for the QA metrics reporter and Currents analytics. */ +export async function attachMetric( + testInfo: TestInfo, + metricName: string, + value: number, + unit?: string, + dimensions?: Record, +): Promise { + await testInfo.attach(`metric:${metricName}`, { + body: JSON.stringify({ value, unit, dimensions }), + }); + + testInfo.annotations.push({ + type: 'currents:metric', + description: JSON.stringify({ + name: metricName, + value, + type: Number.isInteger(value) ? 'integer' : 'float', + ...(unit && { unit: unit.toLowerCase() }), + }), + }); +} diff --git a/packages/quality/testing/playwright/utils/performance-helper.ts b/packages/quality/testing/playwright/utils/performance-helper.ts index 6e1ac87728e..09aee94070c 100644 --- a/packages/quality/testing/playwright/utils/performance-helper.ts +++ b/packages/quality/testing/playwright/utils/performance-helper.ts @@ -6,6 +6,10 @@ import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import type { ReadableStream as NodeReadableStream } from 'node:stream/web'; +import { attachMetric } from './metrics'; + +export { attachMetric }; + interface HeapSnapshotResult { success: boolean; filePath?: string; @@ -45,30 +49,6 @@ export async function getAllPerformanceMetrics(page: Page) { }); } -/** Attach a performance metric for collection by the metrics reporter */ -export async function attachMetric( - testInfo: TestInfo, - metricName: string, - value: number, - unit?: string, - dimensions?: Record, -): Promise { - await testInfo.attach(`metric:${metricName}`, { - body: JSON.stringify({ value, unit, dimensions }), - }); - - // Currents native format — surfaces metrics in their analytics dashboard - testInfo.annotations.push({ - type: 'currents:metric', - description: JSON.stringify({ - name: metricName, - value, - type: Number.isInteger(value) ? 'integer' : 'float', - ...(unit && { unit: unit.toLowerCase() }), - }), - }); -} - export interface StableHeapOptions { maxWaitMs?: number; checkIntervalMs?: number; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 05f2cb90b96..dc411e826c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8086,6 +8086,30 @@ importers: specifier: 'catalog:' version: 3.1.0(typescript@7.0.2)(vitest@5.0.1) + packages/quality/efficiency: + devDependencies: + '@n8n/oxlint-config': + specifier: workspace:* + version: link:../../@n8n/oxlint-config + '@playwright/test': + specifier: catalog:e2e + version: 1.62.1 + n8n-playwright: + specifier: workspace:* + version: link:../testing/playwright + oxlint: + specifier: 'catalog:' + version: 1.78.0(oxlint-tsgolint@0.21.1) + oxlint-tsgolint: + specifier: 'catalog:' + version: 0.21.1 + typescript: + specifier: catalog:typescript + version: 7.0.2 + zod: + specifier: 3.25.76 + version: 3.25.76 + packages/quality/efficiency/microbenchmarks: devDependencies: '@n8n/expression-runtime': @@ -8408,6 +8432,9 @@ importers: eslint-plugin-playwright: specifier: catalog:e2e version: 2.2.2(eslint@9.29.0(jiti@2.6.1)(supports-color@8.1.1)) + execa: + specifier: 10.1.0 + version: 10.1.0 flatted: specifier: 3.4.2 version: 3.4.2 @@ -18387,6 +18414,10 @@ packages: resolution: {integrity: sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==} engines: {node: '>=6'} + execa@10.1.0: + resolution: {integrity: sha512-5HT/600HqXEtmkb9mK/PkTWw+bQSCzPnWiZ4/87wD3Hi0wFzCapOuNT9p4M08PDd+KLHdHvFhKjOWmNx/npRNQ==} + engines: {node: '>=22'} + execa@5.1.1: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} @@ -22195,6 +22226,10 @@ packages: resolution: {integrity: sha512-4yf0QO/sllf/1zbZWYnvWw3NxCQwLXKzIj0G849LSufP15BXKM0rbD2Z3wVnkMfjdn/CB0Dpp444gYAACdsplg==} engines: {node: '>=18'} + pretty-ms@9.3.1: + resolution: {integrity: sha512-HzMy3Geq23nVALD/M2LliU+F+M+gVNsvkQWWqeBZ8HDiCgzo6YPJ/Omrmtq24EFrIsk0a3EkQGEd7bDOo+IhGA==} + engines: {node: '>=18'} + prismjs@1.29.0: resolution: {integrity: sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==} engines: {node: '>=6'} @@ -24854,6 +24889,11 @@ packages: resolution: {integrity: sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw==} engines: {node: '>= 0.4'} + which-command@0.1.0: + resolution: {integrity: sha512-XZyoF5/5hZtXitIwzrU4NKK+Wtbb9aB9CezUEw2Q0wlYK8NUYQxC1rRXgNueYLtBAJwXIb+/tFVk4dozciNJMA==} + engines: {node: '>=22'} + hasBin: true + which-typed-array@1.1.19: resolution: {integrity: sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==} engines: {node: '>= 0.4'} @@ -25117,6 +25157,10 @@ packages: resolution: {integrity: sha512-GQHQqAopRhwU8Kt1DDM8NjibDXHC8eoh1erhGAJPEyveY9qqVeXvVikNKrDz69sHowPMorbPUrH/mx8c50eiBQ==} engines: {node: '>=18'} + yoctocolors@2.2.0: + resolution: {integrity: sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==} + engines: {node: '>=18'} + yuku-ast@0.1.7: resolution: {integrity: sha512-2RiMEWv500TixY5rJy6OZd4fSy9WYZKWh6gGbIJ7y7vAGcuCugWOWwOLGaQcRZrXcPUfqtLtvpaJ3SdXtWlhKA==} @@ -37231,6 +37275,21 @@ snapshots: signal-exit: 3.0.7 strip-eof: 1.0.0 + execa@10.1.0: + dependencies: + '@sindresorhus/merge-streams': 4.0.0 + figures: 6.1.0 + get-stream: 9.0.1 + human-signals: 8.0.1 + is-plain-obj: 4.1.0 + is-stream: 4.0.1 + npm-run-path: 6.0.0 + pretty-ms: 9.3.1 + signal-exit: 4.1.0 + strip-final-newline: 4.0.0 + which-command: 0.1.0 + yoctocolors: 2.2.0 + execa@5.1.1: dependencies: cross-spawn: 7.0.6 @@ -41813,6 +41872,10 @@ snapshots: dependencies: parse-ms: 4.0.0 + pretty-ms@9.3.1: + dependencies: + parse-ms: 4.0.0 + prismjs@1.29.0: {} proc-log@6.1.0: {} @@ -45232,6 +45295,8 @@ snapshots: is-weakmap: 2.0.2 is-weakset: 2.0.4 + which-command@0.1.0: {} + which-typed-array@1.1.19: dependencies: available-typed-arrays: 1.0.7 @@ -45499,6 +45564,8 @@ snapshots: yoctocolors@2.1.1: {} + yoctocolors@2.2.0: {} + yuku-ast@0.1.7: dependencies: '@yuku-toolchain/types': 0.5.43