diff --git a/.github/workflows/encryption-upgrade-test.yml b/.github/workflows/encryption-upgrade-test.yml new file mode 100644 index 00000000000..6d47fe44978 --- /dev/null +++ b/.github/workflows/encryption-upgrade-test.yml @@ -0,0 +1,152 @@ +name: 'Test: Encryption Rollout' + +on: + workflow_dispatch: + inputs: + test: + description: 'Which test to run' + type: choice + options: + - upgrade + - rotation + default: upgrade + db: + description: 'Database backends to run' + type: choice + options: + - both + - sqlite + - postgres + default: both + from-image: + description: 'Old release image to seed on (upgrade only; empty = pinned script default)' + type: string + default: '' + schedule: + # Twice a day, 09:00 and 21:00 UTC. + - cron: '0 9,21 * * *' + # PR events use the workflow file from the branch, so changes to the tests + # verify themselves (both modes) before the merge. + pull_request: + paths: + - 'packages/testing/playwright/tests/infrastructure/encryption/**' + - 'packages/testing/containers/services/n8n.ts' + - 'packages/testing/containers/services/types.ts' + - 'packages/testing/containers/stack.ts' + - '.github/workflows/encryption-upgrade-test.yml' + +permissions: + contents: read + +# A new push to the same PR cancels the runs of the previous one. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + encryption-cycle: + name: ${{ matrix.mode }} cycle (${{ matrix.db }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # PRs run both modes; a dispatch runs the selected one; the cron runs + # the full upgrade cycle. Each database backend is its own job. + mode: ${{ fromJSON(github.event_name == 'pull_request' && '["upgrade","rotation"]' || format('["{0}"]', inputs.test || 'upgrade')) }} + db: ${{ fromJSON((inputs.db == 'sqlite' || inputs.db == 'postgres') && format('["{0}"]', inputs.db) || '["sqlite","postgres"]') }} + env: + MODE: ${{ matrix.mode }} + DB: ${{ matrix.db }} + # Not a hidden dir: upload-artifact skips hidden paths by default. + WORK_ROOT: ${{ github.workspace }}/upgrade-test-work + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # The image under test is the docker build of this checkout + # (n8nio/n8n:local) — the same artifact a release ships. + - name: Setup and build the image under test + uses: ./.github/actions/setup-nodejs + with: + build-command: pnpm build:docker + + - name: Run the test cycle + env: + FROM_IMAGE_INPUT: ${{ inputs.from-image }} + ENCRYPTION_CYCLE_REQUIRED: 'true' + run: | + mkdir -p "$WORK_ROOT" + if [ -n "$FROM_IMAGE_INPUT" ]; then export FROM_IMAGE="$FROM_IMAGE_INPUT"; fi + set -o pipefail + pnpm --filter=n8n-playwright test:encryption:"$MODE" 2>&1 | tee "$WORK_ROOT/run.log" + echo "PASS: ${MODE}-cycle — all requested backends green ($DB)" | tee -a "$WORK_ROOT/run.log" + + - name: Write run summary + if: always() + run: | + LOG="$WORK_ROOT/run.log" + [ -f "$LOG" ] || { echo "## Encryption test ($MODE, $DB)" >> "$GITHUB_STEP_SUMMARY"; echo "No run log was produced." >> "$GITHUB_STEP_SUMMARY"; exit 0; } + { + echo "## Encryption test ($MODE, $DB)" + echo + if grep -q "all requested backends green" "$LOG"; then + echo "**Result: PASS** :white_check_mark:" + else + echo "**Result: FAIL** :x:" + fi + echo + echo '```' + grep -E "PASS \[|PASS:|FAIL:" "$LOG" || true + echo '```' + echo + echo '```' + # Each metrics block ends at the first blank line (it may carry a + # "rotate api:" line after the totals row). + awk '/=== metrics \[/{f=1} f && /^$/{f=0; print ""} f{print}' "$LOG" + echo '```' + if ! grep -q "all requested backends green" "$LOG"; then + echo + echo '
Last 80 log lines' + echo + echo '```' + tail -n 80 "$LOG" + echo '```' + echo '
' + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload run artifacts + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: encryption-test-${{ matrix.mode }}-${{ matrix.db }}-${{ github.run_id }} + retention-days: 14 + if-no-files-found: warn + path: | + ${{ github.workspace }}/upgrade-test-work/run.log + ${{ github.workspace }}/upgrade-test-work/*/metrics.csv + ${{ github.workspace }}/upgrade-test-work/*/n8n.log + + # A scheduled run fails silently otherwise; the job-level gate is required + # because a step-level `if: failure()` never fires on a needs-failure. + notify-on-failure: + name: Notify on scheduled failure + runs-on: ubuntu-slim + needs: [encryption-cycle] + if: ${{ always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: .github/scripts/slack + + - name: Send Slack notification + env: + SLACK_TOKEN: ${{ secrets.QBOT_SLACK_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + node .github/scripts/slack/notify.mjs \ + --channel '#alerts-build' \ + --text "<${RUN_URL}|Scheduled encryption rollout test failed>" diff --git a/package.json b/package.json index ef33528b0be..31121bafa35 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,8 @@ "test:ci:backend:integration": "turbo run test:integration --continue --concurrency=1 --filter='!./packages/frontend/**' --filter='!./packages/modules/*/frontend'", "test:ci:backend:integration:changed": "turbo run test:integration:changed --continue --concurrency=1 --filter='!./packages/frontend/**' --filter='!./packages/modules/*/frontend'", "test:affected": "turbo run test --affected --concurrency=1", + "test:encryption:upgrade": "pnpm --filter=n8n-playwright test:encryption:upgrade", + "test:encryption:rotation": "pnpm --filter=n8n-playwright test:encryption:rotation", "db:schema:docs": "turbo run build --filter=@n8n/db && pnpm --filter=@n8n/db schema:docs", "db:schema:check": "turbo run build --filter=@n8n/db && pnpm --filter=@n8n/db schema:check", "db:schema:check:sqlite": "turbo run build --filter=@n8n/db && pnpm --filter=@n8n/db schema:check:sqlite", diff --git a/packages/testing/containers/services/n8n.ts b/packages/testing/containers/services/n8n.ts index 5b02e9bab2a..1005a5a9852 100644 --- a/packages/testing/containers/services/n8n.ts +++ b/packages/testing/containers/services/n8n.ts @@ -83,6 +83,23 @@ export interface N8NInstancesOptions { filesToMount?: FileToMount[]; coverageHostDir?: string; registerContainer?: (container: StartedTestContainer) => void; + /** + * Override the n8n image for these instances (default: the process-wide + * TEST_IMAGE_N8N resolution). Lets one process boot different releases in + * sequence — the upgrade/downgrade cycles swap images over the same data. + */ + image?: string; + /** + * Host dir bind-mounted as the container's home (`/home/node`), so the + * user folder (settings file, sqlite database) outlives the container and + * a different image can boot on the same data. Disables container reuse. + * Pair with `user` so the files stay owned by the host user. + */ + userHomeHostDir?: string; + /** Run the container as this uid:gid (e.g. the host user for bind mounts). */ + user?: string; + /** Readiness timeout override; an old release migrating a fresh DB can exceed the default. */ + startupTimeoutMs?: number; } export interface N8NInstancesResult { @@ -154,6 +171,10 @@ interface SharedConfig { filesToMount?: FileToMount[]; coverageHostDir?: string; registerContainer?: (container: StartedTestContainer) => void; + image?: string; + userHomeHostDir?: string; + user?: string; + startupTimeoutMs?: number; } interface ContainerStartResult { @@ -182,30 +203,53 @@ async function createContainer( filesToMount, coverageHostDir, registerContainer, + image, + userHomeHostDir, + user, + startupTimeoutMs, } = shared; const { consumer, throwWithLogs, getLogs } = createSilentLogConsumer(); const { strategy: waitStrategy, getLastBody: getLastReadinessBody } = createReadinessProbe( '/healthz/readiness', N8N_READINESS_PORT, - { startupTimeoutMs: N8N_STARTUP_TIMEOUT_MS, readTimeoutMs: N8N_READ_TIMEOUT_MS }, + { + startupTimeoutMs: startupTimeoutMs ?? N8N_STARTUP_TIMEOUT_MS, + readTimeoutMs: N8N_READ_TIMEOUT_MS, + }, ); const containerEnvironment = coverageHostDir ? { ...environment, NODE_V8_COVERAGE: CONTAINER_COVERAGE_DIR } : environment; - let container = new GenericContainer(N8N_IMAGE) + const containerImage = image ?? N8N_IMAGE; + let container = new GenericContainer(containerImage) .withEnvironment(containerEnvironment) .withLabels({ 'com.docker.compose.project': projectName, 'com.docker.compose.service': SERVICE_LABEL[role], instance: instanceNumber.toString(), }) - .withPullPolicy(new N8nImagePullPolicy(N8N_IMAGE)) + .withPullPolicy(new N8nImagePullPolicy(containerImage)) .withName(name) .withLogConsumer(consumer) .withNetwork(network); + if (user) { + container = container.withUser(user); + } + + // withBindMounts REPLACES the mount list, so all mounts go in one call. + const bindMounts: Array<{ source: string; target: string; mode: 'rw' }> = []; + + if (userHomeHostDir) { + // The whole home is mounted (not just ~/.n8n) so the settings file and + // the sqlite database live on the host and a different image can boot + // on the same data later. + mkdirSync(userHomeHostDir, { recursive: true }); + bindMounts.push({ source: userHomeHostDir, target: '/home/node', mode: 'rw' }); + } + if (coverageHostDir) { // Per-container host dir → /cov; n8n flushes V8 here on graceful stop. // Reuse must stay off so the process actually exits and flushes. @@ -215,10 +259,16 @@ async function createContainer( // is direct (no Docker Desktop uid mapping), so make the dir writable by // the container or NODE_V8_COVERAGE silently fails to flush. chmodSync(hostCoverageDir, 0o777); - container = container.withBindMounts([ - { source: hostCoverageDir, target: CONTAINER_COVERAGE_DIR, mode: 'rw' }, - ]); - } else { + bindMounts.push({ source: hostCoverageDir, target: CONTAINER_COVERAGE_DIR, mode: 'rw' }); + } + + if (bindMounts.length > 0) { + container = container.withBindMounts(bindMounts); + } + + // Reuse stays off for a coverage or mounted-home container: the process + // must exit to flush coverage, and cycle data belongs to the cycle. + if (!coverageHostDir && !userHomeHostDir) { container = container.withReuse(); } @@ -285,6 +335,10 @@ export async function createN8NInstances( filesToMount, coverageHostDir, registerContainer, + image, + userHomeHostDir, + user, + startupTimeoutMs, } = options; const log = createElapsedLogger('n8n-instances'); @@ -300,6 +354,10 @@ export async function createN8NInstances( filesToMount, coverageHostDir, registerContainer, + image, + userHomeHostDir, + user, + startupTimeoutMs, }; const workerShared: SharedConfig = { @@ -310,6 +368,9 @@ export async function createN8NInstances( filesToMount, coverageHostDir, registerContainer, + image, + user, + startupTimeoutMs, }; const webhookShared: SharedConfig = { @@ -319,6 +380,9 @@ export async function createN8NInstances( resourceQuota: webhookResourceQuota ?? resourceQuota, filesToMount, registerContainer, + image, + user, + startupTimeoutMs, }; const sharedByRole: Record = { diff --git a/packages/testing/containers/services/types.ts b/packages/testing/containers/services/types.ts index c4e36c715d9..a41245cf04f 100644 --- a/packages/testing/containers/services/types.ts +++ b/packages/testing/containers/services/types.ts @@ -100,6 +100,23 @@ export interface StackConfig { * Opt-in capability for the coverage pipeline; off by default. */ coverageHostDir?: string; + /** + * Override the n8n image for this stack (default: the process-wide + * TEST_IMAGE_N8N resolution). `stack.replaceN8N()` can then swap to a + * different image on the same data — the upgrade/downgrade cycles. + */ + image?: string; + /** + * Host dir bind-mounted as the n8n container's home (`/home/node`), so the + * user folder (settings file, sqlite database) outlives the container and + * `replaceN8N()` can boot another image on the same data. Single-main + * stacks only. Pair with `user` so the files stay owned by the host user. + */ + userHomeHostDir?: string; + /** Run the n8n containers as this uid:gid (e.g. the host user for bind mounts). */ + user?: string; + /** n8n readiness timeout override; an old release migrating a fresh DB can exceed the default. */ + startupTimeoutMs?: number; } export interface Service { diff --git a/packages/testing/containers/stack.ts b/packages/testing/containers/stack.ts index 13bce26c0c2..b413a58d92a 100644 --- a/packages/testing/containers/stack.ts +++ b/packages/testing/containers/stack.ts @@ -36,6 +36,13 @@ const SERVICE_REGISTRY: Record = services; export type N8NConfig = StackConfig; +export interface ReplaceN8NOptions { + /** The image the replacement main boots. */ + image: string; + /** Merged over the stack's original user env (e.g. a feature flag flip). */ + env?: Record; +} + export interface N8NStack { attemptId: string; baseUrl: string; @@ -53,6 +60,15 @@ export interface N8NStack { metrics: ServiceHelpers['observability']['metrics']; findContainers: (namePattern: string | RegExp) => StartedTestContainer[]; stopContainer: (namePattern: string | RegExp) => Promise; + /** + * Stops the current n8n main and boots a replacement with the requested + * image, keeping the service containers, network, host port, and + * readiness/logging/cleanup behavior. Data survives the swap when it lives + * outside the replaced container: in the postgres service, or in a + * `userHomeHostDir` mount (the sqlite file and user folder). Single-main + * stacks only — the substrate of the upgrade/downgrade cycles. + */ + replaceN8N: (options: ReplaceN8NOptions) => Promise; /** Direct URLs to each main instance (bypasses load balancer). Index 0 = main-1, etc. */ mainUrls: string[]; /** @@ -112,8 +128,16 @@ export async function createN8NStack(config: N8NConfig = {}): Promise external = false, networkName, coverageHostDir, + image, + userHomeHostDir, + user, + startupTimeoutMs, } = config; + if (userHomeHostDir && (mains > 1 || workers > 0 || webhooks > 0)) { + throw new Error('userHomeHostDir supports single-main stacks only (one shared home)'); + } + const log = createElapsedLogger('stack'); const isQueueMode = mains > 1 || workers > 0 || webhooks > 0; @@ -308,6 +332,10 @@ export async function createN8NStack(config: N8NConfig = {}): Promise resources.trackContainer(container); if (!containers.includes(container)) containers.push(container); }, + image, + userHomeHostDir, + user, + startupTimeoutMs, }); telemetry.finishStage(); } catch (error) { @@ -481,6 +509,44 @@ export async function createN8NStack(config: N8NConfig = {}): Promise const container = containers.find((c) => regex.test(c.getName())); return container ? await container.stop() : null; }, + async replaceN8N(options: ReplaceN8NOptions): Promise { + if (mains > 1 || needsLoadBalancer) { + throw new Error('replaceN8N supports single-main stacks only'); + } + const current = containers.find((c) => c.getName().endsWith('-n8n')); + if (current) { + await current.stop(); + containers.splice(containers.indexOf(current), 1); + } + const endAcquisition = resources.beginAcquisition(); + try { + await createN8NInstances({ + mains: 1, + workers: 0, + projectName: uniqueProjectName, + network, + serviceEnvironment: environment, + userEnvironment: { ...env, ...options.env }, + usePostgres, + baseUrl, + // The same host port keeps `baseUrl` valid across the swap. + allocatedPort: allocatedMainPort, + resourceQuota, + filesToMount, + coverageHostDir, + registerContainer: (container) => { + resources.trackContainer(container); + if (!containers.includes(container)) containers.push(container); + }, + image: options.image, + userHomeHostDir, + user, + startupTimeoutMs, + }); + } finally { + endAcquisition(); + } + }, mainUrls, internalMainUrls, startupDiagnostics: n8nResult.diagnostics, diff --git a/packages/testing/playwright/package.json b/packages/testing/playwright/package.json index 4551e382985..e37da6bbc67 100644 --- a/packages/testing/playwright/package.json +++ b/packages/testing/playwright/package.json @@ -16,6 +16,8 @@ "bench:canvas:sentinels": "node scripts/canvas-perf-sentinels.mjs test-results.json", "bench:canvas:report": "playwright show-report", "test:infrastructure": "playwright test --project='*:infrastructure'", + "test:encryption:upgrade": "playwright test --project='encryption:infrastructure' tests/infrastructure/encryption/upgrade-cycle.spec.ts", + "test:encryption:rotation": "playwright test --project='encryption:infrastructure' tests/infrastructure/encryption/rotation-cycle.spec.ts", "test:benchmark": "playwright test --project='benchmarking:infrastructure'", "benchmark:sizing-matrix": "tsx scripts/sizing-matrix-aggregate.ts", "test:container:sqlite": "playwright test --project='sqlite:*'", diff --git a/packages/testing/playwright/playwright-projects.ts b/packages/testing/playwright/playwright-projects.ts index b3711382f17..5105396dc17 100644 --- a/packages/testing/playwright/playwright-projects.ts +++ b/packages/testing/playwright/playwright-projects.ts @@ -263,6 +263,20 @@ export function getProjects(): Project[] { use: { containerConfig: BENCHMARKING_DEFAULT_CONFIG }, }); + // API-only, no browser: the specs manage their own stack (they swap n8n + // images mid-test via stack.replaceN8N), so no containerConfig fixture. + projects.push({ + name: 'encryption:infrastructure', + testDir: './tests/infrastructure/encryption', + workers: 1, + // One upgrade cycle boots four instances and pulls the old release. + timeout: 900_000, + retries: 0, + // No browser runs here — the global trace/video/screenshot capture + // only produces empty artifacts for these tests. + use: { trace: 'off', video: 'off', screenshot: 'off' }, + }); + for (const { name, config } of LOCAL_ONLY_BENCHMARK_PROFILES) { projects.push({ name: `benchmark-${name}:infrastructure`, diff --git a/packages/testing/playwright/tests/infrastructure/encryption/README.md b/packages/testing/playwright/tests/infrastructure/encryption/README.md new file mode 100644 index 00000000000..a736d3452f5 --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/README.md @@ -0,0 +1,123 @@ +# Encryption-cycle tests + +Repeatable end-to-end tests for the encryption-key rollout: an API-only +Playwright suite (project `encryption:infrastructure`, no browser) on the +`n8n-containers` Testcontainers stack. Both modes run on both supported +databases: sqlite and postgres. + +```bash +pnpm build:docker # builds n8nio/n8n:local — the image under test +pnpm test:encryption:upgrade # full upgrade cycle (old release + the local image) +pnpm test:encryption:rotation # standalone rotation test (the local image only) +``` + +## Layout + +| File | Responsibility | +|---|---| +| `upgrade-cycle.spec.ts` / `rotation-cycle.spec.ts` | one test per backend; phases render as `test.step()` | +| `spec-helpers.ts` | env parameters, docker/image skip gate, the loud FAIL path, attachments | +| `harness.ts` | phase/step logging, metrics + summary, failure type | +| `instances.ts` | stack boot and per-phase image swaps; log capture | +| `api.ts` | REST client and the encryption assertions (`assertDecrypts`, `rotateKey`, …) | +| `db.ts` | raw DB access: `node:sqlite` on the mounted file / `psql` in the postgres container | +| `journeys.ts` | seed + per-boot checks: an active scheduled workflow bound to the credential | +| `cycles.ts` | the two scenarios: `runUpgradeCycle` (P1–P4), `runRotationCycle` (R1–R3) | + +The stack capabilities this test leans on: `createN8NStack` accepts an +`image`, a `userHomeHostDir` bind mount (the user folder — settings file and +sqlite database — outlives the container), a `user` override so the mounted +files stay host-owned, and a `startupTimeoutMs` for old-release migrations; +`stack.replaceN8N({ image, env })` then swaps the running n8n main for a +different image while the database service, network, user folder, and host +port stay alive. One stack lives through every phase; only the image changes. + +## Upgrade mode (`MODE=upgrade`, the default) + +Proves that an upgrade keeps existing data readable, that a rollback stays +safe while the rotation flag is off, and that flipping the flag switches new +writes to the new `keyId:`-prefixed format without breaking anything written +before. + +| Phase | Image | Checks | +|---|---|---| +| P1 seed | `FROM_IMAGE` (pinned `n8nio/n8n:2.37.10`; bump deliberately) | owner + credential created, decrypt round-trip, scheduled workflow seeded and executing | +| P2 upgrade | `TO_IMAGE`, rotation flag **off** | old credential decrypts; a new write is byte-compatible legacy format (`U2FsdGVkX1…`); key store seeded with exactly 2 rows; the newer release re-activates and executes the workflow the old release wrote | +| P3 downgrade | `FROM_IMAGE` again, same data | the value written by the **newer** instance decrypts on the **older** one; the old release still activates and executes on the migrated-forward database | +| P4 write-on | `TO_IMAGE`, flag **on** | mixed data decrypts; new write is `:…`; rotation via `POST /rest/encryption/keys` moves the write key immediately; all four data generations decrypt | + +## Rotation mode (`MODE=rotation`) + +The standalone test of DB-stored key rotation. Boots only the image under +test on a fresh database with the flag on. + +| Phase | Checks | +|---|---| +| R1 seed | fresh boot seeds exactly 2 `deployment_key` rows; a write is prefixed with the active key id | +| R2 rotate | two rotations via `POST /rest/encryption/keys`; a write after each one uses the newest key id; 4 key rows kept, exactly 1 active | +| R3 restart | a fresh process on the same data: keys reload from the database, a new write keeps the active key id, every generation decrypts | + +## Data journeys + +Each cycle seeds and keeps re-checking on every boot: + +- **A static credential** (`httpHeaderAuth`) — the encrypted payload. The + unique secret lives in the non-password `name` field, so the REST + round-trip (`GET /rest/credentials/:id?includeData=true`) proves the + stored blob decrypts; the raw column is also byte-asserted + (legacy prefix / `keyId:` prefix). +- **An active scheduled workflow** (Schedule Trigger → HTTP Request bound to + the credential, against the instance's own `/healthz`). Every boot must + re-activate it and produce a fresh successful execution — that decrypts + the credential inside the execution engine, so a read-path regression + fails the run even when the credentials endpoint still works. + +Not covered here (license-gated on the published images, and the e2e feature +stub does not ship in the docker dist — tracked separately): variables, SSO +settings (SAML/OIDC secrets in `settings.value`), and end-user (dynamic) +credentials. + +## Metrics + +Every run prints a summary table per backend: per-phase duration, number of +decrypt round-trips, and their average/max end-to-end latency. Runs that +rotate also print a `rotate api:` line with the count and average/max latency +of the rotation call itself, and every run prints a `scheduled executions:` +line — how many boots produced a fresh successful execution and how long the +wait was (activation + schedule fire + engine decrypt, end to end). Raw +samples land in `//metrics.csv`. + +## Failure behavior + +Any failed check throws and fails the Playwright test: a loud block prints +the `FAIL` line with the expected/actual details, the current container's +docker log is appended to `/n8n.log`, the last 40 log lines print +inline, and the stack (n8n, database, network) is torn down. The log and +`metrics.csv` are attached to the test. In CI the job summary shows +**Result: FAIL** with the last 80 run-log lines, the artifacts still upload, +and a failed scheduled run notifies Slack. Without docker (or without the +locally built image) the tests skip — except in the encryption CI workflow, +where `ENCRYPTION_CYCLE_REQUIRED=true` turns that into a failure. + +## Parameters + +| Env | Default | Meaning | +|---|---|---| +| `MODE` | `upgrade` | `upgrade` (full cycle) or `rotation` (standalone rotation test) | +| `DB` | `both` | database backends to run: `sqlite`, `postgres`, or `both` | +| `FROM_IMAGE` | `n8nio/n8n:2.37.10` (pinned) | the "old" release to seed on and downgrade to (upgrade mode only) | +| `TO_IMAGE` | `n8nio/n8n:local` | the image under test (`pnpm build:docker` output) | +| `TEST_IMAGE_POSTGRES` | stack default | the postgres container image | +| `WORK_ROOT` | `mktemp -d` | where the home dirs, logs, and metrics land | + +## CI + +`.github/workflows/encryption-upgrade-test.yml` ("Test: Encryption Rollout") +runs the upgrade cycle on a twice-daily cron and notifies Slack when a +scheduled run fails. A manual `workflow_dispatch` run has a `test` select +(`upgrade` | `rotation`) plus `db` and `from-image` inputs. A PR that touches +the tests or the stack pieces they lean on runs the full matrix — both modes +x both databases, four parallel jobs — with the workflow file taken from the +PR branch. The run summary +carries the result and the metrics; the run log plus `metrics.csv` and +`n8n.log` are uploaded as an artifact per job (mode x database). diff --git a/packages/testing/playwright/tests/infrastructure/encryption/api.ts b/packages/testing/playwright/tests/infrastructure/encryption/api.ts new file mode 100644 index 00000000000..92a39f92570 --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/api.ts @@ -0,0 +1,162 @@ +import type { CycleContext } from './harness'; +import { fail, metric, ok, step } from './harness'; + +const OWNER_EMAIL = 'owner@example.com'; +const OWNER_PASSWORD = 'SuperSecret123'; + +interface RestResponse { + code: number; + ms: number; + body: unknown; + text: string; +} + +/** + * Minimal REST client against one running instance. Keeps its own cookie so a + * login survives only as long as the instance it came from — every boot logs + * in again. + */ +export class RestClient { + private cookie = ''; + + constructor( + private readonly ctx: CycleContext, + readonly baseUrl: string, + ) {} + + async request(method: string, path: string, payload?: unknown): Promise { + const headers: Record = { 'browser-id': 'encryption-cycle' }; + if (this.cookie) headers.cookie = this.cookie; + if (payload !== undefined) headers['content-type'] = 'application/json'; + const started = performance.now(); + const res = await fetch(`${this.baseUrl}${path}`, { + method, + headers, + body: payload === undefined ? undefined : JSON.stringify(payload), + }); + const text = await res.text(); + const ms = performance.now() - started; + const setCookie = res.headers.get('set-cookie'); + if (setCookie) this.cookie = setCookie.split(';')[0]; + let body: unknown = null; + try { + body = JSON.parse(text); + } catch { + // non-JSON bodies stay available via `text` + } + return { code: res.status, ms, body, text }; + } + + private dataOf(res: RestResponse): Record { + return ((res.body as Record)?.data ?? {}) as Record; + } + + async createOwner(): Promise { + step(this.ctx, 'creating the owner account'); + const res = await this.request('POST', '/rest/owner/setup', { + email: OWNER_EMAIL, + firstName: 'Spec', + lastName: 'Owner', + password: OWNER_PASSWORD, + }); + if (res.code !== 200) fail('owner setup failed', `${res.code}: ${res.text.slice(0, 300)}`); + ok(this.ctx, 'owner created'); + } + + /** The login DTO field name differs across releases; try both. */ + async login(): Promise { + step(this.ctx, `logging in as ${OWNER_EMAIL}`); + this.cookie = ''; + let res = await this.request('POST', '/rest/login', { + emailOrLdapLoginId: OWNER_EMAIL, + password: OWNER_PASSWORD, + }); + if (res.code !== 200) { + res = await this.request('POST', '/rest/login', { + email: OWNER_EMAIL, + password: OWNER_PASSWORD, + }); + } + if (res.code !== 200) fail('login failed', `${res.code}: ${res.text.slice(0, 300)}`); + ok(this.ctx, 'logged in'); + } + + async createCredential(name: string, secret: string): Promise { + const res = await this.request('POST', '/rest/credentials', { + name, + type: 'httpHeaderAuth', + data: { name: secret, value: 'spec-header-password' }, + }); + if (res.code !== 200) { + fail(`create credential "${name}" failed`, `${res.code}: ${res.text.slice(0, 300)}`); + } + return String(this.dataOf(res).id); + } + + /** + * The unique secret lives in the NON-password `name` field: the API returns + * it decrypted verbatim, while both fields sit in one encrypted blob — so + * the round-trip proves the stored value decrypts. Records decrypt_ms. + */ + async assertDecrypts(credId: string, expected: string, label: string): Promise { + step(this.ctx, `checking decrypt of credential ${credId} (${label})`); + const res = await this.request('GET', `/rest/credentials/${credId}?includeData=true`); + if (res.code !== 200) { + fail(`${label}: GET credential ${credId} returned ${res.code}`, res.text.slice(0, 300)); + } + const data = this.dataOf(res).data as Record | undefined; + const got = data?.name; + if (got !== expected) { + fail( + `${label}: credential ${credId} did not decrypt to the seeded secret`, + `expected: ${expected}\nactual: ${String(got)}`, + ); + } + metric(this.ctx, 'decrypt_ms', this.ctx.phase, Math.round(res.ms * 10) / 10); + ok( + this.ctx, + `credential ${credId} decrypts to the seeded secret (${(res.ms / 1000).toFixed(3)}s)`, + ); + } + + /** Rotates the data-encryption key; returns the new key id. Records rotate_ms. */ + async rotateKey(oldKeyId: string): Promise { + const res = await this.request('POST', '/rest/encryption/keys', { type: 'data_encryption' }); + if (res.code !== 200) + fail('key rotation via API failed', `${res.code}: ${res.text.slice(0, 300)}`); + metric(this.ctx, 'rotate_ms', this.ctx.phase, Math.round(res.ms * 10) / 10); + const rawId = this.dataOf(res).id; + const newId = typeof rawId === 'string' ? rawId : ''; + if (!newId || newId === oldKeyId) { + fail('rotation did not produce a new key id', `old: ${oldKeyId}, new: ${newId}`); + } + return newId; + } + + async createWorkflow( + payload: Record, + ): Promise<{ id: string; versionId: string }> { + const res = await this.request('POST', '/rest/workflows', payload); + if (res.code !== 200) fail('create workflow failed', `${res.code}: ${res.text.slice(0, 300)}`); + const data = this.dataOf(res); + return { id: String(data.id), versionId: String(data.versionId) }; + } + + /** Activation endpoint differs across releases; try the route, then the update fallback. */ + async activateWorkflow(id: string, versionId: string): Promise { + let res = await this.request('POST', `/rest/workflows/${id}/activate`, { versionId }); + if (res.code === 404 || res.code === 405) { + res = await this.request('PATCH', `/rest/workflows/${id}`, { active: true, versionId }); + } + if (res.code !== 200) { + fail(`activate workflow ${id} failed`, `${res.code}: ${res.text.slice(0, 300)}`); + } + } + + async getWorkflowActive(id: string): Promise { + const res = await this.request('GET', `/rest/workflows/${id}`); + if (res.code !== 200) + fail(`GET workflow ${id} failed`, `${res.code}: ${res.text.slice(0, 300)}`); + return Boolean(this.dataOf(res).active); + } +} diff --git a/packages/testing/playwright/tests/infrastructure/encryption/cycles.ts b/packages/testing/playwright/tests/infrastructure/encryption/cycles.ts new file mode 100644 index 00000000000..86f4f843ba6 --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/cycles.ts @@ -0,0 +1,235 @@ +import type { RestClient } from './api'; +import type { CycleContext } from './harness'; +import { fail, ok, passLine, runPhase, step, summary } from './harness'; +import { bootStack, finishCycle, readInstanceVersion, swapInstance } from './instances'; +import type { SeededJourneys } from './journeys'; +import { + assertJourneys, + assertKeyRowCount, + assertLegacyFormat, + assertPrefixed, + getActiveKeyId, + seedJourneys, +} from './journeys'; + +export interface CycleImages { + /** The old release the upgrade cycle seeds on and downgrades to. */ + from: string; + /** The build under test. */ + to: string; +} + +const secret = (tag: string) => `enc-${tag}-${Date.now()}-${Math.floor(Math.random() * 1e6)}`; + +/** + * MODE=upgrade — P1 seed on the old release, P2 upgrade with the rotation + * flag off (byte-compatible writes), P3 downgrade-read on the old release, + * P4 write-on + rotate. One stack lives through all four phases; only the + * n8n image is swapped (`stack.replaceN8N`), so the database, network, user + * folder, and host port stay. + */ +export async function runUpgradeCycle(ctx: CycleContext, images: CycleImages): Promise { + const SECRET_A = secret('up-A'); + const SECRET_B = secret('up-B'); + const SECRET_C = secret('up-C'); + const SECRET_D = secret('up-D'); + + let api: RestClient; + let fromVersion = '?'; + let credA = ''; + let credB = ''; + let seeded: SeededJourneys; + + await runPhase(ctx, 'P1 seed', async () => { + step(ctx, `seeding on the old release (${images.from})`); + api = await bootStack(ctx, { + image: images.from, + rotationFlag: false, + label: 'the old release', + }); + fromVersion = await readInstanceVersion(ctx); + step(ctx, `FROM version: ${fromVersion}`); + await api.createOwner(); + step(ctx, 'creating credential A (the seed data)'); + credA = await api.createCredential('encryption-cycle cred A (seeded on FROM)', SECRET_A); + ok(ctx, `credential A = ${credA}`); + await api.assertDecrypts(credA, SECRET_A, 'baseline on FROM'); + seeded = await seedJourneys(ctx, api, credA, 'encryption-cycle cred A (seeded on FROM)'); + await assertJourneys(ctx, api, seeded, 'baseline on FROM'); + }); + + await runPhase(ctx, 'P2 upgrade', async () => { + api = await swapInstance(ctx, { + image: images.to, + rotationFlag: false, + label: 'the build under test', + }); + await api.login(); + await api.assertDecrypts(credA, SECRET_A, 'old data survives the upgrade'); + step(ctx, 'creating credential B on the upgraded instance'); + credB = await api.createCredential( + 'encryption-cycle cred B (written on TO, flag off)', + SECRET_B, + ); + ok(ctx, `credential B = ${credB}`); + step(ctx, 'checking B is stored in the LEGACY format (flag off => byte-compatible writes)'); + await assertLegacyFormat(ctx, credB, 'flag-off write'); + await api.assertDecrypts(credB, SECRET_B, 'fresh write on TO'); + step(ctx, 'checking the key store got seeded (exactly 2 deployment_key rows)'); + await assertKeyRowCount(ctx, 2); + await assertJourneys(ctx, api, seeded, 'after the upgrade'); + }); + + await runPhase(ctx, 'P3 downgrade', async () => { + api = await swapInstance(ctx, { + image: images.from, + rotationFlag: false, + label: 'the old release (downgrade)', + }); + await api.login(); + await api.assertDecrypts( + credB, + SECRET_B, + 'value written by the NEWER instance reads on the OLDER one', + ); + await api.assertDecrypts(credA, SECRET_A, 'original data still fine on FROM'); + await assertJourneys(ctx, api, seeded, 'after the downgrade'); + }); + + await runPhase(ctx, 'P4 write-on', async () => { + api = await swapInstance(ctx, { + image: images.to, + rotationFlag: true, + label: 'the build under test (flag on)', + }); + await api.login(); + await api.assertDecrypts(credA, SECRET_A, 'mixed: seeded on FROM'); + await api.assertDecrypts(credB, SECRET_B, 'mixed: legacy written on TO'); + step(ctx, 'reading the active data-encryption key id'); + const activeKeyId = await getActiveKeyId(ctx); + if (!activeKeyId) fail('no active aes-256-gcm deployment_key row'); + ok(ctx, `active key id: ${activeKeyId}`); + step(ctx, 'creating credential C (must be keyId-prefixed)'); + const credC = await api.createCredential('encryption-cycle cred C (flag on)', SECRET_C); + await assertPrefixed(ctx, credC, activeKeyId, 'flag-on write'); + await api.assertDecrypts(credC, SECRET_C, 'prefixed write'); + step(ctx, 'rotating the key via POST /rest/encryption/keys'); + const newKeyId = await api.rotateKey(activeKeyId); + ok(ctx, `rotated: ${activeKeyId} -> ${newKeyId}`); + step(ctx, 'creating credential D (must use the NEW key id)'); + const credD = await api.createCredential('encryption-cycle cred D (after rotate)', SECRET_D); + await assertPrefixed(ctx, credD, newKeyId, 'write after rotation'); + step(ctx, 'final sweep: all four generations must decrypt'); + for (const [id, expected] of [ + [credA, SECRET_A], + [credB, SECRET_B], + [credC, SECRET_C], + [credD, SECRET_D], + ] as const) { + await api.assertDecrypts(id, expected, 'all generations'); + } + await assertJourneys(ctx, api, seeded, 'write-on'); + }); + + await finishCycle(ctx); + summary(ctx); + passLine( + `PASS [${ctx.backend}]: seed(${fromVersion}) -> upgrade(read) -> downgrade-read -> write-on+rotate, all decrypts OK`, + ); +} + +/** + * MODE=rotation — the standalone test of DB-stored key rotation: the build + * under test only, fresh database, flag ON. R1 seed, R2 rotate twice with a + * write after each, R3 restart-read (a fresh process must reload the keys + * from the database). + */ +export async function runRotationCycle(ctx: CycleContext, images: CycleImages): Promise { + const SECRET_A = secret('rot-A'); + const SECRET_B = secret('rot-B'); + const SECRET_C = secret('rot-C'); + const SECRET_D = secret('rot-D'); + + let api: RestClient; + let credA = ''; + let credB = ''; + let credC = ''; + let key1 = ''; + let key3 = ''; + let seeded: SeededJourneys; + + await runPhase(ctx, 'R1 seed', async () => { + step(ctx, 'booting the build under test on a fresh database, rotation flag ON'); + api = await bootStack(ctx, { + image: images.to, + rotationFlag: true, + label: 'the build under test', + }); + await api.createOwner(); + step(ctx, 'checking the key store got seeded (exactly 2 deployment_key rows)'); + await assertKeyRowCount(ctx, 2); + key1 = await getActiveKeyId(ctx); + if (!key1) fail('no active aes-256-gcm deployment_key row'); + ok(ctx, `active key id: ${key1}`); + step(ctx, 'creating credential A (must be keyId-prefixed)'); + credA = await api.createCredential('rotation-cycle cred A (initial key)', SECRET_A); + await assertPrefixed(ctx, credA, key1, 'first write'); + await api.assertDecrypts(credA, SECRET_A, 'first write'); + seeded = await seedJourneys(ctx, api, credA, 'rotation-cycle cred A (initial key)'); + await assertJourneys(ctx, api, seeded, 'seed'); + }); + + await runPhase(ctx, 'R2 rotate', async () => { + step(ctx, 'first rotation via POST /rest/encryption/keys'); + const key2 = await api.rotateKey(key1); + ok(ctx, `rotated: ${key1} -> ${key2}`); + step(ctx, 'creating credential B (must use the NEW key id)'); + credB = await api.createCredential('rotation-cycle cred B (after 1st rotate)', SECRET_B); + await assertPrefixed(ctx, credB, key2, 'write after the 1st rotation'); + await api.assertDecrypts(credB, SECRET_B, 'write after the 1st rotation'); + step(ctx, 'second rotation via POST /rest/encryption/keys'); + key3 = await api.rotateKey(key2); + ok(ctx, `rotated: ${key2} -> ${key3}`); + step(ctx, 'creating credential C (must use the NEWEST key id)'); + credC = await api.createCredential('rotation-cycle cred C (after 2nd rotate)', SECRET_C); + await assertPrefixed(ctx, credC, key3, 'write after the 2nd rotation'); + await api.assertDecrypts(credC, SECRET_C, 'write after the 2nd rotation'); + step(ctx, 'checking the key store keeps every generation (4 rows, exactly 1 active)'); + await assertKeyRowCount(ctx, 4); + const activeRows = await getActiveKeyId(ctx); + if (!activeRows || activeRows.includes('\n')) { + fail('expected exactly 1 active key', `active rows:\n${activeRows}`); + } + ok(ctx, '4 key rows kept, exactly 1 active'); + }); + + await runPhase(ctx, 'R3 restart', async () => { + step(ctx, 'restarting the instance (keys must reload from the database)'); + api = await swapInstance(ctx, { + image: images.to, + rotationFlag: true, + label: 'the build under test (restarted)', + }); + await api.login(); + step( + ctx, + 'creating credential D (the restarted instance must keep writing with the active key)', + ); + const credD = await api.createCredential('rotation-cycle cred D (after restart)', SECRET_D); + await assertPrefixed(ctx, credD, key3, 'write after the restart'); + step(ctx, 'final sweep: every generation must decrypt after the restart'); + for (const [id, expected] of [ + [credA, SECRET_A], + [credB, SECRET_B], + [credC, SECRET_C], + [credD, SECRET_D], + ] as const) { + await api.assertDecrypts(id, expected, 'all generations'); + } + await assertJourneys(ctx, api, seeded, 'after the restart'); + }); + + await finishCycle(ctx); + summary(ctx); + passLine(`PASS [${ctx.backend}]: seed(flag on) -> rotate x2 -> restart-read, all decrypts OK`); +} diff --git a/packages/testing/playwright/tests/infrastructure/encryption/db.ts b/packages/testing/playwright/tests/infrastructure/encryption/db.ts new file mode 100644 index 00000000000..3ff36168961 --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/db.ts @@ -0,0 +1,53 @@ +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +import type { CycleContext } from './harness'; +import { fail } from './harness'; + +/** + * Raw database access for the assertions. sqlite reads the database file on + * the host (the container home is bind-mounted and host-owned); postgres runs + * psql inside the postgres container. Both return pipe-separated rows, so the + * call sites stay backend-agnostic. + */ +export async function dbQuery(ctx: CycleContext, sql: string): Promise { + if (ctx.backend === 'postgres') { + const pg = ctx.stack?.serviceResults.postgres?.container; + if (!pg) return fail('postgres container is not running'); + const result = await pg.exec([ + 'psql', + '-U', + 'n8n_user', + '-d', + 'n8n_db', + '-tA', + '-F|', + '-c', + sql, + ]); + if (result.exitCode !== 0) { + return fail(`psql failed: ${sql}`, result.output); + } + return result.output.trim(); + } + + // The running container writes this file concurrently (WAL mode), so a + // read can hit SQLITE_BUSY: give sqlite a busy timeout and retry the + // whole open+read a few times before failing loudly. + const dbPath = join(ctx.homeDir, '.n8n', 'database.sqlite'); + let lastError: unknown; + for (let attempt = 0; attempt < 5; attempt++) { + if (attempt > 0) await new Promise((resolve) => setTimeout(resolve, 300)); + let db: DatabaseSync | undefined; + try { + db = new DatabaseSync(dbPath, { readOnly: true, timeout: 5000 }); + const rows = db.prepare(sql).all() as Array>; + return rows.map((r) => Object.values(r).join('|')).join('\n'); + } catch (error) { + lastError = error; + } finally { + db?.close(); + } + } + return fail(`sqlite query failed after retries: ${sql}`, String(lastError)); +} diff --git a/packages/testing/playwright/tests/infrastructure/encryption/harness.ts b/packages/testing/playwright/tests/infrastructure/encryption/harness.ts new file mode 100644 index 00000000000..844dcbc4f20 --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/harness.ts @@ -0,0 +1,151 @@ +import { test } from '@playwright/test'; +import { appendFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { N8NStack } from 'n8n-containers/stack'; + +/** One backend run's mutable state: paths, metrics, and the current phase. */ +export interface CycleContext { + mode: 'upgrade' | 'rotation'; + backend: 'sqlite' | 'postgres'; + homeDir: string; + logFile: string; + metricsFile: string; + /** The backend's stack, set by bootStack() and cleared by finishCycle(). */ + stack?: N8NStack; + phase: string; + phaseStartedMs: number; +} + +export function createCycleContext( + mode: 'upgrade' | 'rotation', + backend: 'sqlite' | 'postgres', + workRoot: string, +): CycleContext { + const backendDir = join(workRoot, backend); + const ctx: CycleContext = { + mode, + backend, + homeDir: join(backendDir, 'home'), + logFile: join(backendDir, 'n8n.log'), + metricsFile: join(backendDir, 'metrics.csv'), + phase: 'setup', + phaseStartedMs: Date.now(), + }; + // A reused WORK_ROOT must start clean: a leftover home dir would boot the + // seed phase on the previous run's database. + rmSync(backendDir, { recursive: true, force: true }); + mkdirSync(ctx.homeDir, { recursive: true }); + writeFileSync(ctx.metricsFile, ''); + writeFileSync(ctx.logFile, ''); + return ctx; +} + +const ts = () => new Date().toTimeString().slice(0, 8); + +export function metric(ctx: CycleContext, kind: string, label: string, value: number): void { + appendFileSync(ctx.metricsFile, `${kind},${label},${value}\n`); +} + +/** Rolls the phase over for metrics/logging and runs the body as a test step. */ +export async function runPhase( + ctx: CycleContext, + name: string, + body: () => Promise, +): Promise { + metric(ctx, 'phase_s', ctx.phase, Math.round((Date.now() - ctx.phaseStartedMs) / 1000)); + ctx.phase = name; + ctx.phaseStartedMs = Date.now(); + console.log(''); + console.log(`[${ts()}] ================ [${ctx.backend}] ${name} ================`); + await test.step(name, body); +} + +export function step(ctx: CycleContext, msg: string): void { + console.log(`[${ts()}] [${ctx.backend}/${ctx.phase}] ${msg}`); +} + +export function ok(ctx: CycleContext, msg: string): void { + console.log(`[${ts()}] [${ctx.backend}/${ctx.phase}] OK: ${msg}`); +} + +/** A failed check. run.ts turns it into the loud FAIL output and exit 1. */ +export class CycleFailure extends Error { + constructor( + message: string, + readonly details?: string, + ) { + super(message); + this.name = 'CycleFailure'; + } +} + +export function fail(msg: string, details?: string): never { + throw new CycleFailure(msg, details); +} + +/** Prints the per-backend metrics block the CI summary parses. */ +export function summary(ctx: CycleContext): void { + metric(ctx, 'phase_s', ctx.phase, Math.round((Date.now() - ctx.phaseStartedMs) / 1000)); + + const phaseSeconds = new Map(); + const decrypts = new Map(); + const rotates: number[] = []; + const journeyExecs: number[] = []; + const lines = readFileSync(ctx.metricsFile, 'utf8') + .split('\n') + .filter((l) => l.length > 0); + for (const line of lines) { + const [kind, label, value] = line.split(','); + if (kind === 'phase_s' && label !== 'setup') { + phaseSeconds.set(label, (phaseSeconds.get(label) ?? 0) + Number(value)); + } else if (kind === 'decrypt_ms') { + const list = decrypts.get(label) ?? []; + list.push(Number(value)); + decrypts.set(label, list); + } else if (kind === 'rotate_ms') { + rotates.push(Number(value)); + } else if (kind === 'journey_exec_ms') { + journeyExecs.push(Number(value)); + } + } + + const row = (label: string, secs: number | string, count: string, avg: string, max: string) => + `${label.padEnd(14)} ${String(secs).padStart(8)}s ${count.padStart(15)} ${avg.padStart(8)} ${max.padStart(8)}`; + + console.log(''); + console.log(`=== metrics [${ctx.backend}] ===`); + console.log( + `${'phase'.padEnd(14)} ${'duration'.padStart(9)} ${'decrypt checks'.padStart(15)} ${'avg ms'.padStart(8)} ${'max ms'.padStart(8)}`, + ); + let totalSecs = 0; + const allMs: number[] = []; + for (const [label, secs] of phaseSeconds) { + totalSecs += secs; + const ms = decrypts.get(label) ?? []; + allMs.push(...ms); + const avg = ms.length ? (ms.reduce((a, b) => a + b, 0) / ms.length).toFixed(1) : '-'; + const max = ms.length ? Math.max(...ms).toFixed(1) : '-'; + console.log(row(label, secs, String(ms.length), avg, max)); + } + const avgAll = allMs.length ? (allMs.reduce((a, b) => a + b, 0) / allMs.length).toFixed(1) : '-'; + const maxAll = allMs.length ? Math.max(...allMs).toFixed(1) : '-'; + console.log(row('total', totalSecs, String(allMs.length), avgAll, maxAll)); + if (rotates.length) { + const avg = (rotates.reduce((a, b) => a + b, 0) / rotates.length).toFixed(1); + console.log( + `rotate api: ${rotates.length} calls, avg ${avg} ms, max ${Math.max(...rotates).toFixed(1)} ms`, + ); + } + if (journeyExecs.length) { + const avg = (journeyExecs.reduce((a, b) => a + b, 0) / journeyExecs.length / 1000).toFixed(1); + const max = (Math.max(...journeyExecs) / 1000).toFixed(1); + console.log( + `scheduled executions: ${journeyExecs.length} boots verified, avg ${avg} s, max ${max} s to a fresh success`, + ); + } +} + +export function passLine(msg: string): void { + console.log(''); + console.log(`[${ts()}] ${msg}`); +} diff --git a/packages/testing/playwright/tests/infrastructure/encryption/instances.ts b/packages/testing/playwright/tests/infrastructure/encryption/instances.ts new file mode 100644 index 00000000000..53764eb8d59 --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/instances.ts @@ -0,0 +1,144 @@ +import { appendFileSync, readFileSync } from 'node:fs'; + +import { RestClient } from './api'; +import type { CycleContext } from './harness'; +import { fail, ok, step } from './harness'; +import { N8NStartupError } from 'n8n-containers/services/n8n'; +import { createN8NStack } from 'n8n-containers/stack'; + +/** Env every phase shares; the rotation flag is the only per-phase change. */ +const CYCLE_ENV: Record = { + HOME: '/home/node', + N8N_LOG_LEVEL: 'info', + N8N_RUNNERS_ENABLED: 'false', + N8N_RUNNERS_MODE: 'internal', + N8N_VERSION_NOTIFICATIONS_ENABLED: 'false', + N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS: 'false', +}; + +const flagEnv = (rotationFlag: boolean): Record => ({ + N8N_ENV_FEAT_ENCRYPTION_KEY_ROTATION: rotationFlag ? 'true' : 'false', +}); + +const hostUser = () => `${process.getuid?.() ?? 1000}:${process.getgid?.() ?? 1000}`; + +export interface PhaseInstance { + image: string; + rotationFlag: boolean; + label: string; +} + +/** + * Boots the backend's stack — the database service plus the first n8n image — + * over the context's persistent home dir. The container runs as the host user + * so the mounted files stay host-owned: later phases boot a different image on + * the same data, and the sqlite assertions read the file directly. + */ +export async function bootStack(ctx: CycleContext, options: PhaseInstance): Promise { + const { image, rotationFlag, label } = options; + step(ctx, `starting ${label} (${image}, rotation flag: ${rotationFlag ? 'on' : 'off'})`); + try { + ctx.stack = await createN8NStack({ + postgres: ctx.backend === 'postgres', + projectName: `enc-${ctx.mode}-${ctx.backend}-${process.pid}`, + env: { ...CYCLE_ENV, ...flagEnv(rotationFlag) }, + image, + userHomeHostDir: ctx.homeDir, + user: hostUser(), + // An old release migrating a fresh database can exceed the default 60s. + startupTimeoutMs: 180_000, + }); + } catch (error) { + return bootFailure(ctx, label, error); + } + ok(ctx, `instance is ready (${ctx.stack.baseUrl})`); + return new RestClient(ctx, ctx.stack.baseUrl); +} + +/** + * Swaps the running n8n main for the requested image on the live stack + * (`stack.replaceN8N`): the database, network, user folder, and host port + * stay. Collects the outgoing container's logs first. + */ +export async function swapInstance(ctx: CycleContext, options: PhaseInstance): Promise { + const { image, rotationFlag, label } = options; + const stack = requireStack(ctx); + await collectCurrentLogs(ctx); + step(ctx, `swapping to ${label} (${image}, rotation flag: ${rotationFlag ? 'on' : 'off'})`); + try { + await stack.replaceN8N({ image, env: flagEnv(rotationFlag) }); + } catch (error) { + return bootFailure(ctx, label, error); + } + ok(ctx, `instance is ready (${stack.baseUrl})`); + return new RestClient(ctx, stack.baseUrl); +} + +/** Collects the current main's logs and tears the whole stack down. */ +export async function finishCycle(ctx: CycleContext): Promise { + if (!ctx.stack) return; + step(ctx, `stopping the stack (logs in ${ctx.logFile})`); + await collectCurrentLogs(ctx); + await ctx.stack.stop(); + ctx.stack = undefined; +} + +function requireStack(ctx: CycleContext) { + if (!ctx.stack) return fail('the stack is not running'); + return ctx.stack; +} + +function bootFailure(ctx: CycleContext, label: string, error: unknown): never { + // Keep the boot log readable in the FAIL output: the startup diagnostics + // land in the backend's n8n.log, the FAIL block prints its tail. + if (error instanceof N8NStartupError) { + for (const logs of Object.values(error.diagnostics.logs)) { + appendFileSync(ctx.logFile, logs); + } + const readiness = Object.values(error.diagnostics.readinessPayloads).at(-1); + fail(`${label} did not become ready`, `last readiness payload: ${readiness ?? '(none)'}`); + } + throw error; +} + +/** Appends the running main's docker logs to the backend's n8n.log. */ +export async function collectCurrentLogs(ctx: CycleContext): Promise { + const container = ctx.stack?.findContainers(/-n8n$/)[0]; + if (!container) return; + try { + const stream = await container.logs(); + const chunks: Buffer[] = []; + await new Promise((resolve) => { + stream.on('data', (chunk: Buffer) => chunks.push(Buffer.from(chunk))); + stream.on('end', resolve); + stream.on('error', () => resolve()); + // docker follows the log stream; a short settle window is enough for history. + setTimeout(resolve, 2000); + }); + appendFileSync(ctx.logFile, Buffer.concat(chunks).toString('utf8')); + } catch { + // logs are diagnostics only; never fail the cycle over them + } +} + +export async function readInstanceVersion(ctx: CycleContext): Promise { + const container = ctx.stack?.findContainers(/-n8n$/)[0]; + if (!container) return '?'; + try { + const result = await container.exec(['n8n', '--version']); + const lines = result.output.trim().split('\n'); + return lines[lines.length - 1] ?? '?'; + } catch { + return '?'; + } +} + +/** The last-40-lines tail the FAIL output prints. */ +export function logTail(ctx: CycleContext): string { + try { + const lines = readFileSync(ctx.logFile, 'utf8').split('\n'); + return lines.slice(-40).join('\n'); + } catch { + return '(no log file)'; + } +} diff --git a/packages/testing/playwright/tests/infrastructure/encryption/journeys.ts b/packages/testing/playwright/tests/infrastructure/encryption/journeys.ts new file mode 100644 index 00000000000..56d44b08d5a --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/journeys.ts @@ -0,0 +1,162 @@ +import type { RestClient } from './api'; +import { dbQuery } from './db'; +import type { CycleContext } from './harness'; +import { fail, metric, ok, step } from './harness'; + +/** Ids of the data one cycle seeds and keeps asserting across phases. */ +export interface SeededJourneys { + workflowId: string; +} + +const LEGACY_PREFIX = 'U2FsdGVkX1'; + +async function rawCredentialValue(ctx: CycleContext, credId: string): Promise { + return await dbQuery(ctx, `SELECT data FROM credentials_entity WHERE id='${credId}';`); +} + +export async function getActiveKeyId(ctx: CycleContext): Promise { + return await dbQuery( + ctx, + "SELECT id FROM deployment_key WHERE type='data_encryption' AND status='active' AND algorithm='aes-256-gcm';", + ); +} + +/** The raw column must start with ":". */ +export async function assertPrefixed( + ctx: CycleContext, + credId: string, + keyId: string, + label: string, +): Promise { + const raw = await rawCredentialValue(ctx, credId); + if (!raw.startsWith(`${keyId}:`)) { + fail( + `${label}: value is not prefixed with the expected key id`, + `expected prefix: ${keyId}:\nraw (first 60): ${raw.slice(0, 60)}`, + ); + } + ok(ctx, `${label}: value is prefixed with key id ${keyId}`); +} + +export async function assertLegacyFormat( + ctx: CycleContext, + credId: string, + label: string, +): Promise { + const raw = await rawCredentialValue(ctx, credId); + if (!raw.startsWith(LEGACY_PREFIX)) { + fail(`${label}: new write is not legacy-format`, `raw (first 60): ${raw.slice(0, 60)}`); + } + ok(ctx, `${label}: stored in legacy format (${LEGACY_PREFIX}...)`); +} + +export async function assertKeyRowCount(ctx: CycleContext, expected: number): Promise { + const count = await dbQuery( + ctx, + "SELECT COUNT(*) AS c FROM deployment_key WHERE type='data_encryption';", + ); + if (count !== String(expected)) { + fail(`expected exactly ${expected} deployment_key rows, got ${count}`); + } + ok(ctx, `deployment_key has exactly ${expected} rows`); +} + +/** + * Seeds one ACTIVE scheduled workflow whose HTTP node uses the seeded + * credential against the instance's own /healthz. Every later boot then + * exercises activation and an execution that decrypts the credential inside + * the engine — a read-path regression fails the run even when the + * credentials GET endpoint still works. + * + * Variables and SSO settings are deliberately absent: on the published + * docker images they are license-gated (the e2e feature stub does not ship + * in the docker dist), so those journeys live behind a real license. + */ +export async function seedJourneys( + ctx: CycleContext, + api: RestClient, + credentialId: string, + credentialName: string, +): Promise { + const stamp = `${Date.now()}-${process.pid}`; + + step(ctx, 'seeding an active scheduled workflow bound to the credential'); + const { id: workflowId, versionId } = await api.createWorkflow({ + name: `encryption-cycle scheduled (${stamp})`, + nodes: [ + { + id: 'schedule-1', + name: 'Schedule Trigger', + type: 'n8n-nodes-base.scheduleTrigger', + typeVersion: 1.2, + position: [0, 0], + parameters: { rule: { interval: [{ field: 'seconds', secondsInterval: 2 }] } }, + }, + { + id: 'http-1', + name: 'Self Healthz', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 4.2, + position: [200, 0], + parameters: { + url: 'http://localhost:5678/healthz', + authentication: 'genericCredentialType', + genericAuthType: 'httpHeaderAuth', + }, + credentials: { httpHeaderAuth: { id: credentialId, name: credentialName } }, + }, + ], + connections: { + 'Schedule Trigger': { main: [[{ node: 'Self Healthz', type: 'main', index: 0 }]] }, + }, + settings: { executionOrder: 'v1' }, + active: false, + }); + await api.activateWorkflow(workflowId, versionId); + ok(ctx, `workflow ${workflowId} created and activated`); + + return { workflowId }; +} + +/** + * Per-boot journey checks: the workflow is still active and a fresh + * execution completes — proving the schedule re-armed on this boot and the + * engine decrypted the credential. + */ +export async function assertJourneys( + ctx: CycleContext, + api: RestClient, + seeded: SeededJourneys, + label: string, +): Promise { + step(ctx, `checking the scheduled workflow is active and executes (${label})`); + const active = await api.getWorkflowActive(seeded.workflowId); + if (!active) fail(`${label}: seeded workflow is no longer active`); + + const countSql = `SELECT COUNT(*) AS c FROM execution_entity WHERE "workflowId"='${seeded.workflowId}' AND status='success';`; + const baseline = Number(await dbQuery(ctx, countSql)); + const waitStarted = Date.now(); + const deadline = waitStarted + 30_000; + let current = baseline; + while (current <= baseline && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + current = Number(await dbQuery(ctx, countSql)); + } + if (current <= baseline) { + const lastStatus = await dbQuery( + ctx, + `SELECT status FROM execution_entity WHERE "workflowId"='${seeded.workflowId}' ORDER BY id DESC LIMIT 1;`, + ); + fail( + `${label}: no new successful execution within 30s`, + `successful executions: ${baseline} (unchanged), last status: ${lastStatus || '(none)'}`, + ); + } + // Time from "instance is up" polling start until a fresh execution landed: + // activation + schedule fire + engine decrypt, end to end. + metric(ctx, 'journey_exec_ms', ctx.phase, Date.now() - waitStarted); + ok( + ctx, + `a scheduled execution succeeded on this boot (engine decrypted the credential; ${baseline} -> ${current})`, + ); +} diff --git a/packages/testing/playwright/tests/infrastructure/encryption/node-sqlite.d.ts b/packages/testing/playwright/tests/infrastructure/encryption/node-sqlite.d.ts new file mode 100644 index 00000000000..6867eb8e40d --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/node-sqlite.d.ts @@ -0,0 +1,10 @@ +// The installed @types/node predates the node:sqlite module (available at +// runtime since Node 22.5; engines require >= 24). Remove when @types/node +// ships the built-in declaration. +declare module 'node:sqlite' { + export class DatabaseSync { + constructor(path: string, options?: { readOnly?: boolean; timeout?: number }); + prepare(sql: string): { all(): unknown[] }; + close(): void; + } +} diff --git a/packages/testing/playwright/tests/infrastructure/encryption/rotation-cycle.spec.ts b/packages/testing/playwright/tests/infrastructure/encryption/rotation-cycle.spec.ts new file mode 100644 index 00000000000..4e4535d2b14 --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/rotation-cycle.spec.ts @@ -0,0 +1,16 @@ +import { test } from '@playwright/test'; + +import { backends, runBackendCycle } from './spec-helpers'; + +// API-only suite on the n8n-containers stack — no browser. See README.md. +test.describe('encryption rollout: rotation cycle', () => { + for (const backend of backends()) { + test( + `rotation cycle (${backend})`, + { annotation: [{ type: 'owner', description: 'Identity & Access' }] }, + async () => { + await runBackendCycle('rotation', backend); + }, + ); + } +}); diff --git a/packages/testing/playwright/tests/infrastructure/encryption/spec-helpers.ts b/packages/testing/playwright/tests/infrastructure/encryption/spec-helpers.ts new file mode 100644 index 00000000000..96b9376be1a --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/spec-helpers.ts @@ -0,0 +1,108 @@ +import { test } from '@playwright/test'; +import { execSync } from 'node:child_process'; +import { existsSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { CycleImages } from './cycles'; +import { runRotationCycle, runUpgradeCycle } from './cycles'; +import { createCycleContext, CycleFailure } from './harness'; +import { collectCurrentLogs, logTail } from './instances'; + +export type Backend = 'sqlite' | 'postgres'; +export type Mode = 'upgrade' | 'rotation'; + +/** DB=sqlite|postgres|both (default both) picks the per-backend tests. */ +export function backends(): Backend[] { + const db = process.env.DB ?? 'both'; + if (db === 'sqlite' || db === 'postgres') return [db]; + return ['sqlite', 'postgres']; +} + +export function cycleImages(): CycleImages { + return { + // Pinned so the suite is repeatable; bump deliberately when the + // compatibility baseline changes. + from: process.env.FROM_IMAGE ?? 'n8nio/n8n:2.37.10', + to: process.env.TO_IMAGE ?? 'n8nio/n8n:local', + }; +} + +let cachedWorkRoot: string | undefined; +/** One work root per process; the CI artifact globs point at WORK_ROOT. */ +export function workRoot(): string { + cachedWorkRoot ??= process.env.WORK_ROOT ?? mkdtempSync(join(tmpdir(), 'encryption-cycle-')); + return cachedWorkRoot; +} + +function commandOk(command: string): boolean { + try { + execSync(command, { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +/** + * Skips when the environment cannot run the cycle: no docker, or the default + * locally-built image is absent. The encryption CI workflow builds the image + * and sets ENCRYPTION_CYCLE_REQUIRED=true, so a broken build fails loudly + * there instead of skipping. + */ +function skipUnlessRunnable(images: CycleImages): void { + const required = process.env.ENCRYPTION_CYCLE_REQUIRED === 'true'; + const dockerUp = commandOk('docker info'); + if (required && !dockerUp) throw new Error('docker is not available'); + test.skip(!dockerUp, 'docker is not available'); + + if (images.to === 'n8nio/n8n:local') { + const imagePresent = commandOk(`docker image inspect ${images.to}`); + if (required && !imagePresent) { + throw new Error(`${images.to} not found; run \`pnpm build:docker\` first`); + } + test.skip( + !imagePresent, + `${images.to} not found — run \`pnpm build:docker\`, or set TO_IMAGE to a published tag`, + ); + } +} + +/** + * Runs one backend cycle with the loud failure contract: on any error the + * running container's log is collected, the stack is torn down, the log tail + * is printed, and the log/metrics files are attached to the test. + */ +export async function runBackendCycle(mode: Mode, backend: Backend): Promise { + const images = cycleImages(); + skipUnlessRunnable(images); + + const ctx = createCycleContext(mode, backend, workRoot()); + try { + if (mode === 'rotation') { + await runRotationCycle(ctx, images); + } else { + await runUpgradeCycle(ctx, images); + } + } catch (error) { + const message = + error instanceof CycleFailure + ? `${error.message}${error.details ? `\n${error.details}` : ''}` + : String(error); + console.log(`[${backend}/${ctx.phase}] FAIL: ${mode}-cycle - ${message}`); + await collectCurrentLogs(ctx); + await ctx.stack?.stop().catch(() => {}); + console.log(`--- last 40 log lines (${ctx.logFile}) ---`); + console.log(logTail(ctx)); + throw error; + } finally { + for (const [name, path] of [ + [`${backend}-n8n.log`, ctx.logFile], + [`${backend}-metrics.csv`, ctx.metricsFile], + ] as const) { + if (existsSync(path)) { + await test.info().attach(name, { path, contentType: 'text/plain' }); + } + } + } +} diff --git a/packages/testing/playwright/tests/infrastructure/encryption/upgrade-cycle.spec.ts b/packages/testing/playwright/tests/infrastructure/encryption/upgrade-cycle.spec.ts new file mode 100644 index 00000000000..da746f70137 --- /dev/null +++ b/packages/testing/playwright/tests/infrastructure/encryption/upgrade-cycle.spec.ts @@ -0,0 +1,16 @@ +import { test } from '@playwright/test'; + +import { backends, runBackendCycle } from './spec-helpers'; + +// API-only suite on the n8n-containers stack — no browser. See README.md. +test.describe('encryption rollout: upgrade cycle', () => { + for (const backend of backends()) { + test( + `upgrade cycle (${backend})`, + { annotation: [{ type: 'owner', description: 'Identity & Access' }] }, + async () => { + await runBackendCycle('upgrade', backend); + }, + ); + } +});